From ee293db39957892bff93f07aba05e75760725fc4 Mon Sep 17 00:00:00 2001 From: Tim Voronov Date: Sat, 5 Sep 2026 11:08:17 -0400 Subject: [PATCH 1/2] Remove outdated runtime adapter tests: eliminate redundant test files for runtime allocation, debug, session, and transport logic. --- AGENTS.md | 3 +- README.md | 11 +- client/remote_debug_event.go | 5 +- server/runtime_adapter_allocation_test.go | 103 ----- server/runtime_adapter_debug_test.go | 193 ---------- server/runtime_adapter_debugger_fake_test.go | 204 ---------- server/runtime_adapter_options_fake_test.go | 9 - server/runtime_adapter_run_test.go | 149 -------- server/runtime_adapter_session_fake_test.go | 7 - server/runtime_adapter_session_test.go | 258 ------------- server/runtime_adapter_transport_test.go | 125 ------ server/runtime_allocation_fixture_test.go | 255 ------------- server/runtime_optimization_presence_test.go | 261 ++++++------- server/runtime_release_failure_test.go | 63 --- test/integration/README.md | 137 +++++++ test/integration/allocation_fixture_test.go | 160 ++++++++ .../integration/allocation_race_test.go | 84 +--- .../integration/allocation_test.go | 151 +++----- test/integration/cancellation_test.go | 215 +++++++++++ test/integration/connection_test.go | 198 ++++++++++ test/integration/debugger_test.go | 279 ++++++++++++++ test/integration/errors_test.go | 331 ++++++++++++++++ test/integration/harness/coordination.go | 68 ++++ test/integration/harness/debugger.go | 192 ++++++++++ test/integration/harness/failure.go | 280 ++++++++++++++ test/integration/harness/harness.go | 266 +++++++++++++ test/integration/harness/options.go | 95 +++++ test/integration/harness/plan.go | 83 ++++ test/integration/harness/recorder.go | 179 +++++++++ test/integration/harness/runtime.go | 97 +++++ test/integration/harness/session.go | 44 +++ test/integration/lifecycle_test.go | 361 ++++++++++++++++++ test/integration/plan_test.go | 237 ++++++++++++ test/integration/release_test.go | 122 ++++++ test/integration/runtime_test.go | 211 ++++++++++ test/integration/session_test.go | 138 +++++++ 36 files changed, 3898 insertions(+), 1676 deletions(-) delete mode 100644 server/runtime_adapter_allocation_test.go delete mode 100644 server/runtime_adapter_debug_test.go delete mode 100644 server/runtime_adapter_debugger_fake_test.go delete mode 100644 server/runtime_adapter_run_test.go delete mode 100644 server/runtime_adapter_session_test.go delete mode 100644 server/runtime_adapter_transport_test.go delete mode 100644 server/runtime_allocation_fixture_test.go delete mode 100644 server/runtime_release_failure_test.go create mode 100644 test/integration/README.md create mode 100644 test/integration/allocation_fixture_test.go rename server/runtime_allocation_race_test.go => test/integration/allocation_race_test.go (51%) rename server/runtime_allocation_reclamation_test.go => test/integration/allocation_test.go (56%) create mode 100644 test/integration/cancellation_test.go create mode 100644 test/integration/connection_test.go create mode 100644 test/integration/debugger_test.go create mode 100644 test/integration/errors_test.go create mode 100644 test/integration/harness/coordination.go create mode 100644 test/integration/harness/debugger.go create mode 100644 test/integration/harness/failure.go create mode 100644 test/integration/harness/harness.go create mode 100644 test/integration/harness/options.go create mode 100644 test/integration/harness/plan.go create mode 100644 test/integration/harness/recorder.go create mode 100644 test/integration/harness/runtime.go create mode 100644 test/integration/harness/session.go create mode 100644 test/integration/lifecycle_test.go create mode 100644 test/integration/plan_test.go create mode 100644 test/integration/release_test.go create mode 100644 test/integration/runtime_test.go create mode 100644 test/integration/session_test.go diff --git a/AGENTS.md b/AGENTS.md index 8bd8f12..345fa93 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -221,7 +221,8 @@ contract crosses layers: | Protobuf/API compatibility | Buf lint and breaking checks | | Server request semantics | `server/internal/grpcserver` tests | | Logical ownership and limits | `server/internal/core` lifecycle tests | -| Unified runtime adaptation | `server` integration tests using Unified API fakes | +| Public Universal API round trips | `test/integration` using the public client/server and hosted API spies | +| Low-level facade and protocol integration | `server` integration tests | | Cancellation and cleanup | Lifecycle and integration tests | | Debugger commands and inspection | Core and integration debugger tests | | Client facade and conversions | `client` contract tests | diff --git a/README.md b/README.md index 5a73bf3..cc3ece2 100644 --- a/README.md +++ b/README.md @@ -256,4 +256,13 @@ make test-race make build ``` -Tests use in-memory `bufconn` transports and direct lifecycle coverage. CI invokes these Make targets on Linux, macOS, and Windows; Linux additionally runs the race detector, Buf lint, checked generation, and pull-request breaking checks against the fetched base branch. +The [Universal API integration suite](test/integration/README.md) exercises the +public client/server boundary over real gRPC using an in-memory `bufconn` +transport and hosted API spies. Run it independently with +`go test ./test/integration/...` or `go test -race ./test/integration/...`. +Package-local tests retain component, conversion, and low-level protocol coverage. + +CI invokes these Make targets on Linux, macOS, and Windows; Linux additionally +runs the race detector, Buf lint, checked generation, and pull-request breaking +checks against the fetched base branch. The integration suite is included in +the existing `./...` targets without build tags or extra services. diff --git a/client/remote_debug_event.go b/client/remote_debug_event.go index 0d44e38..f48cd1d 100644 --- a/client/remote_debug_event.go +++ b/client/remote_debug_event.go @@ -21,11 +21,14 @@ func remoteDebuggerEvent(event wiredebugger.Event) (*debugger.Event, bool, error return nil, false, snapshot.Failure case wiredebugger.StateStopped: result := &debugger.Event{ - Error: snapshot.Failure, Reason: snapshot.StopReason, HitBreakpointIDs: append([]debugger.BreakpointID(nil), snapshot.HitBreakpointIDs...), Depth: snapshot.Depth, } + if snapshot.Failure != nil { + result.Error = snapshot.Failure + } + if snapshot.Location != nil { result.Location = *snapshot.Location } diff --git a/server/runtime_adapter_allocation_test.go b/server/runtime_adapter_allocation_test.go deleted file mode 100644 index bb3416c..0000000 --- a/server/runtime_adapter_allocation_test.go +++ /dev/null @@ -1,103 +0,0 @@ -package server_test - -import ( - "context" - "errors" - "testing" - - "github.com/MontFerret/api" - "github.com/MontFerret/wire/client" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -func TestRuntimeAdapterReleasesAllocationsThatRaceCancellation(t *testing.T) { - t.Run("plan", func(t *testing.T) { - started := make(chan struct{}) - finish := make(chan struct{}) - hostedPlan := &contractPlan{} - hosted := &contractRuntime{compile: func(context.Context, api.Source, bool, contractPlanOptions) (api.Plan, error) { - close(started) - <-finish - - return hostedPlan, nil - }} - env := newIntegrationEnv(t, hosted) - remote, err := client.NewRuntime(testContext(t), env.conn) - if err != nil { - t.Fatal(err) - } - - ctx, cancel := context.WithCancel(context.Background()) - result := make(chan error, 1) - go func() { - _, compileErr := remote.Compile(ctx, api.Source{Content: "RETURN 1"}) - result <- compileErr - }() - <-started - cancel() - close(finish) - if err := <-result; !cancellationError(err) { - t.Fatalf("compile cancellation was not preserved: %v", err) - } - hostedPlan.mu.Lock() - closeCalls := hostedPlan.closeCalls - hostedPlan.mu.Unlock() - if closeCalls != 1 { - t.Fatalf("Plan published during cancellation closed %d times", closeCalls) - } - if err := remote.Close(); err != nil { - t.Fatal(err) - } - }) - - t.Run("normal session", func(t *testing.T) { - started := make(chan struct{}) - finish := make(chan struct{}) - hostedSession := &contractSession{} - hostedPlan := &contractPlan{newSession: func(context.Context, apiSessionOptions) (api.Session, error) { - close(started) - <-finish - - return hostedSession, nil - }} - hosted := &contractRuntime{compile: func(context.Context, api.Source, bool, contractPlanOptions) (api.Plan, error) { - return hostedPlan, nil - }} - env := newIntegrationEnv(t, hosted) - remote, err := client.NewRuntime(testContext(t), env.conn) - if err != nil { - t.Fatal(err) - } - plan, err := remote.Compile(testContext(t), api.Source{Content: "RETURN 1"}) - if err != nil { - t.Fatal(err) - } - - ctx, cancel := context.WithCancel(context.Background()) - result := make(chan error, 1) - go func() { - _, createErr := plan.NewSession(ctx) - result <- createErr - }() - <-started - cancel() - close(finish) - if err := <-result; !cancellationError(err) { - t.Fatalf("session cancellation was not preserved: %v", err) - } - if runs, closes := hostedSession.counts(); runs != 0 || closes != 1 { - t.Fatalf("Session published during cancellation leaked: runs=%d closes=%d", runs, closes) - } - if err := plan.Close(); err != nil { - t.Fatal(err) - } - if err := remote.Close(); err != nil { - t.Fatal(err) - } - }) -} - -func cancellationError(err error) bool { - return errors.Is(err, context.Canceled) || status.Code(err) == codes.Canceled -} diff --git a/server/runtime_adapter_debug_test.go b/server/runtime_adapter_debug_test.go deleted file mode 100644 index a2c4409..0000000 --- a/server/runtime_adapter_debug_test.go +++ /dev/null @@ -1,193 +0,0 @@ -package server_test - -import ( - "context" - "errors" - "reflect" - "sort" - "testing" - - "github.com/MontFerret/api" - "github.com/MontFerret/api/debugger" - "github.com/MontFerret/api/source" - "github.com/MontFerret/wire/client" - "github.com/MontFerret/wire/pkg/failure" -) - -func TestRuntimeAdapterDebuggerBridge(t *testing.T) { - hostedDebugger := newContractDebugger() - hostedPlan := &contractPlan{newDebugSession: func(context.Context, apiSessionOptions) (debugger.Session, error) { - return hostedDebugger, nil - }} - hosted := &contractRuntime{compile: func(context.Context, api.Source, bool, contractPlanOptions) (api.Plan, error) { - return hostedPlan, nil - }} - env := newIntegrationEnv(t, hosted) - remote, err := client.NewRuntime(testContext(t), env.conn) - if err != nil { - t.Fatal(err) - } - plan, err := remote.CompileDebug(testContext(t), api.Source{Name: "debug.fql", Content: "RETURN 1"}) - if err != nil { - t.Fatal(err) - } - session, err := plan.NewDebugSession(testContext(t), api.WithOutputContentType("application/json")) - if err != nil { - t.Fatal(err) - } - - entry, err := session.Start(testContext(t)) - if err != nil { - t.Fatal(err) - } - - if entry.Reason != debugger.ReasonEntry || entry.Depth != 2 || entry.Location.SourceName != "debug.fql" { - t.Fatalf("unexpected entry event: %#v", entry) - } - - defaultBreakpoint, err := session.SetBreakpoint(source.Location{ - SourceName: "debug.fql", - Position: source.Position{Line: 2}, - }) - if err != nil { - t.Fatal(err) - } - exactBreakpoint, err := session.SetBreakpointAt( - source.Location{SourceName: "debug.fql", Position: source.Position{Line: 1, Column: 3}}, - debugger.BreakpointOptions{BindingMode: debugger.BreakpointBindExact}, - ) - if err != nil { - t.Fatal(err) - } - functionBreakpoint, err := session.SetBreakpointAt( - source.Location{SourceName: "debug.fql", Position: source.Position{Line: 3}}, - debugger.BreakpointOptions{BindingMode: debugger.BreakpointBindNextExecutableInFunction}, - ) - if err != nil { - t.Fatal(err) - } - - if defaultBreakpoint.BindingMode != debugger.BreakpointBindNextExecutableInSource || - exactBreakpoint.BindingMode != debugger.BreakpointBindExact || - functionBreakpoint.BindingMode != debugger.BreakpointBindNextExecutableInFunction { - t.Fatalf("breakpoint binding modes were not preserved: %#v %#v %#v", defaultBreakpoint, exactBreakpoint, functionBreakpoint) - } - breakpoints := session.Breakpoints() - if len(breakpoints) != 3 || breakpoints[0].ID >= breakpoints[1].ID || breakpoints[1].ID >= breakpoints[2].ID { - t.Fatalf("breakpoint snapshot was not ID ordered: %#v", breakpoints) - } - breakpoints[0].ID = 999 - if session.Breakpoints()[0].ID == 999 { - t.Fatal("breakpoint snapshot was not defensive") - } - - if err := session.DeleteBreakpoint(defaultBreakpoint.ID); err != nil { - t.Fatal(err) - } - - if got := session.Breakpoints(); len(got) != 2 || got[0].ID != exactBreakpoint.ID || got[1].ID != functionBreakpoint.ID { - t.Fatalf("breakpoint cache did not track deletion: %#v", got) - } - - frames, err := session.Frames() - if err != nil || len(frames) != 2 || frames[0].Name != "top" { - t.Fatalf("unexpected frames: %#v, %v", frames, err) - } - locals, err := session.Locals() - if err != nil || len(locals) != 1 || locals[0].Name != "local" { - t.Fatalf("unexpected top-frame locals: %#v, %v", locals, err) - } - - if _, err := session.FrameLocals(1); err != nil { - t.Fatal(err) - } - variables, err := session.Variables(9) - if err != nil || len(variables) != 1 || variables[0].Name != "child" { - t.Fatalf("unexpected variables: %#v, %v", variables, err) - } - value, err := session.Evaluate(testContext(t), "local") - if err != nil || value.Display != "frame-0:local" { - t.Fatalf("unexpected top-frame evaluation: %#v, %v", value, err) - } - value, err = session.EvaluateFrame(testContext(t), 1, "caller") - if err != nil || value.Display != "frame-1:caller" { - t.Fatalf("unexpected indexed evaluation: %#v, %v", value, err) - } - - steppedIn, err := session.StepIn(testContext(t)) - if err != nil || steppedIn.Reason != debugger.ReasonBreakpoint || steppedIn.Depth != 3 || - !reflect.DeepEqual(steppedIn.HitBreakpointIDs, []debugger.BreakpointID{exactBreakpoint.ID}) { - t.Fatalf("step-in breakpoint event was not preserved: %#v, %v", steppedIn, err) - } - steppedOver, err := session.StepOver(testContext(t)) - if err != nil || steppedOver.Reason != debugger.ReasonStep { - t.Fatalf("step-over failed: %#v, %v", steppedOver, err) - } - runtimeError, err := session.StepOut(testContext(t)) - var runtimeFailure *failure.Failure - if err != nil || runtimeError.Reason != debugger.ReasonRuntimeError || - !errors.As(runtimeError.Error, &runtimeFailure) || runtimeFailure.Category != failure.CategoryExecution { - t.Fatalf("runtime-error event was not preserved: %#v, %v", runtimeError, err) - } - - continued := make(chan struct { - event *debugger.Event - err error - }, 1) - go func() { - event, continueErr := session.Continue(testContext(t)) - continued <- struct { - event *debugger.Event - err error - }{event: event, err: continueErr} - }() - <-hostedDebugger.continueStarted - if err := session.Pause(); err != nil { - t.Fatal(err) - } - paused := <-continued - if paused.err != nil || paused.event.Reason != debugger.ReasonPause { - t.Fatalf("Pause did not interrupt active Continue: %#v, %v", paused.event, paused.err) - } - - completed, err := session.Continue(testContext(t)) - if err != nil || completed.Reason != debugger.ReasonCompleted || completed.Output == nil || - string(completed.Output.Content) != `{"done":true}` { - t.Fatalf("completion event was not preserved: %#v, %v", completed, err) - } - - if err := session.Close(); err != nil { - t.Fatal(err) - } - - if err := session.Close(); err != nil { - t.Fatalf("debug Close did not retain its result: %v", err) - } - - hostedDebugger.mu.Lock() - commands := append([]string(nil), hostedDebugger.commands...) - frameLocals := append([]int(nil), hostedDebugger.frameLocals...) - evaluateFrames := append([]int(nil), hostedDebugger.evaluateFrames...) - closeCalls := hostedDebugger.closeCalls - hostedDebugger.mu.Unlock() - sort.Strings(commands) - if !reflect.DeepEqual(commands, []string{"continue", "continue", "start", "step-in", "step-out", "step-over"}) { - t.Fatalf("unexpected debugger commands: %#v", commands) - } - - if !reflect.DeepEqual(frameLocals, []int{0, 1}) || !reflect.DeepEqual(evaluateFrames, []int{0, 1}) { - t.Fatalf("top-frame bridging failed: locals=%#v evaluate=%#v", frameLocals, evaluateFrames) - } - - if closeCalls != 1 { - t.Fatalf("hosted debugger closed %d times", closeCalls) - } - - if err := plan.Close(); err != nil { - t.Fatal(err) - } - - if err := remote.Close(); err != nil { - t.Fatal(err) - } -} diff --git a/server/runtime_adapter_debugger_fake_test.go b/server/runtime_adapter_debugger_fake_test.go deleted file mode 100644 index 6538b21..0000000 --- a/server/runtime_adapter_debugger_fake_test.go +++ /dev/null @@ -1,204 +0,0 @@ -package server_test - -import ( - "context" - "errors" - "sync" - - "github.com/MontFerret/api" - "github.com/MontFerret/api/debugger" - "github.com/MontFerret/api/source" -) - -type contractDebugger struct { - mu sync.Mutex - continueStarted chan struct{} - pauseRequested chan struct{} - breakpoints map[debugger.BreakpointID]debugger.Breakpoint - nextBreakpointID debugger.BreakpointID - commands []string - frameLocals []int - evaluateFrames []int - pauseOnce sync.Once - continueCalls int - closeCalls int -} - -func newContractDebugger() *contractDebugger { - return &contractDebugger{ - continueStarted: make(chan struct{}), - pauseRequested: make(chan struct{}), - breakpoints: make(map[debugger.BreakpointID]debugger.Breakpoint), - } -} - -func (d *contractDebugger) Start(context.Context) (*debugger.Event, error) { - d.recordCommand("start") - - return &debugger.Event{ - Reason: debugger.ReasonEntry, - Location: source.Range{Location: source.Location{ - SourceName: "debug.fql", - Position: source.Position{Line: 1, Column: 1}, - }}, - Depth: 2, - }, nil -} - -func (d *contractDebugger) Continue(context.Context) (*debugger.Event, error) { - d.recordCommand("continue") - d.mu.Lock() - d.continueCalls++ - call := d.continueCalls - d.mu.Unlock() - if call == 1 { - close(d.continueStarted) - <-d.pauseRequested - - return &debugger.Event{Reason: debugger.ReasonPause}, nil - } - - return &debugger.Event{ - Reason: debugger.ReasonCompleted, - Output: &api.Output{ContentType: "application/json", Content: []byte(`{"done":true}`)}, - }, nil -} - -func (d *contractDebugger) StepIn(context.Context) (*debugger.Event, error) { - d.recordCommand("step-in") - - return &debugger.Event{ - Reason: debugger.ReasonBreakpoint, - HitBreakpointIDs: []debugger.BreakpointID{2}, - Location: source.Range{ - Location: source.Location{SourceName: "debug.fql", Position: source.Position{Line: 1, Column: 3}}, - Span: source.Span{Start: 3, End: 4}, - }, - Depth: 3, - }, nil -} - -func (d *contractDebugger) StepOver(context.Context) (*debugger.Event, error) { - d.recordCommand("step-over") - - return &debugger.Event{Reason: debugger.ReasonStep}, nil -} - -func (d *contractDebugger) StepOut(context.Context) (*debugger.Event, error) { - d.recordCommand("step-out") - - return &debugger.Event{Reason: debugger.ReasonRuntimeError, Error: errors.New("runtime error")}, nil -} - -func (d *contractDebugger) Pause() error { - d.pauseOnce.Do(func() { close(d.pauseRequested) }) - - return nil -} - -func (d *contractDebugger) SetBreakpoint(location source.Location) (debugger.Breakpoint, error) { - return d.SetBreakpointAt(location, debugger.BreakpointOptions{}) -} - -func (d *contractDebugger) SetBreakpointAt( - location source.Location, - options debugger.BreakpointOptions, -) (debugger.Breakpoint, error) { - d.mu.Lock() - defer d.mu.Unlock() - - d.nextBreakpointID++ - value := debugger.Breakpoint{ - ID: d.nextBreakpointID, - RequestedLocation: location, - Location: source.Range{Location: location, Span: source.Span{Start: 0, End: 1}}, - PointID: debugger.PointID(10 + d.nextBreakpointID), - FunctionID: 7, - BindingMode: options.BindingMode, - Bound: true, - } - d.breakpoints[value.ID] = value - - return value, nil -} - -func (d *contractDebugger) DeleteBreakpoint(id debugger.BreakpointID) error { - d.mu.Lock() - delete(d.breakpoints, id) - d.mu.Unlock() - - return nil -} - -func (d *contractDebugger) Breakpoints() []debugger.Breakpoint { - d.mu.Lock() - defer d.mu.Unlock() - - result := make([]debugger.Breakpoint, 0, len(d.breakpoints)) - for _, value := range d.breakpoints { - result = append(result, value) - } - - return result -} - -func (d *contractDebugger) Frames() ([]debugger.Frame, error) { - return []debugger.Frame{ - {Name: "top", Location: source.Location{SourceName: "debug.fql", Position: source.Position{Line: 2}}, FunctionID: 7}, - {Name: "caller", Location: source.Location{SourceName: "debug.fql", Position: source.Position{Line: 1}}, FunctionID: 6}, - }, nil -} - -func (d *contractDebugger) Locals() ([]debugger.Variable, error) { - return d.FrameLocals(0) -} - -func (d *contractDebugger) FrameLocals(frame int) ([]debugger.Variable, error) { - d.mu.Lock() - d.frameLocals = append(d.frameLocals, frame) - d.mu.Unlock() - - return []debugger.Variable{{ - Name: "local", - Value: debugger.Value{Type: "object", Display: "{...}", Reference: 9}, - Mutable: true, - }}, nil -} - -func (d *contractDebugger) Variables(reference debugger.ValueReference) ([]debugger.Variable, error) { - if reference != 9 { - return nil, errors.New("unexpected reference") - } - - return []debugger.Variable{{Name: "child", Value: debugger.Value{Type: "string", Display: "value"}}}, nil -} - -func (d *contractDebugger) Evaluate(ctx context.Context, expression string) (debugger.Value, error) { - return d.EvaluateFrame(ctx, 0, expression) -} - -func (d *contractDebugger) EvaluateFrame( - _ context.Context, - frame int, - expression string, -) (debugger.Value, error) { - d.mu.Lock() - d.evaluateFrames = append(d.evaluateFrames, frame) - d.mu.Unlock() - - return debugger.Value{Type: "string", Display: "frame-" + string(rune('0'+frame)) + ":" + expression}, nil -} - -func (d *contractDebugger) Close() error { - d.mu.Lock() - d.closeCalls++ - d.mu.Unlock() - - return nil -} - -func (d *contractDebugger) recordCommand(command string) { - d.mu.Lock() - d.commands = append(d.commands, command) - d.mu.Unlock() -} diff --git a/server/runtime_adapter_options_fake_test.go b/server/runtime_adapter_options_fake_test.go index aa9976d..a9f939b 100644 --- a/server/runtime_adapter_options_fake_test.go +++ b/server/runtime_adapter_options_fake_test.go @@ -30,12 +30,3 @@ func applyAPIOptions(options []api.SessionOption) (apiSessionOptions, error) { return configured, nil } - -func cloneOptimizationLevels(values []contractPlanOptions) []api.OptimizationLevel { - result := make([]api.OptimizationLevel, len(values)) - for index, value := range values { - result[index] = value.optimizationLevel - } - - return result -} diff --git a/server/runtime_adapter_run_test.go b/server/runtime_adapter_run_test.go deleted file mode 100644 index ffa344c..0000000 --- a/server/runtime_adapter_run_test.go +++ /dev/null @@ -1,149 +0,0 @@ -package server_test - -import ( - "context" - "errors" - "reflect" - "sync" - "testing" - "time" - - "github.com/MontFerret/api" - "github.com/MontFerret/wire/client" - "github.com/MontFerret/wire/pkg/failure" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -func TestRuntimeAdapterDirectRunPreservesContractAndBorrowedOwnership(t *testing.T) { - started := make(chan struct{}) - settled := make(chan struct{}) - var cancelOnce sync.Once - hosted := &contractRuntime{} - hosted.run = func(ctx context.Context, src api.Source, options apiSessionOptions) (api.Output, error) { - switch src.Content { - case "partial": - return api.Output{ContentType: "application/json", Content: []byte(`{"partial":true}`)}, errors.New("host failure") - case "cancel": - cancelOnce.Do(func() { close(started) }) - <-ctx.Done() - close(settled) - - return api.Output{ContentType: "text/plain", Content: []byte("partial")}, ctx.Err() - default: - return api.Output{ContentType: options.contentType, Content: []byte(`{"ok":true}`)}, nil - } - } - hosted.compile = func(context.Context, api.Source, bool, contractPlanOptions) (api.Plan, error) { - return &contractPlan{}, nil - } - env := newIntegrationEnv(t, hosted) - remote, err := client.NewRuntime(testContext(t), env.conn) - if err != nil { - t.Fatal(err) - } - - output, err := remote.Run( - testContext(t), - api.Source{Name: "direct.fql", Content: "success"}, - api.WithParam("input", map[string]any{"value": int64(7)}), - api.WithOutputContentType("application/json"), - ) - if err != nil { - t.Fatal(err) - } - - if output.ContentType != "application/json" || string(output.Content) != `{"ok":true}` { - t.Fatalf("unexpected direct runtime output: %#v", output) - } - - partial, err := remote.Run(testContext(t), api.Source{Name: "partial.fql", Content: "partial"}) - var asynchronous *failure.Failure - if !errors.As(err, &asynchronous) || asynchronous.Category != failure.CategoryExecution { - t.Fatalf("unexpected asynchronous failure: %v", err) - } - - if partial.ContentType != "application/json" || string(partial.Content) != `{"partial":true}` { - t.Fatalf("partial output was lost: %#v", partial) - } - - cancelCtx, cancel := context.WithCancel(context.Background()) - cancelled := make(chan error, 1) - go func() { - _, runErr := remote.Run(cancelCtx, api.Source{Name: "cancel.fql", Content: "cancel"}) - cancelled <- runErr - }() - <-started - cancel() - cancelErr := <-cancelled - if !errors.Is(cancelErr, context.Canceled) && status.Code(cancelErr) != codes.Canceled { - t.Fatalf("caller cancellation was not preserved: %v", cancelErr) - } - select { - case <-settled: - case <-time.After(5 * time.Second): - t.Fatalf("cancelled Runtime.Run returned before remote cleanup settled: %T %v", cancelErr, cancelErr) - } - - hosted.mu.Lock() - sources := append([]api.Source(nil), hosted.runSources...) - options := make([]apiSessionOptions, len(hosted.runOptions)) - for index, value := range hosted.runOptions { - options[index] = value.clone() - } - hosted.mu.Unlock() - if len(sources) != 3 || sources[0] != (api.Source{Name: "direct.fql", Content: "success"}) { - t.Fatalf("hosted Runtime.Run did not receive exact source: %#v", sources) - } - - if len(options) != 3 || options[0].contentType != "application/json" || - !reflect.DeepEqual(options[0].params, map[string]any{"input": map[string]any{"value": int64(7)}}) { - t.Fatalf("hosted Runtime.Run did not receive exact options: %#v", options) - } - - if err := remote.Close(); err != nil { - t.Fatal(err) - } - - if err := remote.Close(); err != nil { - t.Fatalf("Runtime.Close did not retain its result: %v", err) - } - - if _, err := env.client.Compile(testContext(t), api.Source{Content: "RETURN 1"}, client.CompileOptions{}); err != nil { - t.Fatalf("Runtime.Close closed the caller-owned transport: %v", err) - } - hosted.mu.Lock() - closeCalls := hosted.closeCalls - hosted.mu.Unlock() - if closeCalls != 0 { - t.Fatalf("Wire closed the borrowed hosted Runtime %d times", closeCalls) - } -} - -func TestRuntimeAdapterCompilePreservesDiagnostics(t *testing.T) { - values := testDiagnostics() - hosted := &contractRuntime{compile: func(context.Context, api.Source, bool, contractPlanOptions) (api.Plan, error) { - return nil, errors.Join(errors.New("runtime compiler secret"), values) - }} - env := newIntegrationEnv(t, hosted) - remote, err := client.NewRuntime(testContext(t), env.conn) - if err != nil { - t.Fatal(err) - } - - for _, compile := range []func(context.Context, api.Source, ...api.PlanOption) (api.Plan, error){ - remote.Compile, - remote.CompileDebug, - } { - _, err := compile(testContext(t), api.Source{Name: "query.fql", Content: "RETURN"}) - var wireErr *client.Error - if !errors.As(err, &wireErr) || wireErr.Category != failure.CategoryCompilation || - !reflect.DeepEqual(wireErr.Diagnostics, values) { - t.Fatalf("portable compile diagnostics changed: %#v", wireErr) - } - } - - if err := remote.Close(); err != nil { - t.Fatal(err) - } -} diff --git a/server/runtime_adapter_session_fake_test.go b/server/runtime_adapter_session_fake_test.go index be249df..8603452 100644 --- a/server/runtime_adapter_session_fake_test.go +++ b/server/runtime_adapter_session_fake_test.go @@ -34,10 +34,3 @@ func (s *contractSession) Close() error { return nil } - -func (s *contractSession) counts() (int, int) { - s.mu.Lock() - defer s.mu.Unlock() - - return s.runCalls, s.closeCalls -} diff --git a/server/runtime_adapter_session_test.go b/server/runtime_adapter_session_test.go deleted file mode 100644 index f0ce3e9..0000000 --- a/server/runtime_adapter_session_test.go +++ /dev/null @@ -1,258 +0,0 @@ -package server_test - -import ( - "context" - "errors" - "reflect" - "sync" - "testing" - - "github.com/MontFerret/api" - "github.com/MontFerret/wire/client" - "github.com/MontFerret/wire/pkg/failure" -) - -func TestRuntimeAdapterCompilesReusableDurableSessions(t *testing.T) { - var sessionsMu sync.Mutex - var sessions []*contractSession - hostedPlan := &contractPlan{ - params: []string{"input"}, - newSession: func(context.Context, apiSessionOptions) (api.Session, error) { - session := &contractSession{run: func(_ context.Context, call int) (api.Output, error) { - return api.Output{ContentType: "text/plain", Content: []byte{byte('0' + call)}}, nil - }} - sessionsMu.Lock() - sessions = append(sessions, session) - sessionsMu.Unlock() - - return session, nil - }, - } - hosted := &contractRuntime{compile: func(_ context.Context, _ api.Source, debug bool, configured contractPlanOptions) (api.Plan, error) { - if debug && configured.hasOptimizationLevel && configured.optimizationLevel == api.OptimizationBasic { - return hostedPlan, nil - } - - return &contractPlan{params: []string{"input"}}, nil - }} - env := newIntegrationEnv(t, hosted) - remote, err := client.NewRuntime(testContext(t), env.conn) - if err != nil { - t.Fatal(err) - } - - var plan api.Plan - for _, level := range []api.OptimizationLevel{ - api.OptimizationNone, - api.OptimizationBasic, - api.OptimizationFull, - api.OptimizationAggressive, - } { - compiledPlan, err := remote.Compile( - testContext(t), - api.Source{Name: "compiled.fql", Content: "RETURN @input"}, - api.WithOptimizationLevel(level), - ) - if err != nil { - t.Fatal(err) - } - if err := compiledPlan.Close(); err != nil { - t.Fatal(err) - } - debugPlan, err := remote.CompileDebug( - testContext(t), - api.Source{Name: "debug.fql", Content: "RETURN @input"}, - api.WithOptimizationLevel(level), - ) - if err != nil { - t.Fatal(err) - } - if level == api.OptimizationBasic { - plan = debugPlan - } else if err := debugPlan.Close(); err != nil { - t.Fatal(err) - } - } - - if plan == nil { - t.Fatal("reusable debug Plan was not retained") - } - params := plan.Params() - params[0] = "mutated" - if !reflect.DeepEqual(plan.Params(), []string{"input"}) { - t.Fatalf("Plan.Params was not defensive: %#v", plan.Params()) - } - - session, err := plan.NewSession( - testContext(t), - api.WithParam("input", int64(42)), - api.WithOutputContentType("text/plain"), - ) - if err != nil { - t.Fatal(err) - } - for index, expected := range []string{"1", "2"} { - output, err := session.Run(testContext(t)) - if err != nil { - t.Fatalf("sequential run %d failed: %v", index, err) - } - if output.ContentType != "text/plain" || string(output.Content) != expected { - t.Fatalf("unexpected sequential output %d: %#v", index, output) - } - } - - if err := session.Close(); err != nil { - t.Fatal(err) - } - - if err := session.Close(); err != nil { - t.Fatalf("Session.Close did not retain its result: %v", err) - } - concurrent := make([]api.Session, 2) - for index := range concurrent { - concurrent[index], err = plan.NewSession(testContext(t), api.WithParam("input", int64(index))) - if err != nil { - t.Fatal(err) - } - } - type sessionRunResult struct { - output api.Output - err error - } - runResults := make(chan sessionRunResult, len(concurrent)) - for _, value := range concurrent { - go func() { - output, runErr := value.Run(testContext(t)) - runResults <- sessionRunResult{output: output, err: runErr} - }() - } - for range concurrent { - result := <-runResults - if result.err != nil || string(result.output.Content) != "1" { - t.Fatalf("concurrent Session run failed: %#v, %v", result.output, result.err) - } - } - for _, value := range concurrent { - if err := value.Close(); err != nil { - t.Fatal(err) - } - } - - sessionsMu.Lock() - createdSessions := append([]*contractSession(nil), sessions...) - sessionsMu.Unlock() - if len(createdSessions) != 3 { - t.Fatalf("three durable remote Sessions created %d hosted Sessions", len(createdSessions)) - } - - if runs, closes := createdSessions[0].counts(); runs != 2 || closes != 1 { - t.Fatalf("unexpected hosted Session lifecycle: runs=%d closes=%d", runs, closes) - } - for index, value := range createdSessions[1:] { - if runs, closes := value.counts(); runs != 1 || closes != 1 { - t.Fatalf("unexpected concurrent hosted Session %d lifecycle: runs=%d closes=%d", index, runs, closes) - } - } - - hosted.mu.Lock() - levels := cloneOptimizationLevels(hosted.compileLevels) - debug := append([]bool(nil), hosted.compileDebug...) - hosted.mu.Unlock() - wantLevels := []api.OptimizationLevel{ - api.OptimizationNone, - api.OptimizationNone, - api.OptimizationBasic, - api.OptimizationBasic, - api.OptimizationFull, - api.OptimizationFull, - api.OptimizationAggressive, - api.OptimizationAggressive, - } - - if !reflect.DeepEqual(levels, wantLevels) || - !reflect.DeepEqual(debug, []bool{false, true, false, true, false, true, false, true}) { - t.Fatalf("compile options were not preserved: levels=%#v debug=%#v", levels, debug) - } - hostedPlan.mu.Lock() - sessionOptions := append([]apiSessionOptions(nil), hostedPlan.sessionOptions...) - hostedPlan.mu.Unlock() - if len(sessionOptions) != 3 || sessionOptions[0].contentType != "text/plain" || - !reflect.DeepEqual(sessionOptions[0].params, map[string]any{"input": int64(42)}) { - t.Fatalf("session options were not preserved: %#v", sessionOptions) - } - - if err := plan.Close(); err != nil { - t.Fatal(err) - } - - if err := remote.Close(); err != nil { - t.Fatal(err) - } -} - -func TestRuntimeAdapterSessionRejectsOverlapAndReopensAfterRelease(t *testing.T) { - started := make(chan struct{}) - hostedSession := &contractSession{run: func(ctx context.Context, call int) (api.Output, error) { - if call == 1 { - close(started) - <-ctx.Done() - - return api.Output{}, ctx.Err() - } - - return api.Output{ContentType: "text/plain", Content: []byte("reused")}, nil - }} - hostedPlan := &contractPlan{newSession: func(context.Context, apiSessionOptions) (api.Session, error) { - return hostedSession, nil - }} - hosted := &contractRuntime{compile: func(context.Context, api.Source, bool, contractPlanOptions) (api.Plan, error) { - return hostedPlan, nil - }} - env := newIntegrationEnv(t, hosted) - remote, err := client.NewRuntime(testContext(t), env.conn) - if err != nil { - t.Fatal(err) - } - plan, err := remote.Compile(testContext(t), api.Source{Content: "RETURN 1"}) - if err != nil { - t.Fatal(err) - } - session, err := plan.NewSession(testContext(t)) - if err != nil { - t.Fatal(err) - } - - firstCtx, cancelFirst := context.WithCancel(context.Background()) - first := make(chan error, 1) - go func() { - _, runErr := session.Run(firstCtx) - first <- runErr - }() - <-started - _, err = session.Run(testContext(t)) - var wireErr *client.Error - if !errors.As(err, &wireErr) || wireErr.Category != failure.CategoryInvalidState { - t.Fatalf("overlapping Session.Run was not rejected: %v", err) - } - cancelFirst() - if err := <-first; !cancellationError(err) { - t.Fatalf("active Session.Run cancellation was not preserved: %v", err) - } - - output, err := session.Run(testContext(t)) - if err != nil || string(output.Content) != "reused" { - t.Fatalf("Session was not reusable after hidden Execution release: %#v, %v", output, err) - } - - if err := session.Close(); err != nil { - t.Fatal(err) - } - - if err := plan.Close(); err != nil { - t.Fatal(err) - } - - if err := remote.Close(); err != nil { - t.Fatal(err) - } -} diff --git a/server/runtime_adapter_transport_test.go b/server/runtime_adapter_transport_test.go deleted file mode 100644 index 206f059..0000000 --- a/server/runtime_adapter_transport_test.go +++ /dev/null @@ -1,125 +0,0 @@ -package server_test - -import ( - "context" - "net" - "sync" - "testing" - "time" - - "github.com/MontFerret/api" - "github.com/MontFerret/wire/client" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/credentials/insecure" - "google.golang.org/grpc/status" - "google.golang.org/grpc/test/bufconn" -) - -func TestNewRuntimePreservesUnavailableStatus(t *testing.T) { - listener := bufconn.Listen(1 << 20) - if err := listener.Close(); err != nil { - t.Fatal(err) - } - connection, err := grpc.NewClient( - "passthrough:///unavailable-wire-test", - grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { - return listener.DialContext(ctx) - }), - ) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = connection.Close() }) - - remote, err := client.NewRuntime(testContext(t), connection) - if status.Code(err) != codes.Unavailable { - t.Fatalf("NewRuntime changed unavailable transport status: %v", err) - } - - if remote != nil { - t.Fatalf("NewRuntime returned a non-nil runtime on failure: %T", remote) - } -} - -func TestTransportLossCleansRuntimeAdapterDescendants(t *testing.T) { - started := make(chan struct{}) - settled := make(chan struct{}) - var runOnce sync.Once - hostedSession := &contractSession{run: func(ctx context.Context, _ int) (api.Output, error) { - runOnce.Do(func() { close(started) }) - <-ctx.Done() - close(settled) - - return api.Output{}, ctx.Err() - }} - hostedPlan := &contractPlan{newSession: func(context.Context, apiSessionOptions) (api.Session, error) { - return hostedSession, nil - }} - hosted := &contractRuntime{compile: func(context.Context, api.Source, bool, contractPlanOptions) (api.Plan, error) { - return hostedPlan, nil - }} - env := newIntegrationEnv(t, hosted) - remote, err := client.NewRuntime(testContext(t), env.conn) - if err != nil { - t.Fatal(err) - } - plan, err := remote.Compile(testContext(t), api.Source{Content: "RETURN 1"}) - if err != nil { - t.Fatal(err) - } - session, err := plan.NewSession(testContext(t)) - if err != nil { - t.Fatal(err) - } - runResult := make(chan error, 1) - go func() { - _, runErr := session.Run(context.Background()) - runResult <- runErr - }() - <-started - - if err := env.conn.Close(); err != nil { - t.Fatal(err) - } - env.shutdown = true - env.transportClosed = true - if err := <-runResult; status.Code(err) != codes.Unavailable && status.Code(err) != codes.Canceled { - t.Fatalf("transport loss status was not preserved: %v", err) - } - select { - case <-settled: - case <-time.After(5 * time.Second): - t.Fatal("transport loss did not cancel hosted Session.Run") - } - - deadline := time.Now().Add(5 * time.Second) - cleanupSettled := false - for time.Now().Before(deadline) { - _, sessionCloseCalls := hostedSession.counts() - hostedPlan.mu.Lock() - planCloseCalls := hostedPlan.closeCalls - hostedPlan.mu.Unlock() - if sessionCloseCalls == 1 && planCloseCalls == 1 { - cleanupSettled = true - - break - } - time.Sleep(time.Millisecond) - } - - if !cleanupSettled { - _, sessionCloseCalls := hostedSession.counts() - hostedPlan.mu.Lock() - planCloseCalls := hostedPlan.closeCalls - hostedPlan.mu.Unlock() - t.Fatalf("transport loss cleanup did not settle: session=%d plan=%d", sessionCloseCalls, planCloseCalls) - } - hosted.mu.Lock() - runtimeCloseCalls := hosted.closeCalls - hosted.mu.Unlock() - if runtimeCloseCalls != 0 { - t.Fatalf("transport loss closed borrowed Runtime %d times", runtimeCloseCalls) - } -} diff --git a/server/runtime_allocation_fixture_test.go b/server/runtime_allocation_fixture_test.go deleted file mode 100644 index 4c34c90..0000000 --- a/server/runtime_allocation_fixture_test.go +++ /dev/null @@ -1,255 +0,0 @@ -package server_test - -import ( - "context" - "errors" - "sync" - "testing" - "time" - - "github.com/MontFerret/api" - "github.com/MontFerret/api/debugger" - "github.com/MontFerret/wire/client" - wirev1 "github.com/MontFerret/wire/gen/ferret/wire/v1" -) - -type ( - allocationOperation struct { - name string - method string - release string - parentRelease string - } - - runtimeAllocationFixture struct { - t *testing.T - env *integrationEnv - gate *allocationResponseGate - remote client.Runtime - plan api.Plan - session api.Session - operation allocationOperation - mu sync.Mutex - plans []*contractPlan - sessions []*contractSession - debuggers []*contractDebugger - expectedCloseError error - } -) - -func allocationOperations() []allocationOperation { - return []allocationOperation{ - {"compile", wirev1.PlanService_Compile_FullMethodName, wirev1.PlanService_ReleasePlan_FullMethodName, wirev1.RuntimeService_CloseConnection_FullMethodName}, - {"compile debug", wirev1.PlanService_CompileDebug_FullMethodName, wirev1.PlanService_ReleasePlan_FullMethodName, wirev1.RuntimeService_CloseConnection_FullMethodName}, - {"session", wirev1.SessionService_CreateSession_FullMethodName, wirev1.SessionService_ReleaseSession_FullMethodName, wirev1.PlanService_ReleasePlan_FullMethodName}, - {"debug session", wirev1.DebugService_CreateDebugSession_FullMethodName, wirev1.DebugService_ReleaseDebugSession_FullMethodName, wirev1.PlanService_ReleasePlan_FullMethodName}, - {"session run", wirev1.ExecutionService_RunSession_FullMethodName, wirev1.ExecutionService_ReleaseExecution_FullMethodName, wirev1.SessionService_ReleaseSession_FullMethodName}, - {"runtime run", wirev1.RuntimeService_Run_FullMethodName, wirev1.ExecutionService_ReleaseExecution_FullMethodName, wirev1.RuntimeService_CloseConnection_FullMethodName}, - } -} - -func newRuntimeAllocationFixture(t *testing.T, operation allocationOperation) *runtimeAllocationFixture { - t.Helper() - f := &runtimeAllocationFixture{t: t, operation: operation} - hosted := &contractRuntime{compile: func(context.Context, api.Source, bool, contractPlanOptions) (api.Plan, error) { - plan := &contractPlan{ - newSession: func(context.Context, apiSessionOptions) (api.Session, error) { - session := &contractSession{} - f.mu.Lock() - f.sessions = append(f.sessions, session) - f.mu.Unlock() - - return session, nil - }, - newDebugSession: func(context.Context, apiSessionOptions) (debugger.Session, error) { - session := newContractDebugger() - f.mu.Lock() - f.debuggers = append(f.debuggers, session) - f.mu.Unlock() - - return session, nil - }, - } - f.mu.Lock() - f.plans = append(f.plans, plan) - f.mu.Unlock() - - return plan, nil - }} - f.env = newIntegrationEnv(t, hosted) - f.gate = &allocationResponseGate{ClientConnInterface: f.env.conn, calls: make(map[string]int), failures: make(map[string]error)} - var err error - f.remote, err = client.NewRuntime(testContext(t), f.gate) - if err != nil { - t.Fatal(err) - } - - t.Cleanup(func() { - if err := f.remote.Close(); err != nil && !errors.Is(err, f.expectedCloseError) { - t.Errorf("close fixture Runtime: %v", err) - } - f.assertAllClosed() - hosted.mu.Lock() - defer hosted.mu.Unlock() - if hosted.closeCalls != 0 { - t.Errorf("closed borrowed Runtime %d times", hosted.closeCalls) - } - }) - - switch operation.name { - case "session", "debug session", "session run": - f.plan, err = f.remote.CompileDebug(testContext(t), api.Source{Content: "RETURN 1"}) - if err != nil { - t.Fatal(err) - } - } - - if operation.name == "session run" { - f.session, err = f.plan.NewSession(testContext(t)) - if err != nil { - t.Fatal(err) - } - } - - return f -} - -func (f *runtimeAllocationFixture) allocate(ctx context.Context, cancelInOption context.CancelFunc) (func() error, error) { - var planOptions []api.PlanOption - var sessionOptions []api.SessionOption - if cancelInOption != nil { - planOptions = []api.PlanOption{func(api.PlanOptions) error { cancelInOption(); return nil }} - sessionOptions = []api.SessionOption{func(api.SessionOptions) error { cancelInOption(); return nil }} - } - - switch f.operation.name { - case "compile", "compile debug": - compile := f.remote.Compile - if f.operation.name == "compile debug" { - compile = f.remote.CompileDebug - } - - plan, err := compile(ctx, api.Source{Content: "RETURN 1"}, planOptions...) - if err != nil { - return nil, err - } - - return plan.Close, nil - case "session": - session, err := f.plan.NewSession(ctx, sessionOptions...) - if err != nil { - return nil, err - } - - return session.Close, nil - case "debug session": - session, err := f.plan.NewDebugSession(ctx, sessionOptions...) - if err != nil { - return nil, err - } - - return session.Close, nil - case "session run": - _, err := f.session.Run(ctx) - - return nil, err - default: - _, err := f.remote.Run(ctx, api.Source{Content: "RETURN 1"}, sessionOptions...) - - return nil, err - } -} - -func (f *runtimeAllocationFixture) awaitCommitted() { - f.t.Helper() - select { - case <-f.gate.committed: - case <-time.After(5 * time.Second): - f.t.Fatal("allocation did not commit") - } -} - -func (f *runtimeAllocationFixture) awaitResult(result <-chan error) error { - f.t.Helper() - select { - case err := <-result: - return err - case <-time.After(5 * time.Second): - f.t.Fatal("allocation or cleanup did not settle") - - return nil - } -} - -func (f *runtimeAllocationFixture) assertAllClosed() { - f.t.Helper() - deadline := time.NewTimer(5 * time.Second) - defer deadline.Stop() - tick := time.NewTicker(time.Millisecond) - defer tick.Stop() - for { - f.mu.Lock() - settled := true - for _, plan := range f.plans { - plan.mu.Lock() - settled = settled && plan.closeCalls == 1 - plan.mu.Unlock() - } - for _, session := range f.sessions { - _, closes := session.counts() - settled = settled && closes == 1 - } - for _, session := range f.debuggers { - session.mu.Lock() - settled = settled && session.closeCalls == 1 - session.mu.Unlock() - } - f.mu.Unlock() - if settled { - return - } - - select { - case <-tick.C: - case <-deadline.C: - f.t.Error("hosted resources were leaked or closed more than once") - - return - } - } -} - -func (f *runtimeAllocationFixture) assertNarrowParentClosed() { - f.t.Helper() - f.mu.Lock() - defer f.mu.Unlock() - - if f.operation.name != "session run" { - f.plans[0].mu.Lock() - closes := f.plans[0].closeCalls - f.plans[0].mu.Unlock() - if closes != 1 { - f.t.Fatalf("narrow reclamation did not close its hosted Plan: closes=%d", closes) - } - } - - switch f.operation.name { - case "session run": - if _, closes := f.sessions[0].counts(); closes != 1 { - f.t.Fatalf("narrow reclamation did not close its hosted Session: closes=%d", closes) - } - case "session": - if _, closes := f.sessions[len(f.sessions)-1].counts(); closes != 1 { - f.t.Fatalf("Plan reclamation did not close the unknown hosted Session: closes=%d", closes) - } - case "debug session": - f.debuggers[0].mu.Lock() - closes := f.debuggers[0].closeCalls - f.debuggers[0].mu.Unlock() - if closes != 1 { - f.t.Fatalf("Plan reclamation did not close the unknown hosted debugger: closes=%d", closes) - } - default: - f.t.Fatalf("narrow-parent assertion cannot inspect %s", f.operation.name) - } -} diff --git a/server/runtime_optimization_presence_test.go b/server/runtime_optimization_presence_test.go index e2d29ce..69ce19b 100644 --- a/server/runtime_optimization_presence_test.go +++ b/server/runtime_optimization_presence_test.go @@ -11,26 +11,11 @@ import ( wirev1 "github.com/MontFerret/wire/gen/ferret/wire/v1" ) -func TestRuntimeOptimizationPresenceRoundTrip(t *testing.T) { +func TestClientOptimizationPresenceRoundTrip(t *testing.T) { hosted := &contractRuntime{} env := newIntegrationEnv(t, hosted) - remote, err := client.NewRuntime(testContext(t), env.conn) - if err != nil { - t.Fatal(err) - } - - t.Cleanup(func() { - if err := remote.Close(); err != nil { - t.Error(err) - } - }) for _, debug := range []bool{false, true} { - compile := remote.Compile - if debug { - compile = remote.CompileDebug - } - for _, test := range []struct { name string present bool @@ -44,19 +29,11 @@ func TestRuntimeOptimizationPresenceRoundTrip(t *testing.T) { } { t.Run(map[bool]string{false: "normal/", true: "debug/"}[debug]+test.name, func(t *testing.T) { var options []api.PlanOption + if test.present { options = append(options, api.WithOptimizationLevel(test.level)) } - plan, err := compile(testContext(t), api.Source{Content: "RETURN 1"}, options...) - if err != nil { - t.Fatal(err) - } - - if err := plan.Close(); err != nil { - t.Fatal(err) - } - lowLevel, err := env.client.Compile(testContext(t), api.Source{Content: "RETURN 1"}, client.CompileOptions{ Debuggable: debug, PlanOptions: options, @@ -72,7 +49,8 @@ func TestRuntimeOptimizationPresenceRoundTrip(t *testing.T) { hosted.mu.Lock() defer hosted.mu.Unlock() - levels := hosted.compileLevels[len(hosted.compileLevels)-2:] + levels := hosted.compileLevels[len(hosted.compileLevels)-1:] + for _, got := range levels { if got.hasOptimizationLevel != test.present || got.optimizationLevel != test.level { t.Fatalf("optimization = %+v, want present=%v level=%v", got, test.present, test.level) @@ -84,151 +62,128 @@ func TestRuntimeOptimizationPresenceRoundTrip(t *testing.T) { } func TestCompileOptionsApplyOnceBeforeDispatch(t *testing.T) { - for _, adapter := range []bool{false, true} { - for _, debug := range []bool{false, true} { - for _, outcome := range []string{"success", "invalid level", "callback errors", "callback cancellation", "already cancelled"} { - name := map[bool]string{false: "Client/", true: "Runtime/"}[adapter] + map[bool]string{false: "normal/", true: "debug/"}[debug] + outcome - t.Run(name, func(t *testing.T) { - hosted := &contractRuntime{} - env := newIntegrationEnv(t, hosted) - gate := &allocationResponseGate{ClientConnInterface: env.conn, calls: make(map[string]int)} - var compile func(context.Context, []api.PlanOption) error - if adapter { - remote, err := client.NewRuntime(testContext(t), gate) - if err != nil { - t.Fatal(err) + for _, debug := range []bool{false, true} { + for _, outcome := range []string{"success", "invalid level", "callback errors", "callback cancellation", "already cancelled"} { + name := "Client/" + map[bool]string{false: "normal/", true: "debug/"}[debug] + outcome + t.Run(name, func(t *testing.T) { + hosted := &contractRuntime{} + env := newIntegrationEnv(t, hosted) + gate := &allocationResponseGate{ClientConnInterface: env.conn, calls: make(map[string]int)} + lowLevel, err := client.New(testContext(t), gate) + if err != nil { + t.Fatal(err) + } + + t.Cleanup(func() { + if err := lowLevel.Close(testContext(t)); err != nil { + t.Error(err) + } + }) + compile := func(ctx context.Context, options []api.PlanOption) error { + plan, err := lowLevel.Compile(ctx, api.Source{Content: "RETURN 1"}, client.CompileOptions{Debuggable: debug, PlanOptions: options}) + if err != nil { + return err + } + + return plan.Close(testContext(t)) + } + + ctx, cancel := context.WithCancel(testContext(t)) + defer cancel() + + firstErr, secondErr := errors.New("first option failed"), errors.New("second option failed") + var order []int + options := []api.PlanOption{ + func(options api.PlanOptions) error { + order = append(order, 1) + + if outcome == "callback cancellation" { + cancel() } - t.Cleanup(func() { - if err := remote.Close(); err != nil { - t.Error(err) - } - }) - compile = func(ctx context.Context, options []api.PlanOption) error { - create := remote.Compile - if debug { - create = remote.CompileDebug - } - - plan, err := create(ctx, api.Source{Content: "RETURN 1"}, options...) - if err != nil { - return err - } - - return plan.Close() + if outcome == "callback errors" { + return firstErr } - } else { - lowLevel, err := client.New(testContext(t), gate) - if err != nil { - t.Fatal(err) + + if outcome == "invalid level" { + return options.SetOptimizationLevel(api.OptimizationLevel(99)) } - t.Cleanup(func() { - if err := lowLevel.Close(testContext(t)); err != nil { - t.Error(err) - } - }) - compile = func(ctx context.Context, options []api.PlanOption) error { - plan, err := lowLevel.Compile(ctx, api.Source{Content: "RETURN 1"}, client.CompileOptions{Debuggable: debug, PlanOptions: options}) - if err != nil { - return err - } - - return plan.Close(testContext(t)) + return options.SetOptimizationLevel(api.OptimizationBasic) + }, + nil, + func(options api.PlanOptions) error { + order = append(order, 2) + + if outcome == "callback errors" { + return secondErr } + + return options.SetOptimizationLevel(api.OptimizationNone) + }, + } + + if outcome == "already cancelled" { + cancel() + } + + err = compile(ctx, options) + + if outcome == "already cancelled" { + if len(order) != 0 { + t.Fatalf("cancelled call applied options: %v", order) } + } else if !reflect.DeepEqual(order, []int{1, 2}) { + t.Fatalf("callbacks did not run once in order: %v", order) + } - ctx, cancel := context.WithCancel(testContext(t)) - defer cancel() - - firstErr, secondErr := errors.New("first option failed"), errors.New("second option failed") - var order []int - options := []api.PlanOption{ - func(options api.PlanOptions) error { - order = append(order, 1) - if outcome == "callback cancellation" { - cancel() - } - - if outcome == "callback errors" { - return firstErr - } - - if outcome == "invalid level" { - return options.SetOptimizationLevel(api.OptimizationLevel(99)) - } - - return options.SetOptimizationLevel(api.OptimizationBasic) - }, - nil, - func(options api.PlanOptions) error { - order = append(order, 2) - if outcome == "callback errors" { - return secondErr - } - - return options.SetOptimizationLevel(api.OptimizationNone) - }, + switch outcome { + case "success": + if err != nil { + t.Fatal(err) } - if outcome == "already cancelled" { - cancel() + case "callback cancellation", "already cancelled": + if !errors.Is(err, context.Canceled) { + t.Fatalf("cancellation was lost: %v", err) } - - err := compile(ctx, options) - if outcome == "already cancelled" { - if len(order) != 0 { - t.Fatalf("cancelled call applied options: %v", order) - } - } else if !reflect.DeepEqual(order, []int{1, 2}) { - t.Fatalf("callbacks did not run once in order: %v", order) + case "callback errors": + if !errors.Is(err, firstErr) || !errors.Is(err, secondErr) { + t.Fatalf("callback errors were not joined: %v", err) } - - switch outcome { - case "success": - if err != nil { - t.Fatal(err) - } - case "callback cancellation", "already cancelled": - if !errors.Is(err, context.Canceled) { - t.Fatalf("cancellation was lost: %v", err) - } - case "callback errors": - if !errors.Is(err, firstErr) || !errors.Is(err, secondErr) { - t.Fatalf("callback errors were not joined: %v", err) - } - default: - if err == nil { - t.Fatal("invalid optimization was accepted") - } + default: + if err == nil { + t.Fatal("invalid optimization was accepted") } + } - method := wirev1.PlanService_Compile_FullMethodName - if debug { - method = wirev1.PlanService_CompileDebug_FullMethodName - } + method := wirev1.PlanService_Compile_FullMethodName - expectedCalls := 0 - if outcome == "success" { - expectedCalls = 1 - } + if debug { + method = wirev1.PlanService_CompileDebug_FullMethodName + } - if calls := gate.count(method); calls != expectedCalls { - t.Fatalf("dispatched %d compile requests, want %d", calls, expectedCalls) - } + expectedCalls := 0 - hosted.mu.Lock() - defer hosted.mu.Unlock() + if outcome == "success" { + expectedCalls = 1 + } - if outcome == "success" { - if len(hosted.compileLevels) != 1 || !hosted.compileLevels[0].hasOptimizationLevel || - hosted.compileLevels[0].optimizationLevel != api.OptimizationNone || hosted.compileDebug[0] != debug { - t.Fatalf("host did not receive final explicit zero option and debug choice: %+v", hosted.compileLevels) - } - } else if len(hosted.compileSources) != 0 { - t.Fatal("failed local options reached the hosted runtime") + if calls := gate.count(method); calls != expectedCalls { + t.Fatalf("dispatched %d compile requests, want %d", calls, expectedCalls) + } + + hosted.mu.Lock() + defer hosted.mu.Unlock() + + if outcome == "success" { + if len(hosted.compileLevels) != 1 || !hosted.compileLevels[0].hasOptimizationLevel || + hosted.compileLevels[0].optimizationLevel != api.OptimizationNone || hosted.compileDebug[0] != debug { + t.Fatalf("host did not receive final explicit zero option and debug choice: %+v", hosted.compileLevels) } - }) - } + } else if len(hosted.compileSources) != 0 { + t.Fatal("failed local options reached the hosted runtime") + } + }) } } } diff --git a/server/runtime_release_failure_test.go b/server/runtime_release_failure_test.go deleted file mode 100644 index 5bb1397..0000000 --- a/server/runtime_release_failure_test.go +++ /dev/null @@ -1,63 +0,0 @@ -package server_test - -import ( - "errors" - "testing" - - "github.com/MontFerret/api" - wirev1 "github.com/MontFerret/wire/gen/ferret/wire/v1" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -func TestRuntimeCompletedExecutionReleaseFailurePreservesParents(t *testing.T) { - for _, index := range []int{4, 5} { - operation := allocationOperations()[index] - for _, acknowledged := range []bool{false, true} { - t.Run(operation.name+map[bool]string{false: "/failed delivery", true: "/lost acknowledgement"}[acknowledged], func(t *testing.T) { - f := newRuntimeAllocationFixture(t, operation) - releaseErr := status.Error(codes.Unavailable, "execution release unavailable") - fail := f.gate.fail - if acknowledged { - fail = f.gate.failResponse - } - - fail(operation.release, releaseErr) - if _, err := f.allocate(testContext(t), nil); !errors.Is(err, releaseErr) { - t.Fatalf("successful execution lost its release failure: %v", err) - } - - if f.gate.count(operation.release) != 1 || f.gate.count(operation.parentRelease) != 0 || - f.gate.count(wirev1.PlanService_ReleasePlan_FullMethodName) != 0 || - f.gate.count(wirev1.RuntimeService_CloseConnection_FullMethodName) != 0 { - t.Fatal("known Execution release failure closed its parent or retried") - } - - fail(operation.release, nil) - if operation.name == "session run" { - _, err := f.session.Run(testContext(t)) - if acknowledged && err != nil { - t.Fatalf("durable Session could not be reused after server committed release: %v", err) - } - - if !acknowledged && status.Code(err) != codes.FailedPrecondition { - t.Fatalf("undelivered release should retain the hosted execution: %v", err) - } - - sibling, err := f.plan.NewSession(testContext(t)) - if err != nil { - t.Fatalf("release failure invalidated Plan: %v", err) - } - - if _, err := sibling.Run(testContext(t)); err != nil { - t.Fatalf("release failure prevented sibling execution: %v", err) - } - } - - if _, err := f.remote.Run(testContext(t), api.Source{Content: "RETURN 3"}); err != nil { - t.Fatalf("release failure invalidated Runtime: %v", err) - } - }) - } - } -} diff --git a/test/integration/README.md b/test/integration/README.md new file mode 100644 index 0000000..367bff2 --- /dev/null +++ b/test/integration/README.md @@ -0,0 +1,137 @@ +# Universal API integration contracts + +These external-consumer tests exercise: + +```text +api.Runtime caller → wire/client → protobuf and real gRPC over bufconn + → wire/server → hosted Universal API spies +``` + +The suite depends on the API version pinned in the root `go.mod` (currently +`v1.0.0-alpha.11`). It imports neither native Ferret nor Wire internal packages. +Existing component, conversion, low-level facade/protocol tests, and benchmarks +remain beside their owning packages. The former server-package Universal API +adapter, allocation, cancellation, and transport tests are consolidated here. + +## Harness + +`harness.New(t)` starts the public Wire server, opens a real gRPC connection, and +returns a canonical runtime through `h.Runtime()`. `h.RuntimeSpy().Recorder()` +provides copied call and lifecycle snapshots. `WithBehavior` configures immutable +runtime/plan/session/debugger behavior before startup; `WithRuntime` accepts a +custom API implementation; `WithServerOptions` applies public server limits. + +The harness separates transport ownership, API doubles, option snapshots, +lifecycle recording, failure injection, and cancellation coordination. Recorded +object IDs identify hosted spy instances and parent relationships, never Wire +resource handles. Every hosted close attempt is counted, including duplicates. +Fake parents never close children on Wire's behalf. + +`NewBlock`, `Await`, and recorder change notifications coordinate operations with +bounded waits. Hooks run outside recorder locks and must honor their context. +Use a fresh Block for each blocked invocation and await every test goroutine's +result. Do not use sleeps, polling, unbounded receives, or global goroutine counts. + +`h.Faults()` wraps the real connection. Allocation gates hold replies only after +the real server has committed and the response has crossed gRPC. Semantic fault +operations support lost/malformed replies, releases that never arrive, lost +release acknowledgements, and watches ending after a real initial snapshot. +Only this fault infrastructure references generated method constants or mutates +received protobuf payloads. Tests never call generated clients or handlers. + +`OpenRuntime` creates another logical client on the same physical connection. +`Shutdown` and `CloseTransport` model distinct lifetime failures. Cleanup removes +faults, closes logical clients, and asserts hosted resources finished closing +exactly once before server shutdown could hide a leak. It then shuts down the +server, closes owned transport and listener resources, and waits for serving to +finish. Expected retained cleanup errors must be registered +with `ExpectCleanupError`; connection-loss tests permit the corresponding +transport/closed-handle errors. Wire must never close the hosted runtime. + +## Interface coverage + +Every current interface method has end-to-end coverage. Names below are Go test +names; each linked file contains the complete scenario and assertions. + +| Interface | Method | Contract test | +| --- | --- | --- | +| `api.Runtime` | `Run` | [TestRuntimeAndSessionOutputRoundTrip](runtime_test.go) | +| | `Compile` | [TestCompileRoundTrip](plan_test.go) | +| | `CompileDebug` | [TestCompileRoundTrip](plan_test.go), [TestDebuggerRoundTrip](debugger_test.go) | +| | `Close` | [TestRuntimeCloseBorrowsTransportAndHostedRuntime](runtime_test.go) | +| `api.Plan` | `Params` | [TestCompileRoundTrip](plan_test.go) | +| | `NewSession` | [TestReusablePlanAndDurableSessions](plan_test.go) | +| | `NewDebugSession` | [TestDebuggerRoundTrip](debugger_test.go) | +| | `Close` | [TestReusablePlanAndDurableSessions](plan_test.go), [TestRecursiveCloseReclaimsActiveDescendants](lifecycle_test.go) | +| `api.Session` | `Run` | [TestReusablePlanAndDurableSessions](plan_test.go), [TestSessionRejectsOverlapAndReopensAfterRelease](session_test.go) | +| | `Close` | [TestReusablePlanAndDurableSessions](plan_test.go), [TestRecursiveCloseReclaimsActiveDescendants](lifecycle_test.go) | +| `debugger.Session` | `Start` | [TestDebuggerRoundTrip](debugger_test.go) | +| | `Continue` | [TestDebuggerRoundTrip](debugger_test.go) | +| | `StepOver` | [TestDebuggerRoundTrip](debugger_test.go) | +| | `StepIn` | [TestDebuggerRoundTrip](debugger_test.go) | +| | `StepOut` | [TestDebuggerRoundTrip](debugger_test.go) | +| | `Pause` | [TestDebuggerRoundTrip](debugger_test.go) | +| | `SetBreakpoint` | [TestDebuggerRoundTrip](debugger_test.go) | +| | `SetBreakpointAt` | [TestDebuggerRoundTrip](debugger_test.go) | +| | `DeleteBreakpoint` | [TestDebuggerRoundTrip](debugger_test.go) | +| | `Breakpoints` | [TestDebuggerRoundTrip](debugger_test.go) | +| | `Frames` | [TestDebuggerRoundTrip](debugger_test.go) | +| | `Locals` | [TestDebuggerRoundTrip](debugger_test.go) | +| | `FrameLocals` | [TestDebuggerRoundTrip](debugger_test.go) | +| | `Variables` | [TestDebuggerRoundTrip](debugger_test.go) | +| | `Evaluate` | [TestDebuggerRoundTrip](debugger_test.go) | +| | `EvaluateFrame` | [TestDebuggerRoundTrip](debugger_test.go) | +| | `Close` | [TestDebuggerRoundTrip](debugger_test.go), [TestRecursiveCloseReclaimsActiveDescendants](lifecycle_test.go) | + +Output has content and content type, without structured metadata. Diagnostics +have an open `Kind`, source, annotations, hint, and note, without a separate +severity field. Debugger frames use indices and function IDs. Convenience +operations may delegate to their indexed/default-binding equivalents; assertions +cover those documented semantic calls. Successful stopped events have nil +`Event.Error`; runtime-error stops preserve a public `failure.Failure`. + +## Lifecycle and failure coverage + +| Contract | Tests | +| --- | --- | +| Portable values, option overrides, output selection | [TestSessionOptionsRoundTrip](runtime_test.go) | +| Optimization presence, option ordering and failure before dispatch | [TestCompileRoundTrip, TestCompileOptionsApplyOnceBeforeDispatch](plan_test.go) | +| Plain and diagnostic-bearing compile/execution/debug failures; sanitization | [TestDiagnosticsAndFailureClassification](errors_test.go) | +| Invalid request/state, not found, expired deadline, remote cancellation, resource limits | [TestErrorFamilies](errors_test.go) | +| Runtime/session/Start/Continue/Evaluate cancellation and execution-slot reuse | [TestCancellationReachesHostedOperations](cancellation_test.go) | +| Detached compile allocation and hosted cancellation on logical shutdown | [TestCompileCancellationPreservesDetachedAllocation, TestLogicalShutdownCancelsHostedCompile](cancellation_test.go) | +| Cancellation before dispatch, after committed allocation, concurrent Close | [allocation_race_test.go](allocation_race_test.go) | +| Unknown plan/session/debugger/execution; nearest parent and escalation | [allocation_test.go](allocation_test.go) | +| Known-ID release delivery failure versus lost acknowledgement; sibling preservation | [release_test.go](release_test.go) | +| Execution completion racing cancellation with exactly one release | [TestSessionCompletionRacesCancellationWithoutDuplicateCleanup](session_test.go) | +| Recursive close, distinct concurrent plans/sessions, mixed execution/debug resources | [lifecycle_test.go](lifecycle_test.go) | +| Unavailable handshake, server shutdown or transport closure during active work | [TestUnavailableServer, TestConnectionLossReclaimsResources](connection_test.go) | +| Initial snapshots, fast completion, debugger transitions, premature EOF/transport errors | [runtime_test.go](runtime_test.go), [debugger_test.go](debugger_test.go), [TestWatchTerminationReturnsError](connection_test.go) | +| Debugger command failure distinct from a runtime-error stop | [TestDebuggerCommandFailure](connection_test.go) | +| Constructor panic, poisoned sessions/debuggers, cleanup panic and sanitization | [TestConstructorPanicPreservesParent, TestPanicContainmentAndResourcePoisoning](errors_test.go) | + +Allocation and release policy is defined in [Client Handles](../../docs/client.md): +caller cancellation does not interrupt an in-flight bounded allocation, and an +eventual known handle is reclaimed before returning cancellation. Unknown IDs +require nearest-owner invalidation; failed release of a known ID does not. +Transport `ResourceExhausted` alone cannot distinguish a quota rejection from an +oversized committed response, so the existing conservative reclamation policy +applies. No test weakens this policy by treating the status alone as proof of +rejected creation. + +Abrupt connection loss returns errors. Graceful server shutdown can deliver a +semantic debugger termination event before transport closure; the suite accepts +that terminal outcome. Premature watch closure must not look like successful +execution. Explicit debugger Close still owns cleanup after a watch-only failure. +No automatic reconnection or protocol changes are introduced. + +## Running + +```sh +go test ./test/integration/... +go test -race ./test/integration/... +go test -race ./test/integration -count=20 -shuffle=on +``` + +The root `make test` and `make test-race` include this suite on existing CI jobs. +Tests need no native runtime, external server, TCP port, or additional dependency. diff --git a/test/integration/allocation_fixture_test.go b/test/integration/allocation_fixture_test.go new file mode 100644 index 0000000..c1a0367 --- /dev/null +++ b/test/integration/allocation_fixture_test.go @@ -0,0 +1,160 @@ +package integration_test + +import ( + "context" + "testing" + + "github.com/MontFerret/api" + "github.com/MontFerret/wire/test/integration/harness" +) + +type ( + allocationOperation struct { + name string + method, release, parentRelease harness.Operation + } + + runtimeAllocationFixture struct { + t *testing.T + h *harness.Harness + gate *harness.Faults + reply *harness.ResponseGate + record *harness.Recorder + remote, other api.Runtime + plan api.Plan + session api.Session + operation allocationOperation + expectedCloseError error + } +) + +func allocationOperations() []allocationOperation { + return []allocationOperation{ + {"compile", harness.Compile, harness.ReleasePlan, harness.CloseRuntime}, + {"compile debug", harness.CompileDebug, harness.ReleasePlan, harness.CloseRuntime}, + {"session", harness.CreateSession, harness.ReleaseSession, harness.ReleasePlan}, + {"debug session", harness.CreateDebugger, harness.ReleaseDebugger, harness.ReleasePlan}, + {"session run", harness.RunSession, harness.ReleaseExecution, harness.ReleaseSession}, + {"runtime run", harness.RunRuntime, harness.ReleaseExecution, harness.CloseRuntime}, + } +} + +func newRuntimeAllocationFixture(t *testing.T, operation allocationOperation) *runtimeAllocationFixture { + t.Helper() + h := harness.New(t) + f := &runtimeAllocationFixture{t: t, h: h, gate: h.Faults(), record: h.RuntimeSpy().Recorder(), remote: h.Runtime(), operation: operation} + var err error + f.other, err = h.OpenRuntime() + if err != nil { + t.Fatal(err) + } + + switch operation.method { + case harness.CreateSession, harness.CreateDebugger, harness.RunSession: + f.plan, err = f.remote.CompileDebug(h.Context(), api.Source{Content: "RETURN 1"}) + if err != nil { + t.Fatal(err) + } + } + + if operation.method == harness.RunSession { + f.session, err = f.plan.NewSession(h.Context()) + if err != nil { + t.Fatal(err) + } + } + + return f +} + +func (f *runtimeAllocationFixture) allocate(ctx context.Context, cancelInOption context.CancelFunc) (func() error, error) { + var planOptions []api.PlanOption + var sessionOptions []api.SessionOption + + if cancelInOption != nil { + planOptions = []api.PlanOption{func(api.PlanOptions) error { + cancelInOption() + + return nil + }} + sessionOptions = []api.SessionOption{func(api.SessionOptions) error { + cancelInOption() + + return nil + }} + } + + switch f.operation.method { + case harness.Compile, harness.CompileDebug: + compile := f.remote.Compile + + if f.operation.method == harness.CompileDebug { + compile = f.remote.CompileDebug + } + + plan, err := compile(ctx, api.Source{Content: "RETURN 1"}, planOptions...) + if err != nil { + return nil, err + } + + return plan.Close, nil + case harness.CreateSession: + session, err := f.plan.NewSession(ctx, sessionOptions...) + if err != nil { + return nil, err + } + + return session.Close, nil + case harness.CreateDebugger: + session, err := f.plan.NewDebugSession(ctx, sessionOptions...) + if err != nil { + return nil, err + } + + return session.Close, nil + case harness.RunSession: + _, err := f.session.Run(ctx) + + return nil, err + default: + _, err := f.remote.Run(ctx, api.Source{Content: "RETURN 1"}, sessionOptions...) + + return nil, err + } +} + +func (f *runtimeAllocationFixture) awaitCommitted() { + f.t.Helper() + harness.Await(f.t, f.reply.Committed) +} + +func (f *runtimeAllocationFixture) awaitResult(result <-chan error) error { + f.t.Helper() + + return harness.Await(f.t, result) +} + +func (f *runtimeAllocationFixture) assertAllClosed() { + f.t.Helper() + f.record.AssertClosed(f.t) +} + +func (f *runtimeAllocationFixture) assertNarrowParentClosed() { + f.t.Helper() + snapshot := f.record.Snapshot() + parent := snapshot.OfKind("plan")[0] + + if f.operation.method == harness.RunSession { + parent = snapshot.OfKind("session")[0] + } + + if got := snapshot.Count(parent.ID, "Close"); got != 1 { + f.t.Fatalf("narrow parent %v closed %d times", parent, got) + } + + for _, resource := range snapshot.Resources { + if resource.Parent == parent.ID && snapshot.Count(resource.ID, "Close") != 1 { + f.t.Fatalf("narrow parent left child %v open", resource) + } + } +} diff --git a/server/runtime_allocation_race_test.go b/test/integration/allocation_race_test.go similarity index 51% rename from server/runtime_allocation_race_test.go rename to test/integration/allocation_race_test.go index 05f71ca..ad89561 100644 --- a/server/runtime_allocation_race_test.go +++ b/test/integration/allocation_race_test.go @@ -1,14 +1,11 @@ -package server_test +package integration_test import ( "context" "errors" - "sync" "testing" - "github.com/MontFerret/api" - "github.com/MontFerret/wire/client" - wirev1 "github.com/MontFerret/wire/gen/ferret/wire/v1" + "github.com/MontFerret/wire/test/integration/harness" ) func TestRuntimeAllocationCancellationBeforeDispatch(t *testing.T) { @@ -20,10 +17,11 @@ func TestRuntimeAllocationCancellationBeforeDispatch(t *testing.T) { t.Run(operation.name+"/"+stage, func(t *testing.T) { f := newRuntimeAllocationFixture(t, operation) - before := f.gate.count(operation.method) - ctx, cancel := context.WithCancel(testContext(t)) + before := f.gate.Count(operation.method) + ctx, cancel := context.WithCancel(harness.Context(t)) defer cancel() var optionCancel context.CancelFunc + if stage == "before call" { cancel() } else { @@ -35,7 +33,7 @@ func TestRuntimeAllocationCancellationBeforeDispatch(t *testing.T) { t.Fatalf("cancelled acquisition returned a handle or lost cancellation: %v", err) } - if calls := f.gate.count(operation.method); calls != before { + if calls := f.gate.Count(operation.method); calls != before { t.Fatalf("cancelled acquisition sent %d RPCs", calls-before) } }) @@ -47,34 +45,37 @@ func TestRuntimeAllocationCancellationAfterCommitReleasesOnlyNewResource(t *test for _, operation := range allocationOperations() { t.Run(operation.name, func(t *testing.T) { f := newRuntimeAllocationFixture(t, operation) - f.gate.arm(operation.method, "success") - ctx, cancel := context.WithCancel(testContext(t)) + f.reply = f.gate.Arm(operation.method, harness.Deliver) + ctx, cancel := context.WithCancel(harness.Context(t)) defer cancel() result := make(chan error, 1) go func() { closeHandle, err := f.allocate(ctx, nil) if closeHandle != nil { + t.Error("cancelled allocation returned a handle") err = errors.Join(err, errors.New("cancelled allocation returned a handle"), closeHandle()) } + result <- err }() f.awaitCommitted() cancel() - close(f.gate.deliver) + f.reply.Deliver() + if err := f.awaitResult(result); !errors.Is(err, context.Canceled) { t.Fatalf("caller cancellation was lost: %v", err) } - if calls := f.gate.count(operation.release); calls != 1 { + if calls := f.gate.Count(operation.release); calls != 1 { t.Fatalf("resource release calls = %d, want 1", calls) } - if calls := f.gate.count(operation.parentRelease); calls != 0 { + if calls := f.gate.Count(operation.parentRelease); calls != 0 { t.Fatalf("healthy parent was released %d times", calls) } if operation.name == "session run" { - if _, err := f.session.Run(testContext(t)); err != nil { + if _, err := f.session.Run(harness.Context(t)); err != nil { t.Fatalf("durable Session was not reusable after cancellation: %v", err) } } @@ -86,16 +87,18 @@ func TestRuntimeAllocationNormalCloseReleasesOnce(t *testing.T) { for _, operation := range allocationOperations() { t.Run(operation.name, func(t *testing.T) { f := newRuntimeAllocationFixture(t, operation) - closeHandle, err := f.allocate(testContext(t), nil) + closeHandle, err := f.allocate(harness.Context(t), nil) if err != nil { t.Fatal(err) } if closeHandle != nil { results := make(chan error, 8) + for range 8 { go func() { results <- closeHandle() }() } + for range 8 { if err := f.awaitResult(results); err != nil { t.Fatal(err) @@ -103,58 +106,9 @@ func TestRuntimeAllocationNormalCloseReleasesOnce(t *testing.T) { } } - if calls := f.gate.count(operation.release); calls != 1 { + if calls := f.gate.Count(operation.release); calls != 1 { t.Fatalf("resource release calls = %d, want 1", calls) } }) } } - -func TestRuntimeSessionCompletionRacesCancellationWithoutDuplicateCleanup(t *testing.T) { - for range 20 { - operation := allocationOperations()[4] - f := newRuntimeAllocationFixture(t, operation) - started, finish := make(chan struct{}), make(chan struct{}) - f.sessions[0].run = func(context.Context, int) (api.Output, error) { - close(started) - <-finish - - return api.Output{}, nil - } - ctx, cancel := context.WithCancel(testContext(t)) - result := make(chan error, 1) - go func() { - _, err := f.session.Run(ctx) - result <- err - }() - select { - case <-started: - case <-ctx.Done(): - t.Fatal("hosted run did not start") - } - - var race sync.WaitGroup - race.Add(2) - go func() { defer race.Done(); cancel() }() - go func() { defer race.Done(); close(finish) }() - race.Wait() - if err := f.awaitResult(result); err != nil && !cancellationError(err) && !errors.Is(err, client.ErrExecutionCancelled) { - t.Fatalf("unexpected completion race error: %v", err) - } - - if f.gate.count(operation.release) != 1 || f.gate.count(wirev1.ExecutionService_CancelExecution_FullMethodName) != 0 { - t.Fatal("adapter did not use exactly one execution release") - } - - f.sessions[0].mu.Lock() - f.sessions[0].run = nil - f.sessions[0].mu.Unlock() - if _, err := f.session.Run(testContext(t)); err != nil { - t.Fatalf("Session retained a leaked execution: %v", err) - } - - if runs, closes := f.sessions[0].counts(); runs != 2 || closes != 0 { - t.Fatalf("durable Session lifecycle changed: runs=%d closes=%d", runs, closes) - } - } -} diff --git a/server/runtime_allocation_reclamation_test.go b/test/integration/allocation_test.go similarity index 56% rename from server/runtime_allocation_reclamation_test.go rename to test/integration/allocation_test.go index e2adbc4..577072b 100644 --- a/server/runtime_allocation_reclamation_test.go +++ b/test/integration/allocation_test.go @@ -1,4 +1,4 @@ -package server_test +package integration_test import ( "context" @@ -8,64 +8,66 @@ import ( "github.com/MontFerret/api" "github.com/MontFerret/wire/client" - wirev1 "github.com/MontFerret/wire/gen/ferret/wire/v1" + "github.com/MontFerret/wire/test/integration/harness" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) func TestRuntimeLostAllocationReclaimsNearestParentAndPreservesSiblings(t *testing.T) { for _, operation := range allocationOperations() { - for _, outcome := range []string{"deadline", "unavailable", "oversized", "transport internal", "malformed"} { - t.Run(operation.name+"/"+outcome, func(t *testing.T) { + for _, outcome := range []harness.Outcome{harness.LostDeadline, harness.LostUnavailable, harness.LostOversized, harness.LostDecode, harness.Malformed} { + t.Run(operation.name+"/"+string(outcome), func(t *testing.T) { f := newRuntimeAllocationFixture(t, operation) - root := operation.parentRelease == wirev1.RuntimeService_CloseConnection_FullMethodName + root := operation.parentRelease == harness.CloseRuntime var sibling api.Session + if !root { parent := f.plan + if operation.name != "session run" { var err error - parent, err = f.remote.Compile(testContext(t), api.Source{Content: "RETURN 2"}) + parent, err = f.remote.Compile(harness.Context(t), api.Source{Content: "RETURN 2"}) if err != nil { t.Fatal(err) } } var err error - sibling, err = parent.NewSession(testContext(t)) + sibling, err = parent.NewSession(harness.Context(t)) if err != nil { t.Fatal(err) } } - f.gate.arm(operation.method, outcome) + f.reply = f.gate.Arm(operation.method, outcome) result := make(chan error, 1) go func() { - _, err := f.allocate(testContext(t), nil) + _, err := f.allocate(harness.Context(t), nil) result <- err }() f.awaitCommitted() - close(f.gate.deliver) + f.reply.Deliver() err := f.awaitResult(result) if err == nil { t.Fatal("lost or malformed allocation response succeeded") } - if outcome == "deadline" && status.Code(err) != codes.DeadlineExceeded { + if outcome == harness.LostDeadline && status.Code(err) != codes.DeadlineExceeded { t.Fatalf("allocation deadline was lost: %v", err) } - if calls := f.gate.count(operation.parentRelease); calls != 1 { + if calls := f.gate.Count(operation.parentRelease); calls != 1 { t.Fatalf("parent release calls = %d, want 1", calls) } if root { - if _, err := f.remote.Run(testContext(t), api.Source{Content: "RETURN 1"}); !errors.Is(err, client.ErrClosed) { + if _, err := f.remote.Run(harness.Context(t), api.Source{Content: "RETURN 1"}); !errors.Is(err, client.ErrClosed) { t.Fatalf("indeterminate root allocation left Runtime usable: %v", err) } f.assertAllClosed() } else { - if f.gate.count(wirev1.RuntimeService_CloseConnection_FullMethodName) != 0 { + if f.gate.Count(harness.CloseRuntime) != 0 { t.Fatal("successful narrow cleanup destroyed Runtime") } @@ -74,29 +76,29 @@ func TestRuntimeLostAllocationReclaimsNearestParentAndPreservesSiblings(t *testi f.assertNarrowParentClosed() if operation.name == "session run" { - if _, err := f.session.Run(testContext(t)); !errors.Is(err, client.ErrClosed) { + if _, err := f.session.Run(harness.Context(t)); !errors.Is(err, client.ErrClosed) { t.Fatalf("Session parent remained usable: %v", err) } - } else if _, err := f.plan.NewSession(testContext(t)); !errors.Is(err, client.ErrClosed) { + } else if _, err := f.plan.NewSession(harness.Context(t)); !errors.Is(err, client.ErrClosed) { t.Fatalf("Plan parent remained usable: %v", err) } - if _, err := sibling.Run(testContext(t)); err != nil { + if _, err := sibling.Run(harness.Context(t)); err != nil { t.Fatalf("unrelated sibling was invalidated: %v", err) } - if _, err := f.remote.Run(testContext(t), api.Source{Content: "RETURN 1"}); err != nil { + if _, err := f.remote.Run(harness.Context(t), api.Source{Content: "RETURN 1"}); err != nil { t.Fatalf("logical Runtime was invalidated: %v", err) } } // This second logical client borrows the same physical connection. - plan, err := f.env.client.Compile(testContext(t), api.Source{Content: "RETURN 3"}, client.CompileOptions{}) + plan, err := f.other.Compile(harness.Context(t), api.Source{Content: "RETURN 3"}) if err != nil { t.Fatalf("caller-owned transport or sibling client was closed: %v", err) } - if err := plan.Close(testContext(t)); err != nil { + if err := plan.Close(); err != nil { t.Fatal(err) } }) @@ -112,26 +114,27 @@ func TestRuntimeCancelledKnownAllocationPreservesParentsOnReleaseFailure(t *test siblingPlan := f.plan if siblingPlan == nil { var err error - siblingPlan, err = f.remote.Compile(testContext(t), api.Source{Content: "RETURN 2"}) + siblingPlan, err = f.remote.Compile(harness.Context(t), api.Source{Content: "RETURN 2"}) if err != nil { t.Fatal(err) } } - sibling, err := siblingPlan.NewSession(testContext(t)) + sibling, err := siblingPlan.NewSession(harness.Context(t)) if err != nil { t.Fatal(err) } releaseErr := status.Error(codes.Unavailable, "resource release unavailable") - fail := f.gate.fail + fail := f.gate.Fail + if acknowledged { - fail = f.gate.failResponse + fail = f.gate.FailResponse } fail(operation.release, releaseErr) - f.gate.arm(operation.method, "success") - ctx, cancel := context.WithCancel(testContext(t)) + f.reply = f.gate.Arm(operation.method, harness.Deliver) + ctx, cancel := context.WithCancel(harness.Context(t)) defer cancel() result := make(chan error, 1) go func() { @@ -140,28 +143,29 @@ func TestRuntimeCancelledKnownAllocationPreservesParentsOnReleaseFailure(t *test }() f.awaitCommitted() cancel() - close(f.gate.deliver) + f.reply.Deliver() err = f.awaitResult(result) if !errors.Is(err, context.Canceled) || !errors.Is(err, releaseErr) { t.Fatalf("cancellation or handle release error was lost: %v", err) } - if f.gate.count(operation.release) != 1 || f.gate.count(operation.parentRelease) != 0 || - f.gate.count(wirev1.RuntimeService_CloseConnection_FullMethodName) != 0 { + if f.gate.Count(operation.release) != 1 || f.gate.Count(operation.parentRelease) != 0 || + f.gate.Count(harness.CloseRuntime) != 0 { t.Fatal("known allocation release failure invalidated an ancestor or retried") } fail(operation.release, nil) - if _, err := sibling.Run(testContext(t)); err != nil { + + if _, err := sibling.Run(harness.Context(t)); err != nil { t.Fatalf("known allocation release failure invalidated its sibling: %v", err) } - if _, err := f.remote.Run(testContext(t), api.Source{Content: "RETURN 3"}); err != nil { + if _, err := f.remote.Run(harness.Context(t), api.Source{Content: "RETURN 3"}); err != nil { t.Fatalf("known allocation release failure invalidated Runtime: %v", err) } if operation.name == "session run" { - _, err := f.session.Run(testContext(t)) + _, err := f.session.Run(harness.Context(t)) if acknowledged && err != nil { t.Fatalf("Session could not run after committed execution release: %v", err) } @@ -178,56 +182,55 @@ func TestRuntimeCancelledKnownAllocationPreservesParentsOnReleaseFailure(t *test func TestRuntimeLostExecutionTriesPlanBeforeRuntime(t *testing.T) { operation := allocationOperations()[4] f := newRuntimeAllocationFixture(t, operation) - siblingPlan, err := f.remote.Compile(testContext(t), api.Source{Content: "RETURN 2"}) + siblingPlan, err := f.remote.Compile(harness.Context(t), api.Source{Content: "RETURN 2"}) if err != nil { t.Fatal(err) } - sibling, err := siblingPlan.NewSession(testContext(t)) + sibling, err := siblingPlan.NewSession(harness.Context(t)) if err != nil { t.Fatal(err) } parentErr := status.Error(codes.Unavailable, "session release unavailable") - f.gate.fail(operation.parentRelease, parentErr) - f.gate.arm(operation.method, "unavailable") + f.gate.Fail(operation.parentRelease, parentErr) + f.reply = f.gate.Arm(operation.method, harness.LostUnavailable) result := make(chan error, 1) go func() { - _, err := f.allocate(testContext(t), nil) + _, err := f.allocate(harness.Context(t), nil) result <- err }() f.awaitCommitted() - close(f.gate.deliver) + f.reply.Deliver() + if err := f.awaitResult(result); !errors.Is(err, parentErr) { t.Fatalf("failed Session cleanup was lost: %v", err) } - methods := f.gate.methodSequence() - sessionRelease := slices.Index(methods, wirev1.SessionService_ReleaseSession_FullMethodName) - planRelease := slices.Index(methods, wirev1.PlanService_ReleasePlan_FullMethodName) + methods := f.gate.Sequence() + sessionRelease := slices.Index(methods, harness.ReleaseSession) + planRelease := slices.Index(methods, harness.ReleasePlan) if sessionRelease < 0 || planRelease <= sessionRelease || - f.gate.count(wirev1.PlanService_ReleasePlan_FullMethodName) != 1 || - f.gate.count(wirev1.RuntimeService_CloseConnection_FullMethodName) != 0 { + f.gate.Count(harness.ReleasePlan) != 1 || + f.gate.Count(harness.CloseRuntime) != 0 { t.Fatalf("reclamation did not stop after Session then Plan: %v", methods) } - if _, err := f.plan.NewSession(testContext(t)); !errors.Is(err, client.ErrClosed) { + if _, err := f.plan.NewSession(harness.Context(t)); !errors.Is(err, client.ErrClosed) { t.Fatalf("owning Plan was not invalidated: %v", err) } f.assertNarrowParentClosed() - f.plans[0].mu.Lock() - planCloses := f.plans[0].closeCalls - f.plans[0].mu.Unlock() + planCloses := f.record.Snapshot().Count(f.record.Snapshot().OfKind("plan")[0].ID, "Close") if planCloses != 1 { t.Fatalf("Plan fallback returned before hosted Plan cleanup: closes=%d", planCloses) } - if _, err := sibling.Run(testContext(t)); err != nil { + if _, err := sibling.Run(harness.Context(t)); err != nil { t.Fatalf("another Plan's Session was invalidated: %v", err) } - if _, err := f.remote.Run(testContext(t), api.Source{Content: "RETURN 3"}); err != nil { + if _, err := f.remote.Run(harness.Context(t), api.Source{Content: "RETURN 3"}); err != nil { t.Fatalf("successful Plan reclamation invalidated Runtime: %v", err) } } @@ -235,29 +238,32 @@ func TestRuntimeLostExecutionTriesPlanBeforeRuntime(t *testing.T) { func TestRuntimeLostAllocationEscalatesFailedParentCleanup(t *testing.T) { for _, index := range []int{2, 3, 4} { operation := allocationOperations()[index] + for _, failConnection := range []bool{false, true} { t.Run(operation.name+map[bool]string{false: "/connection release", true: "/Connect stream"}[failConnection], func(t *testing.T) { f := newRuntimeAllocationFixture(t, operation) parentErr := status.Error(codes.Unavailable, "parent release unavailable") - f.gate.fail(operation.parentRelease, parentErr) + f.gate.Fail(operation.parentRelease, parentErr) planErr := status.Error(codes.Unavailable, "plan release unavailable") + if operation.name == "session run" { - f.gate.fail(wirev1.PlanService_ReleasePlan_FullMethodName, planErr) + f.gate.Fail(harness.ReleasePlan, planErr) } if failConnection { f.expectedCloseError = status.Error(codes.Unavailable, "connection release unavailable") - f.gate.fail(wirev1.RuntimeService_CloseConnection_FullMethodName, f.expectedCloseError) + f.h.ExpectCleanupError(f.expectedCloseError) + f.gate.Fail(harness.CloseRuntime, f.expectedCloseError) } - f.gate.arm(operation.method, "deadline") + f.reply = f.gate.Arm(operation.method, harness.LostDeadline) result := make(chan error, 1) go func() { - _, err := f.allocate(testContext(t), nil) + _, err := f.allocate(harness.Context(t), nil) result <- err }() f.awaitCommitted() - close(f.gate.deliver) + f.reply.Deliver() err := f.awaitResult(result) if !errors.Is(err, parentErr) || (operation.name == "session run" && !errors.Is(err, planErr)) || @@ -265,7 +271,7 @@ func TestRuntimeLostAllocationEscalatesFailedParentCleanup(t *testing.T) { t.Fatalf("cleanup errors were lost: %v", err) } - if calls := f.gate.count(wirev1.RuntimeService_CloseConnection_FullMethodName); calls != 1 { + if calls := f.gate.Count(harness.CloseRuntime); calls != 1 { t.Fatalf("failed parent cleanup did not escalate once: %d", calls) } @@ -274,36 +280,3 @@ func TestRuntimeLostAllocationEscalatesFailedParentCleanup(t *testing.T) { } } } - -func TestRuntimeRejectedAllocationPreservesParent(t *testing.T) { - f := newRuntimeAllocationFixture(t, allocationOperations()[2]) - if _, err := f.remote.Compile(testContext(t), api.Source{}); status.Code(err) != codes.InvalidArgument { - t.Fatalf("invalid source was not rejected: %v", err) - } - - if _, err := f.plan.NewSession(testContext(t), api.WithParam("bad", make(chan int))); err == nil { - t.Fatal("nonportable parameter was accepted") - } - - f.plans[0].mu.Lock() - create := f.plans[0].newSession - f.plans[0].newSession = func(context.Context, apiSessionOptions) (api.Session, error) { - panic("hosted constructor secret") - } - f.plans[0].mu.Unlock() - if _, err := f.plan.NewSession(testContext(t)); status.Code(err) != codes.Internal { - t.Fatalf("constructor panic was not contained: %v", err) - } - - f.plans[0].mu.Lock() - f.plans[0].newSession = create - f.plans[0].mu.Unlock() - session, err := f.plan.NewSession(testContext(t)) - if err != nil { - t.Fatalf("rejected creation invalidated its parent: %v", err) - } - - if err := session.Close(); err != nil { - t.Fatal(err) - } -} diff --git a/test/integration/cancellation_test.go b/test/integration/cancellation_test.go new file mode 100644 index 0000000..0288c73 --- /dev/null +++ b/test/integration/cancellation_test.go @@ -0,0 +1,215 @@ +package integration_test + +import ( + "context" + "errors" + "testing" + + "github.com/MontFerret/api" + "github.com/MontFerret/api/debugger" + "github.com/MontFerret/wire/server" + "github.com/MontFerret/wire/test/integration/harness" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestCancellationReachesHostedOperations(t *testing.T) { + for _, operation := range []string{"runtime", "session", "Start", "Continue", "Evaluate"} { + t.Run(operation, func(t *testing.T) { + block := harness.NewBlock(t) + behavior := harness.RuntimeBehavior{ + Run: func(ctx context.Context, src api.Source, _ harness.SessionOptions) (api.Output, error) { + if src.Content == "RETURN 2" { + return api.Output{}, nil + } + + return api.Output{}, block.Wait(ctx) + }, + Plan: harness.PlanBehavior{ + Session: func(harness.SessionOptions) harness.SessionBehavior { + return harness.SessionBehavior{Run: func(ctx context.Context, _ int) (api.Output, error) { return api.Output{}, block.Wait(ctx) }} + }, + Debugger: harness.DebuggerBehavior{ + Command: func(ctx context.Context, method string, _ int) (*debugger.Event, error) { + if method == operation { + return nil, block.Wait(ctx) + } + + return &debugger.Event{Reason: debugger.ReasonEntry}, nil + }, + Evaluate: func(ctx context.Context, _ int, _ string) (debugger.Value, error) { + return debugger.Value{}, block.Wait(ctx) + }, + }, + }, + } + limits := server.DefaultLimits() + limits.MaxExecutionsPerConnection = 1 + h := harness.New(t, harness.WithBehavior(behavior), harness.WithServerOptions(server.WithLimits(limits))) + ctx, cancel := context.WithCancel(h.Context()) + defer cancel() + run := func() error { + _, err := h.Runtime().Run(ctx, api.Source{Content: "RETURN 1"}) + + return err + } + + if operation != "runtime" { + plan, err := h.Runtime().CompileDebug(h.Context(), api.Source{Content: "RETURN 1"}) + if err != nil { + t.Fatal(err) + } + + if operation == "session" { + session, err := plan.NewSession(h.Context()) + if err != nil { + t.Fatal(err) + } + + run = func() error { + _, err := session.Run(ctx) + + return err + } + } else { + session, err := plan.NewDebugSession(h.Context()) + if err != nil { + t.Fatal(err) + } + + if operation != "Start" { + if _, err := session.Start(h.Context()); err != nil { + t.Fatal(err) + } + } + + switch operation { + case "Start": + run = func() error { + _, err := session.Start(ctx) + + return err + } + case "Continue": + run = func() error { + _, err := session.Continue(ctx) + + return err + } + case "Evaluate": + run = func() error { + _, err := session.Evaluate(ctx, "input") + + return err + } + } + } + } + + result := make(chan error, 1) + go func() { result <- run() }() + harness.Await(t, block.Started) + cancel() + err := harness.Await(t, result) + if !errors.Is(err, context.Canceled) && status.Code(err) != codes.Canceled { + t.Fatalf("caller cancellation=%v", err) + } + + harness.Await(t, block.Cancelled) + harness.Await(t, block.Finished) + + if operation == "Start" || operation == "Continue" { + snapshot := h.RuntimeSpy().Recorder().Snapshot() + if snapshot.Count(snapshot.OfKind("debugger")[0].ID, "Close") != 1 { + t.Fatal("cancelled debugger not closed before return") + } + } + + if operation == "runtime" || operation == "session" { + if h.Faults().Count(harness.ReleaseExecution) != 1 { + t.Fatal("cancelled execution was not released") + } + + if _, err := h.Runtime().Run(h.Context(), api.Source{Content: "RETURN 2"}); err != nil { + t.Fatalf("cancelled execution retained the only execution slot: %v", err) + } + } + }) + } +} + +func TestCompileCancellationPreservesDetachedAllocation(t *testing.T) { + for _, debug := range []bool{false, true} { + t.Run(map[bool]string{false: "normal", true: "debug"}[debug], func(t *testing.T) { + block := harness.NewBlock(t) + hostedContext := make(chan context.Context, 1) + h := harness.New(t, harness.WithBehavior(harness.RuntimeBehavior{Compile: func(ctx context.Context, _ api.Source, _ bool, _ harness.CompileOptions) error { + hostedContext <- ctx + + return block.Wait(ctx) + }})) + compile := h.Runtime().Compile + + if debug { + compile = h.Runtime().CompileDebug + } + + ctx, cancel := context.WithCancel(h.Context()) + defer cancel() + result := make(chan error, 1) + go func() { + plan, err := compile(ctx, api.Source{Content: "RETURN 1"}) + if plan != nil { + t.Error("cancelled compile returned a plan") + err = errors.Join(err, errors.New("cancelled compile returned a plan"), plan.Close()) + } + + result <- err + }() + harness.Await(t, block.Started) + hostCtx := harness.Await(t, hostedContext) + cancel() + + if hostCtx.Err() != nil { + t.Fatalf("allocation incorrectly inherited caller cancellation: %v", hostCtx.Err()) + } + + block.Release() + + if err := harness.Await(t, result); !errors.Is(err, context.Canceled) { + t.Fatalf("compile cancellation=%v", err) + } + + snapshot := h.RuntimeSpy().Recorder().Snapshot() + plans := snapshot.OfKind("plan") + if len(plans) != 1 || snapshot.Count(plans[0].ID, "Close") != 1 || h.Faults().Count(harness.CloseRuntime) != 0 { + t.Fatalf("cancelled allocation was not reclaimed narrowly: %+v", snapshot) + } + }) + } +} + +func TestLogicalShutdownCancelsHostedCompile(t *testing.T) { + block := harness.NewBlock(t) + h := harness.New(t, harness.WithBehavior(harness.RuntimeBehavior{Compile: func(ctx context.Context, _ api.Source, _ bool, _ harness.CompileOptions) error { + return block.Wait(ctx) + }})) + result := make(chan error, 1) + go func() { + _, err := h.Runtime().Compile(h.Context(), api.Source{Content: "RETURN 1"}) + result <- err + }() + harness.Await(t, block.Started) + + if err := h.Runtime().Close(); err != nil { + t.Fatal(err) + } + + if err := harness.Await(t, result); err == nil { + t.Fatal("compile succeeded after logical shutdown") + } + + harness.Await(t, block.Cancelled) + harness.Await(t, block.Finished) + h.RuntimeSpy().Recorder().AssertClosed(t) +} diff --git a/test/integration/connection_test.go b/test/integration/connection_test.go new file mode 100644 index 0000000..19d37dc --- /dev/null +++ b/test/integration/connection_test.go @@ -0,0 +1,198 @@ +package integration_test + +import ( + "context" + "errors" + "io" + "testing" + + "github.com/MontFerret/api" + "github.com/MontFerret/api/debugger" + "github.com/MontFerret/wire/client" + "github.com/MontFerret/wire/pkg/failure" + "github.com/MontFerret/wire/test/integration/harness" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestUnavailableServer(t *testing.T) { + h := harness.New(t, harness.WithUnavailableServer()) + runtime, err := h.OpenRuntime() + if runtime != nil || status.Code(err) != codes.Unavailable { + t.Fatalf("unavailable handshake: runtime=%T err=%v", runtime, err) + } + + var remote *client.Error + if !errors.As(err, &remote) || remote.Category != 0 { + t.Fatalf("transport error acquired a hosted category: %v", err) + } +} + +func TestConnectionLossReclaimsResources(t *testing.T) { + for _, mode := range []string{"runtime", "session", "debugger"} { + for _, shutdown := range []bool{false, true} { + t.Run(mode+map[bool]string{false: "/transport", true: "/server"}[shutdown], func(t *testing.T) { + block := harness.NewBlock(t) + h := harness.New(t, harness.WithBehavior(harness.RuntimeBehavior{ + Run: func(ctx context.Context, _ api.Source, _ harness.SessionOptions) (api.Output, error) { + return api.Output{}, block.Wait(ctx) + }, + Plan: harness.PlanBehavior{ + Session: func(harness.SessionOptions) harness.SessionBehavior { + return harness.SessionBehavior{Run: func(ctx context.Context, _ int) (api.Output, error) { return api.Output{}, block.Wait(ctx) }} + }, + Debugger: harness.DebuggerBehavior{Command: func(ctx context.Context, _ string, _ int) (*debugger.Event, error) { return nil, block.Wait(ctx) }}, + }, + })) + run := func() error { + _, err := h.Runtime().Run(h.Context(), api.Source{Content: "RETURN 1"}) + + return err + } + var terminalEvent *debugger.Event + + if mode != "runtime" { + plan, err := h.Runtime().CompileDebug(h.Context(), api.Source{Content: "RETURN 1"}) + if err != nil { + t.Fatal(err) + } + + if mode == "session" { + session, err := plan.NewSession(h.Context()) + if err != nil { + t.Fatal(err) + } + + run = func() error { + _, err := session.Run(h.Context()) + + return err + } + } else { + session, err := plan.NewDebugSession(h.Context()) + if err != nil { + t.Fatal(err) + } + + run = func() error { + var err error + terminalEvent, err = session.Start(h.Context()) + + return err + } + } + } + + result := make(chan error, 1) + go func() { result <- run() }() + harness.Await(t, block.Started) + stop := h.CloseTransport + + if shutdown { + stop = h.Shutdown + } + + if err := stop(); err != nil { + t.Fatal(err) + } + + err := harness.Await(t, result) + if err == nil && (!shutdown || mode != "debugger" || terminalEvent == nil || terminalEvent.Reason != debugger.ReasonTerminated) { + t.Fatalf("active operation silently succeeded after connection loss: event=%#v", terminalEvent) + } + + if code := status.Code(err); err != nil && code != codes.Unavailable && code != codes.Canceled && !errors.Is(err, client.ErrClosed) && !errors.Is(err, client.ErrExecutionCancelled) { + t.Fatalf("connection failure classification=%v", err) + } + + harness.Await(t, block.Cancelled) + harness.Await(t, block.Finished) + h.RuntimeSpy().Recorder().AssertClosed(t) + }) + } + } +} + +func TestWatchTerminationReturnsError(t *testing.T) { + for _, debug := range []bool{false, true} { + for _, watchErr := range []error{io.EOF, status.Error(codes.Unavailable, "watch transport lost")} { + t.Run(map[bool]string{false: "execution/", true: "debugger/"}[debug]+watchErr.Error(), func(t *testing.T) { + block := harness.NewBlock(t) + h := harness.New(t, harness.WithBehavior(harness.RuntimeBehavior{ + Run: func(ctx context.Context, _ api.Source, _ harness.SessionOptions) (api.Output, error) { + return api.Output{}, block.Wait(ctx) + }, + Plan: harness.PlanBehavior{Debugger: harness.DebuggerBehavior{Command: func(ctx context.Context, _ string, _ int) (*debugger.Event, error) { return nil, block.Wait(ctx) }}}, + })) + operation := harness.WatchExecution + run := func() error { + _, err := h.Runtime().Run(h.Context(), api.Source{Content: "RETURN 1"}) + + return err + } + var closeDebug func() error + + if debug { + plan, err := h.Runtime().CompileDebug(h.Context(), api.Source{Content: "RETURN 1"}) + if err != nil { + t.Fatal(err) + } + + session, err := plan.NewDebugSession(h.Context()) + if err != nil { + t.Fatal(err) + } + + operation = harness.WatchDebugger + run = func() error { + _, err := session.Start(h.Context()) + + return err + } + closeDebug = session.Close + } + + h.Faults().EndWatch(operation, watchErr, block.Started) + err := run() + if err == nil || (watchErr == io.EOF && !errors.Is(err, io.EOF)) || (watchErr != io.EOF && status.Code(err) != codes.Unavailable) { + t.Fatalf("watch failure=%v", err) + } + + if closeDebug != nil { + if err := closeDebug(); err != nil { + t.Fatal(err) + } + } + + harness.Await(t, block.Started) + harness.Await(t, block.Cancelled) + harness.Await(t, block.Finished) + }) + } + } +} + +func TestDebuggerCommandFailure(t *testing.T) { + h := harness.New(t, harness.WithBehavior(harness.RuntimeBehavior{Plan: harness.PlanBehavior{Debugger: harness.DebuggerBehavior{Command: func(context.Context, string, int) (*debugger.Event, error) { + return nil, errors.New("private command failure") + }}}})) + plan, err := h.Runtime().CompileDebug(h.Context(), api.Source{Content: "RETURN 1"}) + if err != nil { + t.Fatal(err) + } + + session, err := plan.NewDebugSession(h.Context()) + if err != nil { + t.Fatal(err) + } + + _, err = session.Start(h.Context()) + var remote *failure.Failure + if !errors.As(err, &remote) || remote.Category != failure.CategoryInternalRuntime || remote.Message != "runtime operation failed" { + t.Fatalf("debug command failure=%v", err) + } + + if err := session.Close(); err != nil { + t.Fatal(err) + } +} diff --git a/test/integration/debugger_test.go b/test/integration/debugger_test.go new file mode 100644 index 0000000..339b7f1 --- /dev/null +++ b/test/integration/debugger_test.go @@ -0,0 +1,279 @@ +package integration_test + +import ( + "context" + "errors" + "reflect" + "testing" + + "github.com/MontFerret/api" + "github.com/MontFerret/api/debugger" + "github.com/MontFerret/api/source" + "github.com/MontFerret/wire/client" + "github.com/MontFerret/wire/pkg/failure" + "github.com/MontFerret/wire/test/integration/harness" +) + +func TestDebuggerRoundTrip(t *testing.T) { + pause := harness.NewBlock(t) + location := source.Range{Location: source.Location{SourceName: "debug.fql", Position: source.Position{Line: 2, Column: 3}}, Span: source.Span{Start: 3, End: 8}} + h := harness.New(t, harness.WithBehavior(harness.RuntimeBehavior{Plan: harness.PlanBehavior{Debugger: harness.DebuggerBehavior{ + Command: func(ctx context.Context, method string, call int) (*debugger.Event, error) { + switch method { + case "Start": + return &debugger.Event{Reason: debugger.ReasonEntry, Location: location, Depth: 2}, nil + case "Continue": + if call == 1 { + return &debugger.Event{Reason: debugger.ReasonBreakpoint, Location: location, Depth: 3, HitBreakpointIDs: []debugger.BreakpointID{2}}, nil + } + + if call == 2 { + if err := pause.Wait(ctx); err != nil { + return nil, err + } + + return &debugger.Event{Reason: debugger.ReasonPause, Location: location, Depth: 2}, nil + } + + return &debugger.Event{Reason: debugger.ReasonCompleted, Output: &api.Output{ContentType: "application/json", Content: []byte(`{"done":true}`)}}, nil + case "StepOut": + return &debugger.Event{Reason: debugger.ReasonRuntimeError, Location: location, Depth: 1, Error: errors.New("hosted debugger secret")}, nil + default: + return &debugger.Event{Reason: debugger.ReasonStep, Location: location, Depth: 2}, nil + } + }, Pause: func() error { + pause.Release() + + return nil + }, + }}})) + plan, err := h.Runtime().CompileDebug(h.Context(), api.Source{Name: "debug.fql", Content: "RETURN @input"}) + if err != nil { + t.Fatal(err) + } + + session, err := plan.NewDebugSession(h.Context(), api.WithParam("input", int64(7)), api.WithOutputContentType("application/json")) + if err != nil { + t.Fatal(err) + } + + var breakpoints []debugger.Breakpoint + + for index, mode := range []debugger.BreakpointBindingMode{debugger.BreakpointBindNextExecutableInSource, debugger.BreakpointBindExact, debugger.BreakpointBindNextExecutableInFunction} { + requested := source.Location{SourceName: "debug.fql", Position: source.Position{Line: index + 1, Column: 3}} + var breakpoint debugger.Breakpoint + + if index == 0 { + breakpoint, err = session.SetBreakpoint(requested) + } else { + breakpoint, err = session.SetBreakpointAt(requested, debugger.BreakpointOptions{BindingMode: mode}) + } + + if err != nil { + t.Fatal(err) + } + + want := debugger.Breakpoint{ID: debugger.BreakpointID(index + 1), RequestedLocation: requested, Location: source.Range{Location: requested, Span: source.Span{Start: 3, End: 8}}, PointID: debugger.PointID(index + 11), FunctionID: 7, BindingMode: mode, Bound: true} + if breakpoint != want { + t.Fatalf("breakpoint=%+v want=%+v", breakpoint, want) + } + + breakpoints = append(breakpoints, breakpoint) + } + + if !reflect.DeepEqual(session.Breakpoints(), breakpoints) { + t.Fatalf("breakpoint snapshot=%+v", session.Breakpoints()) + } + + session.Breakpoints()[0].ID = 999 + + if !reflect.DeepEqual(session.Breakpoints(), breakpoints) { + t.Fatal("breakpoint snapshot not defensive") + } + + if err := session.DeleteBreakpoint(breakpoints[0].ID); err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(session.Breakpoints(), breakpoints[1:]) { + t.Fatal("breakpoint deletion not reflected") + } + + entry, err := session.Start(h.Context()) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(entry, &debugger.Event{Reason: debugger.ReasonEntry, Location: location, Depth: 2}) { + t.Fatalf("entry event changed (including nil Error): %#v", entry) + } + + stopped, err := session.Continue(h.Context()) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(stopped, &debugger.Event{Reason: debugger.ReasonBreakpoint, Location: location, Depth: 3, HitBreakpointIDs: []debugger.BreakpointID{2}}) { + t.Fatalf("breakpoint event=%#v", stopped) + } + + frames, err := session.Frames() + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(frames, []debugger.Frame{ + {Name: "top", Location: source.Location{SourceName: "debug.fql", Position: source.Position{Line: 2, Column: 3}}, FunctionID: 7}, + {Name: "caller", Location: source.Location{SourceName: "caller.fql", Position: source.Position{Line: 4, Column: 5}}, FunctionID: 6}, + }) { + t.Fatalf("frames=%+v", frames) + } + + for index := range 2 { + var locals []debugger.Variable + + if index == 0 { + locals, err = session.Locals() + } else { + locals, err = session.FrameLocals(index) + } + + if err != nil { + t.Fatal(err) + } + + name := []string{"local-0", "local-1"}[index] + if !reflect.DeepEqual(locals, []debugger.Variable{{Name: name, Value: debugger.Value{Type: "object", Display: "{...}", Reference: 9}, Mutable: true, Param: index == 1}}) { + t.Fatalf("locals=%+v", locals) + } + } + + variables, err := session.Variables(9) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(variables, []debugger.Variable{{Name: "child", Value: debugger.Value{Type: "string", Display: "value"}}}) { + t.Fatalf("variables=%+v", variables) + } + + value, err := session.Evaluate(h.Context(), "local") + if err != nil || value != (debugger.Value{Type: "string", Display: "frame-0:local"}) { + t.Fatalf("Evaluate=%+v err=%v", value, err) + } + + value, err = session.EvaluateFrame(h.Context(), 1, "caller") + if err != nil || value != (debugger.Value{Type: "string", Display: "frame-1:caller"}) { + t.Fatalf("EvaluateFrame=%+v err=%v", value, err) + } + + for _, step := range []func(context.Context) (*debugger.Event, error){session.StepOver, session.StepIn} { + event, err := step(h.Context()) + if err != nil || !reflect.DeepEqual(event, &debugger.Event{Reason: debugger.ReasonStep, Location: location, Depth: 2}) { + t.Fatalf("step=%#v err=%v", event, err) + } + } + + runtimeError, err := session.StepOut(h.Context()) + if err != nil { + t.Fatal(err) + } + + var remoteFailure *failure.Failure + if runtimeError.Reason != debugger.ReasonRuntimeError || runtimeError.Location != location || runtimeError.Depth != 1 || !errors.As(runtimeError.Error, &remoteFailure) || remoteFailure.Category != failure.CategoryExecution { + t.Fatalf("runtime-error stop=%#v", runtimeError) + } + + type commandResult struct { + event *debugger.Event + err error + } + continued := make(chan commandResult, 1) + go func() { + event, err := session.Continue(h.Context()) + continued <- commandResult{event, err} + }() + harness.Await(t, pause.Started) + + if err := session.Pause(); err != nil { + t.Fatal(err) + } + + paused := harness.Await(t, continued) + if paused.err != nil || !reflect.DeepEqual(paused.event, &debugger.Event{Reason: debugger.ReasonPause, Location: location, Depth: 2}) { + t.Fatalf("pause=%+v", paused) + } + + completed, err := session.Continue(h.Context()) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(completed, &debugger.Event{Reason: debugger.ReasonCompleted, Output: &api.Output{ContentType: "application/json", Content: []byte(`{"done":true}`)}}) { + t.Fatalf("completion=%#v", completed) + } + + if err := session.Close(); err != nil { + t.Fatal(err) + } + + if err := session.Close(); err != nil { + t.Fatal(err) + } + + if _, err := session.Continue(h.Context()); !errors.Is(err, client.ErrClosed) { + t.Fatalf("command after Close=%v", err) + } + + snapshot := h.RuntimeSpy().Recorder().Snapshot() + id := snapshot.OfKind("debugger")[0].ID + var commands []string + var indices []int + var expressions []string + var requests []harness.BreakpointRequest + + for _, call := range snapshot.Calls { + if call.Resource != id { + continue + } + + switch call.Method { + case "Start", "Continue", "StepOver", "StepIn", "StepOut", "Pause": + commands = append(commands, call.Method) + case "FrameLocals", "EvaluateFrame": + indices = append(indices, call.Index) + + if call.Method == "EvaluateFrame" { + expressions = append(expressions, call.Argument.(string)) + } + case "SetBreakpointAt": + requests = append(requests, call.Argument.(harness.BreakpointRequest)) + case "DeleteBreakpoint": + if call.Argument != debugger.BreakpointID(1) { + t.Fatalf("deleted wrong breakpoint: %+v", call) + } + case "Variables": + if call.Argument != debugger.ValueReference(9) { + t.Fatalf("wrong value reference: %+v", call) + } + } + } + + if !reflect.DeepEqual(commands, []string{"Start", "Continue", "StepOver", "StepIn", "StepOut", "Continue", "Pause", "Continue"}) || !reflect.DeepEqual(indices, []int{0, 1, 0, 1}) || !reflect.DeepEqual(expressions, []string{"local", "caller"}) { + t.Fatalf("commands=%v indices=%v expressions=%v", commands, indices, expressions) + } + + if len(requests) != 3 { + t.Fatalf("breakpoint requests=%v", requests) + } + + for i, request := range requests { + if request.Location != breakpoints[i].RequestedLocation || request.Options.BindingMode != breakpoints[i].BindingMode { + t.Fatalf("breakpoint request changed: %+v", request) + } + } + + if snapshot.Count(id, "Close") != 1 { + t.Fatal("debugger cleanup was not exactly once") + } +} diff --git a/test/integration/errors_test.go b/test/integration/errors_test.go new file mode 100644 index 0000000..b64a345 --- /dev/null +++ b/test/integration/errors_test.go @@ -0,0 +1,331 @@ +package integration_test + +import ( + "context" + "errors" + "reflect" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/MontFerret/api" + "github.com/MontFerret/api/debugger" + "github.com/MontFerret/api/diagnostics" + "github.com/MontFerret/api/source" + "github.com/MontFerret/wire/client" + "github.com/MontFerret/wire/pkg/failure" + "github.com/MontFerret/wire/server" + "github.com/MontFerret/wire/test/integration/harness" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestDiagnosticsAndFailureClassification(t *testing.T) { + for _, mode := range []string{"compile", "compile debug", "runtime", "session", "debugger"} { + for count := range 3 { + t.Run(mode+"/"+[]string{"plain", "single", "multiple"}[count], func(t *testing.T) { + var values diagnostics.Diagnostics + + for index := range count { + values = append(values, diagnostics.Diagnostic{Kind: diagnostics.Kind("CustomKind"), Message: "expected expression", Source: source.New("query.fql", "RETURN @input"), Annotations: []diagnostics.Annotation{ + {Range: source.Range{Location: source.Location{SourceName: "query.fql", Position: source.Position{Line: index + 1, Column: 2}}, Span: source.Span{Start: 1, End: 4}}, Message: "primary span", Primary: true}, + {Range: source.Range{Location: source.Location{SourceName: "query.fql", Position: source.Position{Line: 1, Column: 5}}, Span: source.Span{Start: 4, End: 4}}, Message: "secondary span"}, + }, Hint: "provide a value", Note: "portable note"}) + } + + hostErr := errors.Join(errors.New("private-host-secret"), values.Err()) + behavior := harness.RuntimeBehavior{ + Run: func(context.Context, api.Source, harness.SessionOptions) (api.Output, error) { + return api.Output{}, hostErr + }, + Plan: harness.PlanBehavior{ + Session: func(harness.SessionOptions) harness.SessionBehavior { + return harness.SessionBehavior{Run: func(context.Context, int) (api.Output, error) { return api.Output{}, hostErr }} + }, + Debugger: harness.DebuggerBehavior{Command: func(context.Context, string, int) (*debugger.Event, error) { + return &debugger.Event{Reason: debugger.ReasonRuntimeError, Error: hostErr}, nil + }}, + }, + } + + if strings.HasPrefix(mode, "compile") { + behavior.Compile = func(context.Context, api.Source, bool, harness.CompileOptions) error { return hostErr } + } + + h := harness.New(t, harness.WithBehavior(behavior)) + var err error + src := api.Source{Name: "query.fql", Content: "RETURN @input"} + + switch mode { + case "compile": + _, err = h.Runtime().Compile(h.Context(), src) + case "compile debug": + _, err = h.Runtime().CompileDebug(h.Context(), src) + case "runtime": + _, err = h.Runtime().Run(h.Context(), src) + default: + plan, createErr := h.Runtime().CompileDebug(h.Context(), src) + if createErr != nil { + t.Fatal(createErr) + } + + if mode == "session" { + session, createErr := plan.NewSession(h.Context()) + if createErr != nil { + t.Fatal(createErr) + } + + _, err = session.Run(h.Context()) + } else { + session, createErr := plan.NewDebugSession(h.Context()) + if createErr != nil { + t.Fatal(createErr) + } + + event, commandErr := session.Start(h.Context()) + if commandErr != nil { + t.Fatal(commandErr) + } + + err = event.Error + } + } + + if err == nil || strings.Contains(err.Error(), "private-host-secret") { + t.Fatalf("failure absent or unsanitized: %v", err) + } + + var actual diagnostics.Diagnostics + + if strings.HasPrefix(mode, "compile") { + var remote *client.Error + if !errors.As(err, &remote) || remote.Category != failure.CategoryCompilation || status.Code(err) != codes.InvalidArgument || remote.Message != "compilation failed" { + t.Fatalf("compile classification=%v", err) + } + + actual = remote.Diagnostics + } else { + var remote *failure.Failure + if !errors.As(err, &remote) || remote.Category != failure.CategoryExecution || remote.Message != "runtime operation failed" { + t.Fatalf("execution classification=%v", err) + } + + actual = remote.Diagnostics + } + + if !reflect.DeepEqual(actual, values) { + t.Fatalf("diagnostics=%#v want=%#v", actual, values) + } + }) + } + } +} + +func TestErrorFamilies(t *testing.T) { + t.Run("invalid source and portable value", func(t *testing.T) { + h := harness.New(t) + + if _, err := h.Runtime().Compile(h.Context(), api.Source{}); status.Code(err) != codes.InvalidArgument { + t.Fatalf("invalid source=%v", err) + } + + if _, err := h.Runtime().Run(h.Context(), api.Source{Content: "RETURN 1"}, api.WithParam("bad", make(chan int))); err == nil { + t.Fatal("unsupported value accepted") + } + + if h.Faults().Count(harness.RunRuntime) != 0 { + t.Fatal("local validation dispatched runtime execution") + } + + if _, err := h.Runtime().Run(h.Context(), api.Source{Content: "RETURN 2"}); err != nil { + t.Fatalf("rejection invalidated Runtime: %v", err) + } + }) + t.Run("not found and invalid debug state", func(t *testing.T) { + h := harness.New(t) + plan, err := h.Runtime().CompileDebug(h.Context(), api.Source{Content: "RETURN 1"}) + if err != nil { + t.Fatal(err) + } + + session, err := plan.NewDebugSession(h.Context()) + if err != nil { + t.Fatal(err) + } + + err = session.DeleteBreakpoint(987) + var remote *client.Error + if !errors.As(err, &remote) || remote.Category != failure.CategoryBreakpointNotFound || status.Code(err) != codes.NotFound { + t.Fatalf("not-found classification=%v", err) + } + + if _, err := session.Frames(); !errors.As(err, &remote) || remote.Category != failure.CategoryInvalidState || status.Code(err) != codes.FailedPrecondition { + t.Fatalf("invalid-state classification=%v", err) + } + }) + t.Run("deadline exceeded", func(t *testing.T) { + h := harness.New(t) + ctx, cancel := context.WithDeadline(h.Context(), time.Now().Add(-time.Second)) + defer cancel() + + if _, err := h.Runtime().Run(ctx, api.Source{Content: "RETURN 1"}); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("caller deadline=%v", err) + } + + if h.Faults().Count(harness.RunRuntime) != 0 { + t.Fatal("expired call dispatched") + } + }) + t.Run("remote cancellation differs from caller cancellation", func(t *testing.T) { + h := harness.New(t, harness.WithBehavior(harness.RuntimeBehavior{Run: func(context.Context, api.Source, harness.SessionOptions) (api.Output, error) { + return api.Output{}, context.Canceled + }})) + + if _, err := h.Runtime().Run(h.Context(), api.Source{Content: "RETURN 1"}); !errors.Is(err, client.ErrExecutionCancelled) || errors.Is(err, context.Canceled) { + t.Fatalf("remote cancellation=%v", err) + } + }) + t.Run("protocol resource limit", func(t *testing.T) { + limits := server.DefaultLimits() + limits.MaxPlansPerConnection = 1 + h := harness.New(t, harness.WithServerOptions(server.WithLimits(limits))) + + if _, err := h.Runtime().Compile(h.Context(), api.Source{Content: "RETURN 1"}); err != nil { + t.Fatal(err) + } + + if _, err := h.Runtime().Compile(h.Context(), api.Source{Content: "RETURN 2"}); status.Code(err) != codes.ResourceExhausted { + t.Fatalf("limit rejection=%v", err) + } + + if h.RuntimeSpy().Recorder().Snapshot().Count(h.RuntimeSpy().ID(), "Compile") != 1 { + t.Fatal("rejected allocation reached host") + } + }) +} + +func TestConstructorPanicPreservesParent(t *testing.T) { + var attempts atomic.Int32 + h := harness.New(t, harness.WithBehavior(harness.RuntimeBehavior{Plan: harness.PlanBehavior{NewSession: func(context.Context, harness.SessionOptions) error { + if attempts.Add(1) == 1 { + panic("constructor-secret") + } + + return nil + }}})) + plan, err := h.Runtime().Compile(h.Context(), api.Source{Content: "RETURN 1"}) + if err != nil { + t.Fatal(err) + } + + if _, err := plan.NewSession(h.Context()); status.Code(err) != codes.Internal || strings.Contains(err.Error(), "constructor-secret") { + t.Fatalf("constructor panic=%v", err) + } + + session, err := plan.NewSession(h.Context()) + if err != nil { + t.Fatalf("constructor panic invalidated parent: %v", err) + } + + if _, err := session.Run(h.Context()); err != nil { + t.Fatal(err) + } +} + +func TestPanicContainmentAndResourcePoisoning(t *testing.T) { + for _, mode := range []string{"runtime", "compile", "session", "debugger", "session close"} { + t.Run(mode, func(t *testing.T) { + behavior := harness.RuntimeBehavior{} + + switch mode { + case "runtime": + behavior.Run = func(context.Context, api.Source, harness.SessionOptions) (api.Output, error) { panic("panic-secret") } + case "compile": + behavior.Compile = func(context.Context, api.Source, bool, harness.CompileOptions) error { panic("panic-secret") } + case "session": + behavior.Plan.Session = func(harness.SessionOptions) harness.SessionBehavior { + return harness.SessionBehavior{Run: func(context.Context, int) (api.Output, error) { panic("panic-secret") }} + } + case "debugger": + behavior.Plan.Debugger.Inspect = func(string) error { panic("panic-secret") } + case "session close": + behavior.Plan.Session = func(harness.SessionOptions) harness.SessionBehavior { + return harness.SessionBehavior{Close: func() error { panic("panic-secret") }} + } + } + + h := harness.New(t, harness.WithBehavior(behavior)) + var err error + + switch mode { + case "runtime": + _, err = h.Runtime().Run(h.Context(), api.Source{Content: "RETURN 1"}) + case "compile": + _, err = h.Runtime().Compile(h.Context(), api.Source{Content: "RETURN 1"}) + default: + plan, createErr := h.Runtime().CompileDebug(h.Context(), api.Source{Content: "RETURN 1"}) + if createErr != nil { + t.Fatal(createErr) + } + + if mode == "debugger" { + session, createErr := plan.NewDebugSession(h.Context()) + if createErr != nil { + t.Fatal(createErr) + } + + if _, startErr := session.Start(h.Context()); startErr != nil { + t.Fatal(startErr) + } + + _, err = session.Frames() + + if _, nextErr := session.Frames(); nextErr == nil { + t.Fatal("poisoned debugger was reused") + } + + h.RuntimeSpy().Recorder().Wait(t, "panicked debugger closes", func(s harness.Snapshot) bool { return s.Count(s.OfKind("debugger")[0].ID, "Close") == 1 }) + snapshot := h.RuntimeSpy().Recorder().Snapshot() + if snapshot.Count(snapshot.OfKind("debugger")[0].ID, "Frames") != 1 { + t.Fatal("a poisoned hosted debugger received another inspection call") + } + } else { + session, createErr := plan.NewSession(h.Context()) + if createErr != nil { + t.Fatal(createErr) + } + + if mode == "session close" { + err = session.Close() + + if second := session.Close(); second == nil { + t.Fatal("cleanup panic was not retained") + } + } else { + _, err = session.Run(h.Context()) + + if _, nextErr := session.Run(h.Context()); status.Code(nextErr) != codes.FailedPrecondition { + t.Fatalf("poisoned Session reuse=%v", nextErr) + } + } + } + } + + if err == nil || strings.Contains(err.Error(), "panic-secret") { + t.Fatalf("panic unsanitized or lost: %v", err) + } + + var rpc *client.Error + var terminal *failure.Failure + + if errors.As(err, &rpc) { + if rpc.Category != failure.CategoryInternalRuntime || status.Code(err) != codes.Internal { + t.Fatalf("panic RPC=%+v", rpc) + } + } else if !errors.As(err, &terminal) || terminal.Category != failure.CategoryInternalRuntime { + t.Fatalf("panic classification=%v", err) + } + }) + } +} diff --git a/test/integration/harness/coordination.go b/test/integration/harness/coordination.go new file mode 100644 index 0000000..e657a9c --- /dev/null +++ b/test/integration/harness/coordination.go @@ -0,0 +1,68 @@ +package harness + +import ( + "context" + "sync" + "testing" + "time" +) + +// Block provides observable operation entry, cancellation, and settlement. +// A fixture must use a fresh Block for each independently blocked invocation. +type Block struct { + Started chan struct{} + Cancelled chan struct{} + Finished chan struct{} + release chan struct{} + once sync.Once +} + +func NewBlock(t testing.TB) *Block { + block := &Block{Started: make(chan struct{}), Cancelled: make(chan struct{}), Finished: make(chan struct{}), release: make(chan struct{})} + t.Cleanup(block.Release) + + return block +} + +func (b *Block) Wait(ctx context.Context) error { + close(b.Started) + defer close(b.Finished) + + select { + case <-ctx.Done(): + close(b.Cancelled) + + return ctx.Err() + case <-b.release: + return nil + } +} + +func (b *Block) Release() { + b.once.Do(func() { close(b.release) }) +} + +func Await[T any](t testing.TB, channel <-chan T) T { + t.Helper() + timer := time.NewTimer(10 * time.Second) + defer timer.Stop() + + select { + case value := <-channel: + return value + case <-timer.C: + t.Fatal("timed out waiting for coordinated operation") + + var zero T + + return zero + } +} + +func Context(t testing.TB) context.Context { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + t.Cleanup(cancel) + + return ctx +} diff --git a/test/integration/harness/debugger.go b/test/integration/harness/debugger.go new file mode 100644 index 0000000..8e8a712 --- /dev/null +++ b/test/integration/harness/debugger.go @@ -0,0 +1,192 @@ +package harness + +import ( + "context" + "fmt" + "sort" + "sync" + + "github.com/MontFerret/api/debugger" + "github.com/MontFerret/api/source" +) + +type ( + DebuggerBehavior struct { + Command func(context.Context, string, int) (*debugger.Event, error) + Pause func() error + Inspect func(string) error + Evaluate func(context.Context, int, string) (debugger.Value, error) + Close func() error + } + + BreakpointRequest struct { + Location source.Location + Options debugger.BreakpointOptions + } + + DebuggerSpy struct { + id int + recorder *Recorder + behavior DebuggerBehavior + mu sync.Mutex + breakpoints map[debugger.BreakpointID]debugger.Breakpoint + nextID debugger.BreakpointID + } +) + +var _ debugger.Session = (*DebuggerSpy)(nil) + +func newDebuggerSpy(recorder *Recorder, parent int, behavior DebuggerBehavior) *DebuggerSpy { + return &DebuggerSpy{id: recorder.create("debugger", parent), recorder: recorder, behavior: behavior, breakpoints: make(map[debugger.BreakpointID]debugger.Breakpoint)} +} + +func (d *DebuggerSpy) command(ctx context.Context, method string) (*debugger.Event, error) { + call := d.recorder.record(Call{Resource: d.id, Method: method}) + defer d.recorder.record(Call{Resource: d.id, Method: method + "Finished"}) + + if d.behavior.Command != nil { + return d.behavior.Command(ctx, method, call) + } + + reason := debugger.ReasonStep + + if method == "Start" { + reason = debugger.ReasonEntry + } else if method == "Continue" { + reason = debugger.ReasonCompleted + } + + return &debugger.Event{Reason: reason}, nil +} + +func (d *DebuggerSpy) Start(ctx context.Context) (*debugger.Event, error) { + return d.command(ctx, "Start") +} + +func (d *DebuggerSpy) Continue(ctx context.Context) (*debugger.Event, error) { + return d.command(ctx, "Continue") +} + +func (d *DebuggerSpy) StepOver(ctx context.Context) (*debugger.Event, error) { + return d.command(ctx, "StepOver") +} + +func (d *DebuggerSpy) StepIn(ctx context.Context) (*debugger.Event, error) { + return d.command(ctx, "StepIn") +} + +func (d *DebuggerSpy) StepOut(ctx context.Context) (*debugger.Event, error) { + return d.command(ctx, "StepOut") +} + +func (d *DebuggerSpy) Pause() error { + d.recorder.record(Call{Resource: d.id, Method: "Pause"}) + + if d.behavior.Pause != nil { + return d.behavior.Pause() + } + + return nil +} + +func (d *DebuggerSpy) SetBreakpoint(location source.Location) (debugger.Breakpoint, error) { + return d.SetBreakpointAt(location, debugger.BreakpointOptions{}) +} + +func (d *DebuggerSpy) SetBreakpointAt(location source.Location, options debugger.BreakpointOptions) (debugger.Breakpoint, error) { + d.recorder.record(Call{Resource: d.id, Method: "SetBreakpointAt", Argument: BreakpointRequest{Location: location, Options: options}}) + d.mu.Lock() + defer d.mu.Unlock() + + d.nextID++ + value := debugger.Breakpoint{ID: d.nextID, RequestedLocation: location, Location: source.Range{Location: location, Span: source.Span{Start: 3, End: 8}}, PointID: debugger.PointID(10 + d.nextID), FunctionID: 7, BindingMode: options.BindingMode, Bound: true} + d.breakpoints[value.ID] = value + + return value, nil +} + +func (d *DebuggerSpy) DeleteBreakpoint(id debugger.BreakpointID) error { + d.recorder.record(Call{Resource: d.id, Method: "DeleteBreakpoint", Argument: id}) + d.mu.Lock() + delete(d.breakpoints, id) + d.mu.Unlock() + + return nil +} + +func (d *DebuggerSpy) Breakpoints() []debugger.Breakpoint { + d.recorder.record(Call{Resource: d.id, Method: "Breakpoints"}) + d.mu.Lock() + defer d.mu.Unlock() + + result := make([]debugger.Breakpoint, 0, len(d.breakpoints)) + + for _, value := range d.breakpoints { + result = append(result, value) + } + + sort.Slice(result, func(i, j int) bool { return result[i].ID < result[j].ID }) + + return result +} + +func (d *DebuggerSpy) Frames() ([]debugger.Frame, error) { + d.recorder.record(Call{Resource: d.id, Method: "Frames"}) + + if d.behavior.Inspect != nil { + if err := d.behavior.Inspect("Frames"); err != nil { + return nil, err + } + } + + return []debugger.Frame{ + {Name: "top", Location: source.Location{SourceName: "debug.fql", Position: source.Position{Line: 2, Column: 3}}, FunctionID: 7}, + {Name: "caller", Location: source.Location{SourceName: "caller.fql", Position: source.Position{Line: 4, Column: 5}}, FunctionID: 6}, + }, nil +} + +func (d *DebuggerSpy) Locals() ([]debugger.Variable, error) { + return d.FrameLocals(0) +} + +func (d *DebuggerSpy) FrameLocals(frame int) ([]debugger.Variable, error) { + d.recorder.record(Call{Resource: d.id, Method: "FrameLocals", Index: frame}) + + return []debugger.Variable{{Name: fmt.Sprintf("local-%d", frame), Value: debugger.Value{Type: "object", Display: "{...}", Reference: 9}, Mutable: true, Param: frame == 1}}, nil +} + +func (d *DebuggerSpy) Variables(reference debugger.ValueReference) ([]debugger.Variable, error) { + d.recorder.record(Call{Resource: d.id, Method: "Variables", Argument: reference}) + + if reference != 9 { + return nil, fmt.Errorf("unknown fixture reference %d", reference) + } + + return []debugger.Variable{{Name: "child", Value: debugger.Value{Type: "string", Display: "value"}}}, nil +} + +func (d *DebuggerSpy) Evaluate(ctx context.Context, expression string) (debugger.Value, error) { + return d.EvaluateFrame(ctx, 0, expression) +} + +func (d *DebuggerSpy) EvaluateFrame(ctx context.Context, frame int, expression string) (debugger.Value, error) { + d.recorder.record(Call{Resource: d.id, Method: "EvaluateFrame", Index: frame, Argument: expression}) + defer d.recorder.record(Call{Resource: d.id, Method: "EvaluateFrameFinished"}) + + if d.behavior.Evaluate != nil { + return d.behavior.Evaluate(ctx, frame, expression) + } + + return debugger.Value{Type: "string", Display: fmt.Sprintf("frame-%d:%s", frame, expression)}, nil +} + +func (d *DebuggerSpy) Close() error { + d.recorder.record(Call{Resource: d.id, Method: "Close"}) + defer d.recorder.record(Call{Resource: d.id, Method: "CloseFinished"}) + + if d.behavior.Close != nil { + return d.behavior.Close() + } + + return nil +} diff --git a/test/integration/harness/failure.go b/test/integration/harness/failure.go new file mode 100644 index 0000000..730ed48 --- /dev/null +++ b/test/integration/harness/failure.go @@ -0,0 +1,280 @@ +package harness + +import ( + "context" + "sync" + + wirev1 "github.com/MontFerret/wire/gen/ferret/wire/v1" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" +) + +type ( + // Operation names faults without exposing protobuf services to contract tests. + Operation string + Outcome string + + ResponseGate struct { + Committed chan struct{} + deliver chan struct{} + once sync.Once + outcome Outcome + } + + // Faults forwards real RPCs. Allocation faults only alter already received replies. + Faults struct { + grpc.ClientConnInterface + mu sync.Mutex + gates map[Operation]*ResponseGate + allGates []*ResponseGate + failures map[Operation]error + responseFailures map[Operation]error + watchFailures map[Operation]watchFailure + sequence []Operation + } + + watchFailure struct { + err error + after <-chan struct{} + } + + failingStream struct { + grpc.ClientStream + cancel context.CancelFunc + err error + received bool + after <-chan struct{} + } +) + +const ( + Compile Operation = "compile" + CompileDebug Operation = "compile debug" + CreateSession Operation = "session" + CreateDebugger Operation = "debug session" + RunSession Operation = "session run" + RunRuntime Operation = "runtime run" + ReleasePlan Operation = "release plan" + ReleaseSession Operation = "release session" + ReleaseDebugger Operation = "release debugger" + ReleaseExecution Operation = "release execution" + CloseRuntime Operation = "close runtime" + CancelExecution Operation = "cancel execution" + WatchExecution Operation = "watch execution" + WatchDebugger Operation = "watch debugger" + + Deliver Outcome = "success" + LostDeadline Outcome = "deadline" + LostUnavailable Outcome = "unavailable" + LostOversized Outcome = "oversized" + LostDecode Outcome = "transport internal" + Malformed Outcome = "malformed" +) + +func operationFor(method string) Operation { + switch method { + case wirev1.PlanService_Compile_FullMethodName: + return Compile + case wirev1.PlanService_CompileDebug_FullMethodName: + return CompileDebug + case wirev1.SessionService_CreateSession_FullMethodName: + return CreateSession + case wirev1.DebugService_CreateDebugSession_FullMethodName: + return CreateDebugger + case wirev1.ExecutionService_RunSession_FullMethodName: + return RunSession + case wirev1.RuntimeService_Run_FullMethodName: + return RunRuntime + case wirev1.PlanService_ReleasePlan_FullMethodName: + return ReleasePlan + case wirev1.SessionService_ReleaseSession_FullMethodName: + return ReleaseSession + case wirev1.DebugService_ReleaseDebugSession_FullMethodName: + return ReleaseDebugger + case wirev1.ExecutionService_ReleaseExecution_FullMethodName: + return ReleaseExecution + case wirev1.RuntimeService_CloseConnection_FullMethodName: + return CloseRuntime + case wirev1.ExecutionService_CancelExecution_FullMethodName: + return CancelExecution + case wirev1.ExecutionService_WatchExecution_FullMethodName: + return WatchExecution + case wirev1.DebugService_WatchDebug_FullMethodName: + return WatchDebugger + default: + return Operation(method) + } +} + +func newFaults(connection grpc.ClientConnInterface) *Faults { + return &Faults{ + ClientConnInterface: connection, + gates: make(map[Operation]*ResponseGate), + failures: make(map[Operation]error), + responseFailures: make(map[Operation]error), + watchFailures: make(map[Operation]watchFailure), + } +} + +func (f *Faults) Arm(operation Operation, outcome Outcome) *ResponseGate { + f.mu.Lock() + defer f.mu.Unlock() + + gate := &ResponseGate{Committed: make(chan struct{}), deliver: make(chan struct{}), outcome: outcome} + f.gates[operation] = gate + f.allGates = append(f.allGates, gate) + + return gate +} + +func (g *ResponseGate) Deliver() { + g.once.Do(func() { close(g.deliver) }) +} + +func (f *Faults) Invoke(ctx context.Context, method string, request, response any, options ...grpc.CallOption) error { + operation := operationFor(method) + f.mu.Lock() + f.sequence = append(f.sequence, operation) + gate, failure, responseFailure := f.gates[operation], f.failures[operation], f.responseFailures[operation] + delete(f.gates, operation) + f.mu.Unlock() + + if failure != nil { + return failure + } + + if err := f.ClientConnInterface.Invoke(ctx, method, request, response, options...); err != nil { + return err + } + + if responseFailure != nil { + return responseFailure + } + + if gate == nil { + return nil + } + + close(gate.Committed) + + select { + case <-gate.deliver: + case <-ctx.Done(): + return status.FromContextError(ctx.Err()).Err() + } + + switch gate.outcome { + case LostDeadline: + return status.Error(codes.DeadlineExceeded, "allocation response lost") + case LostUnavailable: + return status.Error(codes.Unavailable, "allocation response lost") + case LostOversized: + return status.Error(codes.ResourceExhausted, "allocation response exceeds receive limit") + case LostDecode: + return status.Error(codes.Internal, "allocation response could not be decoded") + case Malformed: + proto.Reset(response.(proto.Message)) + } + + return nil +} + +func (f *Faults) NewStream(ctx context.Context, description *grpc.StreamDesc, method string, options ...grpc.CallOption) (grpc.ClientStream, error) { + f.mu.Lock() + failure, injected := f.watchFailures[operationFor(method)] + delete(f.watchFailures, operationFor(method)) + f.mu.Unlock() + + if !injected { + return f.ClientConnInterface.NewStream(ctx, description, method, options...) + } + + streamCtx, cancel := context.WithCancel(ctx) + stream, err := f.ClientConnInterface.NewStream(streamCtx, description, method, options...) + if err != nil { + cancel() + + return nil, err + } + + return &failingStream{ClientStream: stream, cancel: cancel, err: failure.err, after: failure.after}, nil +} + +func (s *failingStream) RecvMsg(message any) error { + if s.received { + if s.after != nil { + select { + case <-s.after: + case <-s.Context().Done(): + return status.FromContextError(s.Context().Err()).Err() + } + } + + s.cancel() + + return s.err + } + + s.received = true + + err := s.ClientStream.RecvMsg(message) + if err != nil { + s.cancel() + } + + return err +} + +func (f *Faults) Fail(operation Operation, err error) { + f.mu.Lock() + f.failures[operation] = err + f.mu.Unlock() +} + +func (f *Faults) FailResponse(operation Operation, err error) { + f.mu.Lock() + f.responseFailures[operation] = err + f.mu.Unlock() +} + +// EndWatch delivers a real initial snapshot, then waits for after before failing +// the next receive. A nil signal injects the failure immediately. +func (f *Faults) EndWatch(operation Operation, err error, after <-chan struct{}) { + f.mu.Lock() + f.watchFailures[operation] = watchFailure{err: err, after: after} + f.mu.Unlock() +} + +func (f *Faults) Count(operation Operation) int { + count := 0 + + for _, value := range f.Sequence() { + if value == operation { + count++ + } + } + + return count +} + +func (f *Faults) Sequence() []Operation { + f.mu.Lock() + defer f.mu.Unlock() + + return append([]Operation(nil), f.sequence...) +} + +func (f *Faults) reset() { + f.mu.Lock() + defer f.mu.Unlock() + + for _, gate := range f.allGates { + gate.Deliver() + } + + clear(f.failures) + clear(f.responseFailures) + clear(f.watchFailures) +} diff --git a/test/integration/harness/harness.go b/test/integration/harness/harness.go new file mode 100644 index 0000000..513708b --- /dev/null +++ b/test/integration/harness/harness.go @@ -0,0 +1,266 @@ +// Package harness hosts observable Universal API doubles behind the public Wire +// server and a real in-process gRPC transport. It is test infrastructure only. +package harness + +import ( + "context" + "errors" + "net" + "sync" + "testing" + "time" + + "github.com/MontFerret/api" + "github.com/MontFerret/wire/client" + "github.com/MontFerret/wire/server" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + "google.golang.org/grpc/test/bufconn" +) + +type ( + Option func(*configuration) + + configuration struct { + runtime api.Runtime + behavior RuntimeBehavior + serverOptions []server.Option + unavailable bool + } + + // Harness owns its listener and transport. Wire borrows the hosted Runtime. + Harness struct { + t testing.TB + ctx context.Context + server *server.Server + listener *bufconn.Listener + connection *grpc.ClientConn + faults *Faults + spy *RuntimeSpy + runtime api.Runtime + serveResult chan error + mu sync.Mutex + runtimes []api.Runtime + expected []error + transportClosed bool + stopped bool + } +) + +func WithBehavior(behavior RuntimeBehavior) Option { + return func(c *configuration) { c.behavior = behavior } +} + +func WithRuntime(runtime api.Runtime) Option { + return func(c *configuration) { c.runtime = runtime } +} + +func WithServerOptions(options ...server.Option) Option { + return func(c *configuration) { c.serverOptions = append(c.serverOptions, options...) } +} + +// WithUnavailableServer leaves a closed listener for handshake failure tests. +func WithUnavailableServer() Option { + return func(c *configuration) { c.unavailable = true } +} + +func New(t testing.TB, options ...Option) *Harness { + t.Helper() + var configured configuration + + for _, option := range options { + option(&configured) + } + + h := &Harness{t: t, ctx: Context(t)} + t.Cleanup(h.cleanup) + hosted := configured.runtime + if hosted == nil { + h.spy = NewRuntimeSpy(configured.behavior) + hosted = h.spy + } else { + h.spy, _ = hosted.(*RuntimeSpy) + } + + var err error + h.server, err = server.NewServer(hosted, configured.serverOptions...) + if err != nil { + t.Fatal(err) + } + + h.listener = bufconn.Listen(1 << 20) + + if configured.unavailable { + if err := h.listener.Close(); err != nil { + t.Fatal(err) + } + } else { + h.serveResult = make(chan error, 1) + go func() { h.serveResult <- h.server.Serve(context.Background(), h.listener) }() + } + + h.connection, err = grpc.NewClient( + "passthrough:///wire-integration", + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { + return h.listener.DialContext(ctx) + }), + ) + if err != nil { + t.Fatal(err) + } + + h.faults = newFaults(h.connection) + + if !configured.unavailable { + h.runtime, err = h.OpenRuntime() + if err != nil { + t.Fatal(err) + } + } + + return h +} + +func (h *Harness) Runtime() api.Runtime { + return h.runtime +} + +func (h *Harness) RuntimeSpy() *RuntimeSpy { + return h.spy +} + +func (h *Harness) Context() context.Context { + return h.ctx +} + +func (h *Harness) Faults() *Faults { + return h.faults +} + +func (h *Harness) OpenRuntime() (api.Runtime, error) { + runtime, err := client.NewRuntime(h.ctx, h.faults) + if err != nil { + return nil, err + } + + h.mu.Lock() + h.runtimes = append(h.runtimes, runtime) + h.mu.Unlock() + + return runtime, nil +} + +func (h *Harness) ExpectCleanupError(err error) { + h.mu.Lock() + h.expected = append(h.expected, err) + h.mu.Unlock() +} + +func (h *Harness) Shutdown() error { + h.mu.Lock() + h.stopped = true + h.mu.Unlock() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + return h.server.Shutdown(ctx) +} + +func (h *Harness) CloseTransport() error { + h.mu.Lock() + + if h.transportClosed { + h.mu.Unlock() + + return nil + } + + h.transportClosed = true + h.mu.Unlock() + + return h.connection.Close() +} + +func (h *Harness) expectedError(err error) bool { + if err == nil { + return true + } + + // A matching expected error must not hide another failure in errors.Join. + if joined, ok := err.(interface{ Unwrap() []error }); ok { + for _, cause := range joined.Unwrap() { + if !h.expectedError(cause) { + return false + } + } + + return true + } + + if cause := errors.Unwrap(err); cause != nil { + return h.expectedError(cause) + } + + h.mu.Lock() + defer h.mu.Unlock() + + for _, expected := range h.expected { + if errors.Is(err, expected) { + return true + } + } + + if h.transportClosed || h.stopped { + return errors.Is(err, client.ErrClosed) || status.Code(err) == codes.Unavailable || status.Code(err) == codes.Canceled + } + + return false +} + +func (h *Harness) cleanup() { + if h.faults != nil { + h.faults.reset() + } + + for i := len(h.runtimes) - 1; i >= 0; i-- { + if err := h.runtimes[i].Close(); !h.expectedError(err) { + h.t.Errorf("close logical Runtime: %v", err) + } + } + + // Assert logical-client reclamation before server shutdown could hide a leak. + if h.spy != nil { + h.spy.Recorder().AssertClosed(h.t) + } + + if h.server != nil { + if err := h.Shutdown(); !h.expectedError(err) { + h.t.Errorf("shutdown Wire server: %v", err) + } + } + + if h.connection != nil { + if err := h.CloseTransport(); err != nil { + h.t.Errorf("close transport: %v", err) + } + } + + if h.listener != nil { + if err := h.listener.Close(); err != nil { + h.t.Errorf("close listener: %v", err) + } + } + + if h.serveResult != nil { + if err := Await(h.t, h.serveResult); err != nil { + h.t.Errorf("serve: %v", err) + } + } + + if h.spy != nil { + h.spy.Recorder().AssertClosed(h.t) + } +} diff --git a/test/integration/harness/options.go b/test/integration/harness/options.go new file mode 100644 index 0000000..621ee5f --- /dev/null +++ b/test/integration/harness/options.go @@ -0,0 +1,95 @@ +package harness + +import "github.com/MontFerret/api" + +type ( + CompileOptions struct { + Level api.OptimizationLevel + HasLevel bool + } + + SessionOptions struct { + Params map[string]any + ContentType string + } +) + +func (o *CompileOptions) SetOptimizationLevel(level api.OptimizationLevel) error { + o.Level, o.HasLevel = level, true + + return nil +} + +func (o *SessionOptions) SetParam(name string, value any) error { + if o.Params == nil { + o.Params = make(map[string]any) + } + + o.Params[name] = value + + return nil +} + +func (o *SessionOptions) SetParams(values map[string]any) error { + for name, value := range values { + if err := o.SetParam(name, value); err != nil { + return err + } + } + + return nil +} + +func (o *SessionOptions) SetOutputContentType(value string) error { + o.ContentType = value + + return nil +} + +func (o SessionOptions) clone() SessionOptions { + if o.Params != nil { + o.Params = cloneValue(o.Params).(map[string]any) + } + + return o +} + +// cloneValue copies portable API values without a JSON numeric conversion. +func cloneValue(value any) any { + switch value := value.(type) { + case map[string]any: + result := make(map[string]any, len(value)) + + for key, item := range value { + result[key] = cloneValue(item) + } + + return result + case []any: + result := make([]any, len(value)) + + for index, item := range value { + result[index] = cloneValue(item) + } + + return result + case []byte: + return append([]byte(nil), value...) + default: + return value + } +} + +func applyOptions(options []api.SessionOption) (SessionOptions, error) { + configured := SessionOptions{Params: make(map[string]any)} + + for _, option := range options { + if option != nil { + if err := option(&configured); err != nil { + return SessionOptions{}, err + } + } + } + + return configured, nil +} diff --git a/test/integration/harness/plan.go b/test/integration/harness/plan.go new file mode 100644 index 0000000..b091870 --- /dev/null +++ b/test/integration/harness/plan.go @@ -0,0 +1,83 @@ +package harness + +import ( + "context" + + "github.com/MontFerret/api" + "github.com/MontFerret/api/debugger" +) + +type ( + PlanBehavior struct { + Params []string + NewSession func(context.Context, SessionOptions) error + NewDebugSession func(context.Context, SessionOptions) error + Session func(SessionOptions) SessionBehavior + Debugger DebuggerBehavior + Close func() error + } + + PlanSpy struct { + id int + recorder *Recorder + behavior PlanBehavior + } +) + +var _ api.Plan = (*PlanSpy)(nil) + +func (p *PlanSpy) Params() []string { + p.recorder.record(Call{Resource: p.id, Method: "Params"}) + + return append([]string(nil), p.behavior.Params...) +} + +func (p *PlanSpy) NewSession(ctx context.Context, options ...api.SessionOption) (api.Session, error) { + configured, err := applyOptions(options) + if err != nil { + return nil, err + } + + p.recorder.record(Call{Resource: p.id, Method: "NewSession", Options: configured}) + + if p.behavior.NewSession != nil { + if err := p.behavior.NewSession(ctx, configured); err != nil { + return nil, err + } + } + + var behavior SessionBehavior + if p.behavior.Session != nil { + behavior = p.behavior.Session(configured) + } + + return &SessionSpy{id: p.recorder.create("session", p.id), recorder: p.recorder, behavior: behavior}, nil +} + +func (p *PlanSpy) NewDebugSession(ctx context.Context, options ...api.SessionOption) (debugger.Session, error) { + configured, err := applyOptions(options) + if err != nil { + return nil, err + } + + p.recorder.record(Call{Resource: p.id, Method: "NewDebugSession", Options: configured}) + + if p.behavior.NewDebugSession != nil { + if err := p.behavior.NewDebugSession(ctx, configured); err != nil { + return nil, err + } + } + + return newDebuggerSpy(p.recorder, p.id, p.behavior.Debugger), nil +} + +func (p *PlanSpy) Close() error { + p.recorder.record(Call{Resource: p.id, Method: "Close"}) + defer p.recorder.record(Call{Resource: p.id, Method: "CloseFinished"}) + + if p.behavior.Close != nil { + return p.behavior.Close() + } + + return nil +} diff --git a/test/integration/harness/recorder.go b/test/integration/harness/recorder.go new file mode 100644 index 0000000..c65442f --- /dev/null +++ b/test/integration/harness/recorder.go @@ -0,0 +1,179 @@ +package harness + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/MontFerret/api" +) + +type ( + // Resource identifies a hosted object, never a Wire protocol handle. + Resource struct { + ID, Parent int + Kind string + } + + // Call is an immutable observation of a hosted API invocation. + Call struct { + Resource int + Method string + Source api.Source + Compile CompileOptions + Options SessionOptions + Argument any + Index int + } + + Snapshot struct { + Resources []Resource + Calls []Call + } + + // Recorder broadcasts changes so lifecycle assertions need no polling. + Recorder struct { + mu sync.Mutex + changed chan struct{} + resources []Resource + calls []Call + } +) + +func newRecorder() *Recorder { + return &Recorder{changed: make(chan struct{})} +} + +func (r *Recorder) create(kind string, parent int) int { + r.mu.Lock() + defer r.mu.Unlock() + + id := len(r.resources) + 1 + r.resources = append(r.resources, Resource{ID: id, Parent: parent, Kind: kind}) + r.notify() + + return id +} + +func (r *Recorder) record(call Call) int { + r.mu.Lock() + defer r.mu.Unlock() + + call.Options = call.Options.clone() + r.calls = append(r.calls, call) + r.notify() + count := 0 + + for _, previous := range r.calls { + if previous.Resource == call.Resource && previous.Method == call.Method { + count++ + } + } + + return count +} + +// notify requires mu. Subscribers inspect state and subscribe under the same lock. +func (r *Recorder) notify() { + close(r.changed) + r.changed = make(chan struct{}) +} + +func (r *Recorder) snapshot() Snapshot { + result := Snapshot{Resources: append([]Resource(nil), r.resources...), Calls: append([]Call(nil), r.calls...)} + + for i := range result.Calls { + result.Calls[i].Options = result.Calls[i].Options.clone() + } + + return result +} + +func (r *Recorder) Snapshot() Snapshot { + r.mu.Lock() + defer r.mu.Unlock() + + return r.snapshot() +} + +func (s Snapshot) Count(id int, method string) int { + count := 0 + + for _, call := range s.Calls { + if call.Resource == id && call.Method == method { + count++ + } + } + + return count +} + +func (s Snapshot) OfKind(kind string) []Resource { + var result []Resource + + for _, resource := range s.Resources { + if resource.Kind == kind { + result = append(result, resource) + } + } + + return result +} + +func (r *Recorder) Wait(t testing.TB, description string, predicate func(Snapshot) bool) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + for { + r.mu.Lock() + snapshot, changed := r.snapshot(), r.changed + r.mu.Unlock() + + if predicate(snapshot) { + return + } + + select { + case <-changed: + case <-ctx.Done(): + t.Errorf("%s: timed out; resources=%+v calls=%+v", description, snapshot.Resources, snapshot.Calls) + + return + } + } +} + +func (r *Recorder) AssertClosed(t testing.TB) { + t.Helper() + r.Wait(t, "hosted resource reclamation", func(s Snapshot) bool { + for _, resource := range s.Resources { + if resource.Kind != "runtime" && s.Count(resource.ID, "CloseFinished") == 0 { + return false + } + + for _, method := range []string{"Run", "Compile", "CompileDebug", "Start", "Continue", "StepOver", "StepIn", "StepOut", "EvaluateFrame"} { + if s.Count(resource.ID, method) != s.Count(resource.ID, method+"Finished") { + return false + } + } + } + + return true + }) + + snapshot := r.Snapshot() + + for _, resource := range snapshot.Resources { + want := 1 + + if resource.Kind == "runtime" { + want = 0 + } + + if got := snapshot.Count(resource.ID, "Close"); got != want { + t.Errorf("hosted %s %d Close calls = %d, want %d", resource.Kind, resource.ID, got, want) + } + } +} diff --git a/test/integration/harness/runtime.go b/test/integration/harness/runtime.go new file mode 100644 index 0000000..3aaddfd --- /dev/null +++ b/test/integration/harness/runtime.go @@ -0,0 +1,97 @@ +package harness + +import ( + "context" + + "github.com/MontFerret/api" +) + +type ( + // RuntimeBehavior is configured before the server starts. Hooks run outside locks. + RuntimeBehavior struct { + Run func(context.Context, api.Source, SessionOptions) (api.Output, error) + Compile func(context.Context, api.Source, bool, CompileOptions) error + Plan PlanBehavior + } + + RuntimeSpy struct { + id int + recorder *Recorder + behavior RuntimeBehavior + } +) + +var _ api.Runtime = (*RuntimeSpy)(nil) + +func NewRuntimeSpy(behavior RuntimeBehavior) *RuntimeSpy { + recorder := newRecorder() + + return &RuntimeSpy{id: recorder.create("runtime", 0), recorder: recorder, behavior: behavior} +} + +func (r *RuntimeSpy) Recorder() *Recorder { + return r.recorder +} + +func (r *RuntimeSpy) ID() int { + return r.id +} + +func (r *RuntimeSpy) Run(ctx context.Context, src api.Source, options ...api.SessionOption) (api.Output, error) { + configured, err := applyOptions(options) + if err != nil { + return api.Output{}, err + } + + r.recorder.record(Call{Resource: r.id, Method: "Run", Source: src, Options: configured}) + defer r.recorder.record(Call{Resource: r.id, Method: "RunFinished"}) + + if r.behavior.Run != nil { + return r.behavior.Run(ctx, src, configured) + } + + return api.Output{}, nil +} + +func (r *RuntimeSpy) Compile(ctx context.Context, src api.Source, options ...api.PlanOption) (api.Plan, error) { + return r.compile(ctx, src, false, options) +} + +func (r *RuntimeSpy) CompileDebug(ctx context.Context, src api.Source, options ...api.PlanOption) (api.Plan, error) { + return r.compile(ctx, src, true, options) +} + +func (r *RuntimeSpy) compile(ctx context.Context, src api.Source, debug bool, options []api.PlanOption) (api.Plan, error) { + var configured CompileOptions + + for _, option := range options { + if option != nil { + if err := option(&configured); err != nil { + return nil, err + } + } + } + + method := "Compile" + + if debug { + method = "CompileDebug" + } + + r.recorder.record(Call{Resource: r.id, Method: method, Source: src, Compile: configured}) + defer r.recorder.record(Call{Resource: r.id, Method: method + "Finished"}) + + if r.behavior.Compile != nil { + if err := r.behavior.Compile(ctx, src, debug, configured); err != nil { + return nil, err + } + } + + return &PlanSpy{id: r.recorder.create("plan", r.id), recorder: r.recorder, behavior: r.behavior.Plan}, nil +} + +func (r *RuntimeSpy) Close() error { + r.recorder.record(Call{Resource: r.id, Method: "Close"}) + + return nil +} diff --git a/test/integration/harness/session.go b/test/integration/harness/session.go new file mode 100644 index 0000000..5eb3614 --- /dev/null +++ b/test/integration/harness/session.go @@ -0,0 +1,44 @@ +package harness + +import ( + "context" + + "github.com/MontFerret/api" +) + +type ( + SessionBehavior struct { + Run func(context.Context, int) (api.Output, error) + Close func() error + } + + SessionSpy struct { + id int + recorder *Recorder + behavior SessionBehavior + } +) + +var _ api.Session = (*SessionSpy)(nil) + +func (s *SessionSpy) Run(ctx context.Context) (api.Output, error) { + call := s.recorder.record(Call{Resource: s.id, Method: "Run"}) + defer s.recorder.record(Call{Resource: s.id, Method: "RunFinished"}) + + if s.behavior.Run != nil { + return s.behavior.Run(ctx, call) + } + + return api.Output{}, nil +} + +func (s *SessionSpy) Close() error { + s.recorder.record(Call{Resource: s.id, Method: "Close"}) + defer s.recorder.record(Call{Resource: s.id, Method: "CloseFinished"}) + + if s.behavior.Close != nil { + return s.behavior.Close() + } + + return nil +} diff --git a/test/integration/lifecycle_test.go b/test/integration/lifecycle_test.go new file mode 100644 index 0000000..6a5efab --- /dev/null +++ b/test/integration/lifecycle_test.go @@ -0,0 +1,361 @@ +package integration_test + +import ( + "context" + "fmt" + "sync" + "testing" + + "github.com/MontFerret/api" + "github.com/MontFerret/api/debugger" + "github.com/MontFerret/wire/test/integration/harness" +) + +func TestRecursiveCloseReclaimsActiveDescendants(t *testing.T) { + for _, owner := range []string{"session", "plan", "runtime"} { + t.Run(owner, func(t *testing.T) { + normal, debug, direct := harness.NewBlock(t), harness.NewBlock(t), harness.NewBlock(t) + h := harness.New(t, harness.WithBehavior(harness.RuntimeBehavior{ + Run: func(ctx context.Context, src api.Source, _ harness.SessionOptions) (api.Output, error) { + if src.Content == "blocked" { + return api.Output{}, direct.Wait(ctx) + } + + return api.Output{}, nil + }, + Plan: harness.PlanBehavior{ + Session: func(options harness.SessionOptions) harness.SessionBehavior { + return harness.SessionBehavior{Run: func(ctx context.Context, _ int) (api.Output, error) { + if options.Params["block"] == true { + return api.Output{}, normal.Wait(ctx) + } + + return api.Output{}, nil + }} + }, + Debugger: harness.DebuggerBehavior{Command: func(ctx context.Context, method string, _ int) (*debugger.Event, error) { + if method == "Continue" { + return nil, debug.Wait(ctx) + } + + return &debugger.Event{Reason: debugger.ReasonEntry}, nil + }}, + }, + })) + other, err := h.OpenRuntime() + if err != nil { + t.Fatal(err) + } + + plan, err := h.Runtime().CompileDebug(h.Context(), api.Source{Content: "RETURN 1"}) + if err != nil { + t.Fatal(err) + } + + siblingPlan, err := h.Runtime().Compile(h.Context(), api.Source{Content: "RETURN 2"}) + if err != nil { + t.Fatal(err) + } + + session, err := plan.NewSession(h.Context(), api.WithParam("block", true)) + if err != nil { + t.Fatal(err) + } + + sibling, err := siblingPlan.NewSession(h.Context()) + if err != nil { + t.Fatal(err) + } + + debugSession, err := plan.NewDebugSession(h.Context()) + if err != nil { + t.Fatal(err) + } + + if _, err := debugSession.Start(h.Context()); err != nil { + t.Fatal(err) + } + + normalResult := make(chan error, 1) + go func() { + _, err := session.Run(h.Context()) + normalResult <- err + }() + harness.Await(t, normal.Started) + type debugOutcome struct { + event *debugger.Event + err error + } + debugResult := make(chan debugOutcome, 1) + directResult := make(chan error, 1) + + if owner != "session" { + go func() { + event, err := debugSession.Continue(h.Context()) + debugResult <- debugOutcome{event: event, err: err} + }() + harness.Await(t, debug.Started) + } + + if owner == "runtime" { + go func() { + _, err := h.Runtime().Run(h.Context(), api.Source{Content: "blocked"}) + directResult <- err + }() + harness.Await(t, direct.Started) + } + + closeOwner := session.Close + + if owner == "plan" { + closeOwner = plan.Close + } else if owner == "runtime" { + closeOwner = h.Runtime().Close + } + + if err := closeOwner(); err != nil { + t.Fatal(err) + } + + if err := harness.Await(t, normalResult); err == nil { + t.Fatal("active Run succeeded after recursive close") + } + + harness.Await(t, normal.Cancelled) + harness.Await(t, normal.Finished) + + if owner != "session" { + // A terminated debugger event or a closed-handle error both settle the command. + result := harness.Await(t, debugResult) + if result.err == nil && (result.event == nil || result.event.Reason != debugger.ReasonTerminated) { + t.Fatalf("recursive close returned a successful debugger stop: %#v", result.event) + } + + harness.Await(t, debug.Cancelled) + harness.Await(t, debug.Finished) + } else { + if _, err := debugSession.Frames(); err != nil { + t.Fatalf("Session.Close invalidated sibling debugger: %v", err) + } + } + + if owner == "runtime" { + if err := harness.Await(t, directResult); err == nil { + t.Fatal("direct Run survived Runtime.Close") + } + + harness.Await(t, direct.Cancelled) + harness.Await(t, direct.Finished) + } else { + if _, err := sibling.Run(h.Context()); err != nil { + t.Fatalf("unrelated sibling invalidated: %v", err) + } + } + + if _, err := other.Run(h.Context(), api.Source{Content: "RETURN 3"}); err != nil { + t.Fatalf("other logical Runtime invalidated: %v", err) + } + + snapshot := h.RuntimeSpy().Recorder().Snapshot() + + for _, resource := range snapshot.Resources { + want := 0 + + if resource.Kind == "session" && resource.ID == snapshot.OfKind("session")[0].ID { + want = 1 + } + + if owner == "plan" && (resource.ID == snapshot.OfKind("plan")[0].ID || resource.Kind == "debugger") { + want = 1 + } + + if owner == "runtime" && resource.Kind != "runtime" { + want = 1 + } + + if got := snapshot.Count(resource.ID, "Close"); got != want { + t.Fatalf("%s close: resource=%+v calls=%d want=%d", owner, resource, got, want) + } + } + }) + } +} + +func TestConcurrentSiblingSessionsRemainIndependent(t *testing.T) { + blocks := []*harness.Block{harness.NewBlock(t), harness.NewBlock(t)} + h := harness.New(t, harness.WithBehavior(harness.RuntimeBehavior{Plan: harness.PlanBehavior{Session: func(options harness.SessionOptions) harness.SessionBehavior { + index := int(options.Params["index"].(int64)) + + return harness.SessionBehavior{Run: func(ctx context.Context, _ int) (api.Output, error) { + return api.Output{ContentType: "text/plain", Content: []byte(fmt.Sprint(index))}, blocks[index].Wait(ctx) + }} + }}})) + plan, err := h.Runtime().Compile(h.Context(), api.Source{Content: "RETURN @index"}) + if err != nil { + t.Fatal(err) + } + + var sessions []api.Session + + for index := range 2 { + session, err := plan.NewSession(h.Context(), api.WithParam("index", int64(index))) + if err != nil { + t.Fatal(err) + } + + sessions = append(sessions, session) + } + + type result struct { + output api.Output + err error + } + results := []chan result{make(chan result, 1), make(chan result, 1)} + + for index := range 2 { + go func() { + output, err := sessions[index].Run(h.Context()) + results[index] <- result{output, err} + }() + } + + for _, block := range blocks { + harness.Await(t, block.Started) + } + + if err := sessions[0].Close(); err != nil { + t.Fatal(err) + } + + if first := harness.Await(t, results[0]); first.err == nil { + t.Fatal("closed sibling Run succeeded") + } + + harness.Await(t, blocks[0].Cancelled) + blocks[1].Release() + second := harness.Await(t, results[1]) + if second.err != nil || string(second.output.Content) != "1" { + t.Fatalf("unrelated active sibling affected: %+v", second) + } + + snapshot := h.RuntimeSpy().Recorder().Snapshot() + if len(snapshot.OfKind("plan")) != 1 || len(snapshot.OfKind("session")) != 2 { + t.Fatalf("concurrent sessions changed identity: %+v", snapshot) + } +} + +func TestConcurrentPlanSessionCreation(t *testing.T) { + const workers = 8 + entered := make(chan struct{}, workers) + release := make(chan struct{}) + finish := sync.OnceFunc(func() { close(release) }) + t.Cleanup(finish) + h := harness.New(t, harness.WithBehavior(harness.RuntimeBehavior{Plan: harness.PlanBehavior{NewSession: func(ctx context.Context, _ harness.SessionOptions) error { + entered <- struct{}{} + + select { + case <-ctx.Done(): + return ctx.Err() + case <-release: + return nil + } + }}})) + plan, err := h.Runtime().Compile(h.Context(), api.Source{Content: "RETURN 1"}) + if err != nil { + t.Fatal(err) + } + + results := make(chan error, workers) + + for range workers { + go func() { + session, err := plan.NewSession(h.Context()) + if err == nil { + _, err = session.Run(h.Context()) + } + + results <- err + }() + } + + for range workers { + harness.Await(t, entered) + } + + finish() + + for range workers { + if err := harness.Await(t, results); err != nil { + t.Fatal(err) + } + } + + snapshot := h.RuntimeSpy().Recorder().Snapshot() + if len(snapshot.OfKind("plan")) != 1 || len(snapshot.OfKind("session")) != workers { + t.Fatalf("concurrent plan creation=%+v", snapshot) + } +} + +func TestConcurrentPlansShareRuntime(t *testing.T) { + const workers = 4 + entered := make(chan struct{}, workers) + release := make(chan struct{}) + finish := sync.OnceFunc(func() { close(release) }) + t.Cleanup(finish) + h := harness.New(t, harness.WithBehavior(harness.RuntimeBehavior{Compile: func(ctx context.Context, _ api.Source, _ bool, _ harness.CompileOptions) error { + entered <- struct{}{} + + select { + case <-ctx.Done(): + return ctx.Err() + case <-release: + return nil + } + }})) + results := make(chan error, workers) + + for index := range workers { + go func() { + plan, err := h.Runtime().Compile(h.Context(), api.Source{Name: fmt.Sprintf("plan-%d.fql", index), Content: "RETURN 1"}) + if err != nil { + results <- err + + return + } + + session, err := plan.NewSession(h.Context()) + if err == nil { + _, err = session.Run(h.Context()) + } + + results <- err + }() + } + + for range workers { + harness.Await(t, entered) + } + + finish() + + for range workers { + if err := harness.Await(t, results); err != nil { + t.Fatal(err) + } + } + + snapshot := h.RuntimeSpy().Recorder().Snapshot() + if len(snapshot.OfKind("plan")) != workers || len(snapshot.OfKind("session")) != workers || snapshot.Count(h.RuntimeSpy().ID(), "Compile") != workers { + t.Fatalf("concurrent plans changed identity: %+v", snapshot) + } + + parents := make(map[int]bool) + + for _, session := range snapshot.OfKind("session") { + parents[session.Parent] = true + } + + if len(parents) != workers { + t.Fatal("sessions from distinct compiled plans shared a hosted Plan") + } +} diff --git a/test/integration/plan_test.go b/test/integration/plan_test.go new file mode 100644 index 0000000..5382f38 --- /dev/null +++ b/test/integration/plan_test.go @@ -0,0 +1,237 @@ +package integration_test + +import ( + "context" + "errors" + "reflect" + "testing" + + "github.com/MontFerret/api" + "github.com/MontFerret/wire/client" + "github.com/MontFerret/wire/test/integration/harness" +) + +func TestCompileRoundTrip(t *testing.T) { + for _, debug := range []bool{false, true} { + for _, test := range []struct { + name string + options []api.PlanOption + want harness.CompileOptions + }{ + {"omitted", nil, harness.CompileOptions{}}, + {"none", []api.PlanOption{api.WithOptimizationLevel(api.OptimizationNone)}, harness.CompileOptions{Level: api.OptimizationNone, HasLevel: true}}, + {"basic", []api.PlanOption{api.WithOptimizationLevel(api.OptimizationBasic)}, harness.CompileOptions{Level: api.OptimizationBasic, HasLevel: true}}, + {"full", []api.PlanOption{api.WithOptimizationLevel(api.OptimizationFull)}, harness.CompileOptions{Level: api.OptimizationFull, HasLevel: true}}, + {"aggressive", []api.PlanOption{api.WithOptimizationLevel(api.OptimizationAggressive)}, harness.CompileOptions{Level: api.OptimizationAggressive, HasLevel: true}}, + } { + t.Run(map[bool]string{false: "normal/", true: "debug/"}[debug]+test.name, func(t *testing.T) { + h := harness.New(t, harness.WithBehavior(harness.RuntimeBehavior{Plan: harness.PlanBehavior{Params: []string{"input", "other"}}})) + compile, method := h.Runtime().Compile, "Compile" + + if debug { + compile, method = h.Runtime().CompileDebug, "CompileDebug" + } + + src := api.Source{Name: "folder/query.fql", Content: "RETURN @input + @other\n"} + plan, err := compile(h.Context(), src, test.options...) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(plan.Params(), []string{"input", "other"}) { + t.Fatalf("Params=%v", plan.Params()) + } + + plan.Params()[0] = "changed" + + if plan.Params()[0] != "input" { + t.Fatal("Params was not defensive") + } + + snapshot := h.RuntimeSpy().Recorder().Snapshot() + if snapshot.Count(h.RuntimeSpy().ID(), method) != 1 || len(snapshot.OfKind("plan")) != 1 { + t.Fatalf("unexpected compilation: %+v", snapshot) + } + + for _, call := range snapshot.Calls { + if call.Method == method && (call.Source != src || call.Compile != test.want) { + t.Fatalf("compile call changed: %+v", call) + } + } + }) + } + } +} + +func TestCompileOptionsApplyOnceBeforeDispatch(t *testing.T) { + for _, debug := range []bool{false, true} { + for _, outcome := range []string{"success", "invalid level", "callback errors", "callback cancellation", "already cancelled"} { + t.Run(map[bool]string{false: "normal/", true: "debug/"}[debug]+outcome, func(t *testing.T) { + h := harness.New(t) + ctx, cancel := context.WithCancel(h.Context()) + defer cancel() + + first, second := errors.New("first"), errors.New("second") + var order []int + options := []api.PlanOption{func(o api.PlanOptions) error { + order = append(order, 1) + + if outcome == "callback cancellation" { + cancel() + } + + if outcome == "callback errors" { + return first + } + + if outcome == "invalid level" { + return o.SetOptimizationLevel(99) + } + + return o.SetOptimizationLevel(api.OptimizationBasic) + }, nil, func(o api.PlanOptions) error { + order = append(order, 2) + + if outcome == "callback errors" { + return second + } + + return o.SetOptimizationLevel(api.OptimizationNone) + }} + + if outcome == "already cancelled" { + cancel() + } + + compile, operation := h.Runtime().Compile, harness.Compile + + if debug { + compile, operation = h.Runtime().CompileDebug, harness.CompileDebug + } + + plan, err := compile(ctx, api.Source{Content: "RETURN 1"}, options...) + + if outcome == "already cancelled" { + if len(order) != 0 { + t.Fatalf("cancelled call applied options: %v", order) + } + } else if !reflect.DeepEqual(order, []int{1, 2}) { + t.Fatalf("option order=%v", order) + } + + if outcome == "success" { + if err != nil { + t.Fatal(err) + } + + if err := plan.Close(); err != nil { + t.Fatal(err) + } + + if h.Faults().Count(operation) != 1 { + t.Fatal("expected one compilation") + } + + for _, call := range h.RuntimeSpy().Recorder().Snapshot().Calls { + if call.Method == "Compile" || call.Method == "CompileDebug" { + if call.Compile != (harness.CompileOptions{HasLevel: true, Level: api.OptimizationNone}) { + t.Fatalf("final option changed: %+v", call) + } + } + } + } else { + if err == nil || plan != nil || h.Faults().Count(operation) != 0 { + t.Fatalf("invalid options dispatched: %v", err) + } + + if outcome == "callback errors" && (!errors.Is(err, first) || !errors.Is(err, second)) { + t.Fatalf("option errors not joined: %v", err) + } + + if (outcome == "callback cancellation" || outcome == "already cancelled") && !errors.Is(err, context.Canceled) { + t.Fatalf("cancellation lost: %v", err) + } + } + }) + } + } +} + +func TestReusablePlanAndDurableSessions(t *testing.T) { + h := harness.New(t, harness.WithBehavior(harness.RuntimeBehavior{Plan: harness.PlanBehavior{Session: func(options harness.SessionOptions) harness.SessionBehavior { + return harness.SessionBehavior{Run: func(context.Context, int) (api.Output, error) { + return api.Output{ContentType: options.ContentType, Content: []byte(options.Params["input"].(string))}, nil + }} + }}})) + plan, err := h.Runtime().Compile(h.Context(), api.Source{Content: "RETURN @input"}) + if err != nil { + t.Fatal(err) + } + + var sessions []api.Session + + for _, value := range []string{"first", "second", "third"} { + session, err := plan.NewSession(h.Context(), api.WithParam("input", value), api.WithOutputContentType("text/plain")) + if err != nil { + t.Fatal(err) + } + + sessions = append(sessions, session) + + for range 2 { + output, err := session.Run(h.Context()) + if err != nil || string(output.Content) != value || output.ContentType != "text/plain" { + t.Fatalf("durable output=%+v err=%v", output, err) + } + } + } + + snapshot := h.RuntimeSpy().Recorder().Snapshot() + plans, hostedSessions := snapshot.OfKind("plan"), snapshot.OfKind("session") + if len(plans) != 1 || len(hostedSessions) != 3 || snapshot.Count(h.RuntimeSpy().ID(), "Compile") != 1 { + t.Fatalf("plan or session recreated: %+v", snapshot) + } + + for _, resource := range hostedSessions { + if resource.Parent != plans[0].ID || snapshot.Count(resource.ID, "Run") != 2 || snapshot.Count(resource.ID, "Close") != 0 { + t.Fatalf("durable session changed: %+v", resource) + } + } + + for index, session := range sessions { + output, err := session.Run(h.Context()) + if err != nil || string(output.Content) != []string{"first", "second", "third"}[index] { + t.Fatalf("later session creation changed an earlier session's parameters: %+v, %v", output, err) + } + } + + if err := sessions[0].Close(); err != nil { + t.Fatal(err) + } + + if err := sessions[0].Close(); err != nil { + t.Fatal(err) + } + + if _, err := sessions[0].Run(h.Context()); !errors.Is(err, client.ErrClosed) { + t.Fatalf("Run after Close: %v", err) + } + + if _, err := sessions[1].Run(h.Context()); err != nil { + t.Fatalf("sibling session invalidated: %v", err) + } + + if err := plan.Close(); err != nil { + t.Fatal(err) + } + + if err := plan.Close(); err != nil { + t.Fatal(err) + } + + if _, err := plan.NewSession(h.Context()); !errors.Is(err, client.ErrClosed) { + t.Fatalf("NewSession after Close: %v", err) + } + + h.RuntimeSpy().Recorder().AssertClosed(t) +} diff --git a/test/integration/release_test.go b/test/integration/release_test.go new file mode 100644 index 0000000..03c0433 --- /dev/null +++ b/test/integration/release_test.go @@ -0,0 +1,122 @@ +package integration_test + +import ( + "errors" + "testing" + + "github.com/MontFerret/api" + "github.com/MontFerret/wire/test/integration/harness" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestRuntimeCompletedExecutionReleaseFailurePreservesParents(t *testing.T) { + for _, index := range []int{4, 5} { + operation := allocationOperations()[index] + + for _, acknowledged := range []bool{false, true} { + t.Run(operation.name+map[bool]string{false: "/failed delivery", true: "/lost acknowledgement"}[acknowledged], func(t *testing.T) { + f := newRuntimeAllocationFixture(t, operation) + releaseErr := status.Error(codes.Unavailable, "execution release unavailable") + fail := f.gate.Fail + + if acknowledged { + fail = f.gate.FailResponse + } + + fail(operation.release, releaseErr) + + if _, err := f.allocate(harness.Context(t), nil); !errors.Is(err, releaseErr) { + t.Fatalf("successful execution lost its release failure: %v", err) + } + + if f.gate.Count(operation.release) != 1 || f.gate.Count(operation.parentRelease) != 0 || + f.gate.Count(harness.ReleasePlan) != 0 || + f.gate.Count(harness.CloseRuntime) != 0 { + t.Fatal("known Execution release failure closed its parent or retried") + } + + fail(operation.release, nil) + + if operation.name == "session run" { + _, err := f.session.Run(harness.Context(t)) + if acknowledged && err != nil { + t.Fatalf("durable Session could not be reused after server committed release: %v", err) + } + + if !acknowledged && status.Code(err) != codes.FailedPrecondition { + t.Fatalf("undelivered release should retain the hosted execution: %v", err) + } + + sibling, err := f.plan.NewSession(harness.Context(t)) + if err != nil { + t.Fatalf("release failure invalidated Plan: %v", err) + } + + if _, err := sibling.Run(harness.Context(t)); err != nil { + t.Fatalf("release failure prevented sibling execution: %v", err) + } + } + + if _, err := f.remote.Run(harness.Context(t), api.Source{Content: "RETURN 3"}); err != nil { + t.Fatalf("release failure invalidated Runtime: %v", err) + } + }) + } + } +} + +func TestKnownResourceCloseFailurePreservesSiblings(t *testing.T) { + for _, index := range []int{0, 2, 3} { + operation := allocationOperations()[index] + + for _, acknowledged := range []bool{false, true} { + t.Run(operation.name+map[bool]string{false: "/undelivered", true: "/lost acknowledgement"}[acknowledged], func(t *testing.T) { + f := newRuntimeAllocationFixture(t, operation) + siblingPlan, err := f.remote.Compile(f.h.Context(), api.Source{Content: "RETURN 2"}) + if err != nil { + t.Fatal(err) + } + + sibling, err := siblingPlan.NewSession(f.h.Context()) + if err != nil { + t.Fatal(err) + } + + closeHandle, err := f.allocate(f.h.Context(), nil) + if err != nil { + t.Fatal(err) + } + + releaseErr := status.Error(codes.Unavailable, "known resource release failed") + fail := f.gate.Fail + + if acknowledged { + fail = f.gate.FailResponse + } + + fail(operation.release, releaseErr) + + for range 2 { + if err := closeHandle(); !errors.Is(err, releaseErr) { + t.Fatalf("close did not retain release error: %v", err) + } + } + + if f.gate.Count(operation.release) != 1 || f.gate.Count(operation.parentRelease) != 0 || f.gate.Count(harness.CloseRuntime) != 0 { + t.Fatal("known Close retried or invalidated its parent") + } + + fail(operation.release, nil) + + if _, err := sibling.Run(f.h.Context()); err != nil { + t.Fatalf("known Close failure invalidated sibling: %v", err) + } + + if _, err := f.remote.Run(f.h.Context(), api.Source{Content: "RETURN 3"}); err != nil { + t.Fatalf("known Close failure invalidated Runtime: %v", err) + } + }) + } + } +} diff --git a/test/integration/runtime_test.go b/test/integration/runtime_test.go new file mode 100644 index 0000000..ab24417 --- /dev/null +++ b/test/integration/runtime_test.go @@ -0,0 +1,211 @@ +package integration_test + +import ( + "bytes" + "context" + "errors" + "reflect" + "testing" + + "github.com/MontFerret/api" + "github.com/MontFerret/wire/client" + "github.com/MontFerret/wire/pkg/failure" + "github.com/MontFerret/wire/test/integration/harness" +) + +func TestRuntimeAndSessionOutputRoundTrip(t *testing.T) { + for _, direct := range []bool{true, false} { + for _, test := range []struct { + name string + output api.Output + err error + }{ + {name: "zero"}, + {name: "empty", output: api.Output{ContentType: "text/plain", Content: []byte{}}}, + {name: "text", output: api.Output{ContentType: "text/plain; charset=utf-8", Content: []byte("héllo")}}, + {name: "binary", output: api.Output{ContentType: "application/octet-stream", Content: []byte{0, 255, 3, 0}}}, + {name: "partial", output: api.Output{ContentType: "application/json", Content: []byte(`{"partial":true}`)}, err: errors.New("host secret")}, + } { + t.Run(map[bool]string{true: "runtime/", false: "session/"}[direct]+test.name, func(t *testing.T) { + wantContent := bytes.Clone(test.output.Content) + h := harness.New(t, harness.WithBehavior(harness.RuntimeBehavior{ + Run: func(context.Context, api.Source, harness.SessionOptions) (api.Output, error) { + return test.output, test.err + }, + Plan: harness.PlanBehavior{Session: func(harness.SessionOptions) harness.SessionBehavior { + return harness.SessionBehavior{Run: func(context.Context, int) (api.Output, error) { return test.output, test.err }} + }}, + })) + src := api.Source{Name: "results.fql", Content: "RETURN @input"} + run := func() (api.Output, error) { return h.Runtime().Run(h.Context(), src) } + + if !direct { + plan, err := h.Runtime().Compile(h.Context(), src) + if err != nil { + t.Fatal(err) + } + + session, err := plan.NewSession(h.Context()) + if err != nil { + t.Fatal(err) + } + + run = func() (api.Output, error) { return session.Run(h.Context()) } + } + + for range 2 { + output, err := run() + if test.err == nil && err != nil { + t.Fatal(err) + } + + if test.err != nil { + var remote *failure.Failure + if !errors.As(err, &remote) || remote.Category != failure.CategoryExecution { + t.Fatalf("execution failure changed: %v", err) + } + } + + if output.ContentType != test.output.ContentType || !bytes.Equal(output.Content, wantContent) { + t.Fatalf("output = %#v, want %#v", output, test.output) + } + + if len(output.Content) > 0 { + output.Content[0] ^= 255 + } + + if !bytes.Equal(test.output.Content, wantContent) { + t.Fatal("returned output aliases hosted output bytes") + } + } + + snapshot := h.RuntimeSpy().Recorder().Snapshot() + + if direct { + if len(snapshot.OfKind("plan")) != 0 || len(snapshot.OfKind("session")) != 0 || snapshot.Count(h.RuntimeSpy().ID(), "Run") != 2 { + t.Fatalf("direct Run was composed from other operations: %+v", snapshot) + } + + for _, call := range snapshot.Calls { + if call.Method == "Run" && call.Source != src { + t.Fatalf("source changed: %+v", call) + } + } + } + }) + } + } +} + +func TestSessionOptionsRoundTrip(t *testing.T) { + portable := map[string]any{"null": nil, "bool": true, "integer": int64(-9223372036854775807), "float": 3.5, "text": "héllo", "bytes": []byte{0, 255}, "array": []any{int64(2), map[string]any{"key": "value"}}, "object": map[string]any{"nested": []any{false, nil}}} + + for _, mode := range []string{"runtime", "session", "debugger"} { + for _, test := range []struct { + name string + options []api.SessionOption + want map[string]any + }{ + {"empty", []api.SessionOption{nil, api.WithParams(map[string]any{})}, map[string]any{}}, + {"single", []api.SessionOption{api.WithParam("input", int64(42))}, map[string]any{"input": int64(42)}}, + {"portable", []api.SessionOption{api.WithParams(portable)}, portable}, + {"overrides", []api.SessionOption{api.WithParam("a", int64(1)), api.WithParams(map[string]any{"a": int64(2), "b": true}), api.WithParam("a", int64(3))}, map[string]any{"a": int64(3), "b": true}}, + } { + t.Run(mode+"/"+test.name, func(t *testing.T) { + h := harness.New(t) + calls := 0 + options := append(append([]api.SessionOption(nil), test.options...), func(options api.SessionOptions) error { + calls++ + + return options.SetOutputContentType("application/custom") + }) + method := "Run" + + if mode == "runtime" { + if _, err := h.Runtime().Run(h.Context(), api.Source{Name: "options.fql", Content: "RETURN @input"}, options...); err != nil { + t.Fatal(err) + } + } else { + plan, err := h.Runtime().CompileDebug(h.Context(), api.Source{Content: "RETURN @input"}) + if err != nil { + t.Fatal(err) + } + + method = "NewSession" + + if mode == "session" { + _, err = plan.NewSession(h.Context(), options...) + } else { + method = "NewDebugSession" + _, err = plan.NewDebugSession(h.Context(), options...) + } + + if err != nil { + t.Fatal(err) + } + } + + if calls != 1 { + t.Fatalf("option applied %d times", calls) + } + + found := 0 + + for _, call := range h.RuntimeSpy().Recorder().Snapshot().Calls { + if call.Method == method { + found++ + + if call.Options.ContentType != "application/custom" || !reflect.DeepEqual(call.Options.Params, test.want) { + t.Fatalf("options changed: %+v", call.Options) + } + } + } + + if found != 1 { + t.Fatalf("hosted calls=%d", found) + } + }) + } + } +} + +func TestRuntimeCloseBorrowsTransportAndHostedRuntime(t *testing.T) { + h := harness.New(t) + other, err := h.OpenRuntime() + if err != nil { + t.Fatal(err) + } + + if err := h.Runtime().Close(); err != nil { + t.Fatal(err) + } + + if err := h.Runtime().Close(); err != nil { + t.Fatal(err) + } + + if _, err := h.Runtime().Run(h.Context(), api.Source{Content: "RETURN 1"}); !errors.Is(err, client.ErrClosed) { + t.Fatalf("closed Runtime error=%v", err) + } + + if _, err := other.Run(h.Context(), api.Source{Content: "RETURN 2"}); err != nil { + t.Fatalf("sibling logical Runtime unusable: %v", err) + } + + fresh, err := h.OpenRuntime() + if err != nil { + t.Fatalf("borrowed transport unusable: %v", err) + } + + if _, err := fresh.Run(h.Context(), api.Source{Content: "RETURN 3"}); err != nil { + t.Fatal(err) + } + + if err := h.Shutdown(); err != nil { + t.Fatal(err) + } + + if got := h.RuntimeSpy().Recorder().Snapshot().Count(h.RuntimeSpy().ID(), "Close"); got != 0 { + t.Fatalf("hosted Runtime closed %d times", got) + } +} diff --git a/test/integration/session_test.go b/test/integration/session_test.go new file mode 100644 index 0000000..78b78c3 --- /dev/null +++ b/test/integration/session_test.go @@ -0,0 +1,138 @@ +package integration_test + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" + + "github.com/MontFerret/api" + "github.com/MontFerret/wire/client" + "github.com/MontFerret/wire/pkg/failure" + "github.com/MontFerret/wire/server" + "github.com/MontFerret/wire/test/integration/harness" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestSessionRejectsOverlapAndReopensAfterRelease(t *testing.T) { + block := harness.NewBlock(t) + h := harness.New(t, harness.WithBehavior(harness.RuntimeBehavior{Plan: harness.PlanBehavior{Session: func(harness.SessionOptions) harness.SessionBehavior { + return harness.SessionBehavior{Run: func(ctx context.Context, call int) (api.Output, error) { + if call == 1 { + return api.Output{}, block.Wait(ctx) + } + + return api.Output{ContentType: "text/plain", Content: []byte("reused")}, nil + }} + }}})) + plan, err := h.Runtime().Compile(h.Context(), api.Source{Content: "RETURN 1"}) + if err != nil { + t.Fatal(err) + } + + session, err := plan.NewSession(h.Context()) + if err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithCancel(h.Context()) + defer cancel() + result := make(chan error, 1) + go func() { + _, err := session.Run(ctx) + result <- err + }() + harness.Await(t, block.Started) + _, err = session.Run(h.Context()) + var remote *client.Error + if !errors.As(err, &remote) || remote.Category != failure.CategoryInvalidState || status.Code(err) != codes.FailedPrecondition { + t.Fatalf("overlapping Run=%v", err) + } + + cancel() + + if err := harness.Await(t, result); !errors.Is(err, context.Canceled) { + t.Fatalf("caller cancellation=%v", err) + } + + harness.Await(t, block.Cancelled) + harness.Await(t, block.Finished) + output, err := session.Run(h.Context()) + if err != nil || string(output.Content) != "reused" { + t.Fatalf("durable session not reusable: %+v %v", output, err) + } + + snapshot := h.RuntimeSpy().Recorder().Snapshot() + sessions := snapshot.OfKind("session") + if len(sessions) != 1 || snapshot.Count(sessions[0].ID, "Run") != 2 || snapshot.Count(sessions[0].ID, "Close") != 0 { + t.Fatalf("hosted session lifecycle=%+v", snapshot) + } +} + +func TestSessionCompletionRacesCancellationWithoutDuplicateCleanup(t *testing.T) { + for iteration := range 20 { + t.Run(fmt.Sprint(iteration), func(t *testing.T) { + block := harness.NewBlock(t) + limits := server.DefaultLimits() + limits.MaxExecutionsPerConnection = 1 + h := harness.New(t, harness.WithServerOptions(server.WithLimits(limits)), harness.WithBehavior(harness.RuntimeBehavior{Plan: harness.PlanBehavior{Session: func(harness.SessionOptions) harness.SessionBehavior { + return harness.SessionBehavior{Run: func(ctx context.Context, call int) (api.Output, error) { + if call == 1 { + return api.Output{}, block.Wait(ctx) + } + + return api.Output{}, nil + }} + }}})) + plan, err := h.Runtime().Compile(h.Context(), api.Source{Content: "RETURN 1"}) + if err != nil { + t.Fatal(err) + } + + session, err := plan.NewSession(h.Context()) + if err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithCancel(h.Context()) + defer cancel() + result := make(chan error, 1) + go func() { + _, err := session.Run(ctx) + result <- err + }() + harness.Await(t, block.Started) + var race sync.WaitGroup + race.Add(2) + go func() { + defer race.Done() + cancel() + }() + go func() { + defer race.Done() + block.Release() + }() + race.Wait() + + if err := harness.Await(t, result); err != nil && !errors.Is(err, context.Canceled) && !errors.Is(err, client.ErrExecutionCancelled) { + t.Fatalf("completion race=%v", err) + } + + if h.Faults().Count(harness.ReleaseExecution) != 1 || h.Faults().Count(harness.CancelExecution) != 0 { + t.Fatal("execution cleanup duplicated") + } + + if _, err := session.Run(h.Context()); err != nil { + t.Fatalf("execution capacity leaked: %v", err) + } + + snapshot := h.RuntimeSpy().Recorder().Snapshot() + id := snapshot.OfKind("session")[0].ID + if snapshot.Count(id, "Run") != 2 || snapshot.Count(id, "Close") != 0 { + t.Fatalf("durable session changed: %+v", snapshot) + } + }) + } +} From fd817ccbbc4735b46b764ea2e0d6b22fd3d656f0 Mon Sep 17 00:00:00 2001 From: Tim Voronov Date: Sat, 5 Sep 2026 13:52:38 -0400 Subject: [PATCH 2/2] Classify connection loss errors for integration tests: add `expectedConnectionLoss` utility function, update error handling logic, and introduce new test suite for validating connection loss scenarios. --- test/integration/README.md | 7 +- test/integration/connection_failure_test.go | 76 +++++++++++++++++++++ test/integration/connection_test.go | 43 +++++++++++- 3 files changed, 124 insertions(+), 2 deletions(-) create mode 100644 test/integration/connection_failure_test.go diff --git a/test/integration/README.md b/test/integration/README.md index 367bff2..5fd7b9b 100644 --- a/test/integration/README.md +++ b/test/integration/README.md @@ -121,7 +121,12 @@ rejected creation. Abrupt connection loss returns errors. Graceful server shutdown can deliver a semantic debugger termination event before transport closure; the suite accepts -that terminal outcome. Premature watch closure must not look like successful +that terminal outcome. Runtime and normal-session execution can instead receive +`NotFound` if shutdown removes the logical connection or execution before the +client opens its watch. The suite accepts this only for server shutdown with a +public `ConnectionNotFound` or `ExecutionNotFound` category, and checks each joined +operation and cleanup error independently. Cancellation and exactly-once hosted +cleanup remain required. Premature watch closure must not look like successful execution. Explicit debugger Close still owns cleanup after a watch-only failure. No automatic reconnection or protocol changes are introduced. diff --git a/test/integration/connection_failure_test.go b/test/integration/connection_failure_test.go new file mode 100644 index 0000000..f59856c --- /dev/null +++ b/test/integration/connection_failure_test.go @@ -0,0 +1,76 @@ +package integration_test + +import ( + "errors" + "fmt" + "testing" + + "github.com/MontFerret/wire/client" + "github.com/MontFerret/wire/pkg/failure" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// connectionLossRPCError supplies public client metadata and a transport status +// without coupling contract tests to protobuf error details. +type connectionLossRPCError struct { + cause *client.Error + code codes.Code +} + +func (e *connectionLossRPCError) Error() string { + return e.cause.Error() +} + +func (e *connectionLossRPCError) Unwrap() error { + return e.cause +} + +func (e *connectionLossRPCError) GRPCStatus() *status.Status { + return status.New(e.code, e.Error()) +} + +func TestConnectionLossErrorClassification(t *testing.T) { + connectionGone := &connectionLossRPCError{cause: &client.Error{Category: failure.CategoryConnectionNotFound}, code: codes.NotFound} + executionGone := &connectionLossRPCError{cause: &client.Error{Category: failure.CategoryExecutionNotFound}, code: codes.NotFound} + unavailable := status.Error(codes.Unavailable, "transport closed") + unexpected := errors.New("unexpected cleanup failure") + + for _, mode := range []string{"runtime", "session", "debugger"} { + for _, shutdown := range []bool{false, true} { + allowMissing := shutdown && mode != "debugger" + + for _, test := range []struct { + name string + err error + want bool + }{ + {"no error", nil, true}, + {"transport unavailable", unavailable, true}, + {"transport canceled", status.Error(codes.Canceled, "transport canceled"), true}, + {"closed handle", client.ErrClosed, true}, + {"remote cancellation", client.ErrExecutionCancelled, true}, + {"connection removed", connectionGone, allowMissing}, + {"execution removed", executionGone, allowMissing}, + {"wrapped missing execution", fmt.Errorf("watch: %w", executionGone), allowMissing}, + {"uncategorized not found", status.Error(codes.NotFound, "resource not found"), false}, + {"unrelated missing resource", &connectionLossRPCError{cause: &client.Error{Category: failure.CategoryPlanNotFound}, code: codes.NotFound}, false}, + {"incorrect status", &connectionLossRPCError{cause: &client.Error{Category: failure.CategoryConnectionNotFound}, code: codes.Internal}, false}, + {"unexpected failure", unexpected, false}, + {"expected joined failures", errors.Join(unavailable, client.ErrExecutionCancelled), true}, + {"shutdown joined failures", errors.Join(connectionGone, unavailable), allowMissing}, + {"transport hides failure", errors.Join(unavailable, unexpected), false}, + {"sentinel hides failure", errors.Join(client.ErrClosed, unexpected), false}, + {"shutdown hides failure", errors.Join(connectionGone, unexpected), false}, + {"unexpected first cause", errors.Join(unexpected, unavailable), false}, + {"wrapped nested join", fmt.Errorf("run: %w", errors.Join(unavailable, errors.Join(client.ErrExecutionCancelled, unexpected))), false}, + } { + t.Run(mode+map[bool]string{false: "/transport/", true: "/server/"}[shutdown]+test.name, func(t *testing.T) { + if got := expectedConnectionLoss(test.err, mode, shutdown); got != test.want { + t.Fatalf("accepted %v = %v, want %v", test.err, got, test.want) + } + }) + } + } + } +} diff --git a/test/integration/connection_test.go b/test/integration/connection_test.go index 19d37dc..eae874a 100644 --- a/test/integration/connection_test.go +++ b/test/integration/connection_test.go @@ -101,7 +101,7 @@ func TestConnectionLossReclaimsResources(t *testing.T) { t.Fatalf("active operation silently succeeded after connection loss: event=%#v", terminalEvent) } - if code := status.Code(err); err != nil && code != codes.Unavailable && code != codes.Canceled && !errors.Is(err, client.ErrClosed) && !errors.Is(err, client.ErrExecutionCancelled) { + if !expectedConnectionLoss(err, mode, shutdown) { t.Fatalf("connection failure classification=%v", err) } @@ -113,6 +113,47 @@ func TestConnectionLossReclaimsResources(t *testing.T) { } } +func expectedConnectionLoss(err error, mode string, shutdown bool) bool { + if err == nil { + return true + } + + // A wrapped join still contains independent operation and cleanup failures. + // Inspect every cause before matching a status or sentinel from the tree. + var joined interface{ Unwrap() []error } + if errors.As(err, &joined) { + for _, cause := range joined.Unwrap() { + if !expectedConnectionLoss(cause, mode, shutdown) { + return false + } + } + + return true + } + + if errors.Is(err, client.ErrClosed) || errors.Is(err, client.ErrExecutionCancelled) { + return true + } + + switch status.Code(err) { + case codes.Unavailable, codes.Canceled: + return true + case codes.NotFound: + // Shutdown can remove the connection or execution before Run opens its + // watch. Debugger commands establish their watch before starting work. + if !shutdown || (mode != "runtime" && mode != "session") { + return false + } + + var remote *client.Error + if errors.As(err, &remote) { + return remote.Category == failure.CategoryConnectionNotFound || remote.Category == failure.CategoryExecutionNotFound + } + } + + return false +} + func TestWatchTerminationReturnsError(t *testing.T) { for _, debug := range []bool{false, true} { for _, watchErr := range []error{io.EOF, status.Error(codes.Unavailable, "watch transport lost")} {