From 1ecf9c346a43c6dfd581157bae540f03b6b6b3de Mon Sep 17 00:00:00 2001 From: maya-ots Date: Thu, 29 Oct 2026 18:10:30 +0100 Subject: [PATCH 1/3] feat(plugin): add runtime health failure hook --- cmd/instancer.go | 15 ++ internal/instancer/health_monitor.go | 193 ++++++++++++++++++++++ internal/instancer/health_monitor_test.go | 107 ++++++++++++ internal/instancer/server_test.go | 12 +- internal/plugin/manager.go | 10 ++ internal/plugin/plugin_test.go | 4 +- internal/plugin/protocol.go | 11 ++ 7 files changed, 349 insertions(+), 3 deletions(-) create mode 100644 internal/instancer/health_monitor.go create mode 100644 internal/instancer/health_monitor_test.go diff --git a/cmd/instancer.go b/cmd/instancer.go index a0a2b97..9aa6aaa 100644 --- a/cmd/instancer.go +++ b/cmd/instancer.go @@ -115,6 +115,21 @@ var instancerServeCmd = &cobra.Command{ } }() + healthMonitor := &instancer.HealthMonitor{ + Store: store, + Interval: flagServeInterval, + Event: cfg.Event.Slug, + PluginManager: pm, + Logger: os.Stdout, + } + + go func() { + if err := healthMonitor.Run(ctx); err != nil && + !errors.Is(err, context.Canceled) { + fmt.Fprintf(os.Stderr, "health monitor exited: %v\n", err) + } + }() + httpSrv := &http.Server{Addr: flagServeAddr, Handler: srv.Handler()} errCh := make(chan error, 1) go func() { diff --git a/internal/instancer/health_monitor.go b/internal/instancer/health_monitor.go new file mode 100644 index 0000000..5577c15 --- /dev/null +++ b/internal/instancer/health_monitor.go @@ -0,0 +1,193 @@ +package instancer + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "sync" + "time" + + "github.com/MicroClub-USTHB/mctf-cli/internal/plugin" +) + +// ContainerHealth contains the health information Docker reports for a container. +type ContainerHealth struct { + ID string + Name string + Status string +} + +// HealthMonitor periodically checks instance containers and dispatches a hook +// when a container first becomes unhealthy. +type HealthMonitor struct { + Store *Store + Interval time.Duration + Event string + PluginManager *plugin.Manager + Logger io.Writer + + // Inspect is replaceable in tests so they do not need Docker. + Inspect func(context.Context, string) (ContainerHealth, error) + + // Dispatch overrides PluginManager.Dispatch in tests. + Dispatch func(context.Context, string, any) + + mu sync.Mutex + unhealthy map[string]bool +} + +// Run checks health immediately, then repeats until the context is cancelled. +func (m *HealthMonitor) Run(ctx context.Context) error { + if m.Store == nil { + return fmt.Errorf("health monitor: store is nil") + } + + if m.Interval <= 0 { + m.Interval = 30 * time.Second + } + if m.Logger == nil { + m.Logger = os.Stdout + } + if m.Inspect == nil { + m.Inspect = inspectContainerHealth + } + if m.unhealthy == nil { + m.unhealthy = make(map[string]bool) + } + + ticker := time.NewTicker(m.Interval) + defer ticker.Stop() + + if err := m.Tick(ctx); err != nil && ctx.Err() == nil { + fmt.Fprintf(m.Logger, "health monitor: initial check: %v\n", err) + } + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + if err := m.Tick(ctx); err != nil && ctx.Err() == nil { + fmt.Fprintf(m.Logger, "health monitor: check: %v\n", err) + } + } + } +} + +// Tick checks all containers belonging to currently stored instances. +func (m *HealthMonitor) Tick(ctx context.Context) error { + if m.Store == nil { + return fmt.Errorf("health monitor: store is nil") + } + if m.Inspect == nil { + m.Inspect = inspectContainerHealth + } + if m.Logger == nil { + m.Logger = os.Stdout + } + if m.unhealthy == nil { + m.unhealthy = make(map[string]bool) + } + + instances := m.Store.List() + active := make(map[string]bool) + + for _, inst := range instances { + for _, containerID := range inst.Containers { + if containerID == "" { + continue + } + active[containerID] = true + + health, err := m.Inspect(ctx, containerID) + if err != nil { + fmt.Fprintf(m.Logger, "health monitor: inspect %s: %v\n", containerID, err) + continue + } + + m.mu.Lock() + wasUnhealthy := m.unhealthy[containerID] + + if health.Status != "unhealthy" { + // A recovery (or a container without a health check) resets + // the transition so a later failure can trigger again. + delete(m.unhealthy, containerID) + m.mu.Unlock() + continue + } + + if wasUnhealthy { + m.mu.Unlock() + continue + } + m.unhealthy[containerID] = true + m.mu.Unlock() + + payload := plugin.HealthFailurePayload{ + Event: m.Event, + InstanceID: inst.ID, + ChallengeSlug: inst.ChallengeSlug, + Container: containerID, + Status: health.Status, + Timestamp: time.Now().UTC(), + } + + if m.Dispatch != nil { + m.Dispatch(ctx, plugin.HookOnHealthFailure, payload) + } else if m.PluginManager != nil { + m.PluginManager.Dispatch(ctx, plugin.HookOnHealthFailure, payload) + } + } + } + + // Drop remembered states for containers no longer tracked by the store. + m.mu.Lock() + for id := range m.unhealthy { + if !active[id] { + delete(m.unhealthy, id) + } + } + m.mu.Unlock() + + return nil +} + +// inspectContainerHealth reads Docker's structured health state. +// Containers without a HEALTHCHECK have an empty status and do not trigger +// on_health_failure. +func inspectContainerHealth(ctx context.Context, id string) (ContainerHealth, error) { + cmd := exec.CommandContext(ctx, "docker", "inspect", id) + raw, err := cmd.Output() + if err != nil { + return ContainerHealth{}, fmt.Errorf("docker inspect: %w", err) + } + + var containers []struct { + ID string `json:"Id"` + Name string `json:"Name"` + State struct { + Health *struct { + Status string `json:"Status"` + } `json:"Health"` + } `json:"State"` + } + if err := json.Unmarshal(raw, &containers); err != nil { + return ContainerHealth{}, fmt.Errorf("decode docker inspect output: %w", err) + } + if len(containers) == 0 { + return ContainerHealth{}, fmt.Errorf("docker inspect returned no container for %s", id) + } + + container := containers[0] + result := ContainerHealth{ + ID: container.ID, + Name: container.Name, + } + if container.State.Health != nil { + result.Status = container.State.Health.Status + } + return result, nil +} diff --git a/internal/instancer/health_monitor_test.go b/internal/instancer/health_monitor_test.go new file mode 100644 index 0000000..a25c4a3 --- /dev/null +++ b/internal/instancer/health_monitor_test.go @@ -0,0 +1,107 @@ +package instancer + +import ( + "context" + "path/filepath" + "testing" + + "github.com/MicroClub-USTHB/mctf-cli/internal/plugin" +) + +func TestHealthMonitor_DispatchesOnUnhealthyTransitions(t *testing.T) { + store, err := NewStore(filepath.Join(t.TempDir(), "instances.json")) + if err != nil { + t.Fatal(err) + } + + err = store.Put(Instance{ + ID: "instance-1", + ChallengeSlug: "web", + TeamID: "team-1", + Containers: []string{"container-1"}, + }) + if err != nil { + t.Fatal(err) + } + + status := "unhealthy" + var payloads []plugin.HealthFailurePayload + + monitor := &HealthMonitor{ + Store: store, + Event: "test-event", + Inspect: func(_ context.Context, id string) (ContainerHealth, error) { + return ContainerHealth{ID: id, Status: status}, nil + }, + Dispatch: func(_ context.Context, hook string, payload any) { + if hook != plugin.HookOnHealthFailure { + t.Errorf("unexpected hook: %q", hook) + return + } + + got, ok := payload.(plugin.HealthFailurePayload) + if !ok { + t.Errorf("unexpected payload type: %T", payload) + return + } + payloads = append(payloads, got) + }, + } + + ctx := context.Background() + + // First unhealthy result should trigger the hook. + if err := monitor.Tick(ctx); err != nil { + t.Fatal(err) + } + if len(payloads) != 1 { + t.Fatalf("after first unhealthy check: got %d hooks, want 1", len(payloads)) + } + + // Repeated unhealthy results must not trigger duplicate hooks. + if err := monitor.Tick(ctx); err != nil { + t.Fatal(err) + } + if len(payloads) != 1 { + t.Fatalf("after repeated unhealthy check: got %d hooks, want 1", len(payloads)) + } + + // Recovery resets the transition state. + status = "healthy" + if err := monitor.Tick(ctx); err != nil { + t.Fatal(err) + } + if len(payloads) != 1 { + t.Fatalf("after recovery: got %d hooks, want 1", len(payloads)) + } + + // A new unhealthy transition should trigger the hook again. + status = "unhealthy" + if err := monitor.Tick(ctx); err != nil { + t.Fatal(err) + } + if len(payloads) != 2 { + t.Fatalf("after second unhealthy transition: got %d hooks, want 2", len(payloads)) + } + + first := payloads[0] + if first.Event != "test-event" { + t.Errorf("Event = %q, want test-event", first.Event) + } + if first.InstanceID != "instance-1" { + t.Errorf("InstanceID = %q, want instance-1", first.InstanceID) + } + if first.ChallengeSlug != "web" { + t.Errorf("ChallengeSlug = %q, want web", first.ChallengeSlug) + } + if first.Container != "container-1" { + t.Errorf("Container = %q, want container-1", first.Container) + } + if first.Status != "unhealthy" { + t.Errorf("Status = %q, want unhealthy", first.Status) + } + if first.Timestamp.IsZero() || first.Timestamp.Location() != nil && + first.Timestamp.Location().String() != "UTC" { + t.Errorf("Timestamp = %v, want a non-zero UTC timestamp", first.Timestamp) + } +} diff --git a/internal/instancer/server_test.go b/internal/instancer/server_test.go index aa8ae51..1dd1aae 100644 --- a/internal/instancer/server_test.go +++ b/internal/instancer/server_test.go @@ -11,6 +11,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "runtime" "strings" "sync" "testing" @@ -357,9 +358,18 @@ done t.Fatal(err) } + command := script + var args []string + + if runtime.GOOS == "windows" { + command = "bash" + args = []string{script} + } + raw := []map[string]any{{ "name": "test", - "command": script, + "command": command, + "args": args, }} pm := plugin.Load(context.Background(), raw, plugin.Options{ HostVersion: "test", diff --git a/internal/plugin/manager.go b/internal/plugin/manager.go index 24614da..b09b832 100644 --- a/internal/plugin/manager.go +++ b/internal/plugin/manager.go @@ -6,6 +6,7 @@ import ( "io" "os" "os/exec" + "sync" "time" ) @@ -26,6 +27,7 @@ type Options struct { // A nil *Manager is valid and all methods no-op, so call sites don't branch // when no plugins are configured. type Manager struct { + dispatchMu sync.Mutex plugins []*Plugin opts Options log io.Writer @@ -95,6 +97,10 @@ func (m *Manager) Dispatch(ctx context.Context, hook string, payload any) { if m == nil { return } + + m.dispatchMu.Lock() + defer m.dispatchMu.Unlock() + live := m.plugins[:0] for _, p := range m.plugins { dctx, cancel := context.WithTimeout(ctx, m.opts.DispatchTimeout) @@ -114,6 +120,10 @@ func (m *Manager) Close() { if m == nil { return } + + m.dispatchMu.Lock() + defer m.dispatchMu.Unlock() + for _, p := range m.plugins { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) p.shutdown(ctx) diff --git a/internal/plugin/plugin_test.go b/internal/plugin/plugin_test.go index 7688015..48029b5 100644 --- a/internal/plugin/plugin_test.go +++ b/internal/plugin/plugin_test.go @@ -50,14 +50,14 @@ func TestHelperProcess(t *testing.T) { fmt.Fprintln(os.Stdout, "this is not json") // desync the stream continue } - hooks := []string{"before_deploy", "after_deploy", "before_build", "after_build", "on_instance_spawn"} + hooks := []string{"before_deploy", "after_deploy", "before_build", "after_build", "on_instance_spawn", "on_health_failure"} if mode == "nohooks" { hooks = nil } reply(req.ID, map[string]any{"name": "helper-" + mode, "hooks": hooks}) case "shutdown": os.Exit(0) - case "before_deploy", "after_deploy", "before_build", "after_build", "on_instance_spawn": + case "before_deploy", "after_deploy", "before_build", "after_build", "on_instance_spawn", "on_health_failure": switch mode { case "crash": os.Exit(1) diff --git a/internal/plugin/protocol.go b/internal/plugin/protocol.go index baf26fb..3a396f5 100644 --- a/internal/plugin/protocol.go +++ b/internal/plugin/protocol.go @@ -69,6 +69,7 @@ const ( HookBeforeBuild = "before_build" HookAfterBuild = "after_build" HookOnInstanceSpawn = "on_instance_spawn" + HookOnHealthFailure = "on_health_failure" ) // DeployPayload is the JSON body for the before_/after_deploy hooks. @@ -124,3 +125,13 @@ type InstanceSpawnPayload struct { Containers []string `json:"containers,omitempty"` ExpiresAt time.Time `json:"expires_at"` } + +// HealthFailurePayload is the JSON body for the on_health_failure hook. +type HealthFailurePayload struct { + Event string `json:"event,omitempty"` + InstanceID string `json:"instance_id"` + ChallengeSlug string `json:"challenge_slug"` + Container string `json:"container"` + Status string `json:"status"` + Timestamp time.Time `json:"timestamp"` +} From a2deacea2f78bfcb6c8dbbf04d855b52892870bc Mon Sep 17 00:00:00 2001 From: maya-ots Date: Thu, 29 Oct 2026 18:10:58 +0100 Subject: [PATCH 2/3] docs(plugin): document health failure hook --- docs/BACKLOG.md | 2 +- docs/spec/plugins.md | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/BACKLOG.md b/docs/BACKLOG.md index 2760483..663a61b 100644 --- a/docs/BACKLOG.md +++ b/docs/BACKLOG.md @@ -5,7 +5,7 @@ Open work organized by size. Each line is intentionally GitHub-issue-shaped: tit ## Big rocks (multi-day each) - **K8s deploy mode** — `mctf deploy --mode kubernetes`. Generate K8s Deployment / Service / Ingress manifests from per-challenge compose, `kubectl apply -f`. Honor `cfg.Deploy.Kubernetes.{Kubeconfig, Namespace, IngressClass}`. -- **Plugin system** — subprocess + JSON-RPC over stdio (Terraform-style), language-agnostic, crash-isolated. **Foundation + `before_deploy`/`after_deploy` + `before_build`/`after_build` + `on_instance_spawn` shipped** (`internal/plugin`, `docs/spec/plugins.md`). *Remaining:* `on_health_failure` (needs a health-monitoring loop — largest), dispatch on `down`/kubernetes paths, and a blocking/`required` mode (a plugin that can veto). +- **Plugin system** — subprocess + JSON-RPC over stdio (Terraform-style), language-agnostic, crash-isolated. **Foundation + `before_deploy`/`after_deploy` + `before_build`/`after_build` + `on_instance_spawn` + `on_health_failure` shipped** (`internal/plugin`, `internal/instancer/health_monitor.go`, `docs/spec/plugins.md`). *Remaining:* dispatch on `down`/Kubernetes paths, and a blocking/`required` mode (a plugin that can veto). - ~~**Prometheus / Grafana wiring** — `mctf monitoring up` brings up prom + grafana + cAdvisor with prebuilt dashboards. Generate scrape config from challenge labels.~~ **Shipped:** `mctf monitoring up` / `down` render and run the stack under `monitoring/stack/` (see `docs/spec/monitoring.md`). - **Terraform wrapper** — `mctf infra plan|apply|destroy` for VPS, clusters, networking, DNS. Templates for DigitalOcean, Hetzner, DOKS, EKS, GKE. diff --git a/docs/spec/plugins.md b/docs/spec/plugins.md index 0df9a43..4486fa0 100644 --- a/docs/spec/plugins.md +++ b/docs/spec/plugins.md @@ -80,8 +80,7 @@ killed. (This mirrors the `notify` package.) | `before_build` | build payload (event, sha, dry_run, images) | **shipped** | | `after_build` | build payload + `outcome` (`success`/`failed`) + `error` | **shipped** | | `on_instance_spawn` | instance spawn payload (event, instance_id, challenge_slug, team_id, ports, flag, containers, expires_at) | **shipped** | -| `on_health_failure` | — | planned (needs a health-monitoring loop) | - +| `on_health_failure` | health failure payload (event, instance_id, challenge_slug, container, status, timestamp) | **shipped** | ### Deploy payload ```json From 7fb488ecabbefb40256fdb0f2e6a0e7d5f4acd6e Mon Sep 17 00:00:00 2001 From: maya-ots Date: Fri, 30 Oct 2026 00:02:02 +0100 Subject: [PATCH 3/3] style(instancer): fix lint spelling --- internal/instancer/health_monitor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/instancer/health_monitor.go b/internal/instancer/health_monitor.go index 5577c15..3495ab4 100644 --- a/internal/instancer/health_monitor.go +++ b/internal/instancer/health_monitor.go @@ -39,7 +39,7 @@ type HealthMonitor struct { unhealthy map[string]bool } -// Run checks health immediately, then repeats until the context is cancelled. +// Run checks health immediately, then repeats until the context is canceled. func (m *HealthMonitor) Run(ctx context.Context) error { if m.Store == nil { return fmt.Errorf("health monitor: store is nil")