Feat/plugin health failure hook

cmd/instancer.go

@@ -115,6 +115,21 @@ var instancerServeCmd = &cobra.Command{
 			}
 		}()
 
+		healthMonitor := &instancer.HealthMonitor{
+			Store:         store,
+			Interval:      flagServeInterval,
+			Event:         cfg.Event.Slug,
+			PluginManager: pm,
+			Logger:        os.Stdout,
+		}
+
+		go func() {
+			if err := healthMonitor.Run(ctx); err != nil &&
+				!errors.Is(err, context.Canceled) {
+				fmt.Fprintf(os.Stderr, "health monitor exited: %v\n", err)
+			}
+		}()
+
 		httpSrv := &http.Server{Addr: flagServeAddr, Handler: srv.Handler()}
 		errCh := make(chan error, 1)
 		go func() {

docs/BACKLOG.md

@@ -5,7 +5,7 @@ Open work organized by size. Each line is intentionally GitHub-issue-shaped: tit
 ## Big rocks (multi-day each)
 
 - **K8s deploy mode** — `mctf deploy --mode kubernetes`. Generate K8s Deployment / Service / Ingress manifests from per-challenge compose, `kubectl apply -f`. Honor `cfg.Deploy.Kubernetes.{Kubeconfig, Namespace, IngressClass}`.
-- **Plugin system** — subprocess + JSON-RPC over stdio (Terraform-style), language-agnostic, crash-isolated. **Foundation + `before_deploy`/`after_deploy` + `before_build`/`after_build` + `on_instance_spawn` shipped** (`internal/plugin`, `docs/spec/plugins.md`). *Remaining:* `on_health_failure` (needs a health-monitoring loop — largest), dispatch on `down`/kubernetes paths, and a blocking/`required` mode (a plugin that can veto).
+- **Plugin system** — subprocess + JSON-RPC over stdio (Terraform-style), language-agnostic, crash-isolated. **Foundation + `before_deploy`/`after_deploy` + `before_build`/`after_build` + `on_instance_spawn` + `on_health_failure` shipped** (`internal/plugin`, `internal/instancer/health_monitor.go`, `docs/spec/plugins.md`). *Remaining:* dispatch on `down`/Kubernetes paths, and a blocking/`required` mode (a plugin that can veto).
 - ~~**Prometheus / Grafana wiring** — `mctf monitoring up` brings up prom + grafana + cAdvisor with prebuilt dashboards. Generate scrape config from challenge labels.~~ **Shipped:** `mctf monitoring up` / `down` render and run the stack under `monitoring/stack/` (see `docs/spec/monitoring.md`).
 - **Terraform wrapper** — `mctf infra plan|apply|destroy` for VPS, clusters, networking, DNS. Templates for DigitalOcean, Hetzner, DOKS, EKS, GKE.
 

docs/spec/plugins.md

@@ -80,8 +80,7 @@ killed. (This mirrors the `notify` package.)
 | `before_build` | build payload (event, sha, dry_run, images) | **shipped** |
 | `after_build` | build payload + `outcome` (`success`/`failed`) + `error` | **shipped** |
 | `on_instance_spawn` | instance spawn payload (event, instance_id, challenge_slug, team_id, ports, flag, containers, expires_at) | **shipped** |
-| `on_health_failure` | — | planned (needs a health-monitoring loop) |
-
+| `on_health_failure` | health failure payload (event, instance_id, challenge_slug, container, status, timestamp) | **shipped** |
 ### Deploy payload
 
 ```json

internal/instancer/health_monitor.go

@@ -0,0 +1,193 @@
+package instancer
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"sync"
+	"time"
+
+	"github.com/MicroClub-USTHB/mctf-cli/internal/plugin"
+)
+
+// ContainerHealth contains the health information Docker reports for a container.
+type ContainerHealth struct {
+	ID     string
+	Name   string
+	Status string
+}
+
+// HealthMonitor periodically checks instance containers and dispatches a hook
+// when a container first becomes unhealthy.
+type HealthMonitor struct {
+	Store         *Store
+	Interval      time.Duration
+	Event         string
+	PluginManager *plugin.Manager
+	Logger        io.Writer
+
+	// Inspect is replaceable in tests so they do not need Docker.
+	Inspect func(context.Context, string) (ContainerHealth, error)
+
+	// Dispatch overrides PluginManager.Dispatch in tests.
+	Dispatch func(context.Context, string, any)
+
+	mu        sync.Mutex
+	unhealthy map[string]bool
+}
+
+// Run checks health immediately, then repeats until the context is canceled.
+func (m *HealthMonitor) Run(ctx context.Context) error {
+	if m.Store == nil {
+		return fmt.Errorf("health monitor: store is nil")
+	}
+
+	if m.Interval <= 0 {
+		m.Interval = 30 * time.Second
+	}
+	if m.Logger == nil {
+		m.Logger = os.Stdout
+	}
+	if m.Inspect == nil {
+		m.Inspect = inspectContainerHealth
+	}
+	if m.unhealthy == nil {
+		m.unhealthy = make(map[string]bool)
+	}
+
+	ticker := time.NewTicker(m.Interval)
+	defer ticker.Stop()
+
+	if err := m.Tick(ctx); err != nil && ctx.Err() == nil {
+		fmt.Fprintf(m.Logger, "health monitor: initial check: %v\n", err)
+	}
+
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-ticker.C:
+			if err := m.Tick(ctx); err != nil && ctx.Err() == nil {
+				fmt.Fprintf(m.Logger, "health monitor: check: %v\n", err)
+			}
+		}
+	}
+}
+
+// Tick checks all containers belonging to currently stored instances.
+func (m *HealthMonitor) Tick(ctx context.Context) error {
+	if m.Store == nil {
+		return fmt.Errorf("health monitor: store is nil")
+	}
+	if m.Inspect == nil {
+		m.Inspect = inspectContainerHealth
+	}
+	if m.Logger == nil {
+		m.Logger = os.Stdout
+	}
+	if m.unhealthy == nil {
+		m.unhealthy = make(map[string]bool)
+	}
+
+	instances := m.Store.List()
+	active := make(map[string]bool)
+
+	for _, inst := range instances {
+		for _, containerID := range inst.Containers {
+			if containerID == "" {
+				continue
+			}
+			active[containerID] = true
+
+			health, err := m.Inspect(ctx, containerID)
+			if err != nil {
+				fmt.Fprintf(m.Logger, "health monitor: inspect %s: %v\n", containerID, err)
+				continue
+			}
+
+			m.mu.Lock()
+			wasUnhealthy := m.unhealthy[containerID]
+
+			if health.Status != "unhealthy" {
+				// A recovery (or a container without a health check) resets
+				// the transition so a later failure can trigger again.
+				delete(m.unhealthy, containerID)
+				m.mu.Unlock()
+				continue
+			}
+
+			if wasUnhealthy {
+				m.mu.Unlock()
+				continue
+			}
+			m.unhealthy[containerID] = true
+			m.mu.Unlock()
+
+			payload := plugin.HealthFailurePayload{
+				Event:         m.Event,
+				InstanceID:    inst.ID,
+				ChallengeSlug: inst.ChallengeSlug,
+				Container:     containerID,
+				Status:        health.Status,
+				Timestamp:     time.Now().UTC(),
+			}
+
+			if m.Dispatch != nil {
+				m.Dispatch(ctx, plugin.HookOnHealthFailure, payload)
+			} else if m.PluginManager != nil {
+				m.PluginManager.Dispatch(ctx, plugin.HookOnHealthFailure, payload)
+			}
+		}
+	}
+
+	// Drop remembered states for containers no longer tracked by the store.
+	m.mu.Lock()
+	for id := range m.unhealthy {
+		if !active[id] {
+			delete(m.unhealthy, id)
+		}
+	}
+	m.mu.Unlock()
+
+	return nil
+}
+
+// inspectContainerHealth reads Docker's structured health state.
+// Containers without a HEALTHCHECK have an empty status and do not trigger
+// on_health_failure.
+func inspectContainerHealth(ctx context.Context, id string) (ContainerHealth, error) {
+	cmd := exec.CommandContext(ctx, "docker", "inspect", id)
+	raw, err := cmd.Output()
+	if err != nil {
+		return ContainerHealth{}, fmt.Errorf("docker inspect: %w", err)
+	}
+
+	var containers []struct {
+		ID    string `json:"Id"`
+		Name  string `json:"Name"`
+		State struct {
+			Health *struct {
+				Status string `json:"Status"`
+			} `json:"Health"`
+		} `json:"State"`
+	}
+	if err := json.Unmarshal(raw, &containers); err != nil {
+		return ContainerHealth{}, fmt.Errorf("decode docker inspect output: %w", err)
+	}
+	if len(containers) == 0 {
+		return ContainerHealth{}, fmt.Errorf("docker inspect returned no container for %s", id)
+	}
+
+	container := containers[0]
+	result := ContainerHealth{
+		ID:   container.ID,
+		Name: container.Name,
+	}
+	if container.State.Health != nil {
+		result.Status = container.State.Health.Status
+	}
+	return result, nil
+}

internal/instancer/health_monitor_test.go

@@ -0,0 +1,107 @@
+package instancer
+
+import (
+	"context"
+	"path/filepath"
+	"testing"
+
+	"github.com/MicroClub-USTHB/mctf-cli/internal/plugin"
+)
+
+func TestHealthMonitor_DispatchesOnUnhealthyTransitions(t *testing.T) {
+	store, err := NewStore(filepath.Join(t.TempDir(), "instances.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = store.Put(Instance{
+		ID:            "instance-1",
+		ChallengeSlug: "web",
+		TeamID:        "team-1",
+		Containers:    []string{"container-1"},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	status := "unhealthy"
+	var payloads []plugin.HealthFailurePayload
+
+	monitor := &HealthMonitor{
+		Store: store,
+		Event: "test-event",
+		Inspect: func(_ context.Context, id string) (ContainerHealth, error) {
+			return ContainerHealth{ID: id, Status: status}, nil
+		},
+		Dispatch: func(_ context.Context, hook string, payload any) {
+			if hook != plugin.HookOnHealthFailure {
+				t.Errorf("unexpected hook: %q", hook)
+				return
+			}
+
+			got, ok := payload.(plugin.HealthFailurePayload)
+			if !ok {
+				t.Errorf("unexpected payload type: %T", payload)
+				return
+			}
+			payloads = append(payloads, got)
+		},
+	}
+
+	ctx := context.Background()
+
+	// First unhealthy result should trigger the hook.
+	if err := monitor.Tick(ctx); err != nil {
+		t.Fatal(err)
+	}
+	if len(payloads) != 1 {
+		t.Fatalf("after first unhealthy check: got %d hooks, want 1", len(payloads))
+	}
+
+	// Repeated unhealthy results must not trigger duplicate hooks.
+	if err := monitor.Tick(ctx); err != nil {
+		t.Fatal(err)
+	}
+	if len(payloads) != 1 {
+		t.Fatalf("after repeated unhealthy check: got %d hooks, want 1", len(payloads))
+	}
+
+	// Recovery resets the transition state.
+	status = "healthy"
+	if err := monitor.Tick(ctx); err != nil {
+		t.Fatal(err)
+	}
+	if len(payloads) != 1 {
+		t.Fatalf("after recovery: got %d hooks, want 1", len(payloads))
+	}
+
+	// A new unhealthy transition should trigger the hook again.
+	status = "unhealthy"
+	if err := monitor.Tick(ctx); err != nil {
+		t.Fatal(err)
+	}
+	if len(payloads) != 2 {
+		t.Fatalf("after second unhealthy transition: got %d hooks, want 2", len(payloads))
+	}
+
+	first := payloads[0]
+	if first.Event != "test-event" {
+		t.Errorf("Event = %q, want test-event", first.Event)
+	}
+	if first.InstanceID != "instance-1" {
+		t.Errorf("InstanceID = %q, want instance-1", first.InstanceID)
+	}
+	if first.ChallengeSlug != "web" {
+		t.Errorf("ChallengeSlug = %q, want web", first.ChallengeSlug)
+	}
+	if first.Container != "container-1" {
+		t.Errorf("Container = %q, want container-1", first.Container)
+	}
+	if first.Status != "unhealthy" {
+		t.Errorf("Status = %q, want unhealthy", first.Status)
+	}
+	if first.Timestamp.IsZero() || first.Timestamp.Location() != nil &&
+		first.Timestamp.Location().String() != "UTC" {
+		t.Errorf("Timestamp = %v, want a non-zero UTC timestamp", first.Timestamp)
+	}
+}

internal/instancer/server_test.go

@@ -11,6 +11,7 @@ import (
 	"net/http/httptest"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"sync"
 	"testing"
@@ -357,9 +358,18 @@ done
 		t.Fatal(err)
 	}
 
+	command := script
+	var args []string
+
+	if runtime.GOOS == "windows" {
+		command = "bash"
+		args = []string{script}
+	}
+
 	raw := []map[string]any{{
 		"name":    "test",
-		"command": script,
+		"command": command,
+		"args":    args,
 	}}
 	pm := plugin.Load(context.Background(), raw, plugin.Options{
 		HostVersion:  "test",