Shreesh-Sree · Nov 2, 2026 · Nov 2, 2026 · Nov 2, 2026
@@ -23,6 +23,10 @@ and uses [semantic versioning](https://semver.org/).
 
 ### Changed
 
+- **One compile for the same source** — jobs with the same source, language, files and flags that arrive together share a single compile instead of each starting their own. A source that does not compile fails the same way for all of them; a sandbox error in the one compiling makes the others try for themselves.
+- **`output` mode** — `POST /api/v1/execute` accepts `output`: `full` (default), `preview` or `none`, to keep full stdout and stderr out of the event log, SSE and webhook when only verdicts are needed.
+- **Pool sizes** — `AC_DB_MAX_CONNS`, `AC_REDIS_POOL_SIZE` and `AC_REDIS_MIN_IDLE_CONNS` (defaults 128, 256, 32, as before).
+- **Latency alerts** — Prometheus rules for queue-wait p99, slow database or Redis stages, and a compile cache that almost never hits.
 - **Where the time goes** — new histograms `ac_queue_wait_seconds{stream}` (time between enqueue and a worker reading the job) and `ac_job_stage_seconds{stage}` (`claim`, `load_tests`, `run`, `persist`, `publish`), beside the existing `ac_compile_duration_seconds`.
 - **Lower latency per job** — a worker now fetches a job's tests from Redis while it claims the job in PostgreSQL, instead of one after the other.
 - **One Redis subscription per gateway** — SSE clients share a single pattern subscription instead of one subscription each. A client that falls behind is closed so it reconnects and replays from the event log; no event is dropped silently.

@@ -314,6 +314,7 @@ Submit code for evaluation against test cases.
 | `priority` | string | No | Priority tier: `run` (P0 default), `submit` (P1), or `bulk` (P2) |
 | `time_limit_ms` | integer | No | Per-test CPU time ceiling in milliseconds (100 to 20000) |
 | `memory_limit_kb` | integer | No | Per-test memory ceiling in kilobytes (16384 to 2097152) |
+| `output` | string | No | How much output each test's result carries: `full` (default; stdout and stderr in full plus 256-byte previews), `preview` (previews only) or `none` |
 | `stack_limit_kb` | integer | No | Per-test stack size limit in kilobytes (1024 to 1048576) |
 | `max_processes` | integer | No | Per-test process/thread limit (1 to 512); the language default otherwise |
 | `args` | array | No | Command-line arguments passed to the program on every test, as separate argv entries, never through a shell (max 16, 4 KB in total; not for `sqlite`) |
@@ -622,6 +623,12 @@ If queue wait dominates, add capacity; if `run` dominates, look at the language
 - On a multi-socket host, run one engine plus one go-judge per NUMA node (for example `numactl --cpunodebind=0 --membind=0` for each pair, each with its own listen address and `-parallelism` set to that node's physical cores) rather than one pair on one node. The published benchmark used a single node of a 256-thread host.
 - Raise go-judge's `-pre-fork` toward its parallelism so a burst does not wait for sandbox creation.
 
+**Pools.** The database pool defaults to 128 connections (`AC_DB_MAX_CONNS`) and the Redis pool to 256 (`AC_REDIS_POOL_SIZE`, with `AC_REDIS_MIN_IDLE_CONNS` kept open). Ingest can use up to 512 database connections at once, so under heavy submission traffic raise `AC_DB_MAX_CONNS` (and PostgreSQL's `max_connections`) first; watch `ac_job_stage_seconds{stage="claim"}` and `{stage="persist"}`.
+
+**Compiles.** Jobs with the same source, language, files and flags that arrive together share one compile (the rest wait for it), and later ones hit the compile cache (`AC_COMPILE_CACHE_TTL`). A caller that submits one job per test of the same program therefore compiles once, but a single job with all the tests is cheaper still: one queue entry, one event stream and one compile.
+
+**Output.** A job's full stdout and stderr go through Redis, the event log and every SSE client. Send `"output": "preview"` or `"none"` when only verdicts matter.
+
 **Overload.** `AC_ADMISSION_MAX_QUEUE_DEPTH` bounds how long a job can wait. A job admitted at the limit waits about depth ÷ completion rate, so lower the limit if that exceeds your verdict SLO. Rejected callers receive `Retry-After` computed from the measured completion rate.
 
 ## Repository layout

@@ -175,6 +175,17 @@ components:
             Per-test memory limit override in kilobytes (16384–2097152 = 16 MB–2 GB).
             Omit to use the language default.
           example: 262144
+        output:
+          type: string
+          enum: [full, preview, none]
+          default: full
+          description: |
+            How much of each test's output the VERDICT event, the event log and the
+            webhook carry. `full` includes stdout and stderr in full (up to 8 MB per
+            test) beside the 256-byte previews. `preview` keeps only the previews and
+            `none` drops all output text. A large job's full output goes through Redis
+            and every SSE client, so callers that only need verdicts should not ask for it.
+          example: preview
         stack_limit_kb:
           type: integer
           format: int64

@@ -73,7 +73,7 @@ func main() {
 	slog.Info("go-judge connected", "url", cfg.JudgeURL)
 
 	// Database
-	pool, err := db.Connect(ctx, cfg.DatabaseURL)
+	pool, err := db.ConnectWithMaxConns(ctx, cfg.DatabaseURL, cfg.DBMaxConns)
 	if err != nil {
 		slog.Error("database connection failed", "err", err)
 		os.Exit(1)
@@ -89,8 +89,8 @@ func main() {
 	// Redis
 	rdb := redis.NewClient(&redis.Options{
 		Addr:         cfg.RedisAddr,
-		PoolSize:     256,
-		MinIdleConns: 32,
+		PoolSize:     cfg.RedisPool,
+		MinIdleConns: cfg.RedisMinIdle,
 	})
 	if err := rdb.Ping(ctx).Err(); err != nil {
 		slog.Error("redis connection failed", "addr", cfg.RedisAddr, "err", err)

@@ -78,3 +78,43 @@ groups:
           description: >
             Prometheus cannot scrape the aethercode-exec metrics endpoint.
             The process may have crashed or the host may be unreachable.
+
+      # ── Latency (see README, "Performance Tuning") ──────────────────────────
+      - alert: QueueWaitP99High
+        expr: histogram_quantile(0.99, sum by (le, stream) (rate(ac_queue_wait_seconds_bucket[5m]))) > 30
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Jobs in {{ $labels.stream }} wait more than 30s at the p99"
+          description: >
+            The p99 wait between enqueue and a worker reading the job is
+            {{ $value | humanizeDuration }}. Workers or sandbox slots are the
+            bottleneck: compare AC_WORKER_COUNT with go-judge -parallelism and CPU.
+
+      - alert: JobStageP99High
+        expr: histogram_quantile(0.99, sum by (le, stage) (rate(ac_job_stage_seconds_bucket{stage=~"claim|load_tests|persist|publish"}[5m]))) > 1
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "The {{ $labels.stage }} stage takes more than 1s at the p99"
+          description: >
+            claim, persist and publish are database and Redis round trips and
+            load_tests is a Redis read. Check PostgreSQL and Redis latency and
+            the connection pool sizes (AC_DB_MAX_CONNS, AC_REDIS_POOL_SIZE).
+
+      - alert: CompileCacheMostlyMissing
+        expr: >
+          sum(rate(ac_compile_cache_misses_total[15m]))
+          / (sum(rate(ac_compile_cache_hits_total[15m])) + sum(rate(ac_compile_cache_misses_total[15m]))) > 0.9
+          and sum(rate(ac_compile_cache_misses_total[15m])) > 0.5
+        for: 15m
+        labels:
+          severity: info
+        annotations:
+          summary: "Almost every compile is a cache miss"
+          description: >
+            Expected when every job carries different source. If one source is
+            sent as many jobs (one per test), check that the caller is not
+            defeating the compile cache.
@@ -4,6 +4,10 @@ AC_REDIS_ADDR=127.0.0.1:6379
 AC_JUDGE_URL=http://127.0.0.1:5050
 AC_JUDGE_TOKEN=CHANGEME
 AC_WORKER_COUNT=64
+# Connection pools; raise AC_DB_MAX_CONNS (and PostgreSQL max_connections) under heavy submission traffic.
+# AC_DB_MAX_CONNS=128
+# AC_REDIS_POOL_SIZE=256
+# AC_REDIS_MIN_IDLE_CONNS=32
 AC_LOG_LEVEL=info
 # json | text | gcp (Google Cloud Logging) | cloudwatch (AWS CloudWatch Logs)
 AC_LOG_FORMAT=json

@@ -12,12 +12,18 @@ import (
 )
 
 type Config struct {
-	GatewayAddr   string
-	DatabaseURL   string
-	RedisAddr     string
-	JudgeURL      string
-	JudgeToken    string
-	WorkerCount   int
+	GatewayAddr string
+	DatabaseURL string
+	RedisAddr   string
+	JudgeURL    string
+	JudgeToken  string
+	WorkerCount int
+	// Connection pool sizes. The defaults suit up to about 128 concurrent
+	// database users; ingest alone can use 512 at once (see the admission
+	// semaphore), so raise DBMaxConns before raising ingest concurrency.
+	DBMaxConns    int
+	RedisPool     int
+	RedisMinIdle  int
 	StreamName    string
 	ConsumerGroup string
 	LogLevel      string
@@ -83,6 +89,9 @@ func Load() Config {
 		JudgeURL:      env("AC_JUDGE_URL", "http://127.0.0.1:5050"),
 		JudgeToken:    env("AC_JUDGE_TOKEN", ""),
 		WorkerCount:   envInt("AC_WORKER_COUNT", 8),
+		DBMaxConns:    envInt("AC_DB_MAX_CONNS", 128),
+		RedisPool:     envInt("AC_REDIS_POOL_SIZE", 256),
+		RedisMinIdle:  envInt("AC_REDIS_MIN_IDLE_CONNS", 32),
 		StreamName:    env("AC_STREAM_NAME", "ac:submissions"),
 		ConsumerGroup: env("AC_CONSUMER_GROUP", "workers"),
 		LogLevel:      env("AC_LOG_LEVEL", "info"),
@@ -136,6 +145,9 @@ func (c Config) Validate() error {
 	}
 
 	check(c.WorkerCount > 0 && c.WorkerCount <= 512, "WorkerCount must be 1-512")
+	check(c.DBMaxConns >= 4 && c.DBMaxConns <= 2000, "DBMaxConns must be 4-2000")
+	check(c.RedisPool >= 4 && c.RedisPool <= 10000, "RedisPool must be 4-10000")
+	check(c.RedisMinIdle >= 0 && c.RedisMinIdle <= c.RedisPool, "RedisMinIdle must be 0 up to RedisPool")
 	check(c.SubmitWeight > 0, "SubmitWeight must be > 0")
 	check(c.BulkWeight > 0, "BulkWeight must be > 0")
 	check(c.AdmissionMaxQueueDepth > 0, "AdmissionMaxQueueDepth must be > 0")

@@ -14,6 +14,9 @@ func validConfig() Config {
 		JudgeURL:                 "http://127.0.0.1:5050",
 		JudgeToken:               "a-secret-token",
 		WorkerCount:              8,
+		DBMaxConns:               128,
+		RedisPool:                256,
+		RedisMinIdle:             32,
 		TestParallelism:          4,
 		StreamName:               "ac:submissions",
 		ConsumerGroup:            "workers",
@@ -112,3 +115,22 @@ func TestEnvDuration(t *testing.T) {
 		t.Fatalf("envDuration unset = %v, want fallback 3s", got)
 	}
 }
+
+func TestValidate_PoolSizes(t *testing.T) {
+	for name, mutate := range map[string]func(*Config){
+		"db too small":    func(c *Config) { c.DBMaxConns = 1 },
+		"db too large":    func(c *Config) { c.DBMaxConns = 5000 },
+		"redis too small": func(c *Config) { c.RedisPool = 1 },
+		"idle above pool": func(c *Config) { c.RedisMinIdle = c.RedisPool + 1 },
+		"negative idle":   func(c *Config) { c.RedisMinIdle = -1 },
+	} {
+		cfg := validConfig()
+		mutate(&cfg)
+		if err := cfg.Validate(); err == nil {
+			t.Errorf("%s: Validate accepted it", name)
+		}
+	}
+	if err := validConfig().Validate(); err != nil {
+		t.Errorf("valid config rejected: %v", err)
+	}
+}
@@ -28,13 +28,25 @@ func buildPool(ctx context.Context, config *pgxpool.Config, connectPrefix, pingP
 	return pool, nil
 }
 
+// DefaultMaxConns is the main pool's size unless the caller sets one.
+const DefaultMaxConns = 128
+
 func Connect(ctx context.Context, databaseURL string) (*pgxpool.Pool, error) {
+	return ConnectWithMaxConns(ctx, databaseURL, DefaultMaxConns)
+}
+
+// ConnectWithMaxConns is Connect with the pool's size set by the caller. The
+// pool keeps a sixteenth of it open (at least 4, at most 16).
+func ConnectWithMaxConns(ctx context.Context, databaseURL string, maxConns int) (*pgxpool.Pool, error) {
 	config, err := pgxpool.ParseConfig(databaseURL)
 	if err != nil {
 		return nil, fmt.Errorf("db: parse config: %w", err)
 	}
-	config.MinConns = 16
-	config.MaxConns = 128
+	if maxConns < 4 || maxConns > 2000 {
+		return nil, fmt.Errorf("db: max connections %d outside 4-2000", maxConns)
+	}
+	config.MinConns = int32(max(4, min(16, maxConns/16)))
+	config.MaxConns = int32(maxConns)
 	// Optimize query execution by caching prepared statements and query descriptions
 	config.ConnConfig.DefaultQueryExecMode = pgx.QueryExecModeCacheDescribe
 	return buildPool(ctx, config, "db: connect", "db: ping")

@@ -177,6 +177,8 @@ type executeRequest struct {
 	CallbackSecret string `json:"callback_secret,omitempty"`
 	// Optional priority tier: "run" (default / interactive), "submit" (contest / assessment), or "bulk" (batch re-judge).
 	Priority string `json:"priority"`
+	// Optional output mode: "full" (default), "preview" (256-byte previews only) or "none".
+	Output string `json:"output"`
 
 	// Mode and ProblemVersionID exist only to reject legacy submit-mode requests.
 	Mode             string  `json:"mode"`
@@ -354,6 +356,10 @@ func (h *Handler) HandleExecute(w http.ResponseWriter, r *http.Request) {
 		jsonError(w, "callback_secret exceeds 256 bytes", http.StatusBadRequest)
 		return
 	}
+	if !model.ValidOutputMode(req.Output) {
+		jsonError(w, "output must be one of: full, preview, none", http.StatusBadRequest)
+		return
+	}
 	if !model.ValidPriority(req.Priority) {
 		jsonError(w, "priority must be one of: run, submit, bulk", http.StatusBadRequest)
 		return
@@ -517,6 +523,7 @@ func (h *Handler) HandleExecute(w http.ResponseWriter, r *http.Request) {
 		// CallbackSecret is validated above (length cap). Encryption at rest
 		// before writing to Redis is handled by the pubsub layer (F18/F8).
 		CallbackSecret: req.CallbackSecret,
+		Output:         req.Output,
 	}
 	if err := h.pubsub.PrepareAndEnqueue(ctx, jobID.String(), testSet, queuedEvt, targetStream.StreamName()); err != nil {
 		slog.Error("enqueue failed", "submission_id", jobID, "err", err)

@@ -653,6 +653,9 @@ func TestHandleExecute_RejectsInvalidRunControls(t *testing.T) {
 		{"too many args",
 			`{"language":"c","source_code":"int main(){}","args":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17"],"tests":[{"input":"","expected_output":""}]}`,
 			"args: at most 16 arguments"},
+		{"unknown output mode",
+			`{"language":"c","source_code":"int main(){}","output":"everything","tests":[{"input":"","expected_output":""}]}`,
+			"output must be one of: full, preview, none"},
 		{"args for sqlite",
 			`{"language":"sqlite","source_code":"select 1;","args":["x"],"tests":[{"input":"","expected_output":"1"}]}`,
 			"args are not supported for sqlite"},

@@ -90,6 +90,23 @@ func ValidPriority(s string) bool {
 	return false
 }
 
+// Output modes: how much of each test's output the VERDICT event, the event
+// log and the webhook carry.
+const (
+	OutputFull    = "full"    // stdout and stderr in full, plus 256-byte previews (default)
+	OutputPreview = "preview" // 256-byte previews only
+	OutputNone    = "none"    // no output text at all
+)
+
+// ValidOutputMode reports whether s is an output mode. The empty string is valid and means full.
+func ValidOutputMode(s string) bool {
+	switch s {
+	case "", OutputFull, OutputPreview, OutputNone:
+		return true
+	}
+	return false
+}
+
 // ValidChecker reports whether s names a supported checker. The empty string
 // is valid and means exact. Custom checkers can be specified as "custom" or when checker code is provided.
 func ValidChecker(s string) bool {

@@ -110,6 +110,7 @@ func (p *PubSub) PrepareAndEnqueue(ctx context.Context, jobID string, set TestSe
 			"interactor_language", set.InteractorLanguage,
 			"callback_url", set.CallbackURL,
 			"callback_secret", callbackSecretField,
+			"output_mode", set.Output,
 		)
 		if len(set.Files) > 0 {
 			if filesJSON, err := json.Marshal(set.Files); err == nil {
@@ -379,6 +380,7 @@ func (p *PubSub) GetTests(ctx context.Context, jobID string) (TestSet, error) {
 		CompilerFlags:      compilerFlags,
 		CallbackURL:        fields["callback_url"],
 		CallbackSecret:     callbackSecret,
+		Output:             fields["output_mode"], // absent on jobs enqueued before output modes: empty means full
 	}, nil
 }
 
@@ -402,4 +404,5 @@ type TestSet struct {
 	CompilerFlags      []string          // optional whitelisted compiler flags
 	CallbackURL        string            // optional webhook notification callback URL
 	CallbackSecret     string            // optional HMAC signing secret for webhook
+	Output             string            // how much output the verdict event carries: full (empty), preview or none
 }
@@ -572,6 +572,7 @@ func TestPrepareAndGetTests_RoundTripsRunControls(t *testing.T) {
 	set := TestSet{
 		Tests:  []TestData{{Input: "1", ExpectedOutput: "1"}},
 		Limits: sandbox.RunLimits{CPU: 1_000_000_000, Memory: 64 << 20, Stack: 8 << 20, Procs: 4, Args: []string{"-n", "5"}},
+		Output: "preview",
 	}
 	evt := eventlog.New("job-controls", eventlog.EventQueued, nil)
 	if err := ps.PrepareAndEnqueue(ctx, "job-controls", set, evt, "ac:submissions:run"); err != nil {
@@ -584,6 +585,9 @@ func TestPrepareAndGetTests_RoundTripsRunControls(t *testing.T) {
 	if !reflect.DeepEqual(got.Limits, set.Limits) {
 		t.Errorf("Limits = %+v, want %+v", got.Limits, set.Limits)
 	}
+	if got.Output != "preview" {
+		t.Errorf("Output = %q, want preview", got.Output)
+	}
 }
 
 func TestGetTests_RejectsCorruptArgs(t *testing.T) {