diff --git a/packages/sie_server/models/lightonai__LightOnOCR-3-4B.yaml b/packages/sie_server/models/lightonai__LightOnOCR-3-4B.yaml index d1c60c8cc..40289ac91 100644 --- a/packages/sie_server/models/lightonai__LightOnOCR-3-4B.yaml +++ b/packages/sie_server/models/lightonai__LightOnOCR-3-4B.yaml @@ -43,10 +43,15 @@ tasks: # two trained prompts accepted. The `transformers` profile runs the same contract in-process in the # transformers5 bundle. # -# Throughput: up to 128 pages run in the engine at once. The radix cache is off: every page is a -# new image, so it saves nothing, and with it on SGLang turns overlap scheduling off for this -# hybrid (Gated DeltaNet) model. FlashInfer serves the attention layers. Sampling is unchanged and -# outputs stay within bf16 run-to-run variance. +# Throughput: up to 128 pages run in the engine at once; on an L4 at this memory reservation SGLang +# caps it at 64. The radix cache is off: every page is a new image, so it saves nothing, and with it +# on SGLang turns overlap scheduling off for this hybrid (Gated DeltaNet) model. FlashInfer serves +# the attention layers. Sampling is unchanged and outputs stay within bf16 run-to-run variance. +# +# Memory: SGLang keeps (1 - mem_fraction_static) of the memory free at launch as runtime headroom. +# One page at the 16 MP limit peaks 4.8 GiB above the memory in use after load, in the vision +# encoder. On an L4, 0.75 and 0.85 run out of memory there and the engine exits, which fails every +# page in flight. 0.70 leaves 0.7 GiB free at that peak. max_sequence_length: 24576 profiles: default: @@ -55,7 +60,7 @@ profiles: adapter_path: sie_server.adapters.sglang_vision_extract.adapter:SGLangVisionExtractAdapter adapter_options: loadtime: - mem_fraction_static: 0.85 + mem_fraction_static: 0.70 served_model_name: lightonai/LightOnOCR-3-4B trust_remote_code: false max_concurrent_requests: 128 @@ -78,7 +83,7 @@ profiles: adapter_options: # Profile inheritance replaces loadtime, so retain every engine/sampling option. loadtime: - mem_fraction_static: 0.85 + mem_fraction_static: 0.70 served_model_name: lightonai/LightOnOCR-3-4B trust_remote_code: false max_concurrent_requests: 128 diff --git a/packages/sie_server/tests/config/test_lightonocr3_profiles.py b/packages/sie_server/tests/config/test_lightonocr3_profiles.py index 86c2845d3..c108846e2 100644 --- a/packages/sie_server/tests/config/test_lightonocr3_profiles.py +++ b/packages/sie_server/tests/config/test_lightonocr3_profiles.py @@ -127,6 +127,14 @@ def test_markdown_profile_retains_default_generation_context_and_engine_options( assert adapter._extra_launch_args == ["--disable-radix-cache"] +def test_sglang_profiles_keep_their_l4_calibrated_reservation() -> None: + profiles = yaml.safe_load(_CONFIG.read_text())["profiles"] + for name in ("default", "markdown"): + loadtime = profiles[name]["adapter_options"]["loadtime"] + assert loadtime["mem_fraction_static"] == 0.70 + assert loadtime["max_concurrent_requests"] == 128 + + def test_default_routes_to_sglang_vision_extract_bundle_only() -> None: assert _ADAPTER in yaml.safe_load((_BUNDLES / "transformers5.yaml").read_text())["adapters"] for bundle in sorted(_BUNDLES.glob("*.yaml")): diff --git a/packages/sie_server/tests/config/test_sglang_memory_profiles.py b/packages/sie_server/tests/config/test_sglang_memory_profiles.py index c3175e34a..6bfaeb4a2 100644 --- a/packages/sie_server/tests/config/test_sglang_memory_profiles.py +++ b/packages/sie_server/tests/config/test_sglang_memory_profiles.py @@ -128,9 +128,7 @@ def test_extending_profile_forwards_its_resolved_kv_budget(): assert adapter._memory_budget.kv_budget_tokens == 65536 -@pytest.mark.parametrize( - "name", ["Qwen__Qwen3.5-4B", "lightonai__LightOnOCR-3-4B", "Qwen__Qwen3-VL-8B-Instruct", "zai-org__GLM-OCR"] -) +@pytest.mark.parametrize("name", ["Qwen__Qwen3.5-4B", "Qwen__Qwen3-VL-8B-Instruct", "zai-org__GLM-OCR"]) def test_specialized_profiles_retain_calibrated_reservation(name): config = load_model_config(MODELS / f"{name}.yaml") assert config.resolve_profile("default").loadtime["mem_fraction_static"] == 0.85