diff --git a/models/mistral-small-4-119b-2603/metadata.yaml b/models/mistral-small-4-119b-2603/metadata.yaml new file mode 100644 index 0000000..647b099 --- /dev/null +++ b/models/mistral-small-4-119b-2603/metadata.yaml @@ -0,0 +1,23 @@ +apiVersion: catalog.swiss/v1 +name: mistral-small-4-119b-2603 +displayName: Mistral-Small-4-119B-2603 +description: Mistral-Small-4-119B-2603 on 2 H100 GPUs. Configs require the + measured tokenizer at /tokenizer. +family: mistral +tags: + - chat + - reasoning + - tool-use +source: + hf: mistralai/Mistral-Small-4-119B-2603 +tuning: + - version: 1.0.0 + baseline: sglang-tp2-h100-baseline + optimized: sglang-tp2-h100-optimized + uplift: 52.5 + workloads: + - name: 50k + 1.5k + uplift: 52.5 + - name: 8k + 1k + uplift: 32.1 + report: mistral-small-4-119b-2603-h100-report.html diff --git a/models/mistral-small-4-119b-2603/mistral-small-4-119b-2603-1.0.0.yaml b/models/mistral-small-4-119b-2603/mistral-small-4-119b-2603-1.0.0.yaml new file mode 100644 index 0000000..7851aa7 --- /dev/null +++ b/models/mistral-small-4-119b-2603/mistral-small-4-119b-2603-1.0.0.yaml @@ -0,0 +1,151 @@ +# Serving configs measured, and where marked tuned, by LLM AutoTune. +apiVersion: catalog.swiss/v1 +name: mistral-small-4-119b-2603 +version: 1.0.0 +servedName: Mistral-Small-4-119B-2603 +variants: + - id: sglang-tp2-h100-baseline + description: 'Baseline: TP2 on two H100 GPUs, single node.' + engine: sglang + chart: + name: sglang + version: ">=0.7.1" + image: + repository: harbor.4pd.io/hardcore-tech/sglang + tag: v0.5.19-cu129 + requires: + gpus: 2 + topology: single-node + gpuProduct: + - NVIDIA-H100-80GB-HBM3 + values: + model: + hostPathType: Directory + contextLength: '' + modelCheck: + enabled: true + requiredGlobs: + - config.json + - '*.safetensors' + extraArgs: + - --load-format=mistral + - --model-config-parser=mistral + - --reasoning-parser=mistral + - --tokenizer-path=/tokenizer # Mount the measured tokenizer at /tokenizer via the site profile. + - --tp-size=2 + env: + - name: NVIDIA_DISABLE_REQUIRE + value: '1' + - name: PYTORCH_ALLOC_CONF + value: expandable_segments:True + volumes: + - name: dot-cache + emptyDir: + sizeLimit: 10Gi + - name: dshm + emptyDir: + medium: Memory + sizeLimit: 32Gi + volumeMounts: + - name: dot-cache + mountPath: /root/.cache + - name: dshm + mountPath: /dev/shm + resources: + requests: + ephemeral-storage: 16Gi + limits: + ephemeral-storage: 16Gi + progressDeadlineSeconds: 7200 + startupProbe: + httpGet: + path: /health + port: http + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 180 + terminationGracePeriodSeconds: 3600 + lifecycle: + preStop: + drainSeconds: 600 + pollIntervalSeconds: 5 + forceShutdown: true + preStopKill: true + hangWatcher: + enabled: true + - id: sglang-tp2-h100-optimized + default: true + description: 'Tuned by LLM AutoTune: TP2 on two H100 GPUs, single node. + Normalized throughput within the SLO vs the baseline variant: +52.5% (50k + + 1.5k), +32.1% (8k + 1k).' + engine: sglang + chart: + name: sglang + version: ">=0.7.1" + image: + repository: harbor.4pd.io/hardcore-tech/sglang + tag: v0.5.19-cu129 + digest: sha256:b7a3051b9e15939a5afc74de3564a72fd5b0b42bef80bbaacea9649d09f63d76 + requires: + gpus: 2 + topology: single-node + gpuProduct: + - NVIDIA-H100-80GB-HBM3 + values: + model: + hostPathType: Directory + contextLength: '' + modelCheck: + enabled: true + requiredGlobs: + - config.json + - '*.safetensors' + extraArgs: + - --chunked-prefill-size=8192 + - --load-format=mistral + - --mem-fraction-static=0.86 + - --model-config-parser=mistral + - --reasoning-parser=mistral + - --schedule-conservativeness=1.3 + - --tokenizer-path=/tokenizer # Mount the measured tokenizer at /tokenizer via the site profile. + - --tp-size=2 + env: + - name: NVIDIA_DISABLE_REQUIRE + value: '1' + - name: PYTORCH_ALLOC_CONF + value: expandable_segments:True + volumes: + - name: dot-cache + emptyDir: + sizeLimit: 10Gi + - name: dshm + emptyDir: + medium: Memory + sizeLimit: 32Gi + volumeMounts: + - name: dot-cache + mountPath: /root/.cache + - name: dshm + mountPath: /dev/shm + resources: + requests: + ephemeral-storage: 16Gi + limits: + ephemeral-storage: 16Gi + progressDeadlineSeconds: 7200 + startupProbe: + httpGet: + path: /health + port: http + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 180 + terminationGracePeriodSeconds: 3600 + lifecycle: + preStop: + drainSeconds: 600 + pollIntervalSeconds: 5 + forceShutdown: true + preStopKill: true + hangWatcher: + enabled: true diff --git a/models/mistral-small-4-119b-2603/mistral-small-4-119b-2603-h100-report.html b/models/mistral-small-4-119b-2603/mistral-small-4-119b-2603-h100-report.html new file mode 100644 index 0000000..a4c91f6 --- /dev/null +++ b/models/mistral-small-4-119b-2603/mistral-small-4-119b-2603-h100-report.html @@ -0,0 +1,75 @@ + + +
+ + +