Merge pull request #2430 from cosmincc/feat/llama-cpp-backends

llama-cpp: support every Linux prebuilt backend (CUDA, ROCm, SYCL, OpenVINO)

Author: MickLesk

ct/llama-cpp.sh

@@ -19,6 +19,35 @@ var_gpu="${var_gpu:-yes}"
 var_unprivileged="${var_unprivileged:-1}"
 var_testurl="${var_testurl:-https://github.com/community-scripts/DevScripts/issues/2124}"
 
+# Asked here because the installer runs under lxc-attach, where it can never
+# reach a terminal. Updates run inside the container and keep the recorded build.
+if [[ -z "${var_backend:-}" ]] && command -v pveversion >/dev/null 2>&1 && [[ -t 0 ]]; then
+  LLAMA_BACKENDS=(
+    "auto" "Vulkan if a GPU is passed through, else CPU"
+    "cpu" "CPU only"
+    "vulkan" "Vulkan (Intel, AMD and NVIDIA GPUs)"
+    "cuda" "NVIDIA CUDA (12 or 13, from the host driver)"
+  )
+  if [[ "$(dpkg --print-architecture 2>/dev/null)" == "amd64" ]]; then
+    LLAMA_BACKENDS+=(
+      "rocm" "AMD ROCm"
+      "sycl-fp16" "Intel GPU via oneAPI SYCL (FP16)"
+      "sycl-fp32" "Intel GPU via oneAPI SYCL (FP32)"
+      "openvino" "Intel CPU/GPU via OpenVINO"
+    )
+  fi
+  var_backend=$(whiptail --backtitle "Proxmox VE Helper Scripts" --title "LLAMA.CPP BUILD" --default-item "auto" \
+    --menu "Choose the llama.cpp build:" $((${#LLAMA_BACKENDS[@]} / 2 + 8)) 72 $((${#LLAMA_BACKENDS[@]} / 2)) \
+    "${LLAMA_BACKENDS[@]}" 3>&1 1>&2 2>&3) || exit_script
+fi
+
+export var_backend="${var_backend:-}"
+export var_model_repo="${var_model_repo:-}"
+export var_ctx_size="${var_ctx_size:-}"
+export var_offload_layers="${var_offload_layers:-}"
+export var_api_key="${var_api_key:-}"
+export var_port="${var_port:-}"
+
 header_info "$APP"
 variables
 color
@@ -34,26 +63,79 @@ function update_script() {
     exit
   fi
 
-  if check_for_gh_release "llama-cpp" "ggml-org/llama.cpp"; then
+  # Follow the channel picked at install unless GH_INCLUDE_PRERELEASE is given
+  # for this run: the bNNNN prerelease builds, or the build a stable vX.Y.Z
+  # release names in its nightly-tag.txt (stable releases carry no binaries).
+  # Installs that predate the record all tracked the bNNNN builds, and reading
+  # them as stable would downgrade them.
+  GH_INCLUDE_PRERELEASE="${GH_INCLUDE_PRERELEASE:-$(cat /opt/llama-cpp_data/.prerelease 2>/dev/null || echo 1)}"
+  export GH_INCLUDE_PRERELEASE
+  LLAMA_RELEASE=""
+  if [[ "$GH_INCLUDE_PRERELEASE" == "1" ]]; then
+    if check_for_gh_release "llama-cpp" "ggml-org/llama.cpp" "" "" "b"; then
+      LLAMA_RELEASE="$CHECK_UPDATE_RELEASE"
+    fi
+  elif check_for_gh_release "llama-cpp-stable" "ggml-org/llama.cpp"; then
+    USE_ORIGINAL_FILENAME=true fetch_and_deploy_gh_release "llama-cpp-stable" "ggml-org/llama.cpp" "singlefile" "$CHECK_UPDATE_RELEASE" "/opt/llama-cpp_data" "nightly-tag.txt"
+    LLAMA_RELEASE="$(</opt/llama-cpp_data/nightly-tag.txt)"
+  fi
+
+  if [[ -n "$LLAMA_RELEASE" ]]; then
     msg_info "Stopping Service"
     systemctl stop llama-cpp
     msg_ok "Stopped Service"
 
     # Keep the build variant the install picked; the CPU tarball would otherwise
-    # overwrite a Vulkan install and quietly end GPU offload.
-    #
-    # rocm reads as vulkan: upstream publishes no Linux ROCm build, so anything
-    # recorded as rocm predates that being noticed and would otherwise fail
-    # every update on an asset that does not exist.
+    # overwrite a GPU build and quietly end offload.
     LLAMA_BACKEND="$(cat /opt/llama-cpp_data/.backend 2>/dev/null || echo cpu)"
+    LLAMA_ARCH="$(arch_resolve x64 arm64)"
+    # Installs from before the ROCm build existed recorded rocm while actually
+    # running the Vulkan build.
+    [[ "$LLAMA_BACKEND" == "rocm" && ! -d /opt/llama-cpp_data/rocm ]] && LLAMA_BACKEND="vulkan"
+
+    LLAMA_CUDART_ASSET=""
     case "$LLAMA_BACKEND" in
-    vulkan | rocm) LLAMA_ASSET="llama-*-bin-ubuntu-vulkan-$(arch_resolve x64 arm64).tar.gz" ;;
-    *) LLAMA_ASSET="llama-*-bin-ubuntu-$(arch_resolve x64 arm64).tar.gz" ;;
+    vulkan) LLAMA_ASSET="llama-*-bin-ubuntu-vulkan-${LLAMA_ARCH}.tar.gz" ;;
+    cuda-12 | cuda-13)
+      LLAMA_ASSET="llama-*-bin-ubuntu-cuda-${LLAMA_BACKEND#cuda-}.*-${LLAMA_ARCH}.tar.gz"
+      LLAMA_CUDART_ASSET="cudart-${LLAMA_ASSET}"
+      ;;
+    rocm) LLAMA_ASSET="llama-*-bin-ubuntu-rocm-*-x64.tar.gz" ;;
+    sycl-fp16 | sycl-fp32) LLAMA_ASSET="llama-*-bin-ubuntu-${LLAMA_BACKEND}-x64.tar.gz" ;;
+    openvino) LLAMA_ASSET="llama-*-bin-ubuntu-openvino-*x64.tar.gz" ;;
+    *) LLAMA_ASSET="llama-*-bin-ubuntu-${LLAMA_ARCH}.tar.gz" ;;
     esac
 
-    CLEAN_INSTALL=1 fetch_and_deploy_gh_release "llama-cpp" "ggml-org/llama.cpp" "prebuild" "latest" "/opt/llama-cpp" "$LLAMA_ASSET"
+    CLEAN_INSTALL=1 fetch_and_deploy_gh_release "llama-cpp" "ggml-org/llama.cpp" "prebuild" "$LLAMA_RELEASE" "/opt/llama-cpp" "$LLAMA_ASSET"
+    # The wipe above also removed the CUDA runtime, so it always goes back in.
+    if [[ -n "$LLAMA_CUDART_ASSET" ]]; then
+      rm -f ~/.llama-cpp-cudart
+      fetch_and_deploy_gh_release "llama-cpp-cudart" "ggml-org/llama.cpp" "prebuild" "$LLAMA_RELEASE" "/opt/llama-cpp" "$LLAMA_CUDART_ASSET"
+    fi
     chmod +x /opt/llama-cpp/llama-*
 
+    case "$LLAMA_BACKEND" in
+    sycl-* | openvino)
+      fetch_and_deploy_gh_release "level-zero" "oneapi-src/level-zero" "binary" "latest" "" "libze1_*u24.04_amd64.deb" || true
+      if [[ -d /dev/dri ]]; then
+        fetch_and_deploy_gh_release "intel-level-zero-gpu" "intel/compute-runtime" "binary" "latest" "" "libze-intel-gpu1_*_amd64.deb" || true
+      fi
+      ;;
+    rocm)
+      # The ROCm build links against the ROCm release it was built with, so
+      # follow it when upstream moves to a newer one.
+      ROCM_VERSION="$(jq -r 'if type == "array" then .[0] else . end | .assets[].name' "$TOOLS_GH_REL_JSON" 2>/dev/null | grep -oP 'bin-ubuntu-rocm-\K[0-9.]+(?=-x64)' | head -1)"
+      if [[ -n "$ROCM_VERSION" && "$ROCM_VERSION" != "$(cat /opt/llama-cpp_data/rocm/.version 2>/dev/null)" ]]; then
+        msg_info "Updating ROCm Runtime to ${ROCM_VERSION}"
+        $STD uv pip install --python /opt/llama-cpp_data/rocm --index-url https://stable.repo.amd.com/rocm/whl-next/ "rocm[libraries]==${ROCM_VERSION}.*"
+        echo "$ROCM_VERSION" >/opt/llama-cpp_data/rocm/.version
+        msg_ok "Updated ROCm Runtime to ${ROCM_VERSION}"
+      fi
+      ;;
+    esac
+
+    echo "$GH_INCLUDE_PRERELEASE" >/opt/llama-cpp_data/.prerelease
+
     msg_info "Starting Service"
     systemctl start llama-cpp
     msg_ok "Started Service"

install/llama-cpp-install.sh

@@ -13,62 +13,147 @@ setting_up_container
 network_check
 update_os
 
-var_backend="${var_backend:-auto}"
-var_model_repo="${var_model_repo:-ggml-org/gemma-3-1b-it-GGUF}"
+var_model_repo="${var_model_repo:-ggml-org/gemma-4-E2B-it-GGUF:Q4_0}"
 var_port="${var_port:-8080}"
 var_ctx_size="${var_ctx_size:-4096}"
 var_offload_layers="${var_offload_layers:-}"
 var_api_key="${var_api_key:-}"
 
-# Which prebuilt upstream ships decides what the container can offload to, and
-# for Linux that list is short: ubuntu-x64, ubuntu-arm64, ubuntu-vulkan-x64,
-# ubuntu-vulkan-arm64, plus the Intel sycl and openvino builds. There is no
-# Linux CUDA build and no Linux ROCm build either -- the only ROCm asset
-# upstream publishes is win-rocm-*-x64.zip. Vulkan is therefore the GPU path for
-# every vendor here, AMD included, and it reaches an AMD card through RADV.
-#
-# Asking for a ubuntu-rocm asset failed the install outright with "No asset
-# matching 'llama--bin-ubuntu-rocm--x64.tar.gz' found" on any host where the
-# engine had set up /dev/kfd.
+# The Linux prebuilts upstream publishes (see .github/workflows/release.yml):
+#   cpu       ubuntu-{x64,arm64}
+#   vulkan    ubuntu-vulkan-{x64,arm64}       any GPU vendor (AMD through RADV)
+#   cuda      ubuntu-cuda-{12.x,13.x}-x64, ubuntu-cuda-13.x-arm64, plus a cudart tarball
+#   rocm      ubuntu-rocm-<ver>-x64           needs the matching ROCm (TheRock) runtime
+#   sycl      ubuntu-sycl-{fp16,fp32}-x64     Intel GPUs, needs oneAPI + Level Zero
+#   openvino  ubuntu-openvino-<ver>-x64       Intel CPU/GPU, OpenVINO runtime is bundled
+var_backend="${var_backend:-auto}"
+
 if [[ "$var_backend" == "auto" ]]; then
   if [[ -e /dev/kfd || -e /dev/nvidia0 || -d /dev/dri ]]; then
     var_backend="vulkan"
   else
     var_backend="cpu"
   fi
 fi
-
-# rocm was a documented value, so keep accepting it rather than failing anyone
-# who pinned it -- but say where they are actually being sent.
-if [[ "$var_backend" == "rocm" ]]; then
-  msg_warn "llama.cpp publishes no Linux ROCm build - using the Vulkan build, which drives an AMD card through RADV"
-  var_backend="vulkan"
-fi
+[[ "$var_backend" == "sycl" ]] && var_backend="sycl-fp16"
 
 LLAMA_ARCH="$(arch_resolve x64 arm64)"
-
 case "$var_backend" in
-vulkan) LLAMA_ASSET="llama-*-bin-ubuntu-vulkan-${LLAMA_ARCH}.tar.gz" ;;
+rocm | sycl-fp16 | sycl-fp32 | openvino)
+  if [[ "$LLAMA_ARCH" != "x64" ]]; then
+    msg_error "llama.cpp publishes the ${var_backend} build for x64 only"
+    exit 1
+  fi
+  ;;
+cpu | vulkan | cuda) ;;
 *)
-  var_backend="cpu"
-  LLAMA_ASSET="llama-*-bin-ubuntu-${LLAMA_ARCH}.tar.gz"
+  msg_error "Unknown var_backend '${var_backend}' (expected auto, cpu, vulkan, cuda, rocm, sycl-fp16, sycl-fp32 or openvino)"
+  exit 1
   ;;
 esac
 
 msg_info "Installing Dependencies"
 LLAMA_DEPS=(libgomp1)
-[[ "$var_backend" == "vulkan" ]] && LLAMA_DEPS+=(libvulkan1 mesa-vulkan-drivers vulkan-tools)
+case "$var_backend" in
+vulkan) LLAMA_DEPS+=(libvulkan1 mesa-vulkan-drivers vulkan-tools) ;;
+openvino) LLAMA_DEPS+=(libtbb12 ocl-icd-libopencl1) ;;
+sycl-*) LLAMA_DEPS+=(ocl-icd-libopencl1) ;;
+esac
 $STD apt install -y "${LLAMA_DEPS[@]}"
 msg_ok "Installed Dependencies"
 
 setup_hwaccel
 
-fetch_and_deploy_gh_release "llama-cpp" "ggml-org/llama.cpp" "prebuild" "latest" "/opt/llama-cpp" "$LLAMA_ASSET"
+# SYCL and OpenVINO reach an Intel GPU through Level Zero. release.yml builds
+# SYCL against the loader from oneapi-src/level-zero, so use the same package.
+if [[ "$var_backend" == sycl-* || "$var_backend" == "openvino" ]]; then
+  fetch_and_deploy_gh_release "level-zero" "oneapi-src/level-zero" "binary" "latest" "" "libze1_*u24.04_amd64.deb"
+  if [[ -d /dev/dri ]]; then
+    fetch_and_deploy_gh_release "intel-level-zero-gpu" "intel/compute-runtime" "binary" "latest" "" "libze-intel-gpu1_*_amd64.deb" || true
+  fi
+fi
+
+# The SYCL tarball ships only llama.cpp itself; the DPC++/oneMKL/oneDNN/TBB
+# libraries it links against come from oneAPI. release.yml builds with the full
+# toolkit, but running needs only the runtime libraries.
+if [[ "$var_backend" == sycl-* ]]; then
+  setup_deb822_repo \
+    "oneapi" \
+    "https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB" \
+    "https://apt.repos.intel.com/oneapi" \
+    "all" \
+    "main"
+  msg_info "Installing Intel oneAPI Runtime"
+  $STD apt install -y intel-oneapi-runtime-libs
+  # The runtime packages unpack under /opt/intel/oneapi without telling the
+  # dynamic linker, so register every directory that holds one of them.
+  find /opt/intel/oneapi -type f \( -name 'libsycl.so*' -o -name 'libur_loader.so*' -o -name 'libmkl_core.so*' \
+    -o -name 'libdnnl.so*' -o -name 'libtbb.so*' -o -name 'libsvml.so' -o -name 'libiomp5.so' -o -name 'libumf.so*' \) \
+    -printf '%h\n' 2>/dev/null | sort -u >/etc/ld.so.conf.d/oneapi-llama-cpp.conf
+  ldconfig
+  msg_ok "Installed Intel oneAPI Runtime"
+fi
+
+LLAMA_CUDART_ASSET=""
+case "$var_backend" in
+vulkan) LLAMA_ASSET="llama-*-bin-ubuntu-vulkan-${LLAMA_ARCH}.tar.gz" ;;
+cuda)
+  # CUDA 13 needs a 580+ host driver; anything older gets the CUDA 12 build,
+  # which upstream publishes for x64 only.
+  NVIDIA_DRIVER_MAJOR="$(grep -oP '\d{3,}(?=\.\d+)' /proc/driver/nvidia/version 2>/dev/null | head -1)"
+  if [[ "$LLAMA_ARCH" == "x64" && -n "$NVIDIA_DRIVER_MAJOR" && "$NVIDIA_DRIVER_MAJOR" -lt 580 ]]; then
+    var_backend="cuda-12"
+  else
+    var_backend="cuda-13"
+  fi
+  [[ -z "$NVIDIA_DRIVER_MAJOR" ]] && msg_warn "No NVIDIA host driver visible - installing the CUDA 13 build, which needs driver 580 or newer"
+  LLAMA_ASSET="llama-*-bin-ubuntu-cuda-${var_backend#cuda-}.*-${LLAMA_ARCH}.tar.gz"
+  LLAMA_CUDART_ASSET="cudart-${LLAMA_ASSET}"
+  ;;
+rocm) LLAMA_ASSET="llama-*-bin-ubuntu-rocm-*-x64.tar.gz" ;;
+sycl-fp16 | sycl-fp32) LLAMA_ASSET="llama-*-bin-ubuntu-${var_backend}-x64.tar.gz" ;;
+openvino) LLAMA_ASSET="llama-*-bin-ubuntu-openvino-*x64.tar.gz" ;;
+*) LLAMA_ASSET="llama-*-bin-ubuntu-${LLAMA_ARCH}.tar.gz" ;;
+esac
+
+# Only the bNNNN builds carry binaries, and upstream marks them prerelease. A
+# stable vX.Y.Z release ships just nightly-tag.txt, naming the build it was cut
+# from, so without GH_INCLUDE_PRERELEASE=1 that build is the one deployed.
+mkdir -p /opt/llama-cpp_data/models
+if [[ "${GH_INCLUDE_PRERELEASE:-0}" == "1" ]]; then
+  fetch_and_deploy_gh_release "llama-cpp" "ggml-org/llama.cpp" "prebuild" "latest" "/opt/llama-cpp" "$LLAMA_ASSET" "b"
+  LLAMA_RELEASE="$(cat ~/.llama-cpp)"
+else
+  USE_ORIGINAL_FILENAME=true fetch_and_deploy_gh_release "llama-cpp-stable" "ggml-org/llama.cpp" "singlefile" "latest" "/opt/llama-cpp_data" "nightly-tag.txt"
+  LLAMA_RELEASE="$(</opt/llama-cpp_data/nightly-tag.txt)"
+  fetch_and_deploy_gh_release "llama-cpp" "ggml-org/llama.cpp" "prebuild" "$LLAMA_RELEASE" "/opt/llama-cpp" "$LLAMA_ASSET"
+fi
+if [[ -n "$LLAMA_CUDART_ASSET" ]]; then
+  fetch_and_deploy_gh_release "llama-cpp-cudart" "ggml-org/llama.cpp" "prebuild" "$LLAMA_RELEASE" "/opt/llama-cpp" "$LLAMA_CUDART_ASSET"
+fi
+LLAMA_LIBPATH="/opt/llama-cpp"
+
+# The ROCm build carries no ROCm libraries. release.yml builds it against AMD's
+# TheRock wheels, so install the same ROCm release of those as the runtime.
+if [[ "$var_backend" == "rocm" ]]; then
+  ROCM_VERSION="$(jq -r --arg t "$LLAMA_RELEASE" 'if type == "array" then .[] else . end | select(.tag_name == $t) | .assets[].name' "$TOOLS_GH_REL_JSON" 2>/dev/null | grep -oP 'bin-ubuntu-rocm-\K[0-9.]+(?=-x64)' | head -1)"
+  if [[ -z "$ROCM_VERSION" ]]; then
+    msg_error "Could not read the ROCm release the llama.cpp ${LLAMA_RELEASE} build targets"
+    exit 1
+  fi
+  PYTHON_VERSION="3.12" setup_uv
+  msg_info "Installing ROCm ${ROCM_VERSION} Runtime"
+  $STD uv venv --python 3.12 /opt/llama-cpp_data/rocm
+  $STD uv pip install --python /opt/llama-cpp_data/rocm --index-url https://stable.repo.amd.com/rocm/whl-next/ "rocm[libraries]==${ROCM_VERSION}.*"
+  echo "$ROCM_VERSION" >/opt/llama-cpp_data/rocm/.version
+  LLAMA_LIBPATH="${LLAMA_LIBPATH}:$(/opt/llama-cpp_data/rocm/bin/rocm-sdk path --root)/lib"
+  msg_ok "Installed ROCm ${ROCM_VERSION} Runtime"
+fi
 
 msg_info "Configuring llama.cpp"
 chmod +x /opt/llama-cpp/llama-*
-mkdir -p /opt/llama-cpp_data/models
 echo "$var_backend" >/opt/llama-cpp_data/.backend
+echo "${GH_INCLUDE_PRERELEASE:-0}" >/opt/llama-cpp_data/.prerelease
 if [[ -z "$var_offload_layers" ]]; then
   [[ "$var_backend" == "cpu" ]] && var_offload_layers=0 || var_offload_layers=999
 fi
@@ -80,11 +165,36 @@ LLAMA_ARG_PORT=${var_port}
 #LLAMA_ARG_MODEL=/opt/llama-cpp_data/models/your-model.gguf
 # ...or let llama-server pull one from Hugging Face on first start:
 LLAMA_ARG_HF_REPO=${var_model_repo}
+# Model switching: comment out LLAMA_ARG_HF_REPO (and LLAMA_ARG_MODEL) to serve
+# every model in this folder and pick one per request from the web UI or API:
+LLAMA_ARG_MODELS_DIR=/opt/llama-cpp_data/models
 
 LLAMA_CACHE=/opt/llama-cpp_data/models
 LLAMA_ARG_CTX_SIZE=${var_ctx_size}
 LLAMA_ARG_N_GPU_LAYERS=${var_offload_layers}
 EOF
+case "$var_backend" in
+sycl-*)
+  cat <<EOF >>/opt/llama-cpp.env
+
+# SYCL: the persistent kernel cache speeds up restarts but can stop models
+# from loading on some Intel GPUs, so it is off by default
+#SYCL_CACHE_PERSISTENT=1
+#SYCL_CACHE_DIR=/opt/llama-cpp_data/sycl-cache
+#ONEAPI_DEVICE_SELECTOR=level_zero:0
+EOF
+  ;;
+openvino)
+  OV_DEVICE="CPU"
+  compgen -G "/dev/dri/renderD*" >/dev/null && OV_DEVICE="GPU"
+  cat <<EOF >>/opt/llama-cpp.env
+
+# OpenVINO device: CPU, GPU (GPU.0, GPU.1, ...) or NPU
+GGML_OPENVINO_DEVICE=${OV_DEVICE}
+GGML_OPENVINO_CACHE_DIR=/opt/llama-cpp_data/openvino-cache
+EOF
+  ;;
+esac
 # An empty LLAMA_ARG_API_KEY makes llama-server demand an empty bearer token.
 if [[ -n "$var_api_key" ]]; then
   echo "LLAMA_ARG_API_KEY=${var_api_key}" >>/opt/llama-cpp.env
@@ -95,10 +205,14 @@ chmod 600 /opt/llama-cpp.env
 msg_ok "Configured llama.cpp (${var_backend} build)"
 
 if [[ "$var_backend" != "cpu" ]]; then
-  if LD_LIBRARY_PATH=/opt/llama-cpp ldd /opt/llama-cpp/llama-server 2>/dev/null | grep -q "not found"; then
+  if LD_LIBRARY_PATH="$LLAMA_LIBPATH" ldd /opt/llama-cpp/llama-server /opt/llama-cpp/libggml-*.so 2>/dev/null | grep -q "not found"; then
     msg_warn "The ${var_backend} build is missing shared libraries in this container - it will fall back to the CPU"
   elif [[ "$var_backend" == "vulkan" ]] && ! vulkaninfo --summary >/dev/null 2>&1; then
     msg_warn "Vulkan build installed but no Vulkan device is visible - check GPU passthrough and the driver libraries in the container"
+  elif [[ "$var_backend" == cuda-* ]] && ! ldconfig -p | grep -q 'libcuda[.]so'; then
+    msg_warn "CUDA build installed but the NVIDIA driver library (libcuda) is missing - check GPU passthrough"
+  elif [[ "$var_backend" == sycl-* || "$var_backend" == "rocm" ]] && [[ ! -d /dev/dri && ! -e /dev/kfd ]]; then
+    msg_warn "${var_backend} build installed but no GPU is passed through - it will fall back to the CPU"
   fi
 fi
 
@@ -114,7 +228,7 @@ Type=simple
 User=root
 WorkingDirectory=/opt/llama-cpp
 EnvironmentFile=/opt/llama-cpp.env
-Environment=LD_LIBRARY_PATH=/opt/llama-cpp
+Environment=LD_LIBRARY_PATH=${LLAMA_LIBPATH}
 ExecStart=/opt/llama-cpp/llama-server
 Restart=on-failure
 RestartSec=10