Octo: ggml inference engine (diffusion + L1/proprio), LIBERO client, and open-loop evaluation

.gitignore

@@ -16,6 +16,7 @@ _deps/
 
 # Models / data
 *.gguf
+*.gguf.tensor_map.json
 *.bin
 *.safetensors
 *.pt

CMakeLists.txt

@@ -100,6 +100,29 @@ function(vla_exclude_fetched_targets dir)
 endfunction()
 vla_exclude_fetched_targets(${llama_SOURCE_DIR})
 
+# Octo is the one arch that tokenizes in-process, so it is also the one that
+# needs SentencePiece and, through it, a protobuf. Optional so a build that does
+# not want Octo does not inherit that dependency.
+option(VLA_OCTO "Build the Octo arch (fetches SentencePiece, needs system protobuf)" ON)
+if(VLA_OCTO)
+    # The system protobuf, not sentencepiece's vendored protobuf-lite: vla-server
+    # aborts at static-init if two protobuf runtimes reach the same binary.
+    set(SPM_ENABLE_SHARED    OFF        CACHE BOOL   "" FORCE)
+    set(SPM_BUILD_TEST       OFF        CACHE BOOL   "" FORCE)
+    set(SPM_ENABLE_TCMALLOC  OFF        CACHE BOOL   "" FORCE)
+    set(SPM_PROTOBUF_PROVIDER "package" CACHE STRING "" FORCE)
+    FetchContent_Declare(sentencepiece
+        GIT_REPOSITORY https://github.com/google/sentencepiece
+        GIT_TAG        v0.2.0
+        GIT_SHALLOW    TRUE
+    )
+    # sentencepiece v0.2.0 still asks for cmake_minimum_required(VERSION 3.1),
+    # which CMake 4 refuses outright.
+    set(CMAKE_POLICY_VERSION_MINIMUM 3.5)
+    FetchContent_MakeAvailable(sentencepiece)
+    unset(CMAKE_POLICY_VERSION_MINIMUM)
+endif()
+
 add_library(vla_core
     src/model.cpp
     src/loader.cpp
@@ -129,6 +152,12 @@ target_include_directories(vla_core
 )
 # The VLA archs call no llama_* API; only vlm_core needs llama.
 target_link_libraries(vla_core PUBLIC ggml)
+if(VLA_OCTO)
+    target_sources(vla_core PRIVATE src/models/octo.cpp)
+    target_include_directories(vla_core PRIVATE ${sentencepiece_SOURCE_DIR}/src)
+    target_link_libraries(vla_core PRIVATE sentencepiece-static)
+    target_compile_definitions(vla_core PUBLIC VLA_USE_OCTO)
+endif()
 
 if(GGML_CUDA)
     target_compile_definitions(vla_core PUBLIC GGML_USE_CUDA)

README.md

@@ -325,6 +325,7 @@ supported (released and benchmarked), `~` = in progress, `-` = planned.
 | [VLA-Adapter](https://hf.co/vrfai/vla-adapter-libero-gguf)     | Y | Y | ~ | Y | Y | - | 
 | [OpenVLA-OFT](https://hf.co/vrfai/openvla-oft-libero-gguf)     | Y | Y | - | Y | Y | - | 
 | [VLA-JEPA](https://hf.co/vrfai/vla-jepa-libero)                | Y | Y | - | Y | Y | - | 
+| [Octo-Small](https://hf.co/vrfai/octo-small-libero-gguf)       | Y | Y | Y | Y | - | - | 
 
 ---
 
@@ -362,6 +363,7 @@ Supported VLA models:
 - [OpenVLA-OFT](https://github.com/moojink/openvla-oft) - Moo Jin Kim et al.
 - [GR00T N1.x](https://github.com/NVIDIA/Isaac-GR00T) - NVIDIA Isaac.
 - [VLA-JEPA](https://github.com/ginwind/VLA-JEPA) - Jingwen Sun et al.
+- [Octo](https://github.com/octo-models/octo) - Octo Model Team, UC Berkeley RAIL.
 
 Built on:
 

eval/client/adapters.py

@@ -16,13 +16,25 @@
 from typing import Any
 import numpy as np
 import torch
+from PIL import Image
 from tree import map_structure
 
 from lerobot.envs.utils import preprocess_observation
 from lerobot.processor.env_processor import LiberoProcessorStep
 from lerobot.processor.pipeline import PolicyProcessorPipeline
 from lerobot.utils.constants import ACTION
 
+def octo_preprocess_image(frame: np.ndarray, image_size: int = 256) -> np.ndarray:
+    # Rotate 180 like every other LIBERO adapter here (the off-screen render comes
+    # out upside-down), then resize. LIBERO renders at 256 and Octo's primary
+    # tokenizer wants 256, so the resize only bites if the camera is reconfigured.
+    rotated = np.ascontiguousarray(frame[::-1, ::-1])
+    if rotated.shape[0] == image_size and rotated.shape[1] == image_size:
+        return rotated
+    resized = Image.fromarray(rotated).resize((image_size, image_size), resample=Image.LANCZOS)
+    return np.asarray(resized, dtype=np.uint8)
+
+
 class BasePipelineAdapter:
     def __init__(self, client: Any = None):
         self._client = client
@@ -153,3 +165,21 @@ class Gr00tN15PipelineAdapter(Gr00tPipelineAdapter):
 
     def parse_action(self, action: np.ndarray) -> np.ndarray:
         return np.asarray(action[:7], dtype=np.float32).copy()
+
+# The LIBERO finetunes are single-camera: their image_obs_keys never held a wrist
+# key, so sending the primary view alone is what the checkpoint trained on. Octo's
+# observation tokenizers are image-only, so there is no state to send either.
+class OctoPipelineAdapter(BasePipelineAdapter):
+
+    def parse_observation(self, obs: dict[str, Any]) -> dict[str, Any]:
+        return {
+            "observation.images.image": octo_preprocess_image(obs["pixels"]["image"], image_size=256),
+            "task": obs.get("task_description", ""),
+        }
+
+    def parse_action(self, action: np.ndarray) -> np.ndarray:
+        # The server already returned world units. Only the gripper needs Octo's
+        # +1=open/0=close turned into LIBERO's -1=open/+1=close.
+        action = np.asarray(action[:7], dtype=np.float32).copy()
+        action[6] = -1.0 if action[6] > 0.5 else 1.0
+        return action

eval/client/run_sim_client_direct.py

@@ -29,6 +29,7 @@
     Evo1PipelineAdapter,
     Gr00tPipelineAdapter,
     Gr00tN15PipelineAdapter,
+    OctoPipelineAdapter,
 )
 
 ARCH_CHOICES = sorted(ARCH_PRESETS)
@@ -127,6 +128,8 @@
     elif args.arch in ("gr00t_n1_6", "gr00t_n1_7"):
 
         client = Gr00tPipelineAdapter(client=client)
+    elif args.arch == "octo":
+        client = OctoPipelineAdapter(client=client)
     else:
         client = LeRobotPipelineAdapter(client=client)
 

eval/client/vla_cpp_client.py

@@ -55,6 +55,9 @@
                    "max_state_dim": 64, "trust_remote_code": True},
 
     "gr00t_n1_6": {"image_size": 224, "tokenizer": None, "max_state_dim": 128, "trust_remote_code": True},
+
+    # Single-camera LIBERO finetune, image-only observations, so no state.
+    "octo": {"image_size": 256, "tokenizer": "t5-base", "max_state_dim": 0, "max_length": 16},
 }
 
 BITVLA_N_PATCHES_PER_VIEW = 256
@@ -509,6 +512,8 @@ def get_action(self, observations: dict[str, Any]) -> np.ndarray:
                 chunk = self._predict_chunk_vla_adapter(observations)
             elif self.arch == "openvla_oft":
                 chunk = self._predict_chunk_openvla_oft(observations)
+            elif self.arch == "octo":
+                chunk = self._predict_chunk_octo(observations)
             else:
                 chunk = self._predict_chunk(observations)
             for row in chunk[: self.n_action_steps, : self.real_action_dim]:
@@ -846,6 +851,60 @@ def _predict_chunk_evo1(self, observations: dict[str, Any]) -> np.ndarray:
         return (np.array(resp.action_chunk, dtype=np.float32)
                   .reshape(resp.chunk_size, resp.action_dim))
 
+    def _predict_chunk_octo(self, observations: dict[str, Any]) -> np.ndarray:
+        # OctoPipelineAdapter has already rotated and resized these. A wrist view is
+        # sent when the observation carries one; a checkpoint without one just
+        # leaves that slot out of the sequence.
+        images_u8: list[np.ndarray] = []
+        for key in self.image_keys[:2]:
+            if key not in observations:
+                continue
+            img = observations[key]
+            if isinstance(img, torch.Tensor):
+                img = img.numpy()
+            img = np.asarray(img, dtype=np.uint8)
+            if img.ndim != 3 or img.shape[2] != 3:
+                raise ValueError(f"octo: {key} expected HWC uint8 [H,W,3], got {img.shape}")
+            images_u8.append(np.ascontiguousarray(img, dtype=np.uint8))
+        if not images_u8:
+            raise KeyError(f"octo: no image keys found in observations; got {list(observations.keys())}")
+
+        task = observations.get("task", "")
+        if isinstance(task, bytes):
+            task = task.decode()
+        # The checkpoint's own text_processor: t5-base, max_length=16,
+        # padding="max_length", truncation=True. Octo's T5 encoder needs the real
+        # padding mask, so it is sent alongside the ids.
+        toks = self.tok(task, return_tensors="np", padding="max_length",
+                        truncation=True, max_length=self.max_length)
+        input_ids = toks["input_ids"][0].astype(np.int32)
+        attn_mask = toks["attention_mask"][0].astype(np.int32)
+
+        req = self.pb.PredictRequest()
+        req.request_id = self._step
+        self._step += 1
+        for img in images_u8:
+            ip = req.images.add()
+            ip.encoding = self.pb.Image.RGB_U8
+            ip.height = img.shape[0]
+            ip.width  = img.shape[1]
+            ip.data   = img.tobytes()
+        req.lang_tokens.extend(input_ids.tolist())
+        req.attention_mask.extend(attn_mask.tolist())
+
+        self.sock.send(req.SerializeToString())
+        body = self.sock.recv()
+        resp = self.pb.PredictResponse()
+        resp.ParseFromString(body)
+        if resp.error:
+            raise RuntimeError(f"vla-server error: {resp.error}")
+        self._last_response = resp
+        # World units already: Octo's dataset_statistics lives inside the checkpoint
+        # GGUF, so the server un-normalizes rather than handing the client a
+        # --stats-json it would have to extract from a multi-hundred-MB file.
+        return (np.array(resp.action_chunk, dtype=np.float32)
+                  .reshape(resp.chunk_size, resp.action_dim))
+
     def _predict_chunk_bitvla(self, observations: dict[str, Any]) -> np.ndarray:
 
         images_u8: list[np.ndarray] = []