From bd0bf9fa3d2dacfead078c3870e2f7a7c7bb7f73 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 14:58:55 +0900 Subject: [PATCH 1/2] Fix: Close the gaps Ollama and OpenAI clients hit HEAD / now answers 200 for health checks; /api/version returns the real version (1.0.0, above every Ollama 0.x) instead of 0.0.0. Ollama names qwen3, qwen3:latest, qwen3:0.6b and qwen3:0.6b-fp16 map to Qwen3-0.6B, and an unknown name lists the names that exist. The Ollama chat, generate and embed routes fetch the catalogue model on first use, like the app and the OpenAI API, so 'ollama run qwen3' works on a fresh install. think: false (Ollama) and reasoning_effort none/minimal or chat_template_kwargs (OpenAI) reach the chat template as enable_thinking. The READMEs and the server guide show how to point the ollama CLI and OpenAI SDKs at Gemstone. --- README.md | 14 +++ api/src/main/engine.py | 7 +- api/src/main/ollama_api.py | 38 +++--- api/src/main/openai_api.py | 31 ++++- api/src/main/registry.py | 22 +++- api/src/main/server.py | 8 +- api/src/main/version.py | 6 + api/tests/test_client_compat.py | 202 ++++++++++++++++++++++++++++++++ docs/SPEC.md | 33 +++++- docs/guide/guide-server.html | 8 ++ docs/locale/README_ko.md | 13 ++ 11 files changed, 352 insertions(+), 30 deletions(-) create mode 100644 api/src/main/version.py create mode 100644 api/tests/test_client_compat.py diff --git a/README.md b/README.md index faa3dea0..53e9aa66 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,20 @@ The server listens on `127.0.0.1` only. To serve other machines, set `GEMSTONE_H Clients then send `Authorization: Bearer `. `GEMSTONE_ORIGINS` allows more browser origins and `--reload` (or `GEMSTONE_DEV=1`) turns on auto-reload for development. +**Point existing clients at Gemstone** (it listens on `23100`, not Ollama's `11434` or OpenAI's HTTPS host): + +```bash +OLLAMA_HOST=http://127.0.0.1:23100 ollama run qwen3 # the ollama CLI and Ollama clients +OPENAI_BASE_URL=http://127.0.0.1:23100/v1 OPENAI_API_KEY=x python my_script.py # OpenAI SDKs +``` + +`qwen3`, `qwen3:latest` and `qwen3:0.6b` are the built-in model (fetched on first use); any Hugging Face +model is pulled by its id, e.g. `ollama pull Org/Model`. Other Ollama library names (`llama3.2`) are not +available. When `GEMSTONE_API_KEY` is set on the server, send it as the key: `OPENAI_API_KEY=` for +OpenAI SDKs, or an `Authorization: Bearer ` header for other clients (the stock `ollama` CLI sends none, +so use it on a loopback server without a key). `think: false` (Ollama) and `reasoning_effort: "none"` +(OpenAI) switch Qwen3's reasoning off. + **3. Run a native client** ```bash diff --git a/api/src/main/engine.py b/api/src/main/engine.py index acabc48e..cd362c0d 100644 --- a/api/src/main/engine.py +++ b/api/src/main/engine.py @@ -14,7 +14,7 @@ `model.generate` at a time, with no batched request running. See `Engine.__call__`. """ from contextlib import contextmanager -from typing import Dict, Generator, List, Optional, Union +from typing import Any, Dict, Generator, List, Optional, Union import itertools import logging import os @@ -199,11 +199,15 @@ def __call__( repeat_penalty: float = 1.0, seed: Optional[int] = None, cancel: Optional[threading.Event] = None, + chat_template_kwargs: Optional[Dict[str, Any]] = None, **kwargs ) -> Generator[str, None, None]: """ Stream the reply to `messages` as text chunks. + `chat_template_kwargs` are extra variables for the chat template (`enable_thinking=False` + switches Qwen3's reasoning off). Unlike `kwargs` they do not make the request exclusive. + `stream` is accepted for compatibility with the model layer and ignored: the reply is always produced as a stream. `max_new_tokens <= 0` means "as many as fit": up to the context length, but when the request can join the batch, up to what the paged KV cache holds @@ -223,6 +227,7 @@ def __call__( chat_template=self.chat_template, add_generation_prompt=True, tokenize=False, + **(chat_template_kwargs or {}), ) input_ids = self.tokenizer(prompt, add_special_tokens=False)["input_ids"] prompt_length = len(input_ids) diff --git a/api/src/main/ollama_api.py b/api/src/main/ollama_api.py index fc13b319..838ea5af 100644 --- a/api/src/main/ollama_api.py +++ b/api/src/main/ollama_api.py @@ -12,7 +12,6 @@ `Registry.resolve` (derived models included); residency and keep_alive are handled by `registry.Registry`. """ -from importlib import metadata from typing import Iterator, List, Optional import itertools import json @@ -25,9 +24,10 @@ from starlette.concurrency import run_in_threadpool from . import registry as registry_module +from .version import VERSION from .leases import LeasedStreamingResponse, acquire, once -from .registry import (DERIVED_PARAMETERS, ModelBusy, ModelUnavailable, UnsupportedModel, alias_key, format_parameters, is_builtin_name, now_iso, - parse_keep_alive) +from .registry import (DERIVED_PARAMETERS, ModelBusy, ModelUnavailable, UnsupportedModel, alias_key, format_parameters, + in_catalogue, is_builtin_name, now_iso, parse_keep_alive) router = APIRouter() @@ -246,14 +246,19 @@ def _count_tokens(engine, text: str) -> Optional[int]: return None -def _prompt_tokens(engine, messages, tools) -> Optional[int]: +def _template_kwargs(think: Optional[bool]) -> dict: + """ `think` -> the chat template's `enable_thinking` (Qwen3 and others); unset leaves the template's default. """ + return {} if think is None else {"chat_template_kwargs": {"enable_thinking": think}} + + +def _prompt_tokens(engine, messages, tools, template_kwargs=None) -> Optional[int]: tokenizer = getattr(engine, "tokenizer", None) if tokenizer is None: return None try: prompt = tokenizer.apply_chat_template( messages, tools=tools or None, chat_template=getattr(engine, "chat_template", None), - add_generation_prompt=True, tokenize=False) + add_generation_prompt=True, tokenize=False, **(template_kwargs or {})) except Exception: return None return _count_tokens(engine, prompt) @@ -288,8 +293,9 @@ def _generate(lease, messages, tools, kwargs, stops, think, cancel, started, rel first_token = None chunks = None try: - prompt_eval_count = _prompt_tokens(engine, messages, tools) - chunks = engine(messages, tools=tools or None, cancel=cancel, **kwargs) + template = _template_kwargs(think) + prompt_eval_count = _prompt_tokens(engine, messages, tools, template.get("chat_template_kwargs")) + chunks = engine(messages, tools=tools or None, cancel=cancel, **template, **kwargs) for chunk in chunks: if first_token is None: first_token = time.perf_counter() @@ -381,11 +387,11 @@ async def _run(body: dict, messages: list, tools, render, finish): if keep_alive == 0: await run_in_threadpool(reg.unload, hf_id) return JSONResponse(finish(name, "unload")) - lease = await acquire(reg, hf_id, keep_alive) + lease = await acquire(reg, hf_id, keep_alive, in_catalogue(name)) reg.release(lease) return JSONResponse(finish(name, "load")) - lease = await acquire(reg, hf_id, keep_alive) + lease = await acquire(reg, hf_id, keep_alive, in_catalogue(name)) cancel = threading.Event() release = once(lambda: reg.release(lease)) events = _generate(lease, messages, tools, kwargs, stops, think, cancel, started, release) @@ -785,7 +791,7 @@ async def _embed(body: dict, inputs: List[str]) -> tuple: keep_alive = parse_keep_alive(body.get("keep_alive")) truncate = body.get("truncate", True) is not False reg = _registry() - lease = await acquire(reg, reg.resolve(name), keep_alive) + lease = await acquire(reg, reg.resolve(name), keep_alive, in_catalogue(name)) try: embed = getattr(lease.engine, "embed", None) if embed is None: @@ -828,10 +834,10 @@ async def embeddings(request: Request): return {"embedding": vectors[0] if vectors else []} -@router.get("/api/version") +@router.api_route("/api/version", methods=["GET", "HEAD"]) async def version(): - """ The Gemstone version (clients use it to check that the server is up). """ - try: - return {"version": metadata.version("gemstone")} - except metadata.PackageNotFoundError: - return {"version": "0.0.0"} + """ + The Gemstone version (SPEC S1.15). It is a plain `MAJOR.MINOR.PATCH` string, which is what Ollama + clients compare as semver; it is above every Ollama 0.x, so a "needs Ollama >= 0.x" check passes. + """ + return {"version": VERSION} diff --git a/api/src/main/openai_api.py b/api/src/main/openai_api.py index 35b9e31c..f3677d51 100644 --- a/api/src/main/openai_api.py +++ b/api/src/main/openai_api.py @@ -24,7 +24,7 @@ from . import registry as registry_module from .leases import LeasedStreamingResponse, acquire, once -from .registry import ModelBusy, ModelUnavailable, in_catalogue, parse_keep_alive +from .registry import ModelBusy, ModelUnavailable, in_catalogue, parse_keep_alive, unknown_model_message router = APIRouter(prefix="/v1") @@ -58,6 +58,21 @@ class ChatCompletionRequest(Schema): stream: bool = False stream_options: Optional[Dict[str, Any]] = None n: Optional[int] = None + reasoning_effort: Optional[str] = None # "none" / "minimal" switch Qwen3-style thinking off + chat_template_kwargs: Optional[Dict[str, Any]] = None # extension, as in vLLM + + +def _template_kwargs(req: "ChatCompletionRequest") -> Dict[str, Any]: + """ + Extra variables for the chat template. `reasoning_effort` (OpenAI's field) maps to + `enable_thinking`: `none` / `minimal` is off, any other value on. `chat_template_kwargs` + (vLLM's extension) is passed through and wins. Neither set: the template's own default. + """ + out: Dict[str, Any] = {} + if req.reasoning_effort is not None: + out["enable_thinking"] = req.reasoning_effort.strip().lower() not in ("none", "minimal") + out.update(req.chat_template_kwargs or {}) + return out class APIError(Exception): @@ -264,11 +279,14 @@ class Generation: or None once the reply is complete; `close` stops generation and releases the engine. """ - def __init__(self, engine, messages, tools, parameters: Dict[str, Any], stops: List[str]): + def __init__(self, engine, messages, tools, parameters: Dict[str, Any], stops: List[str], + template_kwargs: Optional[Dict[str, Any]] = None): self.engine, self.messages, self.tools = engine, messages, tools + self.template_kwargs = template_kwargs or {} + extra = dict(chat_template_kwargs=self.template_kwargs) if self.template_kwargs else {} self.max_new_tokens = parameters.get("max_new_tokens", 0) self.cancel = threading.Event() - self.tokens: Iterator[str] = engine(messages, tools=tools, cancel=self.cancel, **parameters) + self.tokens: Iterator[str] = engine(messages, tools=tools, cancel=self.cancel, **extra, **parameters) self.parser = OutputParser(parse_tools=bool(tools)) self.stopper = _StopMatcher(stops) self.raw: List[str] = [] @@ -319,7 +337,7 @@ def _usage(self) -> Dict[str, int]: completion = len(tokenizer.encode("".join(self.raw), add_special_tokens=False)) text = tokenizer.apply_chat_template( self.messages, tools=self.tools, chat_template=getattr(self.engine, "chat_template", None), - add_generation_prompt=True, tokenize=False, + add_generation_prompt=True, tokenize=False, **self.template_kwargs, ) prompt = len(tokenizer.encode(text, add_special_tokens=False)) except Exception: # usage is informative; never fail a reply over it @@ -398,7 +416,8 @@ async def _chat_completions(request: Request): try: hf_id = registry.resolve(req.model) # derived models (`/api/create`, `/api/copy`) too except LookupError: - raise APIError(404, f"The model '{req.model}' does not exist.", param="model", code="model_not_found") + raise APIError(404, f"The {unknown_model_message(req.model)}", + param="model", code="model_not_found") derived = registry.derived(req.model) or {} messages = _engine_messages(req.messages) @@ -446,7 +465,7 @@ async def _complete(req: ChatCompletionRequest, lease, release, messages, tools, parameters["seed"] = req.seed stops = [req.stop] if isinstance(req.stop, str) else list(req.stop or derived.get("stop") or []) - generation = Generation(engine, messages, tools, parameters, stops) + generation = Generation(engine, messages, tools, parameters, stops, _template_kwargs(req)) def finish(): try: diff --git a/api/src/main/registry.py b/api/src/main/registry.py index 6673abd5..02aef348 100644 --- a/api/src/main/registry.py +++ b/api/src/main/registry.py @@ -55,6 +55,7 @@ class CatalogueEntry: **{name: entry.hf_id for name, entry in CATALOGUE.items()}, # qwen3, default "qwen3:0.6b": "Qwen/Qwen3-0.6B", # Ollama library name "qwen3:latest": "Qwen/Qwen3-0.6B", + "qwen3:0.6b-fp16": "Qwen/Qwen3-0.6B", # Ollama's unquantised tag; the weights are served as published "smollm2:135m": "HuggingFaceTB/SmolLM2-135M-Instruct", # Ollama's smollm2:135m is the instruct model } @@ -79,12 +80,27 @@ def resolve_model_name(name: str) -> str: repo, _, tag = name.partition(":") if "/" in repo and tag in ("", "latest") and re.fullmatch(r"[\w.-]+/[\w.-]+", repo): return repo - raise LookupError(f"model '{name}' not found") + raise LookupError(unknown_model_message(name)) + + +def unknown_model_message(name: str) -> str: + """ The 404 text for a name that is neither known nor a Hugging Face id: what exists and how to add more. """ + names = ", ".join(sorted(n for n in MODEL_ALIASES if n != "default")) + return (f"model '{name}' not found. Gemstone serves these names: {names}. " + "Any Hugging Face model can be pulled by its id (`Org/Model`), " + "for example `ollama pull HuggingFaceTB/SmolLM2-360M-Instruct`; Ollama library names such as " + "`llama3.2` are not available.") def in_catalogue(name: str) -> bool: - """ True for a Gemstone model name (`qwen3`, `default`). """ - return isinstance(name, str) and name.strip().lower() in CATALOGUE + """ + True for a Gemstone catalogue model, by its own name (`qwen3`, `default`) or an Ollama name that + maps to it (`qwen3:latest`, `qwen3:0.6b`): such a model is fetched on first use (S1.14). + """ + if not isinstance(name, str): + return False + key = name.strip().lower() + return key in CATALOGUE or MODEL_ALIASES.get(key) in {e.hf_id for e in CATALOGUE.values()} _plain_classes: dict = {} diff --git a/api/src/main/server.py b/api/src/main/server.py index 2727000c..d999eb8c 100644 --- a/api/src/main/server.py +++ b/api/src/main/server.py @@ -1,5 +1,5 @@ from fastapi import FastAPI, WebSocket, WebSocketDisconnect, HTTPException -from fastapi.responses import RedirectResponse, FileResponse +from fastapi.responses import RedirectResponse, FileResponse, Response from fastapi.staticfiles import StaticFiles from starlette.concurrency import run_in_threadpool import anyio @@ -40,6 +40,12 @@ def root(): return FileResponse(os.path.join(WEBPACK_DIR, "gemstone.html")) +@app.head("/") +def root_head(): + """ Health check, as Ollama answers `HEAD /` (SPEC S1.15): 200 with no body and no key. """ + return Response(status_code=200, media_type="text/plain") + + @app.get("/composeResources/{path:path}") def resource(path: str): """ Redirect to the chat page """ diff --git a/api/src/main/version.py b/api/src/main/version.py new file mode 100644 index 00000000..206cc9cc --- /dev/null +++ b/api/src/main/version.py @@ -0,0 +1,6 @@ +""" +The Gemstone version, defined once. The project is not installed as a package (no build system), +so `importlib.metadata` cannot supply it; `pyproject.toml` must carry the same string +(`api/tests/test_client_compat.py` checks that). +""" +VERSION = "1.0.0" diff --git a/api/tests/test_client_compat.py b/api/tests/test_client_compat.py new file mode 100644 index 00000000..c5990f76 --- /dev/null +++ b/api/tests/test_client_compat.py @@ -0,0 +1,202 @@ +""" +Compatibility with stock Ollama and OpenAI clients (SPEC S1.10, S1.15; issue #118): health checks, +the version, Ollama library names, first-use fetch on the Ollama routes, and the thinking switch. + +No model is loaded and nothing is downloaded: a fake engine and a fake model store. +""" +import pathlib +import re +import threading + +import pytest +from fastapi.testclient import TestClient + +from api.src.main import registry as registry_module +from api.src.main import server +from api.src.main.engine import Engine +from api.src.main.registry import Registry, resolve_model_name +from api.src.main.version import VERSION +from api.tests.test_ollama_api import QWEN, SMOL, FakeEngine, FakeStore + +ROOT = pathlib.Path(__file__).resolve().parents[2] +USER = [{"role": "user", "content": "Hi"}] + + +@pytest.fixture(autouse=True) +def no_key(monkeypatch): + monkeypatch.delenv("GEMSTONE_API_KEY", raising=False) + + +def serve(monkeypatch, *cached): + engine = FakeEngine(QWEN, ["Hello", " there", "!"]) + store = FakeStore(*cached) + reg = Registry(loader=lambda hf_id: engine, store=store, close=lambda e: None) + monkeypatch.setattr(registry_module, "registry", reg) + return engine, store, reg + + +@pytest.fixture +def cached(monkeypatch): + engine, store, reg = serve(monkeypatch, QWEN, SMOL) + with TestClient(server.app) as client: + yield dict(client=client, engine=engine, store=store) + reg.unload() + + +@pytest.fixture +def fresh(monkeypatch): + """ A fresh install: nothing in the local store. """ + engine, store, reg = serve(monkeypatch) + with TestClient(server.app) as client: + yield dict(client=client, engine=engine, store=store) + reg.unload() + + +# -- health ---------------------------------------------------------------------------------------- + +def test_head_root_is_200(cached): + r = cached["client"].head("/") + assert r.status_code == 200 and r.content == b"" + + +def test_head_root_and_version_need_no_key(cached, monkeypatch): + monkeypatch.setenv("GEMSTONE_API_KEY", "s3cret") + assert cached["client"].head("/").status_code == 200 + assert cached["client"].head("/api/version").status_code == 200 + assert cached["client"].get("/api/tags").status_code == 401 + + +def test_get_root_still_serves_the_web_client(cached): + assert any(getattr(r, "path", None) == "/" and "GET" in r.methods for r in server.app.routes) + + +# -- version --------------------------------------------------------------------------------------- + +def test_version_is_the_pyproject_version_without_installing(): + declared = re.search(r'^version\s*=\s*"([^"]+)"', (ROOT / "pyproject.toml").read_text(), re.M).group(1) + assert VERSION == declared + + +def test_api_version_is_real_semver(cached): + body = cached["client"].get("/api/version").json() + assert body == {"version": VERSION} and body["version"] != "0.0.0" + assert re.fullmatch(r"\d+\.\d+\.\d+", body["version"]) + + +# -- Ollama names ---------------------------------------------------------------------------------- + +@pytest.mark.parametrize("name", ["qwen3", "qwen3:latest", "qwen3:0.6b", "Qwen3:0.6B", "qwen3:0.6b-fp16", "default"]) +def test_ollama_names_for_the_catalogue_model(name): + assert resolve_model_name(name) == QWEN + + +def test_unknown_ollama_name_says_what_exists(cached): + r = cached["client"].post("/api/chat", json={"model": "llama3.2", "messages": USER}) + assert r.status_code == 404 + error = r.json()["error"] + assert "llama3.2" in error and "qwen3" in error and "Org/Model" in error and "Hugging Face" in error + assert cached["store"].downloads == [] + + +def test_pull_of_unknown_ollama_name_is_404_with_the_same_hint(cached): + r = cached["client"].post("/api/pull", json={"model": "llama3.2", "stream": False}) + assert r.status_code == 404 and "Org/Model" in r.json()["error"] + + +def test_openai_unknown_name_says_what_exists(cached): + r = cached["client"].post("/v1/chat/completions", json={"model": "llama3.2", "messages": USER}) + assert r.status_code == 404 + assert "Org/Model" in r.json()["error"]["message"] and "qwen3" in r.json()["error"]["message"] + + +# -- first use ------------------------------------------------------------------------------------- + +@pytest.mark.parametrize("route, body", [ + ("/api/chat", {"messages": USER}), + ("/api/generate", {"prompt": "Hi"}), + ("/api/embed", {"input": "Hi"}), +]) +@pytest.mark.parametrize("name", ["qwen3", "qwen3:latest", "qwen3:0.6b"]) +def test_ollama_routes_fetch_a_catalogue_model_on_first_use(fresh, route, body, name): + fresh["engine"].embed = lambda texts, truncate=True: [[0.0] for _ in texts] + r = fresh["client"].post(route, json={"model": name, "stream": False, **body}) + assert r.status_code == 200, r.text + assert fresh["store"].downloads == [QWEN] + + +def test_ollama_does_not_fetch_other_models(fresh): + for name in ("someone/not-pulled", "smollm2:135m"): + r = fresh["client"].post("/api/chat", json={"model": name, "messages": USER}) + assert r.status_code == 404 + assert fresh["store"].downloads == [] + + +# -- thinking -------------------------------------------------------------------------------------- + +def ollama_chat(client, **body): + r = client.post("/api/chat", json={"model": "qwen3", "messages": USER, "stream": False, **body}) + assert r.status_code == 200, r.text + + +def test_ollama_think_false_disables_thinking_in_the_template(cached): + ollama_chat(cached["client"], think=False) + assert cached["engine"].calls[-1]["chat_template_kwargs"] == {"enable_thinking": False} + + +def test_ollama_think_true_enables_it_and_unset_changes_nothing(cached): + ollama_chat(cached["client"], think=True) + assert cached["engine"].calls[-1]["chat_template_kwargs"] == {"enable_thinking": True} + ollama_chat(cached["client"]) + assert "chat_template_kwargs" not in cached["engine"].calls[-1] + + +def test_ollama_generate_passes_think_too(cached): + cached["client"].post("/api/generate", json={"model": "qwen3", "prompt": "Hi", "stream": False, "think": False}) + assert cached["engine"].calls[-1]["chat_template_kwargs"] == {"enable_thinking": False} + + +def openai_chat(client, **body): + r = client.post("/v1/chat/completions", json={"model": "qwen3", "messages": USER, **body}) + assert r.status_code == 200, r.text + + +@pytest.mark.parametrize("effort, expected", [("none", False), ("minimal", False), ("low", True), ("high", True)]) +def test_openai_reasoning_effort_sets_enable_thinking(cached, effort, expected): + openai_chat(cached["client"], reasoning_effort=effort) + assert cached["engine"].calls[-1]["chat_template_kwargs"] == {"enable_thinking": expected} + + +def test_openai_chat_template_kwargs_pass_through_and_win(cached): + openai_chat(cached["client"], reasoning_effort="none", chat_template_kwargs={"enable_thinking": True, "x": 1}) + assert cached["engine"].calls[-1]["chat_template_kwargs"] == {"enable_thinking": True, "x": 1} + + +def test_openai_without_a_reasoning_field_changes_nothing(cached): + openai_chat(cached["client"]) + assert "chat_template_kwargs" not in cached["engine"].calls[-1] + + +def test_openai_bad_chat_template_kwargs_is_400(cached): + r = cached["client"].post("/v1/chat/completions", + json={"model": "qwen3", "messages": USER, "chat_template_kwargs": "no"}) + assert r.status_code == 400 + + +# -- the engine ------------------------------------------------------------------------------------ + +def test_engine_gives_chat_template_kwargs_to_the_template_and_stays_batchable(): + seen = {} + + class Tokenizer: + def apply_chat_template(self, messages, **kwargs): + seen.update(kwargs) + return "prompt" + + def __call__(self, text, add_special_tokens=False): + return {"input_ids": [1, 2]} + + engine = Engine.__new__(Engine) + engine.tokenizer, engine.chat_template, engine._batcher, engine.context_length = Tokenizer(), None, None, 100 + engine._generate_exclusive = lambda *args: iter(["ok"]) + assert list(engine(USER, max_new_tokens=5, chat_template_kwargs={"enable_thinking": False})) == ["ok"] + assert seen["enable_thinking"] is False and seen["add_generation_prompt"] is True diff --git a/docs/SPEC.md b/docs/SPEC.md index 4d028505..88c8beda 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -228,6 +228,13 @@ Implemented (#65), `api/src/main/openai_api.py`: recognised. `finish_reason` is `tool_calls` when any call was returned, else `length` when the token limit was reached, else `stop`. With `tool_choice: "none"` the tools are not sent and no output is parsed as a call. A block that is not valid JSON is returned as content. +- `reasoning_effort` switches thinking: `none` or `minimal` passes `enable_thinking=False` to the + chat template (Qwen3 then skips its reasoning), any other value passes `True`. The vLLM extension + `chat_template_kwargs` (an object) is passed to the template as well and wins over + `reasoning_effort`. Neither set: nothing is passed and the template's default applies (Qwen3 + thinks). The engine takes these as `chat_template_kwargs=`; unlike other extra keywords they do + not make a request exclusive (S1.12). An unknown model's 404 message lists the names that exist + and says Hugging Face ids can be pulled (S1.15). - Reasoning between `` and `` is removed from `content` and returned as `reasoning_content` (message field, or delta field when streaming), the extension used by DeepSeek, vLLM and others. Leading and trailing whitespace around content and reasoning is @@ -512,7 +519,7 @@ OpenAI-compatible API (S1.10). Code: `api/src/main/ollama_api.py` (an `APIRouter Implemented (M3 scope, #66; the rest of the commands, #75): - `POST /api/chat`, `POST /api/generate`, `GET /api/tags`, `POST /api/show`, `POST /api/pull`, - `DELETE /api/delete`, `GET /api/ps`, `GET /api/version` (the Gemstone package version). + `DELETE /api/delete`, `GET /api/ps`, `GET /api/version` (below). `GET /api/ps` shows every resident model whichever API loaded it (S1.14). - `POST /api/copy`, `POST /api/create`, `POST /api/embed`, `POST /api/embeddings` and `POST /api/push` (below). @@ -521,12 +528,32 @@ Implemented (M3 scope, #66; the rest of the commands, #75): not offer, 503 when a load timed out waiting for room (S1.14) or a model cannot be downloaded because the machine is offline or the download failed (the message names the model, #114; `/api/pull` too), 500 for a pull of an unknown repository. An error after streaming has started arrives as a final `{"error": ...}` line. A stream whose client is gone before it starts still releases the model (S1.14). +- **Health and version (#118).** `HEAD /` answers 200 with no body and needs no API key, as Ollama's + does; `GET /` keeps serving the bundled web client. `HEAD` and `GET /api/version` return + `{"version": ""}`, the single constant `VERSION` in `api/src/main/version.py` + (equal to `pyproject.toml`, checked by a test; the project is not installed, so + `importlib.metadata` cannot be used). It is a plain `MAJOR.MINOR.PATCH` string, which is what Ollama + clients compare as semver; Gemstone's `1.x` is above every Ollama `0.x`, so a "requires Ollama >= + 0.x" gate passes. +- **Names (#118).** The names that resolve to the catalogue model (`Qwen/Qwen3-0.6B`) are `qwen3`, + `qwen3:latest`, `qwen3:0.6b`, `qwen3:0.6b-fp16` and `default` (case-insensitive); `smollm2:135m` maps + to `HuggingFaceTB/SmolLM2-135M-Instruct`; `Org/Model[:latest]` is a Hugging Face id. Nothing else + is mapped: Ollama library names such as `llama3.2` are not available. Such a name is a 404 + whose message lists the names above and says a Hugging Face id (`Org/Model`) can be pulled. The + same text is used for `/api/pull`, `/api/show` and the OpenAI API. +- **First use (#118).** `/api/chat`, `/api/generate` (also load-only) and `/api/embed`/`/api/embeddings` + download a catalogue model that is not in the cache (`qwen3` and the Ollama names that map to it), + as the app and the OpenAI API do, so `ollama run qwen3` works on a fresh install. Any other model + that is not pulled stays a 404; an offline or failed download is a 503 (#114). - **Chat.** `messages` (`role`, `content`, `tool_calls`, `tool_name`); `images` are dropped (no vision support). `tools` go to the chat template; a `{json}` block the model emits becomes `message.tool_calls: [{function: {name, arguments}}]` and is never executed (pass-through, S1.10). A block that is not valid JSON stays in the content as text. -- **Thinking.** `think: true` returns the `` block as `message.thinking`; `think: false` - drops it from the reply (the model still generates it); unset leaves it in the content as written. +- **Thinking (#118).** `think: false` passes `enable_thinking=False` to the chat template, so a + Qwen3-style model skips its reasoning, and drops any `` block from the reply; `think: true` + (or a level string such as `"high"`) passes `enable_thinking=True` and returns the block as + `message.thinking`; unset passes nothing: the template's default applies (Qwen3 thinks) and the + `` block stays in `content` as the model wrote it. `/api/chat` and `/api/generate` behave alike. - **Options.** `temperature`, `top_p`, `top_k`, `min_p`, `typical_p`, `repeat_penalty` and `seed` go to the engine unchanged; `num_predict` becomes `max_new_tokens` (unset or ≤ 0 means up to the context length); `stop` cuts the output at the first stop string and stops generation (best diff --git a/docs/guide/guide-server.html b/docs/guide/guide-server.html index abca9975..6db11510 100644 --- a/docs/guide/guide-server.html +++ b/docs/guide/guide-server.html @@ -69,6 +69,14 @@

OpenAI-compatible API

OpenAI 호환 AP -d '{"model": "qwen3", "messages": [{"role": "user", "content": "Hi"}]}'

Tool calls are returned, not run: when the model calls a tool, the reply carries tool_calls and finish_reason: "tool_calls". Your client runs the tool and sends the result back as a tool message. The model's reasoning (<think>) comes back separately as reasoning_content, not in content. The chat app's WebSocket is unchanged and still runs Gemstone's built-in tools on the server.

도구 호출은 실행되지 않고 반환됩니다: 모델이 도구를 부르면 응답에 tool_calls 와 finish_reason: "tool_calls" 가 담깁니다. 클라이언트가 도구를 실행하고 그 결과를 tool 메시지로 다시 보냅니다. 모델의 추론(<think>)은 content 가 아니라 reasoning_content 로 따로 옵니다. 채팅 앱의 WebSocket 은 그대로이며, 여전히 Gemstone 내장 도구를 서버에서 실행합니다.

+

Pointing clients at Gemstone

클라이언트를 Gemstone 에 연결하기

+

Gemstone listens on port 23100, not Ollama's 11434. Set the address the client reads:

Gemstone 은 Ollama 의 11434 가 아니라 23100 포트에서 듣습니다. 클라이언트가 읽는 주소를 설정하세요.

+
OLLAMA_HOST=http://127.0.0.1:23100 ollama run qwen3            # the ollama CLI, Ollama clients
+OPENAI_BASE_URL=http://127.0.0.1:23100/v1 OPENAI_API_KEY=x python app.py  # OpenAI SDKs
+

With GEMSTONE_API_KEY set on the server, give OpenAI SDKs that value as OPENAI_API_KEY, and other clients as an Authorization: Bearer header; the stock ollama CLI sends no key, so use it against a loopback server without one. HEAD / and GET /api/version work without a key, for health checks.

서버에 GEMSTONE_API_KEY 를 설정했다면 OpenAI SDK 에는 그 값을 OPENAI_API_KEY 로, 다른 클라이언트에는 Authorization: Bearer 헤더로 주세요. 기본 ollama CLI 는 키를 보내지 않으므로 키 없는 loopback 서버에서 쓰세요. 상태 확인용 HEAD / 와 GET /api/version 은 키 없이 동작합니다.

+

Model names: qwen3, qwen3:latest, qwen3:0.6b and qwen3:0.6b-fp16 are the built-in model and are downloaded on first use, by the app, the OpenAI API and the Ollama API alike. smollm2:135m must be pulled first. Any Hugging Face model is pulled by its id (Org/Model); other Ollama library names such as llama3.2 answer 404 with this list. GET /api/version returns Gemstone's own version (1.0.0), a plain MAJOR.MINOR.PATCH that Ollama clients compare as semver.

모델 이름: qwen3, qwen3:latest, qwen3:0.6b, qwen3:0.6b-fp16 는 기본 제공 모델이며 앱, OpenAI API, Ollama API 모두 처음 쓸 때 내려받습니다. smollm2:135m 은 먼저 pull 해야 합니다. Hugging Face 모델은 id(Org/Model)로 pull 합니다. llama3.2 같은 다른 Ollama 라이브러리 이름은 이 목록과 함께 404 를 돌려줍니다. GET /api/version 은 Gemstone 자신의 버전(1.0.0)을 돌려주며, Ollama 클라이언트가 semver 로 비교하는 MAJOR.MINOR.PATCH 형식입니다.

+

Thinking: Qwen3 reasons by default. Ollama think: false and OpenAI reasoning_effort: "none" (or "minimal") pass enable_thinking=False to the chat template, so the model skips the reasoning; vLLM's chat_template_kwargs is passed through. With none of these set the template's default applies and the <think> block stays in the Ollama content as written (OpenAI returns it as reasoning_content).

추론: Qwen3 는 기본적으로 추론합니다. Ollama 의 think: false 와 OpenAI 의 reasoning_effort: "none"(또는 "minimal")은 채팅 템플릿에 enable_thinking=False 를 넘겨 모델이 추론을 건너뜁니다. vLLM 의 chat_template_kwargs 도 그대로 전달됩니다. 아무것도 설정하지 않으면 템플릿 기본값이 적용되고, Ollama 의 content 에는 <think> 블록이 그대로 남습니다(OpenAI 는 reasoning_content 로 돌려줍니다).

+

Keys

키

Web search uses SerpApi when a key is present and falls back to scraping Bing otherwise. Put the key in a .env file at the repository root (never commit it):

웹 검색은 키가 있으면 SerpApi 를, 없으면 Bing 스크래핑을 씁니다. 키는 저장소 루트의 .env 파일에 둡니다 (절대 커밋하지 마세요):

cp .env.example .env
diff --git a/docs/locale/README_ko.md b/docs/locale/README_ko.md
index 7beca81b..604eaccb 100644
--- a/docs/locale/README_ko.md
+++ b/docs/locale/README_ko.md
@@ -69,6 +69,19 @@ python -m api run server
 클라이언트는 `Authorization: Bearer ` 를 보냅니다. `GEMSTONE_ORIGINS` 는 브라우저 origin 을 더 허용하고,
 `--reload` (또는 `GEMSTONE_DEV=1`) 는 개발용 자동 재시작을 켭니다.
 
+**기존 클라이언트를 Gemstone 에 연결하기** (포트는 Ollama 의 `11434` 가 아니라 `23100` 입니다):
+
+```bash
+OLLAMA_HOST=http://127.0.0.1:23100 ollama run qwen3          # ollama CLI 와 Ollama 클라이언트
+OPENAI_BASE_URL=http://127.0.0.1:23100/v1 OPENAI_API_KEY=x python my_script.py   # OpenAI SDK
+```
+
+`qwen3`, `qwen3:latest`, `qwen3:0.6b` 는 기본 제공 모델이며 처음 쓸 때 내려받습니다. Hugging Face 모델은 id 로
+받습니다(`ollama pull Org/Model`). 그 밖의 Ollama 라이브러리 이름(`llama3.2`)은 쓸 수 없습니다. 서버에
+`GEMSTONE_API_KEY` 를 설정했다면 그 값을 키로 보내세요. OpenAI SDK 는 `OPENAI_API_KEY=`, 다른 클라이언트는
+`Authorization: Bearer ` 헤더입니다(기본 `ollama` CLI 는 키를 보내지 않으므로 키 없는 loopback 서버에서 쓰세요).
+`think: false`(Ollama), `reasoning_effort: "none"`(OpenAI)은 Qwen3 의 추론을 끕니다.
+
 **3. 네이티브 클라이언트 실행**
 
 ```bash

From 9fb8be6ccb27c72be83dc5765365dd163633afcd Mon Sep 17 00:00:00 2001
From: BREW <81044362+b-re-w@users.noreply.github.com>
Date: Sat, 3 Oct 2026 15:00:01 +0900
Subject: [PATCH 2/2] Docs: Do not claim what the ollama CLI sends

---
 README.md                    | 4 ++--
 docs/guide/guide-server.html | 2 +-
 docs/locale/README_ko.md     | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 53e9aa66..482294f8 100644
--- a/README.md
+++ b/README.md
@@ -79,8 +79,8 @@ OPENAI_BASE_URL=http://127.0.0.1:23100/v1 OPENAI_API_KEY=x python my_script.py
 `qwen3`, `qwen3:latest` and `qwen3:0.6b` are the built-in model (fetched on first use); any Hugging Face
 model is pulled by its id, e.g. `ollama pull Org/Model`. Other Ollama library names (`llama3.2`) are not
 available. When `GEMSTONE_API_KEY` is set on the server, send it as the key: `OPENAI_API_KEY=` for
-OpenAI SDKs, or an `Authorization: Bearer ` header for other clients (the stock `ollama` CLI sends none,
-so use it on a loopback server without a key). `think: false` (Ollama) and `reasoning_effort: "none"`
+OpenAI SDKs, or an `Authorization: Bearer ` header for other clients (if your Ollama client cannot send
+a header, use it on a loopback server without a key). `think: false` (Ollama) and `reasoning_effort: "none"`
 (OpenAI) switch Qwen3's reasoning off.
 
 **3. Run a native client**
diff --git a/docs/guide/guide-server.html b/docs/guide/guide-server.html
index 6db11510..bfa7fe1b 100644
--- a/docs/guide/guide-server.html
+++ b/docs/guide/guide-server.html
@@ -73,7 +73,7 @@ 
Pointing clients at Gemstone
클라이
   
Gemstone listens on port 23100, not Ollama's 11434. Set the address the client reads:
Gemstone 은 Ollama 의 11434 가 아니라 23100 포트에서 듣습니다. 클라이언트가 읽는 주소를 설정하세요.

 
OLLAMA_HOST=http://127.0.0.1:23100 ollama run qwen3            # the ollama CLI, Ollama clients
 OPENAI_BASE_URL=http://127.0.0.1:23100/v1 OPENAI_API_KEY=x python app.py  # OpenAI SDKs

-  
With GEMSTONE_API_KEY set on the server, give OpenAI SDKs that value as OPENAI_API_KEY, and other clients as an Authorization: Bearer header; the stock ollama CLI sends no key, so use it against a loopback server without one. HEAD / and GET /api/version work without a key, for health checks.
서버에 GEMSTONE_API_KEY 를 설정했다면 OpenAI SDK 에는 그 값을 OPENAI_API_KEY 로, 다른 클라이언트에는 Authorization: Bearer 헤더로 주세요. 기본 ollama CLI 는 키를 보내지 않으므로 키 없는 loopback 서버에서 쓰세요. 상태 확인용 HEAD / 와 GET /api/version 은 키 없이 동작합니다.

+  
With GEMSTONE_API_KEY set on the server, give OpenAI SDKs that value as OPENAI_API_KEY, and other clients as an Authorization: Bearer header; an Ollama client that cannot send that header should be used against a loopback server without a key. HEAD / and GET /api/version work without a key, for health checks.
서버에 GEMSTONE_API_KEY 를 설정했다면 OpenAI SDK 에는 그 값을 OPENAI_API_KEY 로, 다른 클라이언트에는 Authorization: Bearer 헤더로 주세요. 그 헤더를 보낼 수 없는 Ollama 클라이언트는 키 없는 loopback 서버에서 쓰세요. 상태 확인용 HEAD / 와 GET /api/version 은 키 없이 동작합니다.

   
Model names: qwen3, qwen3:latest, qwen3:0.6b and qwen3:0.6b-fp16 are the built-in model and are downloaded on first use, by the app, the OpenAI API and the Ollama API alike. smollm2:135m must be pulled first. Any Hugging Face model is pulled by its id (Org/Model); other Ollama library names such as llama3.2 answer 404 with this list. GET /api/version returns Gemstone's own version (1.0.0), a plain MAJOR.MINOR.PATCH that Ollama clients compare as semver.
모델 이름: qwen3, qwen3:latest, qwen3:0.6b, qwen3:0.6b-fp16 는 기본 제공 모델이며 앱, OpenAI API, Ollama API 모두 처음 쓸 때 내려받습니다. smollm2:135m 은 먼저 pull 해야 합니다. Hugging Face 모델은 id(Org/Model)로 pull 합니다. llama3.2 같은 다른 Ollama 라이브러리 이름은 이 목록과 함께 404 를 돌려줍니다. GET /api/version 은 Gemstone 자신의 버전(1.0.0)을 돌려주며, Ollama 클라이언트가 semver 로 비교하는 MAJOR.MINOR.PATCH 형식입니다.

   
Thinking: Qwen3 reasons by default. Ollama think: false and OpenAI reasoning_effort: "none" (or "minimal") pass enable_thinking=False to the chat template, so the model skips the reasoning; vLLM's chat_template_kwargs is passed through. With none of these set the template's default applies and the <think> block stays in the Ollama content as written (OpenAI returns it as reasoning_content).
추론: Qwen3 는 기본적으로 추론합니다. Ollama 의 think: false 와 OpenAI 의 reasoning_effort: "none"(또는 "minimal")은 채팅 템플릿에 enable_thinking=False 를 넘겨 모델이 추론을 건너뜁니다. vLLM 의 chat_template_kwargs 도 그대로 전달됩니다. 아무것도 설정하지 않으면 템플릿 기본값이 적용되고, Ollama 의 content 에는 <think> 블록이 그대로 남습니다(OpenAI 는 reasoning_content 로 돌려줍니다).

 
diff --git a/docs/locale/README_ko.md b/docs/locale/README_ko.md
index 604eaccb..173c97ec 100644
--- a/docs/locale/README_ko.md
+++ b/docs/locale/README_ko.md
@@ -79,7 +79,7 @@ OPENAI_BASE_URL=http://127.0.0.1:23100/v1 OPENAI_API_KEY=x python my_script.py
 `qwen3`, `qwen3:latest`, `qwen3:0.6b` 는 기본 제공 모델이며 처음 쓸 때 내려받습니다. Hugging Face 모델은 id 로
 받습니다(`ollama pull Org/Model`). 그 밖의 Ollama 라이브러리 이름(`llama3.2`)은 쓸 수 없습니다. 서버에
 `GEMSTONE_API_KEY` 를 설정했다면 그 값을 키로 보내세요. OpenAI SDK 는 `OPENAI_API_KEY=`, 다른 클라이언트는
-`Authorization: Bearer ` 헤더입니다(기본 `ollama` CLI 는 키를 보내지 않으므로 키 없는 loopback 서버에서 쓰세요).
+`Authorization: Bearer ` 헤더입니다(헤더를 보낼 수 없는 Ollama 클라이언트는 키 없는 loopback 서버에서 쓰세요).
 `think: false`(Ollama), `reasoning_effort: "none"`(OpenAI)은 Qwen3 의 추론을 끕니다.
 
 **3. 네이티브 클라이언트 실행**