`. `GEMSTONE_ORIGINS` allows more browser origins and
`--reload` (or `GEMSTONE_DEV=1`) turns on auto-reload for development.
diff --git a/api/__main__.py b/api/__main__.py
index fc999ba6..dbcdc548 100644
--- a/api/__main__.py
+++ b/api/__main__.py
@@ -11,11 +11,12 @@ def main(argv):
sys.exit("usage: python -m api run server [host] [port] [--reload]\n"
"env: GEMSTONE_HOST=host[:port], GEMSTONE_API_KEY, GEMSTONE_ORIGINS, GEMSTONE_DEV=1")
try:
- from api.src.main import security
+ from api.src.main import security, settings
server_object = "api.src.main.server:app"
except ImportError:
- from src.main import security
+ from src.main import security, settings
server_object = "src.main.server:app"
+ settings.configure_logging()
config = security.resolve_server_config(argv)
security.warn_if_open(config.host)
run_uvicorn(
diff --git a/api/src/main/models/base.py b/api/src/main/models/base.py
index 43f1c730..9307d620 100644
--- a/api/src/main/models/base.py
+++ b/api/src/main/models/base.py
@@ -1,4 +1,4 @@
-import traceback
+import logging
from re import finditer, DOTALL
from dataclasses import dataclass
from typing import Generator, Optional, List, Dict, Union
@@ -7,6 +7,8 @@
from ..engine import Engine, ModelBusy
from ..utils import FunctionCalling, FunctionCallResult
+log = logging.getLogger("gemstone.model")
+
@dataclass
class Tags:
@@ -92,24 +94,21 @@ def parse_tool_calling(
)
# Finalize the tool calls
- print("\n")
- spinner = ['⠋','⠙','⠹','⠸','⠼','⠴','⠦','⠧','⠇','⠏']
- stat = 0
+ waiting = False
while True:
queued = len(result_obj.job_list)
final_result = result_obj.finalize(
chat_history,
- (self.special_tags.TOOLCALL, self.special_tags.TOOLCALL_END),
- print_output=print_output
+ (self.special_tags.TOOLCALL, self.special_tags.TOOLCALL_END)
)
- if queued > 0 and final_result is False:
- print(f"\r{spinner[stat % len(spinner)]} Waiting for tool calls to finish...", end="", flush=True)
+ if queued > 0 and final_result is False and not waiting:
+ waiting = True
+ log.debug("Waiting for %d tool call(s) to finish", queued)
if final_result is False:
- stat += 1
- result_obj.wait(0.1) # sleeps until a tool finishes; wakes to animate the spinner
+ result_obj.wait(0.1) # sleeps until a tool finishes
continue
if queued > 0:
- print("\r[✔] Tool calls are finalized successfully.", flush=True)
+ log.debug("Tool calls are finalized")
if stream:
yield final_result
@@ -150,7 +149,8 @@ def chat(
stream (bool, optional): Stream. Defaults to True.
max_new_tokens (int, optional): Max new tokens. Defaults to 1024.
repeat_penalty (float, optional): Repeat penalty. Defaults to 1.0.
- print_output (bool, optional): Print output. Defaults to False.
+ print_output (bool, optional): Ignored, kept for callers. Prompts and answers are logged at DEBUG
+ (GEMSTONE_LOG_LEVEL=DEBUG), never printed.
tool_call_caches (dict, optional): The session's tool-result cache (call id -> result).
**kwargs: Additional arguments
"""
@@ -184,11 +184,8 @@ def adaptive_special_tag_buffering(outs, wait_tokens_for=6):
prompt = prompt[:-1] # Remove the last user prompt if it's None
user_prompt = None # Reset user prompt to None after appending
- if print_output and initial_operation:
- print("PROMPT:")
- for line in prompt:
- print(line)
- print("\nANSWER:")
+ if initial_operation and log.isEnabledFor(logging.DEBUG):
+ log.debug("PROMPT:\n%s", "\n".join(str(line) for line in prompt))
if initial_operation:
initial_operation = False
# TODO: Add kv cache control for tool-calling here
@@ -213,36 +210,37 @@ def adaptive_special_tag_buffering(outs, wait_tokens_for=6):
chat_history=chat_history,
tools=tools,
stream=stream,
- print_output=print_output,
tool_call_caches=tool_call_caches
)
if stream:
+ answer = []
try:
for word in outputs:
if word:
if self.special_tags.TOOLCALL in word and self.special_tags.TOOLCALL_END in word:
function_called = True # flag on
- if print_output: print(word, end="", flush=True)
+ answer.append(word)
yield word
except ValueError as e: # Over token limit error
- traceback.print_exc()
+ log.exception("Chat failed")
if "token" in str(e) and "limit" in str(e):
message = "\n\nERROR: Chat is unexpectedly terminated due to token limit. Please shorten your prompt or chat history."
else:
message = f"\n\nERROR: {type(e)} - Something went wrong while processing the chat. Please try again later.\n{e}"
- if print_output: print(message, end="", flush=True)
+ answer.append(message)
yield message
except ModelBusy:
raise # the server maps it to 503 / WebSocket close 1013, so it must not become text
except Exception as e: # never let a failure kill the stream without a message
- traceback.print_exc()
+ log.exception("Chat failed")
message = f"\n\nERROR: {type(e).__name__} - Something went wrong while processing the chat.\n{e}"
- if print_output: print(message, end="", flush=True)
+ answer.append(message)
yield message
finally:
- print()
+ if log.isEnabledFor(logging.DEBUG):
+ log.debug("ANSWER:\n%s", "".join(answer))
else:
if self.special_tags.TOOLCALL in outputs and self.special_tags.TOOLCALL_END in outputs:
function_called = True # flag on
- print(outputs, flush=True)
+ log.debug("ANSWER:\n%s", outputs)
diff --git a/api/src/main/models/qwen3/model.py b/api/src/main/models/qwen3/model.py
index 4bb10302..14d2d72c 100644
--- a/api/src/main/models/qwen3/model.py
+++ b/api/src/main/models/qwen3/model.py
@@ -1,3 +1,4 @@
+import logging
from typing import List, Dict, Union, Generator, Optional
from ..base import ChatHistory, FunctionCalling, BaseModel
@@ -42,7 +43,7 @@
3. Call a new tool only when there is no relevant cache or the cached data is insufficient.
Remember: Your role is to be a reliable, knowledgeable professional assistant who thinks carefully before responding and actively seeks current information when needed."""
-print("INFO: Use default system prompt -", system_prompt)
+logging.getLogger("gemstone.model").debug("Use default system prompt - %s", system_prompt)
class Qwen3Model(BaseModel):
diff --git a/api/src/main/server.py b/api/src/main/server.py
index d999eb8c..2c96c473 100644
--- a/api/src/main/server.py
+++ b/api/src/main/server.py
@@ -5,12 +5,13 @@
import anyio
import uvicorn
-import traceback
+import contextlib
+import logging
import threading
import json
import os
-from .settings import STATIC_DIR, WEBPACK_DIR, Session
+from .settings import STATIC_DIR, WEBPACK_DIR, Session, configure_logging
from . import registry as registry_module
from .leases import acquire
from .registry import DEFAULT_KEEP_ALIVE, ModelBusy, ModelUnavailable, in_catalogue
@@ -20,7 +21,23 @@
from . import security
-app = FastAPI()
+configure_logging() # GEMSTONE_LOG_LEVEL
+log = logging.getLogger("gemstone.server")
+
+
+@contextlib.asynccontextmanager
+async def lifespan(_app):
+ """ On shutdown close every loaded engine (stops the batching loops, frees the memory). """
+ yield
+ log.info("Shutting down: unloading models")
+ try:
+ # Read the module attribute at call time: the registry may be replaced (tests).
+ await run_in_threadpool(registry_module.registry.unload)
+ except Exception:
+ log.exception("Unloading models on shutdown failed")
+
+
+app = FastAPI(lifespan=lifespan)
security.install(app) # Origin policy and the optional API key (SPEC S1.1)
app.include_router(ollama_router) # Ollama-compatible API (SPEC S1.15), including POST /api/chat
app.mount("/static", StaticFiles(directory=STATIC_DIR, html=True), name="static")
@@ -105,7 +122,7 @@ async def chat_with_streaming(websocket: WebSocket):
session = Session(session_id=session_id)
hf_id = registry_module.registry.resolve(session.model_id) # derived models too
except Exception:
- traceback.print_exc()
+ log.exception("WebSocket chat failed")
await websocket.close(code=1008, reason="Invalid session ID or model not found.")
return
@@ -126,7 +143,7 @@ async def chat_with_streaming(websocket: WebSocket):
await websocket.close(code=1011, reason=_reason(e.reason))
return
except Exception:
- traceback.print_exc()
+ log.exception("Loading the model failed")
await websocket.close(code=1008, reason="Model not found or failed to load.")
return
@@ -138,7 +155,7 @@ async def chat_with_streaming(websocket: WebSocket):
done = object()
try:
tokens = lease.model.chat( # a generator: nothing runs until the first `next`
- chat_history, user_prompt, print_output=True, cancel=cancel, tool_call_caches=session.tool_call_caches
+ chat_history, user_prompt, cancel=cancel, tool_call_caches=session.tool_call_caches
)
while (token := await run_in_threadpool(next, tokens, done)) is not done:
await websocket.send_text(token)
diff --git a/api/src/main/settings.py b/api/src/main/settings.py
index 40544ef5..59ccaf32 100644
--- a/api/src/main/settings.py
+++ b/api/src/main/settings.py
@@ -1,6 +1,38 @@
from datetime import datetime
from uuid import uuid4
+import logging
import os
+import time
+
+logger = logging.getLogger("gemstone.session")
+
+# Idle seconds after which a session expires (GEMSTONE_SESSION_TTL; 0 = never). Default: 24 hours.
+DEFAULT_SESSION_TTL = 24 * 3600.0
+clock = time.monotonic # replaced by a fake clock in tests
+
+
+def configure_logging() -> None:
+ """
+ Set the level of the `gemstone` loggers from GEMSTONE_LOG_LEVEL (default INFO; DEBUG also logs
+ prompts and answers) and make sure they reach the console in uvicorn's format.
+ """
+ name = os.environ.get("GEMSTONE_LOG_LEVEL", "INFO").strip().upper()
+ level = logging.getLevelName(name)
+ log = logging.getLogger("gemstone")
+ log.setLevel(level if isinstance(level, int) else logging.INFO)
+ if not any(getattr(h, "_gemstone", False) for h in log.handlers):
+ handler = logging.StreamHandler()
+ handler._gemstone = True
+ handler.setFormatter(logging.Formatter("%(levelname)s: %(name)s - %(message)s"))
+ log.addHandler(handler)
+ log.propagate = False
+
+
+def session_ttl() -> float:
+ try:
+ return max(0.0, float(os.environ.get("GEMSTONE_SESSION_TTL", DEFAULT_SESSION_TTL)))
+ except ValueError:
+ return DEFAULT_SESSION_TTL
# The model catalogue lives in the registry (registry.CATALOGUE, SPEC S1.2).
@@ -19,8 +51,11 @@ class Session:
def __new__(cls, model_id: str = None, session_id: str = None):
""" Create a new session or return an existing one """
+ cls.sweep()
if session_id and session_id in cls.__sessions:
- return cls.__sessions[session_id]
+ session = cls.__sessions[session_id]
+ session.last_used = clock()
+ return session
else:
return super().__new__(cls)
@@ -34,18 +69,27 @@ def __init__(self, model_id: str = None, session_id: str = None):
self.model_id = model_id
# The random suffix keeps sessions created in the same second apart (each has its own cache)
self.session_id = f"{model_id}_{datetime.now().strftime('%Y%m%d%H%M%S')}_{uuid4().hex[:8]}"
- print("INFO: Session", self.session_id, "is CREATED for model", model_id)
+ self.last_used = clock()
self.__sessions[self.session_id] = self
- print("INFO: Current sessions:", list(self.__sessions))
+ logger.info("Session %s is CREATED for model %s (%d open)", self.session_id, model_id, len(self.__sessions))
self.tool_call_caches: dict[str, str] = {} # tool call id -> result (S1.6)
+ @classmethod
+ def sweep(cls) -> None:
+ """ Drop the sessions idle longer than the TTL, with their tool caches. """
+ ttl = session_ttl()
+ if ttl <= 0:
+ return
+ now = clock()
+ for sid in [sid for sid, s in cls.__sessions.items() if now - s.last_used > ttl]:
+ logger.info("Session %s EXPIRED after %.0f s idle", sid, ttl)
+ cls.__sessions.pop(sid).tool_call_caches.clear()
+
@classmethod
def close(cls, session_id: str):
""" Close the session. The model stays resident for its keep_alive (S1.14). """
if session_id in cls.__sessions:
- print("INFO: Session", session_id, "is DELETED for model", cls.__sessions[session_id].model_id)
cls.__sessions[session_id].tool_call_caches.clear()
- del cls.__sessions[session_id]
- print("INFO: Current sessions:", list(cls.__sessions))
+ logger.info("Session %s is DELETED for model %s", session_id, cls.__sessions.pop(session_id).model_id)
else:
raise ValueError(f"Session {session_id} not found")
diff --git a/api/src/main/utils/web_search.py b/api/src/main/utils/web_search.py
index 183d248a..400f96da 100644
--- a/api/src/main/utils/web_search.py
+++ b/api/src/main/utils/web_search.py
@@ -13,15 +13,6 @@
pass
-# Load environment variables from .env file
-try:
- from dotenv import load_dotenv
- load_dotenv()
-except ImportError:
- # dotenv not installed, skip
- pass
-
-
class WebSearchAPI:
"""Web search API wrapper using SerpApi with robust fallbacks."""
diff --git a/api/tests/test_lease_release.py b/api/tests/test_lease_release.py
index c75dbc5e..80f13a8b 100644
--- a/api/tests/test_lease_release.py
+++ b/api/tests/test_lease_release.py
@@ -208,7 +208,7 @@ def test_a_finished_reply_releases_the_lease_once(setup, route, stream):
response = client.post(path, json={**body, "stream": stream})
assert response.status_code == 200
assert "Hello" in response.text
- assert_released_once_and_switch_completes(setup)
+ assert_released_once_and_switch_completes(setup) # before shutdown unloads the model
@pytest.mark.parametrize("route", list(STREAM_ROUTES))
diff --git a/api/tests/test_ops_quickstart.py b/api/tests/test_ops_quickstart.py
new file mode 100644
index 00000000..3e88a6af
--- /dev/null
+++ b/api/tests/test_ops_quickstart.py
@@ -0,0 +1,181 @@
+"""
+Operations (issue #120): logging instead of prints, session idle expiry, shutdown, the documented
+quick start and web_search's imports. Fake engines and a fake clock stand in for the model.
+"""
+import importlib
+import logging
+import re
+from pathlib import Path
+
+import pytest
+from fastapi.testclient import TestClient
+
+from api.src.main import registry as registry_module
+from api.src.main import server, settings
+from api.src.main.registry import Registry
+from api.tests.test_ollama_api import SMOL, FakeEngine, FakeStore
+from api.tests.test_residency import new_session, ws_chat
+
+ROOT = Path(__file__).resolve().parents[2]
+START = "uv run --extra torch python -m api run server"
+SECRET_PROMPT = "tell me the secret-word-xyzzy"
+
+
+@pytest.fixture
+def client(monkeypatch):
+ reg = Registry(loader=lambda hf_id: FakeEngine(hf_id, ["Hello", " there", "!"]), store=FakeStore(SMOL))
+ monkeypatch.setattr(registry_module, "registry", reg)
+ with TestClient(server.app) as c:
+ yield c
+ reg.unload()
+
+
+# -- logging ---------------------------------------------------------------------------------------
+
+def test_no_prompt_or_answer_on_stdout_or_info(client, capsys, caplog):
+ caplog.set_level(logging.INFO, logger="gemstone")
+ sid = new_session(client)
+ assert ws_chat(client, sid, SECRET_PROMPT) == "Hello there!"
+ out = capsys.readouterr()
+ text = out.out + out.err + "\n".join(r.getMessage() for r in caplog.records)
+ assert "secret-word-xyzzy" not in text
+ assert "Hello there" not in text
+ assert "PROMPT:" not in text
+ assert any(r.name.startswith("gemstone") for r in caplog.records)
+
+
+def test_prompt_and_answer_at_debug(client, caplog, capsys):
+ caplog.set_level(logging.DEBUG, logger="gemstone")
+ sid = new_session(client)
+ ws_chat(client, sid, SECRET_PROMPT)
+ text = "\n".join(r.getMessage() for r in caplog.records if r.name.startswith("gemstone"))
+ assert "secret-word-xyzzy" in text
+ assert "Hello" in text
+ assert "secret-word-xyzzy" not in capsys.readouterr().out
+
+
+def test_importing_qwen3_prints_nothing(capsys):
+ from api.src.main.models.qwen3 import model
+ importlib.reload(model)
+ assert capsys.readouterr().out == ""
+
+
+@pytest.mark.parametrize("value,level", [("DEBUG", logging.DEBUG), ("warning", logging.WARNING),
+ (None, logging.INFO), ("bogus", logging.INFO)])
+def test_log_level_from_env(monkeypatch, value, level):
+ if value is None:
+ monkeypatch.delenv("GEMSTONE_LOG_LEVEL", raising=False)
+ else:
+ monkeypatch.setenv("GEMSTONE_LOG_LEVEL", value)
+ old = logging.getLogger("gemstone").level
+ try:
+ settings.configure_logging()
+ assert logging.getLogger("gemstone").level == level
+ finally:
+ logging.getLogger("gemstone").setLevel(old)
+
+
+# -- session idle expiry ---------------------------------------------------------------------------
+
+class Clock:
+ def __init__(self):
+ self.now = 1000.0
+
+ def __call__(self):
+ return self.now
+
+
+@pytest.fixture
+def clock(monkeypatch):
+ c = Clock()
+ monkeypatch.setattr(settings, "clock", c)
+ monkeypatch.delenv("GEMSTONE_SESSION_TTL", raising=False)
+ yield c
+ for sid in list(settings.Session._Session__sessions):
+ settings.Session.close(sid)
+
+
+def test_session_expires_after_idle_ttl(clock, monkeypatch):
+ monkeypatch.setenv("GEMSTONE_SESSION_TTL", "60")
+ s = settings.Session(model_id="m")
+ s.tool_call_caches["a"] = "result"
+ clock.now += 59
+ assert settings.Session(session_id=s.session_id) is s # a lookup counts as use
+ clock.now += 59
+ assert settings.Session(session_id=s.session_id) is s
+ clock.now += 61
+ with pytest.raises(ValueError):
+ settings.Session(session_id=s.session_id)
+ assert s.tool_call_caches == {}
+ with pytest.raises(ValueError):
+ settings.Session.close(s.session_id)
+
+
+def test_session_sweep_drops_idle_sessions_on_create(clock, monkeypatch):
+ monkeypatch.setenv("GEMSTONE_SESSION_TTL", "10")
+ old = settings.Session(model_id="m")
+ clock.now += 11
+ new = settings.Session(model_id="m")
+ assert old.session_id not in settings.Session._Session__sessions
+ assert new.session_id in settings.Session._Session__sessions
+
+
+def test_session_ttl_zero_never_expires(clock, monkeypatch):
+ monkeypatch.setenv("GEMSTONE_SESSION_TTL", "0")
+ s = settings.Session(model_id="m")
+ clock.now += 10 ** 9
+ assert settings.Session(session_id=s.session_id) is s
+
+
+def test_session_default_ttl_is_a_day(clock):
+ s = settings.Session(model_id="m")
+ clock.now += 23 * 3600
+ assert settings.Session(session_id=s.session_id) is s
+ clock.now += 25 * 3600
+ with pytest.raises(ValueError):
+ settings.Session(session_id=s.session_id)
+
+
+# -- shutdown --------------------------------------------------------------------------------------
+
+def test_shutdown_closes_loaded_engines(monkeypatch):
+ engines = []
+
+ def loader(hf_id):
+ engines.append(FakeEngine(hf_id, ["x"]))
+ return engines[-1]
+
+ reg = Registry(loader=loader, store=FakeStore(SMOL))
+ monkeypatch.setattr(registry_module, "registry", reg)
+ with TestClient(server.app) as c:
+ ws_chat(c, new_session(c), "hi")
+ assert engines and not engines[0].closed_engine
+ assert engines[0].closed_engine
+ assert reg.loaded() == []
+
+
+# -- web_search and dependencies -------------------------------------------------------------------
+
+def test_web_search_loads_dotenv_once():
+ src = (ROOT / "api/src/main/utils/web_search.py").read_text()
+ assert src.count("load_dotenv()") == 1
+
+
+def test_requests_is_declared():
+ assert re.search(r'^\s*"requests[>=<~ "]', (ROOT / "pyproject.toml").read_text(), re.M)
+
+
+# -- the documented quick start --------------------------------------------------------------------
+
+@pytest.mark.parametrize("path", ["README.md", "docs/locale/README_ko.md", "docs/guide/getting-started.html"])
+def test_quick_start_works_without_an_activated_venv(path):
+ text = (ROOT / path).read_text()
+ assert START in text
+ for line in text.splitlines():
+ if "python -m api run server" in line and "GEMSTONE" not in line:
+ assert "uv run" in line, f"{path}: bare python in: {line}"
+
+
+def test_ci_runs_the_documented_command():
+ wf = (ROOT / ".github/workflows/api-tests.yml").read_text()
+ assert START in wf and "/api/version" in wf
diff --git a/docs/SPEC.md b/docs/SPEC.md
index 88c8beda..95c82198 100644
--- a/docs/SPEC.md
+++ b/docs/SPEC.md
@@ -30,7 +30,7 @@ and S3.4 also record the maintainer's decisions of the same day ([`serving/engin
### S1.1 Server process — `implemented` · G5
-- `python -m api run server [host] [port] [--reload]` starts a uvicorn server on `127.0.0.1:23100` by
+- `uv run --extra torch python -m api run server [host] [port] [--reload]` starts a uvicorn server on `127.0.0.1:23100` by
default, with a 300 s WebSocket ping interval and timeout. `GEMSTONE_HOST=host[:port]` overrides the
default (arguments win over it). Auto-reload only with `--reload` or `GEMSTONE_DEV=1` (#116).
- CORS and the WebSocket accept localhost, `127.0.0.1`, `[::1]` origins (any port) and the server's own
diff --git a/docs/guide/getting-started.html b/docs/guide/getting-started.html
index ba3d28cd..f74d0494 100644
--- a/docs/guide/getting-started.html
+++ b/docs/guide/getting-started.html
@@ -52,8 +52,9 @@ 1. Install the server
1. 서버 설치
That installs upstream PyTorch. To use torchnative instead, run uv sync --extra torchnative. The two extras cannot be installed together.
이렇게 하면 upstream PyTorch 가 설치됩니다. 대신 torchnative 를 쓰려면 uv sync --extra torchnative 를 실행하세요. 두 extra 는 함께 설치할 수 없습니다.
2. Start it
2. 실행
-python -m api run server # 127.0.0.1:23100 (loopback only)
-python -m api run server 127.0.0.1 23100 # explicit host and port
+uv run --extra torch python -m api run server # 127.0.0.1:23100 (loopback only)
+uv run --extra torch python -m api run server 127.0.0.1 23100 # explicit host and port
+ uv run uses the environment uv sync created, so there is no virtual environment to activate. Use --extra torchnative if you installed that one. Logs go to the console at INFO; set GEMSTONE_LOG_LEVEL=DEBUG to also log prompts and answers.
uv run 은 uv sync 가 만든 환경을 쓰므로 가상환경을 따로 활성화할 필요가 없습니다. torchnative 를 설치했다면 --extra torchnative 를 쓰세요. 로그는 INFO 수준으로 콘솔에 나가며, GEMSTONE_LOG_LEVEL=DEBUG 를 설정하면 프롬프트와 답변도 기록합니다.
The model downloads from Hugging Face the first time a session uses it — several gigabytes, once.
모델은 세션이 처음 사용할 때 Hugging Face 에서 내려받습니다 — 수 GB, 한 번만.
3. Open a client
3. 클라이언트 열기
@@ -64,7 +65,7 @@ 3. Open a client
3. 클라이언트
If something goes wrong
문제가 생기면
- python -m api without run server crashes with IndexError — always pass both words.run server 없이 python -m api 를 실행하면 IndexError 로 죽습니다 — 두 단어를 꼭 넘기세요.ModuleNotFoundError from a bare python -m api run server means that is the system Python, not the synced environment — start it with uv run --extra torch as above.- 그냥
python -m api run server 에서 ModuleNotFoundError 가 나면 동기화된 환경이 아니라 시스템 Python 인 것입니다 — 위처럼 uv run --extra torch 로 실행하세요.
diff --git a/docs/guide/guide-clients.html b/docs/guide/guide-clients.html
index 61ba8a6c..3d97448f 100644
--- a/docs/guide/guide-clients.html
+++ b/docs/guide/guide-clients.html
@@ -55,7 +55,7 @@ Android
Android
The Android client has no address setting yet, so adb reverse is the simplest way to reach a server on the same computer from a phone or an emulator. The server listens on 127.0.0.1 by default, which is enough for this.
Android 클라이언트에는 아직 주소 설정이 없으므로, 휴대폰이나 에뮬레이터에서 같은 컴퓨터의 서버로 가는 가장 간단한 방법은 adb reverse 입니다. 서버는 기본적으로 127.0.0.1 에서 받으며, 이것으로 충분합니다.
Serving other machines (LAN)
다른 기기에 서비스하기 (LAN)
-GEMSTONE_HOST=0.0.0.0 GEMSTONE_API_KEY=change-me python -m api run server
+GEMSTONE_HOST=0.0.0.0 GEMSTONE_API_KEY=change-me uv run --extra torch python -m api run server
GEMSTONE_API_KEY=change-me ./gradlew :app:run --args="--server http://192.168.0.10:23100"
GEMSTONE_HOST sets the server's bind address (host or host:port); the default is 127.0.0.1. Binding beyond loopback without GEMSTONE_API_KEY prints one warning at start-up but is allowed, as in Ollama. A browser on another machine may need its origin listed in GEMSTONE_ORIGINS. A phone on the LAN cannot be pointed at the server until Android gets an address setting; meanwhile forward a port from a machine that can reach it and use adb reverse as above.
GEMSTONE_HOST 는 서버가 받는 주소(host 또는 host:port)를 정하며 기본값은 127.0.0.1 입니다. 루프백 밖으로 열면서 GEMSTONE_API_KEY 가 없으면 시작할 때 경고를 한 번 출력하지만, Ollama 와 같게 막지는 않습니다. 다른 기기의 브라우저는 그 출처를 GEMSTONE_ORIGINS 에 적어야 할 수 있습니다. Android 에 주소 설정이 생기기 전까지는 LAN 의 휴대폰이 서버를 가리키게 할 수 없습니다. 그동안은 서버에 닿는 컴퓨터에서 포트를 전달하고 위처럼 adb reverse 를 쓰세요.
diff --git a/docs/guide/guide-server.html b/docs/guide/guide-server.html
index bfa7fe1b..dafbd2af 100644
--- a/docs/guide/guide-server.html
+++ b/docs/guide/guide-server.html
@@ -38,13 +38,13 @@ Run and configure the server
서버
Host and port
호스트와 포트
-
python -m api run server [host] [port] [--reload] # defaults: 127.0.0.1 23100
-GEMSTONE_HOST=0.0.0.0:11434 python -m api run server # host[:port], like OLLAMA_HOST
+
uv run --extra torch python -m api run server [host] [port] [--reload] # defaults: 127.0.0.1 23100
+GEMSTONE_HOST=0.0.0.0:11434 uv run --extra torch python -m api run server # host[:port], like OLLAMA_HOST
The server listens on loopback only. GEMSTONE_HOST overrides the address; the host and port arguments win over it. It reloads on code changes only with --reload or GEMSTONE_DEV=1, and keeps WebSockets alive for 300 seconds of silence — long enough for a slow first token.
서버는 loopback 에서만 듣습니다. GEMSTONE_HOST 로 주소를 바꾸며, host·port 인자가 우선합니다. 코드가 바뀔 때의 재시작은 --reload 또는 GEMSTONE_DEV=1 일 때만 하고, WebSocket 을 300초 침묵까지 유지합니다 — 느린 첫 토큰에도 충분합니다.
Origins and API key
Origin 과 API 키
Browsers may call the server from localhost, 127.0.0.1, [::1] (any port) and the server's own origin. Add others with GEMSTONE_ORIGINS (comma-separated, * allowed). A foreign origin gets no CORS headers, a state-changing request from it is answered 403, and its WebSocket is refused. Requests without an Origin header (the Ollama CLI, the SDKs) always pass.
브라우저는 localhost, 127.0.0.1, [::1] (모든 포트)과 서버 자신의 origin 에서 서버를 부를 수 있습니다. 다른 origin 은 GEMSTONE_ORIGINS (쉼표로 구분, * 허용)로 더합니다. 외부 origin 에는 CORS 헤더가 붙지 않고, 상태를 바꾸는 요청은 403, WebSocket 은 거절됩니다. Origin 헤더가 없는 요청(Ollama CLI, SDK)은 항상 통과합니다.
-
GEMSTONE_API_KEY=s3cret python -m api run server
+GEMSTONE_API_KEY=s3cret uv run --extra torch python -m api run server
curl -H "Authorization: Bearer s3cret" http://127.0.0.1:23100/api/models
With GEMSTONE_API_KEY set, every route needs Authorization: Bearer <key>; a WebSocket may instead pass ?api_key=<key>. The web assets, HEAD / and GET /api/version stay open. Without a key on a non-loopback bind the server logs one warning at start-up and still serves, for Ollama compatibility.
API 키를 설정하면 모든 경로에 Authorization: Bearer <key> 가 필요하며, WebSocket 은 대신 ?api_key=<key> 를 쓸 수 있습니다. 웹 에셋, HEAD /, GET /api/version 은 열려 있습니다. loopback 이 아닌 주소에 키 없이 띄우면 시작 시 경고를 한 번 남기고 Ollama 호환을 위해 그대로 서비스합니다.
diff --git a/docs/locale/README_ko.md b/docs/locale/README_ko.md
index 173c97ec..f925d3d6 100644
--- a/docs/locale/README_ko.md
+++ b/docs/locale/README_ko.md
@@ -59,13 +59,13 @@ uv sync --extra torch # 또는 --extra torchnative (둘은 함께 설
**2. 모델 서버 실행** (포트 `23100`, 모델은 처음 쓸 때 내려받습니다)
```bash
-python -m api run server
+uv run --extra torch python -m api run server
```
를 열면 서버에 함께 들어 있는 웹 클라이언트가 뜹니다.
서버는 `127.0.0.1` 에서만 듣습니다. 다른 기기에 열려면 `OLLAMA_HOST` 처럼 `GEMSTONE_HOST` 와 API 키를 설정하세요:
-`GEMSTONE_HOST=0.0.0.0:23100 GEMSTONE_API_KEY= python -m api run server`.
+`GEMSTONE_HOST=0.0.0.0:23100 GEMSTONE_API_KEY= uv run --extra torch python -m api run server`.
클라이언트는 `Authorization: Bearer ` 를 보냅니다. `GEMSTONE_ORIGINS` 는 브라우저 origin 을 더 허용하고,
`--reload` (또는 `GEMSTONE_DEV=1`) 는 개발용 자동 재시작을 켭니다.
diff --git a/pyproject.toml b/pyproject.toml
index f2dd5ceb..df344a57 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -12,6 +12,7 @@ dependencies = [
"bs4>=0.0.2",
"serpapi>=0.1.5",
"python-dotenv>=1.1.0",
+ "requests>=2.32", # utils/web_search.py
"huggingface_hub>=0.31.2",
"transformers>=5.18", # the continuous-batching API the engine uses was read and tested against 5.18
"psutil>=7.0", # transformers sizes the paged KV cache on CPU from psutil; without it continuous batching cannot start