From 494938ae7c62e97f73cba38e7382993eed3dc80a Mon Sep 17 00:00:00 2001 From: phanindra-max Date: Mon, 14 Sep 2026 00:30:09 -0400 Subject: [PATCH 1/6] feat(llm): decode deterministically by default on every backend Re-lands the decoding half of #426, which #428 reverted, ported onto the current router and with the problems its review found fixed. Addresses the non-determinism question in JOSS review #424. Every backend previously inherited its provider's default decoding: vLLM, OpenAI and Anthropic sample at temperature 1.0, llama.cpp used 0.2, and a Transformers model used whatever its checkpoint's generation_config.json said. Qwen2.5-0.5B-Instruct, for example, ships do_sample=true and temperature=0.7, so the local CPU path sampled even though nothing asked it to. - config: DEFAULT_TEMPERATURE = 0.0 and DEFAULT_TOP_P = 1.0, overridable via LAISER_TEMPERATURE / LAISER_TOP_P. GENERATION_SEED (42) already existed. - Gemini, OpenAI, Anthropic, vLLM, llama.cpp and Transformers entry points all default to those values. The seed is sent to Gemini and llama.cpp only when the installed client accepts one, so older releases do not break. - Transformers: at temperature 0.0, do_sample=False is passed explicitly so a checkpoint's own config cannot re-enable sampling. Above 0.0 the seed is applied before generation. - SkillExtractorRefactored, SkillExtractionService and LLMRouter take temperature and seed and pass them through. Fixed relative to #426: - LLMRouter.generate always sent the router's seed to llama.cpp, ignoring a caller's seed. Per-call temperature= and seed= now reach every backend. - #426 routed Transformers-loaded models to vLLM with no engine; that was fixed separately in #442, and decoding settings now follow both paths. - Gemini's max_output_tokens is kept (#426 dropped it). - The determinism tests read LAISER_TEMPERATURE at import, so a runner with that variable set failed them. They now reload config under a controlled environment, compare entry points against config rather than literals, and apply the `determinism` marker that #426 registered but never used. tests/test_determinism.py: shipped defaults and env overrides; every public entry point's temperature and seed defaults; the decoding parameters each backend actually receives (OpenAI and Anthropic request bodies, Gemini GenerateContentConfig, Transformers generate kwargs, vLLM SamplingParams, llama.cpp with and without seed support); router forwarding and per-call overrides on all five routes; extractor-to-router pass-through; and three alignment-stability tests on the real skills index. tests/test_llm_router.py is updated for the new keyword arguments. Co-Authored-By: Claude Opus 5 --- laiser/config.py | 10 + laiser/llm_models/anthropic.py | 13 + laiser/llm_models/gemini.py | 25 +- laiser/llm_models/hugging_face_llm.py | 56 ++- laiser/llm_models/llama_cpp_handler.py | 33 +- laiser/llm_models/llm_router.py | 37 +- laiser/llm_models/openai.py | 11 +- laiser/services.py | 8 + laiser/skill_extractor_refactored.py | 15 +- pytest.ini | 3 +- tests/test_determinism.py | 458 +++++++++++++++++++++++++ tests/test_llm_router.py | 3 +- 12 files changed, 648 insertions(+), 24 deletions(-) create mode 100644 tests/test_determinism.py diff --git a/laiser/config.py b/laiser/config.py index 03945a7..f8037a0 100644 --- a/laiser/config.py +++ b/laiser/config.py @@ -92,6 +92,16 @@ MAX_NEW_TOKENS = 1000 GENERATION_SEED = 42 +# Decoding defaults. Every backend decodes greedily unless told otherwise, so that +# repeated runs over the same input produce the same generation. A temperature of +# 0.0 always takes the most probable token; top_p = 1.0 leaves nucleus filtering +# off so temperature alone controls decoding. GENERATION_SEED is passed to every +# backend that accepts one, which keeps runs reproducible when a caller samples +# deliberately with a higher temperature. Override per extractor with +# temperature= and seed=, or through these environment variables. +DEFAULT_TEMPERATURE = float(os.getenv("LAISER_TEMPERATURE", "0.0")) +DEFAULT_TOP_P = float(os.getenv("LAISER_TOP_P", "1.0")) + # SCQF Level Descriptors SCQF_LEVELS: Dict[int, str] = { 1: "Basic awareness of simple concepts.", diff --git a/laiser/llm_models/anthropic.py b/laiser/llm_models/anthropic.py index ce85a4b..75ab6f7 100644 --- a/laiser/llm_models/anthropic.py +++ b/laiser/llm_models/anthropic.py @@ -4,6 +4,8 @@ import requests +from laiser.config import DEFAULT_TEMPERATURE, DEFAULT_TOP_P + _CODE_FENCE_RE = re.compile(r"```(?:json)?|```", re.IGNORECASE) @@ -21,7 +23,15 @@ def anthropic_generate( model: str = "claude-haiku-4-5-20251001", timeout: int = 60, max_tokens: int = 4096, + temperature: float = DEFAULT_TEMPERATURE, + top_p: float = DEFAULT_TOP_P, ) -> str: + """Generate text with the Anthropic Messages API. + + Decoding is greedy by default (temperature 0.0). ``top_p`` is only sent when + it differs from 1.0, because the API recommends adjusting temperature or + top_p but not both. The API accepts no seed. + """ api_key = api_key or os.getenv("ANTHROPIC_API_KEY") if not api_key: raise ValueError("Anthropic API key not provided") @@ -35,8 +45,11 @@ def anthropic_generate( payload = { "model": model, "max_tokens": max_tokens, + "temperature": temperature, "messages": [{"role": "user", "content": prompt}], } + if top_p is not None and top_p != 1.0: + payload["top_p"] = top_p try: resp = requests.post(url, headers=headers, json=payload, timeout=timeout) diff --git a/laiser/llm_models/gemini.py b/laiser/llm_models/gemini.py index 929c56a..8066cd4 100644 --- a/laiser/llm_models/gemini.py +++ b/laiser/llm_models/gemini.py @@ -9,10 +9,17 @@ from google.genai import errors as genai_errors from google.genai import types +from laiser.config import DEFAULT_TEMPERATURE, GENERATION_SEED + DEFAULT_GEMINI_MODEL = os.getenv("LAISER_GEMINI_MODEL", "gemini-2.5-flash") DEFAULT_GEMINI_TIMEOUT = float(os.getenv("LAISER_GEMINI_TIMEOUT", "60")) DEFAULT_MAX_OUTPUT_TOKENS = int(os.getenv("LAISER_GEMINI_MAX_OUTPUT_TOKENS", "4096")) +# Older google-genai releases have no seed field on GenerateContentConfig, and +# passing one there fails validation. Check once so a seed is only sent when the +# installed release accepts it. +_SUPPORTS_SEED = "seed" in getattr(types.GenerateContentConfig, "model_fields", {}) + class GeminiAPI: """Small wrapper to keep backward compatibility with older imports.""" @@ -23,6 +30,8 @@ def __init__( model_name: Optional[str] = None, timeout: float = DEFAULT_GEMINI_TIMEOUT, max_output_tokens: int = DEFAULT_MAX_OUTPUT_TOKENS, + temperature: float = DEFAULT_TEMPERATURE, + seed: Optional[int] = GENERATION_SEED, ): if not api_key: raise ValueError("Gemini API key is required") @@ -30,6 +39,8 @@ def __init__( self.model_name = model_name or DEFAULT_GEMINI_MODEL self.timeout = timeout self.max_output_tokens = max_output_tokens + self.temperature = temperature + self.seed = seed self.client = genai.Client(api_key=api_key) def generate(self, prompt: str) -> str: @@ -43,7 +54,9 @@ def generate_with_config( response_schema: Optional[Any] = None, ) -> str: try: - config_kwargs = {"temperature": 0.0, "max_output_tokens": self.max_output_tokens} + config_kwargs = {"temperature": self.temperature, "max_output_tokens": self.max_output_tokens} + if self.seed is not None and _SUPPORTS_SEED: + config_kwargs["seed"] = self.seed if response_mime_type: config_kwargs["response_mime_type"] = response_mime_type if response_schema is not None: @@ -76,13 +89,21 @@ def gemini_generate( max_output_tokens: int = DEFAULT_MAX_OUTPUT_TOKENS, response_mime_type: Optional[str] = None, response_schema: Optional[Any] = None, + temperature: float = DEFAULT_TEMPERATURE, + seed: Optional[int] = GENERATION_SEED, ) -> str: - """Send `prompt` to Gemini and return generated text.""" + """Send `prompt` to Gemini and return generated text. + + Decoding is greedy by default (temperature 0.0), and ``seed`` is sent + whenever the installed google-genai release accepts one. + """ client = GeminiAPI( api_key=api_key, model_name=model_name, timeout=timeout, max_output_tokens=max_output_tokens, + temperature=temperature, + seed=seed, ) return client.generate_with_config( prompt, diff --git a/laiser/llm_models/hugging_face_llm.py b/laiser/llm_models/hugging_face_llm.py index ac74ee0..a142b08 100644 --- a/laiser/llm_models/hugging_face_llm.py +++ b/laiser/llm_models/hugging_face_llm.py @@ -50,7 +50,11 @@ """ -from laiser.config import MAX_NEW_TOKENS +from typing import Optional + +import torch + +from laiser.config import DEFAULT_TEMPERATURE, DEFAULT_TOP_P, GENERATION_SEED, MAX_NEW_TOKENS from laiser.llm_models.model_loader import load_model_from_transformer try: @@ -63,13 +67,26 @@ def llm_generate( - prompt: str, tokenizer, model, model_id: str, use_gpu: bool, max_new_tokens: int = MAX_NEW_TOKENS + prompt: str, + tokenizer, + model, + model_id: str, + use_gpu: bool, + max_new_tokens: int = MAX_NEW_TOKENS, + temperature: float = DEFAULT_TEMPERATURE, + seed: Optional[int] = GENERATION_SEED, ) -> str: """Generate a completion with a transformers model. Returns only the newly generated text. ``model.generate`` returns the prompt tokens followed by the completion; decoding all of it hands the prompt's own example JSON to the response parser, which can match it instead of the answer. + + Decoding is greedy by default. At temperature 0.0 sampling is switched off + explicitly rather than left to the checkpoint, because some checkpoints ship a + generation_config.json that enables sampling and would make repeated runs + differ. Above 0.0 the model samples, and ``seed`` is applied first so the run + is still reproducible. """ if tokenizer is None or model is None: tokenizer, model = load_model_from_transformer(model_id, use_gpu=use_gpu) @@ -84,22 +101,41 @@ def llm_generate( inputs = tokenizer(text, return_tensors="pt").to(model.device) pad_token_id = tokenizer.pad_token_id if tokenizer.pad_token_id is not None else tokenizer.eos_token_id - outputs = model.generate( - **inputs, - max_new_tokens=max_new_tokens, - pad_token_id=pad_token_id, - eos_token_id=tokenizer.eos_token_id, - ) + do_sample = temperature is not None and temperature > 0.0 + generation_kwargs = { + "max_new_tokens": max_new_tokens, + "pad_token_id": pad_token_id, + "eos_token_id": tokenizer.eos_token_id, + "do_sample": do_sample, + } + if do_sample: + generation_kwargs["temperature"] = temperature + generation_kwargs["top_p"] = DEFAULT_TOP_P + if seed is not None: + torch.manual_seed(seed) + + outputs = model.generate(**inputs, **generation_kwargs) completion = outputs[0][inputs["input_ids"].shape[1] :] return tokenizer.decode(completion, skip_special_tokens=True).strip() -def llm_generate_vllm(prompt, llm): +def llm_generate_vllm( + prompt, + llm, + temperature: float = DEFAULT_TEMPERATURE, + seed: Optional[int] = GENERATION_SEED, + max_tokens: int = 200, +): + """Generate a completion with a vLLM engine. + + Temperature defaults to 0.0 (greedy) rather than vLLM's own default of 1.0, + and ``seed`` keeps sampled runs reproducible. + """ if not VLLM_AVAILABLE: raise ImportError("vLLM is not installed. Please install it to use this function.") - sampling_params = SamplingParams(max_tokens=200, seed=42) + sampling_params = SamplingParams(max_tokens=max_tokens, temperature=temperature, top_p=DEFAULT_TOP_P, seed=seed) result = llm.generate([prompt], sampling_params=sampling_params) raw_text = result[0].outputs[0].text.strip() return raw_text diff --git a/laiser/llm_models/llama_cpp_handler.py b/laiser/llm_models/llama_cpp_handler.py index 4d40a7f..b7b0f92 100644 --- a/laiser/llm_models/llama_cpp_handler.py +++ b/laiser/llm_models/llama_cpp_handler.py @@ -1,9 +1,12 @@ import gc +import inspect import os import re from pathlib import Path from typing import Any, List, Optional +from laiser.config import DEFAULT_TEMPERATURE, DEFAULT_TOP_P, GENERATION_SEED + try: from llama_cpp import Llama # type: ignore except ImportError: # pragma: no cover @@ -19,6 +22,18 @@ def _strip_fences(text: str) -> str: return text +def _llama_accepts_seed(llama: Any) -> bool: + """Report whether this llama-cpp-python build accepts a per-call seed. + + Older releases have no ``seed`` argument on ``create_chat_completion``; + passing one there raises TypeError, so check the signature first. + """ + try: + return "seed" in inspect.signature(llama.create_chat_completion).parameters + except (TypeError, ValueError): + return False + + class LlamaCppBackend: def __init__( self, @@ -26,8 +41,9 @@ def __init__( n_ctx: int = 4096, n_threads: Optional[int] = None, n_gpu_layers: int = -1, - temperature: float = 0.2, + temperature: float = DEFAULT_TEMPERATURE, chat_format: str = "chatml", + seed: Optional[int] = GENERATION_SEED, ): if Llama is None: raise ImportError("llama-cpp-python is not installed. Install it to use the llama_cpp backend.") @@ -42,6 +58,7 @@ def __init__( raise ValueError(f"Model path does not exist: {model_path}") self.temperature = temperature + self.seed = seed self.llm = Llama( model_path=model_path, @@ -83,8 +100,10 @@ def generate( max_tokens: Optional[int] = None, stop: Optional[List[str]] = None, temperature: Optional[float] = None, + seed: Optional[int] = None, ) -> str: temperature = self.temperature if temperature is None else temperature + seed = self.seed if seed is None else seed return llama_cpp_chat( prompt, self.llm, @@ -92,6 +111,7 @@ def generate( max_tokens=max_tokens, stop=stop, temperature=temperature, + seed=seed, ) @@ -102,8 +122,14 @@ def llama_cpp_chat( system: str = "You are a helpful assistant that outputs in JSON.", max_tokens: Optional[int] = None, stop: Optional[List[str]] = None, - temperature: Optional[float] = None, + temperature: Optional[float] = DEFAULT_TEMPERATURE, + seed: Optional[int] = GENERATION_SEED, ) -> str: + """Generate a chat completion with a local llama.cpp model. + + Decoding is greedy by default (temperature 0.0), and ``seed`` is sent + whenever the installed llama-cpp-python accepts one. + """ if llama is None: raise ValueError("llama is None; expected an initialized llama_cpp.Llama instance.") @@ -119,6 +145,9 @@ def llama_cpp_chat( kwargs["stop"] = stop if temperature is not None: kwargs["temperature"] = temperature + kwargs["top_p"] = DEFAULT_TOP_P + if seed is not None and _llama_accepts_seed(llama): + kwargs["seed"] = seed resp = llama.create_chat_completion(**kwargs) return _strip_fences(resp["choices"][0]["message"]["content"]) diff --git a/laiser/llm_models/llm_router.py b/laiser/llm_models/llm_router.py index 42956af..0aa0a8f 100644 --- a/laiser/llm_models/llm_router.py +++ b/laiser/llm_models/llm_router.py @@ -53,6 +53,7 @@ import torch +from laiser.config import DEFAULT_TEMPERATURE, GENERATION_SEED from laiser.exceptions import LAiSERError from laiser.llm_models.llama_cpp_handler import llama_cpp_chat from laiser.llm_models.model_loader import load_model_from_transformer, load_model_from_vllm @@ -90,12 +91,24 @@ def llm_generate_vllm(*args, **kwargs): class LLMRouter: - def __init__(self, model_id: str, use_gpu: bool, hf_token=None, api_key=None, backend=None): + def __init__( + self, + model_id: str, + use_gpu: bool, + hf_token=None, + api_key=None, + backend=None, + temperature: float = DEFAULT_TEMPERATURE, + seed=GENERATION_SEED, + ): self.model_id = model_id self.use_gpu = use_gpu self.hf_token = hf_token self.api_key = api_key self.backend = backend + # Applied to every backend; see DEFAULT_TEMPERATURE in laiser/config.py. + self.temperature = temperature + self.seed = seed self.llm = None self.model = None @@ -106,25 +119,37 @@ def __init__(self, model_id: str, use_gpu: bool, hf_token=None, api_key=None, ba # ---------------- ROUTER ---------------- def generate(self, prompt: str, **kwargs): + """Send a prompt to the active backend. + + The router's temperature and seed apply to every backend. Passing + ``temperature=`` or ``seed=`` overrides them for one request, on every + backend that accepts that setting. + """ + temperature = kwargs.pop("temperature", self.temperature) + seed = kwargs.pop("seed", self.seed) + if self.model_id == "gemini": - return gemini_generate(prompt, self.api_key, **kwargs) + return gemini_generate(prompt, self.api_key, temperature=temperature, seed=seed, **kwargs) if self.model_id == "openai": - return openai_generate(prompt, self.api_key) + # The OpenAI Responses API accepts no seed. + return openai_generate(prompt, self.api_key, temperature=temperature) # If a local GGUF model was loaded with llama-cpp-python, use it if self.backend == "llama_cpp": print("LLMRouter: routing request to llama_cpp backend") - return llama_cpp_chat(prompt, self.llm) + return llama_cpp_chat(prompt, self.llm, temperature=temperature, seed=seed) # A model loaded through transformers (always on CPU, and on GPU when vLLM # fails) has no vLLM engine in self.llm, so it must not go to vLLM. if self.llm is None and self.model is not None and self.tokenizer is not None: print("LLMRouter: routing request to transformers backend") - return llm_generate(prompt, self.tokenizer, self.model, self.model_id, self.use_gpu) + return llm_generate( + prompt, self.tokenizer, self.model, self.model_id, self.use_gpu, temperature=temperature, seed=seed + ) print("LLMRouter: routing request to vLLM backend") - return llm_generate_vllm(prompt, self.llm) + return llm_generate_vllm(prompt, self.llm, temperature=temperature, seed=seed) # ---------------- INIT ---------------- def _initialize_components(self): diff --git a/laiser/llm_models/openai.py b/laiser/llm_models/openai.py index 7da53a1..e9fc83e 100644 --- a/laiser/llm_models/openai.py +++ b/laiser/llm_models/openai.py @@ -4,6 +4,8 @@ import requests +from laiser.config import DEFAULT_TEMPERATURE, DEFAULT_TOP_P + _CODE_FENCE_RE = re.compile(r"```(?:json)?|```", re.IGNORECASE) @@ -20,7 +22,14 @@ def openai_generate( api_key: Optional[str] = None, model: str = "gpt-4.1-mini", timeout: int = 60, + temperature: float = DEFAULT_TEMPERATURE, + top_p: float = DEFAULT_TOP_P, ) -> str: + """Generate text with the OpenAI Responses API. + + Decoding is greedy by default (temperature 0.0). The Responses API accepts no + seed, so on this backend reproducibility rests on greedy decoding alone. + """ api_key = api_key or os.getenv("OPENAI_API_KEY") if not api_key: raise ValueError("OpenAI API key not provided") @@ -31,7 +40,7 @@ def openai_generate( "Content-Type": "application/json", } - payload = {"model": model, "input": prompt} + payload = {"model": model, "input": prompt, "temperature": temperature, "top_p": top_p} try: resp = requests.post(url, headers=headers, json=payload, timeout=timeout) diff --git a/laiser/services.py b/laiser/services.py index f7ea7d8..48533d7 100644 --- a/laiser/services.py +++ b/laiser/services.py @@ -17,7 +17,9 @@ COMBINED_EXTRACTION_PROMPT, DEFAULT_BATCH_SIZE, DEFAULT_SIMILARITY_THRESHOLDS, + DEFAULT_TEMPERATURE, DEFAULT_TOP_K, + GENERATION_SEED, KSA_DETAILS_PROMPT, KSA_EXTRACTION_PROMPT, KT_FROM_SKILLS_PROMPT, @@ -628,12 +630,16 @@ def __init__( api_key: Optional[str] = None, use_gpu: Optional[bool] = None, backend: Optional[str] = None, + temperature: float = DEFAULT_TEMPERATURE, + seed: Optional[int] = GENERATION_SEED, ): self.model_id = model_id self.hf_token = hf_token self.api_key = api_key self.use_gpu = use_gpu if use_gpu is not None else torch.cuda.is_available() self.backend = backend + self.temperature = temperature + self.seed = seed self.llm = None self.tokenizer = None self.model = None @@ -662,6 +668,8 @@ def __init__( self.hf_token, self.api_key, backend=self.backend, + temperature=self.temperature, + seed=self.seed, ) # Initialize skills FAISS index diff --git a/laiser/skill_extractor_refactored.py b/laiser/skill_extractor_refactored.py index b6d517b..32a0a90 100644 --- a/laiser/skill_extractor_refactored.py +++ b/laiser/skill_extractor_refactored.py @@ -61,7 +61,7 @@ import pandas as pd -from laiser.config import DEFAULT_BATCH_SIZE +from laiser.config import DEFAULT_BATCH_SIZE, DEFAULT_TEMPERATURE, GENERATION_SEED from laiser.services import SkillExtractionService @@ -119,6 +119,8 @@ def __init__( api_key: Optional[str] = None, use_gpu: Optional[bool] = None, backend: Optional[str] = None, + temperature: float = DEFAULT_TEMPERATURE, + seed: Optional[int] = GENERATION_SEED, ): """ Initialize the skill extractor. @@ -135,6 +137,15 @@ def __init__( Whether to use GPU for model inference backend : str, optional Backend to use for LLM inference (e.g., "llama_cpp", "huggingface", "openai", "gemini") + temperature : float, optional + Decoding temperature for whichever backend is used. Defaults to 0.0 + (greedy decoding), so repeated runs over the same input produce the + same extraction. Raise it only when varied output is wanted. + seed : int, optional + Seed for backends that accept one: vLLM, llama.cpp, Gemini, and local + Transformers models when temperature is above 0. Defaults to 42; pass + None to leave it unset. The OpenAI API accepts no seed, so there + reproducibility rests on greedy decoding alone. """ # Initialize service layer self.skill_service = SkillExtractionService( @@ -143,6 +154,8 @@ def __init__( hf_token=hf_token, use_gpu=use_gpu, backend=backend, + temperature=temperature, + seed=seed, ) def extract_concepts( diff --git a/pytest.ini b/pytest.ini index 8d58ba0..c230a14 100644 --- a/pytest.ini +++ b/pytest.ini @@ -5,4 +5,5 @@ markers = library: entire library llm_cloud: cloud llms test openai: openai - anthropic: anthropic \ No newline at end of file + anthropic: anthropic + determinism: decoding determinism and alignment stability \ No newline at end of file diff --git a/tests/test_determinism.py b/tests/test_determinism.py new file mode 100644 index 0000000..1f45d10 --- /dev/null +++ b/tests/test_determinism.py @@ -0,0 +1,458 @@ +"""Decoding determinism and alignment stability. + +Language models sample their output, so the same input can produce different +generations across runs and across backends (JOSS review #424). LAiSER limits +this in two ways, and these tests check both without a GPU, a model download or +a paid API call: + +- Decoding: every backend defaults to greedy decoding and a fixed seed. Checked + against each entry point's defaults and against the parameters each backend + actually receives. +- Alignment: generated phrases are matched to a fixed taxonomy by exact search, + so repeated alignment of the same phrases returns an identical result. +""" + +import importlib +import inspect +from types import SimpleNamespace + +import pandas as pd +import pytest +import torch + +from laiser import config + +pytestmark = pytest.mark.determinism + + +def _import_or_skip(module_path): + """Import an optional backend module, skipping when its dependency is absent. + + pytest.importorskip only skips on ModuleNotFoundError, but ``from google import + genai`` without google-genai installed raises a plain ImportError. + """ + try: + return importlib.import_module(module_path) + except ImportError as e: + pytest.skip(f"{module_path} unavailable in this environment: {e}") + + +@pytest.fixture +def reload_config(monkeypatch): + """Reload laiser.config under a controlled environment, then restore it. + + laiser.config reads LAISER_TEMPERATURE and LAISER_TOP_P at import. Reloading + lets a test set or clear them without depending on the runner's environment; + teardown restores the variables and reloads again. + """ + yield lambda: importlib.reload(config) + monkeypatch.undo() + importlib.reload(config) + + +# --------------------------------------------------------------------------- +# 1. Configuration +# --------------------------------------------------------------------------- + + +def test_shipped_defaults_are_greedy(monkeypatch, reload_config): + monkeypatch.delenv("LAISER_TEMPERATURE", raising=False) + monkeypatch.delenv("LAISER_TOP_P", raising=False) + + fresh = reload_config() + + assert fresh.DEFAULT_TEMPERATURE == 0.0 + assert fresh.DEFAULT_TOP_P == 1.0 + assert fresh.GENERATION_SEED == 42 + + +def test_environment_can_override_decoding_defaults(monkeypatch, reload_config): + monkeypatch.setenv("LAISER_TEMPERATURE", "0.7") + monkeypatch.setenv("LAISER_TOP_P", "0.9") + + fresh = reload_config() + + assert fresh.DEFAULT_TEMPERATURE == 0.7 + assert fresh.DEFAULT_TOP_P == 0.9 + + +# --------------------------------------------------------------------------- +# 2. Entry-point defaults +# --------------------------------------------------------------------------- + +SEEDABLE_ENTRY_POINTS = [ + ("laiser.llm_models.gemini", "gemini_generate"), + ("laiser.llm_models.gemini", "GeminiAPI"), + ("laiser.llm_models.hugging_face_llm", "llm_generate"), + ("laiser.llm_models.hugging_face_llm", "llm_generate_vllm"), + ("laiser.llm_models.llama_cpp_handler", "llama_cpp_chat"), + ("laiser.llm_models.llama_cpp_handler", "LlamaCppBackend"), + ("laiser.llm_models.llm_router", "LLMRouter"), + ("laiser.services", "SkillExtractionService"), + ("laiser.skill_extractor_refactored", "SkillExtractorRefactored"), +] + +# The OpenAI Responses API and the Anthropic Messages API accept no seed. +ENTRY_POINTS = SEEDABLE_ENTRY_POINTS + [ + ("laiser.llm_models.openai", "openai_generate"), + ("laiser.llm_models.anthropic", "anthropic_generate"), +] + + +@pytest.mark.parametrize("module_path, name", ENTRY_POINTS) +def test_entry_point_defaults_to_greedy_decoding(module_path, name): + """Guards against a backend inheriting its provider's default temperature, usually 1.0.""" + params = inspect.signature(getattr(_import_or_skip(module_path), name)).parameters + + assert "temperature" in params, f"{name} has no temperature parameter" + assert params["temperature"].default == config.DEFAULT_TEMPERATURE + + +@pytest.mark.parametrize("module_path, name", SEEDABLE_ENTRY_POINTS) +def test_seedable_entry_point_defaults_to_generation_seed(module_path, name): + params = inspect.signature(getattr(_import_or_skip(module_path), name)).parameters + + assert "seed" in params, f"{name} has no seed parameter" + assert params["seed"].default == config.GENERATION_SEED + + +# --------------------------------------------------------------------------- +# 3. Decoding parameters each backend actually receives +# --------------------------------------------------------------------------- + + +class _JSONResponse: + status_code = 200 + text = "" + + def __init__(self, body): + self._body = body + + def raise_for_status(self): + return None + + def json(self): + return self._body + + +def _capture_post(monkeypatch, module, body): + sent = {} + + def fake_post(url, headers=None, json=None, timeout=None): + sent.update(json or {}) + return _JSONResponse(body) + + monkeypatch.setattr(module.requests, "post", fake_post) + return sent + + +def test_openai_request_uses_greedy_decoding(monkeypatch): + openai_backend = _import_or_skip("laiser.llm_models.openai") + sent = _capture_post(monkeypatch, openai_backend, {"output_text": "ok"}) + + openai_backend.openai_generate("prompt", api_key="test-key") + + assert sent["temperature"] == config.DEFAULT_TEMPERATURE + assert sent["top_p"] == config.DEFAULT_TOP_P + assert "seed" not in sent + + +def test_anthropic_request_uses_greedy_decoding(monkeypatch): + anthropic_backend = _import_or_skip("laiser.llm_models.anthropic") + sent = _capture_post(monkeypatch, anthropic_backend, {"content": [{"type": "text", "text": "ok"}]}) + + anthropic_backend.anthropic_generate("prompt", api_key="test-key") + + assert sent["temperature"] == config.DEFAULT_TEMPERATURE + # top_p is only sent when it has been moved off 1.0. + assert ("top_p" in sent) == (config.DEFAULT_TOP_P != 1.0) + + +def test_gemini_request_uses_greedy_decoding_and_seed(monkeypatch): + gemini = _import_or_skip("laiser.llm_models.gemini") + captured = {} + + class FakeModels: + def generate_content(self, model, contents, config): + captured["config"] = config + return SimpleNamespace(text="ok", parsed=None) + + class FakeClient: + def __init__(self, api_key): + self.models = FakeModels() + + monkeypatch.setattr(gemini.genai, "Client", FakeClient) + + gemini.gemini_generate("prompt", api_key="test-key") + + sent = captured["config"] + assert sent.temperature == config.DEFAULT_TEMPERATURE + assert sent.max_output_tokens == gemini.DEFAULT_MAX_OUTPUT_TOKENS + if gemini._SUPPORTS_SEED: + assert sent.seed == config.GENERATION_SEED + + +class _Encoded(dict): + def to(self, device): + return self + + +class _FakeTokenizer: + chat_template = None + pad_token_id = 0 + eos_token_id = 9 + + def __call__(self, text, return_tensors): + return _Encoded(input_ids=torch.tensor([[1, 2, 3]])) + + def decode(self, ids, skip_special_tokens): + return "ok" + + +class _FakeModel: + device = "cpu" + + def __init__(self): + self.kwargs = None + + def generate(self, **kwargs): + self.kwargs = kwargs + return torch.tensor([[1, 2, 3, 4]]) + + +def test_transformers_at_temperature_zero_turns_sampling_off(monkeypatch): + hf = _import_or_skip("laiser.llm_models.hugging_face_llm") + seeded = [] + monkeypatch.setattr(hf.torch, "manual_seed", seeded.append) + model = _FakeModel() + + hf.llm_generate("prompt", _FakeTokenizer(), model, "some/model", use_gpu=False, temperature=0.0) + + assert model.kwargs["do_sample"] is False + assert "temperature" not in model.kwargs + assert seeded == [] # nothing is sampled, so there is nothing to seed + + +def test_transformers_sampling_is_seeded(monkeypatch): + hf = _import_or_skip("laiser.llm_models.hugging_face_llm") + seeded = [] + monkeypatch.setattr(hf.torch, "manual_seed", seeded.append) + model = _FakeModel() + + hf.llm_generate("prompt", _FakeTokenizer(), model, "some/model", use_gpu=False, temperature=0.7, seed=123) + + assert model.kwargs["do_sample"] is True + assert model.kwargs["temperature"] == 0.7 + assert seeded == [123] + + +def test_vllm_sampling_params_are_greedy_and_seeded(monkeypatch): + hf = _import_or_skip("laiser.llm_models.hugging_face_llm") + recorded = {} + monkeypatch.setattr(hf, "VLLM_AVAILABLE", True) + monkeypatch.setattr(hf, "SamplingParams", lambda **kwargs: recorded.update(kwargs) or kwargs) + engine = SimpleNamespace( + generate=lambda prompts, sampling_params: [SimpleNamespace(outputs=[SimpleNamespace(text=" ok ")])] + ) + + assert hf.llm_generate_vllm("prompt", engine) == "ok" + assert recorded["temperature"] == config.DEFAULT_TEMPERATURE + assert recorded["top_p"] == config.DEFAULT_TOP_P + assert recorded["seed"] == config.GENERATION_SEED + + +class _SeedAwareLlama: + def __init__(self): + self.kwargs = None + + def create_chat_completion(self, messages, max_tokens=None, stop=None, temperature=None, top_p=None, seed=None): + self.kwargs = {"temperature": temperature, "top_p": top_p, "seed": seed} + return {"choices": [{"message": {"content": "ok"}}]} + + +class _LlamaWithoutSeed: + def __init__(self): + self.kwargs = None + + def create_chat_completion(self, messages, max_tokens=None, stop=None, temperature=None, top_p=None): + self.kwargs = {"temperature": temperature, "top_p": top_p} + return {"choices": [{"message": {"content": "ok"}}]} + + +def test_llama_cpp_passes_greedy_decoding_and_seed(): + llama_cpp = _import_or_skip("laiser.llm_models.llama_cpp_handler") + llama = _SeedAwareLlama() + + llama_cpp.llama_cpp_chat("prompt", llama) + + assert llama.kwargs == { + "temperature": config.DEFAULT_TEMPERATURE, + "top_p": config.DEFAULT_TOP_P, + "seed": config.GENERATION_SEED, + } + + +def test_llama_cpp_omits_seed_on_builds_that_do_not_accept_one(): + llama_cpp = _import_or_skip("laiser.llm_models.llama_cpp_handler") + llama = _LlamaWithoutSeed() + + llama_cpp.llama_cpp_chat("prompt", llama) # would raise TypeError if a seed were sent + + assert llama.kwargs["temperature"] == config.DEFAULT_TEMPERATURE + + +# --------------------------------------------------------------------------- +# 4. Forwarding through the router and the public extractor +# --------------------------------------------------------------------------- + +ROUTES = [ + # (route, router attributes, function the router calls, whether it takes a seed) + ("vllm", {"llm": "engine"}, "llm_generate_vllm", True), + ("transformers", {"model": "model", "tokenizer": "tokenizer"}, "llm_generate", True), + ("llama_cpp", {"backend": "llama_cpp", "llm": "engine"}, "llama_cpp_chat", True), + ("gemini", {"model_id": "gemini"}, "gemini_generate", True), + ("openai", {"model_id": "openai"}, "openai_generate", False), +] + + +def _router(monkeypatch, attrs): + router_module = _import_or_skip("laiser.llm_models.llm_router") + monkeypatch.setattr(router_module.LLMRouter, "_initialize_components", lambda self: None) + attrs = dict(attrs) + router = router_module.LLMRouter( + attrs.pop("model_id", "some/model"), use_gpu=False, backend=attrs.pop("backend", None) + ) + for key, value in attrs.items(): + setattr(router, key, value) + return router_module, router + + +def _record_calls(monkeypatch, module, name): + calls = [] + + def record(*args, **kwargs): + calls.append(kwargs) + return "ok" + + monkeypatch.setattr(module, name, record) + return calls + + +@pytest.mark.parametrize("route, attrs, target, takes_seed", ROUTES, ids=[r[0] for r in ROUTES]) +def test_router_forwards_decoding_defaults(monkeypatch, route, attrs, target, takes_seed): + router_module, router = _router(monkeypatch, attrs) + calls = _record_calls(monkeypatch, router_module, target) + + router.generate("prompt") + + assert calls[0]["temperature"] == config.DEFAULT_TEMPERATURE + if takes_seed: + assert calls[0]["seed"] == config.GENERATION_SEED + + +@pytest.mark.parametrize("route, attrs, target, takes_seed", ROUTES, ids=[r[0] for r in ROUTES]) +def test_router_honours_per_call_overrides(monkeypatch, route, attrs, target, takes_seed): + """Review of #426 found llama.cpp ignored a caller's seed; overrides must reach every backend.""" + router_module, router = _router(monkeypatch, attrs) + calls = _record_calls(monkeypatch, router_module, target) + + router.generate("prompt", temperature=0.5, seed=7) + + assert calls[0]["temperature"] == 0.5 + if takes_seed: + assert calls[0]["seed"] == 7 + + +def test_extractor_passes_decoding_settings_to_the_router(monkeypatch): + services = _import_or_skip("laiser.services") + extractor_module = _import_or_skip("laiser.skill_extractor_refactored") + + class NoOp: + def __init__(self, *args, **kwargs): + pass + + def initialize_index(self, *args, **kwargs): + return None + + for name in ("DataAccessLayer", "FAISSIndexManager", "KnowledgeFAISSIndexManager", "TaskFAISSIndexManager"): + monkeypatch.setattr(services, name, NoOp) + monkeypatch.setattr(services, "AlignmentService", NoOp) + + received = {} + + class RecordingRouter: + def __init__(self, *args, **kwargs): + received.update(kwargs) + + monkeypatch.setattr(services, "LLMRouter", RecordingRouter) + + extractor_module.SkillExtractorRefactored(model_id="some/model", use_gpu=False, temperature=0.3, seed=9) + + assert received["temperature"] == 0.3 + assert received["seed"] == 9 + + +# --------------------------------------------------------------------------- +# 5. Alignment stability +# --------------------------------------------------------------------------- + +RAW_SKILLS = [ + "Python programming", + "python programming skills", + "Experience programming in Python", + "statistical analysis", + "Data visualization techniques", + "machine learning model development", +] + + +@pytest.fixture(scope="module") +def alignment_service(): + data_access = _import_or_skip("laiser.data_access") + services = _import_or_skip("laiser.services") + + da = data_access.DataAccessLayer() + fm = data_access.FAISSIndexManager(da) + try: + fm.initialize_index(force_rebuild=False) + except Exception as e: # pragma: no cover - environment dependent + pytest.skip(f"Taxonomy index unavailable: {e!r}") + return services.SkillAlignmentService(data_access=da, faiss_manager=fm) + + +@pytest.mark.alignment +def test_alignment_is_identical_across_repeated_runs(alignment_service): + """Exact search over fixed embeddings: repeated runs must match, scores included.""" + runs = [alignment_service.align_skills_to_taxonomy(list(RAW_SKILLS), document_id="doc-1") for _ in range(3)] + + first = runs[0].reset_index(drop=True) + for i, other in enumerate(runs[1:], start=2): + pd.testing.assert_frame_equal( + first, other.reset_index(drop=True), check_exact=True, obj=f"alignment run 1 vs run {i}" + ) + + +@pytest.mark.alignment +def test_aligned_output_is_drawn_from_the_taxonomy(alignment_service): + """Whatever a model generates, the reported entry already exists in the taxonomy.""" + aligned = alignment_service.align_skills_to_taxonomy(list(RAW_SKILLS), document_id="doc-2") + if aligned.empty: # pragma: no cover - environment dependent + pytest.skip("Alignment returned no rows") + + vocabulary = set(alignment_service.faiss_manager.get_metadata()["skill"].astype(str).str.strip()) + + unknown = set(aligned["Taxonomy Skill"]) - vocabulary + assert not unknown, f"Aligned output contains entries absent from the taxonomy: {unknown}" + + +@pytest.mark.alignment +def test_the_same_phrase_always_resolves_to_the_same_entry(alignment_service): + aligned = alignment_service.align_skills_to_taxonomy( + ["statistical analysis", "Python programming", "statistical analysis"], document_id="doc-3" + ) + if aligned.empty: # pragma: no cover - environment dependent + pytest.skip("Alignment returned no rows") + + for raw, group in aligned.groupby("Raw Skill"): + assert group["Taxonomy Skill"].nunique() == 1, f"{raw!r} resolved to several entries" + assert group["Correlation Coefficient"].nunique() == 1, f"{raw!r} produced different scores" diff --git a/tests/test_llm_router.py b/tests/test_llm_router.py index eb4502f..8875595 100644 --- a/tests/test_llm_router.py +++ b/tests/test_llm_router.py @@ -51,6 +51,7 @@ def _bare_router(**attrs): router.use_gpu = False router.hf_token = router.api_key = router.backend = None router.llm = router.model = router.tokenizer = router.nlp = None + router.temperature, router.seed = 0.0, 42 for key, value in attrs.items(): setattr(router, key, value) return router @@ -69,7 +70,7 @@ def test_transformers_model_is_routed_to_transformers_generate(monkeypatch): def test_vllm_engine_still_routes_to_vllm(monkeypatch): monkeypatch.setattr(llm_router, "llm_generate", lambda *args, **kwargs: pytest.fail("routed to transformers")) - monkeypatch.setattr(llm_router, "llm_generate_vllm", lambda prompt, llm: f"vllm:{llm}") + monkeypatch.setattr(llm_router, "llm_generate_vllm", lambda prompt, llm, **kwargs: f"vllm:{llm}") assert _bare_router(llm="engine").generate("prompt") == "vllm:engine" From 26a7b88941de6abc8715b0650cdc211e8c1e6327 Mon Sep 17 00:00:00 2001 From: phanindra-max Date: Mon, 14 Sep 2026 00:30:10 -0400 Subject: [PATCH 2/6] docs: document reproducible results and the decoding defaults - New Reproducibility page: per-backend temperature and seed, why sampling is switched off explicitly for Transformers, how to change the defaults, a reference configuration, what is not guaranteed (hosted models change, different models extract different phrases, hardware and library versions), and what tests/test_determinism.py checks. - README: temperature and seed in the runtime options table, and a "Reproducible results" section with the reference configuration, which the reply on #424 described but which was never committed. - Model providers page links to Reproducibility. The API reference picks up the new temperature and seed parameters from the docstrings. Co-Authored-By: Claude Opus 5 --- README.md | 27 ++++++++++++ docs/providers.md | 3 ++ docs/reproducibility.md | 91 +++++++++++++++++++++++++++++++++++++++++ mkdocs.yml | 1 + 4 files changed, 122 insertions(+) create mode 100644 docs/reproducibility.md diff --git a/README.md b/README.md index 9289f15..c488e98 100644 --- a/README.md +++ b/README.md @@ -192,6 +192,8 @@ print(results.head()) | `api_key` | API key for hosted providers | | `use_gpu` | run local models on GPU where available | | `backend` | `"llama_cpp"` to run a local GGUF model | +| `temperature` | decoding temperature for every backend (default `0.0`, greedy) | +| `seed` | seed for backends that accept one (default `42`) | | `concepts` | `["skills"]` (default), or add `"knowledge"` and `"tasks"` | | `allowed_sources` | taxonomies to align against: `"esco"`, `"onet"`, `"osn"`, `"ukos"` | | `top_k` | maximum aligned matches per concept type, per document (default 25) | @@ -200,6 +202,31 @@ print(results.head()) Every option is described in the [usage guide](https://laiser-software.github.io/extract-module/usage/), and more snippets are in [docs/examples.md](docs/examples.md). +### Reproducible results + +Every backend decodes greedily by default (temperature `0.0`) and uses a fixed seed where the backend accepts one, so repeated runs over the same input give the same result. To make results reproducible for others, pin every setting that affects them: + +```python +extractor = SkillExtractorRefactored( + model_id="Qwen/Qwen2.5-0.5B-Instruct", # pin the model + use_gpu=False, + temperature=0.0, # the default: greedy decoding + seed=42, # the default +) + +results = extractor.extract_concepts( + data=data, + id_column="Research ID", + text_columns=["description"], + input_type="job_desc", + concepts=["skills"], + allowed_sources=["esco", "onet", "osn", "ukos"], + similarity_thresholds={"skill": 0.60, "knowledge": 0.50, "task": 0.50}, # the defaults +) +``` + +When reporting results, also record the LAiSER version: it fixes the bundled taxonomy data and the embedding model used for alignment (`sentence-transformers/all-MiniLM-L6-v2`). Hosted providers do not guarantee identical output even at temperature `0.0`. See [Reproducibility](https://laiser-software.github.io/extract-module/reproducibility/) for details. + ## Try it in Google Colab The [cookbook](https://github.com/LAiSER-Software/laiser-cookbook) has complete analyses that open directly in Colab: diff --git a/docs/providers.md b/docs/providers.md index 39dea1b..aaea90e 100644 --- a/docs/providers.md +++ b/docs/providers.md @@ -13,6 +13,9 @@ Choose the model when you construct the extractor. `hf_token` is only needed for gated or private Hugging Face models. +Every provider decodes greedily by default, with a fixed seed wherever it accepts one — see +[Reproducibility](reproducibility.md). + ## Gemini ```python diff --git a/docs/reproducibility.md b/docs/reproducibility.md new file mode 100644 index 0000000..75b5509 --- /dev/null +++ b/docs/reproducibility.md @@ -0,0 +1,91 @@ +# Reproducibility + +Language models generate text by sampling, so left to their providers' defaults two runs over the same input +can return different skills. LAiSER is configured so that, by default, they do not. + +## Defaults + +Every backend decodes greedily: at temperature `0.0` the model always takes its most probable next token, so +the same prompt produces the same text. A fixed seed (`42`) is also supplied wherever a backend accepts one, +which keeps runs reproducible if you deliberately raise the temperature. + +| Backend | Temperature | Seed | +|---|---|---| +| Gemini | `0.0` | `42`, on google-genai releases that accept a seed | +| OpenAI | `0.0` | not accepted by the API | +| Local model through Transformers | greedy — sampling switched off | `42`, used when temperature is above `0.0` | +| Local model through vLLM | `0.0` | `42` | +| llama.cpp | `0.0` | `42`, on llama-cpp-python releases that accept a seed | + +For Transformers models, sampling is turned off explicitly rather than left to the checkpoint. Some checkpoints +ship a `generation_config.json` that enables sampling, which would otherwise make repeated runs differ. + +Alignment is deterministic as well. Each extracted phrase is embedded with a fixed sentence-transformer model +and matched by exact search against the bundled taxonomy index, so the same phrase always resolves to the same +taxonomy entry with the same similarity score. + +## Changing the defaults + +Pass `temperature` and `seed` when creating an extractor: + +```python +extractor = SkillExtractorRefactored(model_id="gemini", api_key=api_key, temperature=0.7, seed=123) +``` + +Pass `seed=None` to send no seed. The defaults can also be changed for a whole process through environment +variables, which LAiSER reads when it is imported: + +| Variable | Default | Effect | +|---|---|---| +| `LAISER_TEMPERATURE` | `0.0` | default temperature for every backend | +| `LAISER_TOP_P` | `1.0` | nucleus sampling cut-off; `1.0` leaves it off | + +## Reference configuration + +To let others reproduce your results, pin every setting that affects them: + +```python +from laiser.skill_extractor_refactored import SkillExtractorRefactored + +extractor = SkillExtractorRefactored( + model_id="Qwen/Qwen2.5-0.5B-Instruct", # pin the model + use_gpu=False, + temperature=0.0, # the default: greedy decoding + seed=42, # the default +) + +results = extractor.extract_concepts( + data=data, + id_column="Research ID", + text_columns=["description"], + input_type="job_desc", + concepts=["skills"], + allowed_sources=["esco", "onet", "osn", "ukos"], + similarity_thresholds={"skill": 0.60, "knowledge": 0.50, "task": 0.50}, # the defaults +) +``` + +Also record: + +- **the LAiSER version** — it fixes the bundled taxonomy data and the embedding model used for alignment, + `sentence-transformers/all-MiniLM-L6-v2`; +- **the model revision**, for Hugging Face models that may be updated under the same name. + +## What this does not guarantee + +- **Hosted models can change.** A provider can update the model behind a name, and providers do not + guarantee identical output even at temperature `0.0`. +- **Different models extract different phrases.** Alignment maps different wordings of a skill onto the same + taxonomy entry, but it cannot recover a skill a model never extracted. Compare models on the aligned columns, + and measure their agreement rather than assuming it. +- **Hardware and library versions matter for local models.** Floating-point results can differ between + devices and library releases. + +## How it is tested + +`tests/test_determinism.py` runs in CI without a GPU, a model download or an API key. It checks that: + +- every public entry point defaults to temperature `0.0`, and every seedable one to seed `42`; +- the decoding parameters each backend actually receives match those defaults, and per-call overrides reach + every backend; +- repeated alignment of the same phrases returns an identical table, similarity scores included. diff --git a/mkdocs.yml b/mkdocs.yml index a29232a..a1ab995 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -51,6 +51,7 @@ nav: - Guides: - Usage: usage.md - Model providers: providers.md + - Reproducibility: reproducibility.md - Taxonomies: taxonomies.md - Examples: examples.md - Cookbook notebooks: cookbook.md From 5b7d302ab7c61725aef9192f8fefff4c67305edd Mon Sep 17 00:00:00 2001 From: phanindra-max Date: Mon, 14 Sep 2026 00:30:10 -0400 Subject: [PATCH 3/6] docs(paper): respond to JOSS review #424 Abbreviations. Every abbreviation is now spelled out at or before first use: ESCO, LLM, AI, NER, FAISS, API, GPU, KSA, SCQF, MIT, GW and CCAS; spaCy is described as a natural language processing library and EMSI as now Lightcast. The ESCO and FAISS expansions had existed in earlier drafts but were cut when the paper was shortened in be462ee. Summary. Rewritten for a non-technical reader: it opens on the problem in plain terms, explains what a skill taxonomy is before naming one, and says who the software is for. Pipeline internals, the SCQF sentence, and the licence and installation line move out of the Summary; the licence and installation line now closes Software design, and the FAISS and SCQF citations move to where those terms are now first used. Non-determinism. Software design regains the explanation #428 removed, with corrections: decoding is greedy with a fixed seed where accepted; alignment projects generations onto a closed taxonomy vocabulary by exact search; every backend shares one output schema; and residual variability is stated, now including that hosted models change and providers do not guarantee identical output at temperature zero. Accuracy fixes made while rewriting those paragraphs: - "ESCO's 13,000+ skill entries" replaced by the bundled figure, about 27,000 skill entries across the four taxonomies. - The backend list now includes llama.cpp and OpenAI where it was missing, and "automatic fallback" is stated precisely: on a GPU, a model vLLM cannot load falls back to Transformers. - The output description no longer lists "optional SCQF proficiency levels", which extract_concepts does not produce. Paper length is 1,628 words excluding references. Co-Authored-By: Claude Opus 5 --- paper.md | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/paper.md b/paper.md index a4f4a8b..231116c 100644 --- a/paper.md +++ b/paper.md @@ -29,7 +29,9 @@ bibliography: paper.bib # Summary -LAiSER is an open-source Python framework for extracting, standardizing, and aligning skill information from unstructured text with established skill taxonomies. The system addresses the lack of standardized, machine-readable skill information needed to facilitate communication between learners, educators, and employers. LAiSER employs a two-stage pipeline combining large language models (LLMs) with semantic vector search using FAISS [@Johnson2019] for high-precision skill extraction and taxonomy alignment to standards such as ESCO [@ESCO2020]. The framework supports multiple computational backends, including GPU-accelerated inference, cloud APIs, and CPU-only environments, making it accessible across diverse research and operational contexts. LAiSER also supports extraction within the Knowledge, Skills, and Abilities (KSA) framework with automatic proficiency level classification using the Scottish Credit and Qualifications Framework (SCQF) [@SCQF2019]. The source code is available at [https://github.com/LAiSER-Software/extract-module](https://github.com/LAiSER-Software/extract-module) under the BSD 3-Clause License, installable via `pip install laiser`, or `pip install laiser[gpu]` for GPU inference. +Job postings, course syllabi, and credential descriptions all describe skills, but each in its own words, so the same ability can be phrased many different ways. That makes it hard to count which skills employers ask for, or to check whether a course teaches what jobs require. Skill taxonomies address this by giving each skill one agreed name and identifier; the European Commission's European Skills, Competences, Qualifications and Occupations classification (ESCO) [@ESCO2020] is a widely used example. + +LAiSER is open-source software that reads such documents and links each skill they mention to its entry in a taxonomy. It uses a large language model (LLM), a kind of artificial intelligence (AI) system that reads and writes text, to find the skills a document describes, then matches each one to the closest taxonomy entry, so that differently worded mentions of the same skill are counted together. It is designed for researchers studying labor markets, curriculum designers comparing programs with employer demand, and organizations that issue digital credentials. # Statement of need @@ -37,23 +39,29 @@ The contemporary labor market is characterized by rapid technological change, ev # State of the field -Several open-source tools address parts of the skill extraction problem. SkillNER [@SkillNER2022] uses rule-based matching against the EMSI skills database via spaCy, offering fast lookups but limited ability to detect novel or context-dependent skills not present in its dictionary. Nesta's Skills Extractor Library [@Nesta2023] trains a spaCy NER model on labeled job advertisements and maps extracted phrases to ESCO or Lightcast taxonomies using sentence-transformer embeddings. While effective for UK job advertisements, its NER model requires domain-specific labeled training data and does not generalize well to other text types such as syllabi or credential descriptions. The esco-skill-extractor [@EscoExtractor2024] takes a simpler approach, embedding full input texts with a transformer and matching against ESCO entries by cosine similarity, but it lacks preprocessing, multi-model flexibility, and proficiency-level classification. +Several open-source tools address parts of the skill extraction problem. SkillNER [@SkillNER2022] uses rule-based matching against the EMSI (now Lightcast) skills database via spaCy, an open-source natural language processing library, offering fast lookups but limited ability to detect novel or context-dependent skills not present in its dictionary. Nesta's Skills Extractor Library [@Nesta2023] trains a spaCy named entity recognition (NER) model on labeled job advertisements and maps extracted phrases to ESCO or Lightcast taxonomies using sentence-transformer embeddings. While effective for UK job advertisements, its NER model requires domain-specific labeled training data and does not generalize well to other text types such as syllabi or credential descriptions. The esco-skill-extractor [@EscoExtractor2024] takes a simpler approach, embedding full input texts with a transformer and matching against ESCO entries by cosine similarity, but it lacks preprocessing, multi-model flexibility, and proficiency-level classification. -LAiSER differs from these tools in several key respects. First, it uses LLMs for skill extraction rather than rule-based or NER-based approaches, enabling it to identify skills expressed in varied natural language without requiring labeled training data. Second, LAiSER provides domain-aware preprocessing that removes extraneous content (company branding, legal boilerplate, benefits sections) while preserving task-relevant skill language, an important step that existing tools omit. Third, the framework supports multiple LLM backends (vLLM, HuggingFace Transformers, Gemini API) with automatic fallback, allowing deployment across GPU clusters, cloud APIs, or CPU-only laptops. Finally, LAiSER uniquely supports KSA decomposition with SCQF proficiency-level classification, a capability absent from all comparable tools. These design choices reflect a deliberate decision to build a new framework rather than extend existing tools, whose architectures are not designed to accommodate LLM-based extraction or multi-taxonomy proficiency alignment. +LAiSER differs from these tools in several key respects. First, it uses LLMs for skill extraction rather than rule-based or NER-based approaches, enabling it to identify skills expressed in varied natural language without requiring labeled training data. Second, LAiSER provides domain-aware preprocessing that removes extraneous content (company branding, legal boilerplate, benefits sections) while preserving task-relevant skill language, an important step that existing tools omit. Third, the framework supports multiple LLM backends (vLLM, HuggingFace Transformers, llama.cpp, and the Gemini and OpenAI application programming interfaces, or APIs), allowing deployment on graphics processing unit (GPU) clusters, hosted APIs, or ordinary laptops. Finally, LAiSER uniquely supports Knowledge, Skills, and Abilities (KSA) decomposition with proficiency-level classification under the Scottish Credit and Qualifications Framework (SCQF) [@SCQF2019], a capability absent from all comparable tools. These design choices reflect a deliberate decision to build a new framework rather than extend existing tools, whose architectures are not designed to accommodate LLM-based extraction or multi-taxonomy proficiency alignment. # Software design LAiSER's architecture is organized into three loosely coupled layers: data access, skill extraction, and taxonomy alignment. This separation was chosen to allow each component to evolve independently and to support diverse deployment scenarios without requiring changes to the overall pipeline. -The extraction layer posed the most significant design trade-off. Training a custom NER model, as Nesta's library does, would yield fast inference but would require substantial labeled data for each new domain (syllabi, credentials, job postings). Instead, LAiSER delegates extraction to LLMs via a prompt engineering approach, trading some inference speed for broad domain generalizability without retraining. To mitigate vendor lock-in and hardware constraints, the extraction layer abstracts over multiple backends (vLLM for GPU clusters, HuggingFace Transformers for local models, llama.cpp for local GGUF models, and the Gemini and OpenAI APIs for cloud-based inference), with automatic fallback when a backend is unavailable. +The extraction layer posed the most significant design trade-off. Training a custom NER model, as Nesta's library does, would yield fast inference but would require substantial labeled data for each new domain (syllabi, credentials, job postings). Instead, LAiSER delegates extraction to LLMs via a prompt engineering approach, trading some inference speed for broad domain generalizability without retraining. To mitigate vendor lock-in and hardware constraints, the extraction layer abstracts over multiple backends: vLLM for GPU clusters, HuggingFace Transformers and llama.cpp for models run locally, and the Gemini and OpenAI APIs for hosted inference. The backend is chosen when an extractor is created, from the requested model and the available hardware; on a GPU, a model that vLLM cannot load falls back to Transformers. + +For taxonomy alignment, the framework embeds both the extracted phrases and the taxonomy entries with a sentence-transformer model [@Reimers2019] and retrieves each phrase's closest entry from a precomputed index built with FAISS (Facebook AI Similarity Search) [@Johnson2019]. The index is a flat inner-product index over normalized embeddings, which makes retrieval exhaustive and exact rather than approximate. Exactness was a deliberate choice: at the scale of the bundled taxonomies, about 27,000 skill entries, an exhaustive scan is inexpensive relative to LLM inference in the preceding stage, and it guarantees that a given query returns the same entry and the same similarity score on every run, a precondition for the reproducibility properties described below. Because retrieval is encapsulated behind an index manager, an approximate index could be substituted without changes to the alignment logic should taxonomy size later require it. The system accepts pandas DataFrames as input and returns one row per aligned phrase, containing the extracted text, the matched taxonomy entry, its source taxonomy and link, and the similarity score. + +Because LLMs sample their output, identical inputs can yield different generations across runs, and different backends can yield different generations from the same input. LAiSER addresses this in three ways. First, decoding is deterministic by default: every backend decodes greedily at temperature zero, and a fixed seed is supplied to every backend that accepts one, so runs stay reproducible even when a user deliberately raises the temperature. Temperature and seed are constructor arguments, so sampled variation is available when wanted but is never the silent default. Second, the alignment stage acts as a normalizing projection. Generated phrases are embedded with a fixed sentence-transformer model and matched by exact search against a precomputed index, so surface variation in the generated text collapses onto a closed, finite vocabulary of taxonomy entries, and different phrasings of the same competency converge on the same aligned record. Third, every backend produces the same output schema, which makes results directly comparable across models and lets cross-model agreement be measured on the aligned columns. The test suite checks each backend's decoding defaults, the decoding parameters each backend actually receives, and that repeated alignment of a fixed input returns an identical result, similarity scores included. + +These measures reduce variability but do not eliminate it. Hosted models can change behind an API, and providers do not guarantee identical output even at temperature zero. Alignment normalizes how a skill is expressed, but it cannot recover a skill that a model failed to emit, so the number and identity of extracted skills can still differ across models. We therefore recommend that users pin the model identifier, temperature, seed, and similarity thresholds when reporting results, and treat cross-model agreement as an empirical quantity to be measured rather than assumed. -For taxonomy alignment, the framework embeds extracted phrases and taxonomy entries with a sentence-transformer model [@Reimers2019] and retrieves matches from a precomputed FAISS index using a flat inner-product index (exact search over normalized embeddings). This choice keeps alignment behavior stable and reproducible at ESCO scale while remaining inexpensive relative to LLM inference. The system accepts pandas DataFrames as input and produces structured output including raw extracted skills, taxonomy-aligned canonical skills, taxonomy identifiers (e.g., ESCO codes), semantic similarity scores, and optional SCQF proficiency levels. +The source code is available at [https://github.com/LAiSER-Software/extract-module](https://github.com/LAiSER-Software/extract-module) under the BSD 3-Clause License, and can be installed with `pip install laiser`, or `pip install laiser[gpu]` for GPU inference. # Research impact statement -LAiSER has demonstrated realized impact across multiple institutions and funding initiatives. The framework's early prototypes contributed to securing a $250,000 grant from the Walmart Foundation and a subsequent $750,000 grant from the Gates Foundation, both supporting the broader Program on Skills, Credentials and Workforce Policy at The George Washington University. LAiSER is integrated into MIT's Digital Credentials Consortium Credential Co-writer tool [@DCC2025], where it provides skill extraction capabilities for generating standards-aligned Open Badges 3.0 credential templates in collaboration with Western Governors University, OneOrigin, and Open edX. Northeastern University has adopted LAiSER for internal academic projects involving skills analysis. The framework has been applied to a large-scale Texas syllabi skills extraction project using the OpenSyllabus Analytics API, analyzing thousands of course documents to map curriculum content to workforce skill demands. +LAiSER has demonstrated realized impact across multiple institutions and funding initiatives. The framework's early prototypes contributed to securing a $250,000 grant from the Walmart Foundation and a subsequent $750,000 grant from the Gates Foundation, both supporting the broader Program on Skills, Credentials and Workforce Policy at The George Washington University (GW). LAiSER is integrated into the Credential Co-writer tool of the Massachusetts Institute of Technology (MIT) Digital Credentials Consortium [@DCC2025], where it provides skill extraction capabilities for generating standards-aligned Open Badges 3.0 credential templates in collaboration with Western Governors University, OneOrigin, and Open edX. Northeastern University has adopted LAiSER for internal academic projects involving skills analysis. The framework has been applied to a large-scale Texas syllabi skills extraction project using the OpenSyllabus Analytics API, analyzing thousands of course documents to map curriculum content to workforce skill demands. -LAiSER received the Bronze Award in the AI in Education (Higher Education Institutions) category at the QS Reimagine Education Awards 2025 [@QSReimagine2025], selected from over 1,600 global submissions. The project has been presented at Badge Summit 2025 (Colorado) and showcased at the GW Open Source Conference, GW CCAS Poster Day, and GW Innovation Fest. LAiSER is listed in the George Washington University Open Source Program Office project registry. A companion cookbook repository provides reproducible use-case notebooks, and a multi-session bootcamp series has been delivered to train researchers and practitioners on the framework. +LAiSER received the Bronze Award in the AI in Education (Higher Education Institutions) category at the QS Reimagine Education Awards 2025 [@QSReimagine2025], selected from over 1,600 global submissions. The project has been presented at Badge Summit 2025 (Colorado) and showcased at the GW Open Source Conference, GW Columbian College of Arts and Sciences (CCAS) Poster Day, and GW Innovation Fest. LAiSER is listed in the George Washington University Open Source Program Office project registry. A companion cookbook repository provides reproducible use-case notebooks, and a multi-session bootcamp series has been delivered to train researchers and practitioners on the framework. # AI usage disclosure From 9e6ea340d7bde1029bff07caf0326bba8ee7db1f Mon Sep 17 00:00:00 2001 From: phanindra-max Date: Mon, 14 Sep 2026 00:35:38 -0400 Subject: [PATCH 4/6] docs(quickstart): show the greedy output for the local example The example table came from a run in #442 where Qwen2.5-0.5B-Instruct was still sampling, because its generation_config.json enables sampling and nothing turned it off. With greedy decoding now the default, the same input returns three skills rather than two, identically on every run (verified in two separate processes). The table and the note beneath it now match. Co-Authored-By: Claude Opus 5 --- docs/quickstart.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/quickstart.md b/docs/quickstart.md index dd3d765..c0c18c1 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -95,10 +95,12 @@ For the input above, the local 0.5B model on a laptop CPU returned: | Type | Raw Concept | Taxonomy Concept | Taxonomy Source | Correlation Coefficient | |---|---|---|---|---:| | skill | Python programming | Program in Python | `ukos` | 0.70 | +| skill | Machine learning | Use machine learning to create or improve solutions | `ukos` | 0.66 | | skill | Docker | Docker | `onet` | 0.75 | -Model output varies between runs and between models. A hosted or larger model typically extracts more -concepts from the same text — this small model missed "machine learning" and "AWS". +Decoding is greedy by default, so running this again returns the same rows — see +[Reproducibility](reproducibility.md). Different models extract different concepts: a hosted or larger model +typically finds more, and this small model missed "AWS". ## Next steps From c1fd5b76338e043362fabe903aab0f22adfa3969 Mon Sep 17 00:00:00 2001 From: phanindra-max Date: Tue, 15 Sep 2026 02:15:37 -0400 Subject: [PATCH 5/6] fix: address review findings on #444 Code - Gemini ignored LAISER_TOP_P: GenerateContentConfig was built without top_p, so the documented process-wide override reached every other backend but not Gemini. GeminiAPI and gemini_generate now take top_p (default DEFAULT_TOP_P) and send it; the Gemini request test asserts it. Tests - _import_or_skip skipped on any ImportError, so a broken import inside a backend module would have turned its tests into silent skips. It now skips only when ImportError.name names a genuinely optional package (google, vllm, llama_cpp) and re-raises everything else. - The module docstring claimed no model download. The alignment tests use the real sentence-transformers embedding model, which Hugging Face downloads on first use, as tests/test_alignment_core.py already does. The claim is corrected rather than the model stubbed, because repeatable embeddings are part of what those tests check. Docs and paper - Reproducibility page: the same no-download claim is corrected, and alignment "can" map different wordings to one entry. Each phrase is matched independently, so that is likely rather than guaranteed. - Paper and providers page: the GPU fallback has two steps. load_model_from_vllm retries DEFAULT_VLLM_MODEL_ID before the router falls back to Transformers; both now say so. - Paper: preprocessing is described as the extraction prompts instructing the model to disregard branding, boilerplate and benefits, not as a separate removal step. Convergence of different phrasings is softened the same way as on the Reproducibility page. Not changed: the State of the field claim that LAiSER supports SCQF proficiency classification, which the public API does not currently expose. That is left for the authors to decide. Co-Authored-By: Claude Opus 5 --- docs/providers.md | 3 ++- docs/reproducibility.md | 10 ++++++---- laiser/llm_models/gemini.py | 16 ++++++++++++---- paper.md | 8 ++++---- tests/test_determinism.py | 29 ++++++++++++++++++++++++----- 5 files changed, 48 insertions(+), 18 deletions(-) diff --git a/docs/providers.md b/docs/providers.md index 4a4803d..a8a0753 100644 --- a/docs/providers.md +++ b/docs/providers.md @@ -73,7 +73,8 @@ because LAiSER formats the prompt with the model's chat template when it has one ## Local model on GPU With `use_gpu=True` on a machine with CUDA, LAiSER first tries to load the model with vLLM. If vLLM cannot -load it, LAiSER loads it with transformers instead, using 8-bit quantization. Install the GPU extra, which +load it, LAiSER retries vLLM once with `TheBloke/Mixtral-7B-Instruct-v0.1-AWQ`. Only if that also fails does it +load the requested model with transformers instead, using 8-bit quantization. Install the GPU extra, which includes vLLM plus the `bitsandbytes` and `accelerate` packages that 8-bit loading needs: ```bash diff --git a/docs/reproducibility.md b/docs/reproducibility.md index 75b5509..f69ab06 100644 --- a/docs/reproducibility.md +++ b/docs/reproducibility.md @@ -75,15 +75,17 @@ Also record: - **Hosted models can change.** A provider can update the model behind a name, and providers do not guarantee identical output even at temperature `0.0`. -- **Different models extract different phrases.** Alignment maps different wordings of a skill onto the same - taxonomy entry, but it cannot recover a skill a model never extracted. Compare models on the aligned columns, - and measure their agreement rather than assuming it. +- **Different models extract different phrases.** Alignment can map different wordings of a skill onto the + same taxonomy entry, but each phrase is matched on its own, so that is likely rather than guaranteed — and + alignment cannot recover a skill a model never extracted. Compare models on the aligned columns, and measure + their agreement rather than assuming it. - **Hardware and library versions matter for local models.** Floating-point results can differ between devices and library releases. ## How it is tested -`tests/test_determinism.py` runs in CI without a GPU, a model download or an API key. It checks that: +`tests/test_determinism.py` runs in CI without a GPU, an API key or a language model download. The alignment +checks use the real embedding model, which Hugging Face downloads on first use. It checks that: - every public entry point defaults to temperature `0.0`, and every seedable one to seed `42`; - the decoding parameters each backend actually receives match those defaults, and per-call overrides reach diff --git a/laiser/llm_models/gemini.py b/laiser/llm_models/gemini.py index 8066cd4..c3c6e55 100644 --- a/laiser/llm_models/gemini.py +++ b/laiser/llm_models/gemini.py @@ -9,7 +9,7 @@ from google.genai import errors as genai_errors from google.genai import types -from laiser.config import DEFAULT_TEMPERATURE, GENERATION_SEED +from laiser.config import DEFAULT_TEMPERATURE, DEFAULT_TOP_P, GENERATION_SEED DEFAULT_GEMINI_MODEL = os.getenv("LAISER_GEMINI_MODEL", "gemini-2.5-flash") DEFAULT_GEMINI_TIMEOUT = float(os.getenv("LAISER_GEMINI_TIMEOUT", "60")) @@ -32,6 +32,7 @@ def __init__( max_output_tokens: int = DEFAULT_MAX_OUTPUT_TOKENS, temperature: float = DEFAULT_TEMPERATURE, seed: Optional[int] = GENERATION_SEED, + top_p: float = DEFAULT_TOP_P, ): if not api_key: raise ValueError("Gemini API key is required") @@ -41,6 +42,7 @@ def __init__( self.max_output_tokens = max_output_tokens self.temperature = temperature self.seed = seed + self.top_p = top_p self.client = genai.Client(api_key=api_key) def generate(self, prompt: str) -> str: @@ -54,7 +56,11 @@ def generate_with_config( response_schema: Optional[Any] = None, ) -> str: try: - config_kwargs = {"temperature": self.temperature, "max_output_tokens": self.max_output_tokens} + config_kwargs = { + "temperature": self.temperature, + "top_p": self.top_p, + "max_output_tokens": self.max_output_tokens, + } if self.seed is not None and _SUPPORTS_SEED: config_kwargs["seed"] = self.seed if response_mime_type: @@ -91,11 +97,12 @@ def gemini_generate( response_schema: Optional[Any] = None, temperature: float = DEFAULT_TEMPERATURE, seed: Optional[int] = GENERATION_SEED, + top_p: float = DEFAULT_TOP_P, ) -> str: """Send `prompt` to Gemini and return generated text. - Decoding is greedy by default (temperature 0.0), and ``seed`` is sent - whenever the installed google-genai release accepts one. + Decoding is greedy by default (temperature 0.0, top_p 1.0), and ``seed`` is + sent whenever the installed google-genai release accepts one. """ client = GeminiAPI( api_key=api_key, @@ -104,6 +111,7 @@ def gemini_generate( max_output_tokens=max_output_tokens, temperature=temperature, seed=seed, + top_p=top_p, ) return client.generate_with_config( prompt, diff --git a/paper.md b/paper.md index 231116c..003639d 100644 --- a/paper.md +++ b/paper.md @@ -41,19 +41,19 @@ The contemporary labor market is characterized by rapid technological change, ev Several open-source tools address parts of the skill extraction problem. SkillNER [@SkillNER2022] uses rule-based matching against the EMSI (now Lightcast) skills database via spaCy, an open-source natural language processing library, offering fast lookups but limited ability to detect novel or context-dependent skills not present in its dictionary. Nesta's Skills Extractor Library [@Nesta2023] trains a spaCy named entity recognition (NER) model on labeled job advertisements and maps extracted phrases to ESCO or Lightcast taxonomies using sentence-transformer embeddings. While effective for UK job advertisements, its NER model requires domain-specific labeled training data and does not generalize well to other text types such as syllabi or credential descriptions. The esco-skill-extractor [@EscoExtractor2024] takes a simpler approach, embedding full input texts with a transformer and matching against ESCO entries by cosine similarity, but it lacks preprocessing, multi-model flexibility, and proficiency-level classification. -LAiSER differs from these tools in several key respects. First, it uses LLMs for skill extraction rather than rule-based or NER-based approaches, enabling it to identify skills expressed in varied natural language without requiring labeled training data. Second, LAiSER provides domain-aware preprocessing that removes extraneous content (company branding, legal boilerplate, benefits sections) while preserving task-relevant skill language, an important step that existing tools omit. Third, the framework supports multiple LLM backends (vLLM, HuggingFace Transformers, llama.cpp, and the Gemini and OpenAI application programming interfaces, or APIs), allowing deployment on graphics processing unit (GPU) clusters, hosted APIs, or ordinary laptops. Finally, LAiSER uniquely supports Knowledge, Skills, and Abilities (KSA) decomposition with proficiency-level classification under the Scottish Credit and Qualifications Framework (SCQF) [@SCQF2019], a capability absent from all comparable tools. These design choices reflect a deliberate decision to build a new framework rather than extend existing tools, whose architectures are not designed to accommodate LLM-based extraction or multi-taxonomy proficiency alignment. +LAiSER differs from these tools in several key respects. First, it uses LLMs for skill extraction rather than rule-based or NER-based approaches, enabling it to identify skills expressed in varied natural language without requiring labeled training data. Second, LAiSER's extraction prompts instruct the model to disregard extraneous content (company branding, legal boilerplate, benefits sections) while preserving task-relevant skill language, an important step that existing tools omit. Third, the framework supports multiple LLM backends (vLLM, HuggingFace Transformers, llama.cpp, and the Gemini and OpenAI application programming interfaces, or APIs), allowing deployment on graphics processing unit (GPU) clusters, hosted APIs, or ordinary laptops. Finally, LAiSER uniquely supports Knowledge, Skills, and Abilities (KSA) decomposition with proficiency-level classification under the Scottish Credit and Qualifications Framework (SCQF) [@SCQF2019], a capability absent from all comparable tools. These design choices reflect a deliberate decision to build a new framework rather than extend existing tools, whose architectures are not designed to accommodate LLM-based extraction or multi-taxonomy proficiency alignment. # Software design LAiSER's architecture is organized into three loosely coupled layers: data access, skill extraction, and taxonomy alignment. This separation was chosen to allow each component to evolve independently and to support diverse deployment scenarios without requiring changes to the overall pipeline. -The extraction layer posed the most significant design trade-off. Training a custom NER model, as Nesta's library does, would yield fast inference but would require substantial labeled data for each new domain (syllabi, credentials, job postings). Instead, LAiSER delegates extraction to LLMs via a prompt engineering approach, trading some inference speed for broad domain generalizability without retraining. To mitigate vendor lock-in and hardware constraints, the extraction layer abstracts over multiple backends: vLLM for GPU clusters, HuggingFace Transformers and llama.cpp for models run locally, and the Gemini and OpenAI APIs for hosted inference. The backend is chosen when an extractor is created, from the requested model and the available hardware; on a GPU, a model that vLLM cannot load falls back to Transformers. +The extraction layer posed the most significant design trade-off. Training a custom NER model, as Nesta's library does, would yield fast inference but would require substantial labeled data for each new domain (syllabi, credentials, job postings). Instead, LAiSER delegates extraction to LLMs via a prompt engineering approach, trading some inference speed for broad domain generalizability without retraining. To mitigate vendor lock-in and hardware constraints, the extraction layer abstracts over multiple backends: vLLM for GPU clusters, HuggingFace Transformers and llama.cpp for models run locally, and the Gemini and OpenAI APIs for hosted inference. The backend is chosen when an extractor is created, from the requested model and the available hardware. On a GPU, if vLLM cannot load the requested model it first retries with a default checkpoint, and only if that also fails does it fall back to Transformers. For taxonomy alignment, the framework embeds both the extracted phrases and the taxonomy entries with a sentence-transformer model [@Reimers2019] and retrieves each phrase's closest entry from a precomputed index built with FAISS (Facebook AI Similarity Search) [@Johnson2019]. The index is a flat inner-product index over normalized embeddings, which makes retrieval exhaustive and exact rather than approximate. Exactness was a deliberate choice: at the scale of the bundled taxonomies, about 27,000 skill entries, an exhaustive scan is inexpensive relative to LLM inference in the preceding stage, and it guarantees that a given query returns the same entry and the same similarity score on every run, a precondition for the reproducibility properties described below. Because retrieval is encapsulated behind an index manager, an approximate index could be substituted without changes to the alignment logic should taxonomy size later require it. The system accepts pandas DataFrames as input and returns one row per aligned phrase, containing the extracted text, the matched taxonomy entry, its source taxonomy and link, and the similarity score. -Because LLMs sample their output, identical inputs can yield different generations across runs, and different backends can yield different generations from the same input. LAiSER addresses this in three ways. First, decoding is deterministic by default: every backend decodes greedily at temperature zero, and a fixed seed is supplied to every backend that accepts one, so runs stay reproducible even when a user deliberately raises the temperature. Temperature and seed are constructor arguments, so sampled variation is available when wanted but is never the silent default. Second, the alignment stage acts as a normalizing projection. Generated phrases are embedded with a fixed sentence-transformer model and matched by exact search against a precomputed index, so surface variation in the generated text collapses onto a closed, finite vocabulary of taxonomy entries, and different phrasings of the same competency converge on the same aligned record. Third, every backend produces the same output schema, which makes results directly comparable across models and lets cross-model agreement be measured on the aligned columns. The test suite checks each backend's decoding defaults, the decoding parameters each backend actually receives, and that repeated alignment of a fixed input returns an identical result, similarity scores included. +Because LLMs sample their output, identical inputs can yield different generations across runs, and different backends can yield different generations from the same input. LAiSER addresses this in three ways. First, decoding is deterministic by default: every backend decodes greedily at temperature zero, and a fixed seed is supplied to every backend that accepts one, so runs stay reproducible even when a user deliberately raises the temperature. Temperature and seed are constructor arguments, so sampled variation is available when wanted but is never the silent default. Second, the alignment stage acts as a normalizing projection. Generated phrases are embedded with a fixed sentence-transformer model and matched by exact search against a precomputed index, so surface variation in the generated text collapses onto a closed, finite vocabulary of taxonomy entries, and different phrasings of the same competency can converge on the same aligned record, although because each phrase is matched independently this is likely rather than guaranteed. Third, every backend produces the same output schema, which makes results directly comparable across models and lets cross-model agreement be measured on the aligned columns. The test suite checks each backend's decoding defaults, the decoding parameters each backend actually receives, and that repeated alignment of a fixed input returns an identical result, similarity scores included. -These measures reduce variability but do not eliminate it. Hosted models can change behind an API, and providers do not guarantee identical output even at temperature zero. Alignment normalizes how a skill is expressed, but it cannot recover a skill that a model failed to emit, so the number and identity of extracted skills can still differ across models. We therefore recommend that users pin the model identifier, temperature, seed, and similarity thresholds when reporting results, and treat cross-model agreement as an empirical quantity to be measured rather than assumed. +These measures reduce variability but do not eliminate it. Hosted models can change behind an API, and providers do not guarantee identical output even at temperature zero. Alignment can normalize how a skill is expressed, but it cannot recover a skill that a model failed to emit, so the number and identity of extracted skills can still differ across models. We therefore recommend that users pin the model identifier, temperature, seed, and similarity thresholds when reporting results, and treat cross-model agreement as an empirical quantity to be measured rather than assumed. The source code is available at [https://github.com/LAiSER-Software/extract-module](https://github.com/LAiSER-Software/extract-module) under the BSD 3-Clause License, and can be installed with `pip install laiser`, or `pip install laiser[gpu]` for GPU inference. diff --git a/tests/test_determinism.py b/tests/test_determinism.py index 1f45d10..9a3298c 100644 --- a/tests/test_determinism.py +++ b/tests/test_determinism.py @@ -2,8 +2,10 @@ Language models sample their output, so the same input can produce different generations across runs and across backends (JOSS review #424). LAiSER limits -this in two ways, and these tests check both without a GPU, a model download or -a paid API call: +this in two ways, and these tests check both without a GPU, an API key or a +language model download. The alignment tests do load the sentence-transformers +embedding model used for alignment, which Hugging Face downloads on first use, +as tests/test_alignment_core.py already does: - Decoding: every backend defaults to greedy decoding and a fixed seed. Checked against each entry point's defaults and against the parameters each backend @@ -25,16 +27,26 @@ pytestmark = pytest.mark.determinism +# Third-party packages a backend module may legitimately lack in a given install. +# ImportError.name holds the package that failed to import, so a missing optional +# dependency skips while a broken import inside LAiSER itself still fails. +_OPTIONAL_PACKAGES = {"google", "vllm", "llama_cpp"} + + def _import_or_skip(module_path): - """Import an optional backend module, skipping when its dependency is absent. + """Import a backend module, skipping only when an optional dependency is absent. pytest.importorskip only skips on ModuleNotFoundError, but ``from google import - genai`` without google-genai installed raises a plain ImportError. + genai`` without google-genai installed raises a plain ImportError. Any other + import failure is re-raised, so a regression cannot turn into a silent skip. """ try: return importlib.import_module(module_path) except ImportError as e: - pytest.skip(f"{module_path} unavailable in this environment: {e}") + missing = (e.name or "").split(".")[0] + if missing not in _OPTIONAL_PACKAGES: + raise + pytest.skip(f"{module_path} needs {missing}, which is not installed: {e}") @pytest.fixture @@ -187,6 +199,7 @@ def __init__(self, api_key): sent = captured["config"] assert sent.temperature == config.DEFAULT_TEMPERATURE + assert sent.top_p == config.DEFAULT_TOP_P assert sent.max_output_tokens == gemini.DEFAULT_MAX_OUTPUT_TOKENS if gemini._SUPPORTS_SEED: assert sent.seed == config.GENERATION_SEED @@ -408,6 +421,12 @@ def __init__(self, *args, **kwargs): @pytest.fixture(scope="module") def alignment_service(): + """The real skills alignment service. + + Uses the bundled FAISS index and the real sentence-transformers embedding model, + which Hugging Face downloads on first use. The model is not stubbed: repeatable + embeddings are part of what these tests check. + """ data_access = _import_or_skip("laiser.data_access") services = _import_or_skip("laiser.services") From 05bc8bdcb2234ea923409371efddbd420bfd290a Mon Sep 17 00:00:00 2001 From: phanindra-max Date: Tue, 15 Sep 2026 02:18:15 -0400 Subject: [PATCH 6/6] docs(paper): remove the unsupported SCQF proficiency claim State of the field claimed LAiSER uniquely supports KSA decomposition with SCQF proficiency-level classification. The public API does not reach that capability: the KSA/SCQF prompt builders and parsers in services.py are never called, and extract_concepts(levels=...) has no effect. The sentence is removed, along with the related 'proficiency-level classification' and 'multi-taxonomy proficiency alignment' wording, so the paper only claims what the code does. Raised in Copilot's review of #444. Co-Authored-By: Claude Opus 5 --- paper.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paper.md b/paper.md index 003639d..ef039e7 100644 --- a/paper.md +++ b/paper.md @@ -39,9 +39,9 @@ The contemporary labor market is characterized by rapid technological change, ev # State of the field -Several open-source tools address parts of the skill extraction problem. SkillNER [@SkillNER2022] uses rule-based matching against the EMSI (now Lightcast) skills database via spaCy, an open-source natural language processing library, offering fast lookups but limited ability to detect novel or context-dependent skills not present in its dictionary. Nesta's Skills Extractor Library [@Nesta2023] trains a spaCy named entity recognition (NER) model on labeled job advertisements and maps extracted phrases to ESCO or Lightcast taxonomies using sentence-transformer embeddings. While effective for UK job advertisements, its NER model requires domain-specific labeled training data and does not generalize well to other text types such as syllabi or credential descriptions. The esco-skill-extractor [@EscoExtractor2024] takes a simpler approach, embedding full input texts with a transformer and matching against ESCO entries by cosine similarity, but it lacks preprocessing, multi-model flexibility, and proficiency-level classification. +Several open-source tools address parts of the skill extraction problem. SkillNER [@SkillNER2022] uses rule-based matching against the EMSI (now Lightcast) skills database via spaCy, an open-source natural language processing library, offering fast lookups but limited ability to detect novel or context-dependent skills not present in its dictionary. Nesta's Skills Extractor Library [@Nesta2023] trains a spaCy named entity recognition (NER) model on labeled job advertisements and maps extracted phrases to ESCO or Lightcast taxonomies using sentence-transformer embeddings. While effective for UK job advertisements, its NER model requires domain-specific labeled training data and does not generalize well to other text types such as syllabi or credential descriptions. The esco-skill-extractor [@EscoExtractor2024] takes a simpler approach, embedding full input texts with a transformer and matching against ESCO entries by cosine similarity, but it lacks preprocessing and multi-model flexibility. -LAiSER differs from these tools in several key respects. First, it uses LLMs for skill extraction rather than rule-based or NER-based approaches, enabling it to identify skills expressed in varied natural language without requiring labeled training data. Second, LAiSER's extraction prompts instruct the model to disregard extraneous content (company branding, legal boilerplate, benefits sections) while preserving task-relevant skill language, an important step that existing tools omit. Third, the framework supports multiple LLM backends (vLLM, HuggingFace Transformers, llama.cpp, and the Gemini and OpenAI application programming interfaces, or APIs), allowing deployment on graphics processing unit (GPU) clusters, hosted APIs, or ordinary laptops. Finally, LAiSER uniquely supports Knowledge, Skills, and Abilities (KSA) decomposition with proficiency-level classification under the Scottish Credit and Qualifications Framework (SCQF) [@SCQF2019], a capability absent from all comparable tools. These design choices reflect a deliberate decision to build a new framework rather than extend existing tools, whose architectures are not designed to accommodate LLM-based extraction or multi-taxonomy proficiency alignment. +LAiSER differs from these tools in several key respects. First, it uses LLMs for skill extraction rather than rule-based or NER-based approaches, enabling it to identify skills expressed in varied natural language without requiring labeled training data. Second, LAiSER's extraction prompts instruct the model to disregard extraneous content (company branding, legal boilerplate, benefits sections) while preserving task-relevant skill language, an important step that existing tools omit. Third, the framework supports multiple LLM backends (vLLM, HuggingFace Transformers, llama.cpp, and the Gemini and OpenAI application programming interfaces, or APIs), allowing deployment on graphics processing unit (GPU) clusters, hosted APIs, or ordinary laptops. These design choices reflect a deliberate decision to build a new framework rather than extend existing tools, whose architectures are not designed to accommodate LLM-based extraction or alignment across multiple taxonomies. # Software design