diff --git a/README.md b/README.md index 0aae28a8..380f7f63 100644 --- a/README.md +++ b/README.md @@ -367,6 +367,8 @@ More details can be found in the [lm_eval](https://github.com/EleutherAI/lm-eval Please note, for tasks such as NER, the automated evaluation is based on a specific pattern. This might fail to extract relevant information in zero-shot settings, resulting in relatively lower performance compared to previous human-annotated results. +**OpenAI** + ```bash export OPENAI_API_SECRET_KEY=YOUR_KEY_HERE python eval.py \ @@ -374,6 +376,17 @@ python eval.py \ --tasks flare_ner,flare_sm_acl,flare_fpb ``` +**MiniMax** + +[MiniMax](https://www.minimaxi.com/) provides large language models (M2.7, M2.5) with 204K context windows via an OpenAI-compatible API. Supported models: `MiniMax-M2.7`, `MiniMax-M2.7-highspeed`, `MiniMax-M2.5`, `MiniMax-M2.5-highspeed`. + +```bash +export MINIMAX_API_KEY=YOUR_MINIMAX_KEY_HERE +python eval.py \ + --model MiniMax-M2.7 \ + --tasks flare_ner,flare_sm_acl,flare_fpb +``` + 3. Self-Hosted Evaluation To run inference backend: diff --git a/src/chatlm.py b/src/chatlm.py index 31117666..ab90f9b1 100644 --- a/src/chatlm.py +++ b/src/chatlm.py @@ -28,7 +28,7 @@ async def single_chat(client, **kwargs): async def oa_completion(**kwargs): - """Query OpenAI API for completion. + """Query OpenAI-compatible API for completion. Retry with back-off until they respond """ @@ -50,6 +50,10 @@ async def oa_completion(**kwargs): class ChatLM(BaseLM): REQ_CHUNK_SIZE = 20 + # Default API configuration (OpenAI) + API_BASE_URL = "https://api.openai.com/v1/chat/completions" + API_KEY_ENV = "OPENAI_API_SECRET_KEY" + def __init__(self, model, truncate=False): """ @@ -59,12 +63,9 @@ def __init__(self, model, truncate=False): """ super().__init__() - import openai - self.model = model self.truncate = truncate - # Read from environment variable OPENAI_API_SECRET_KEY - api_key = os.environ["OPENAI_API_SECRET_KEY"] + api_key = os.environ[self.API_KEY_ENV] self.tokenizer = transformers.GPT2TokenizerFast.from_pretrained("gpt2") self.headers = { "Content-Type": "application/json", @@ -136,12 +137,12 @@ def sameuntil_chunks(xs, size): inps.append(context[0]) responses = asyncio.run(oa_completion( - url="https://api.openai.com/v1/chat/completions", + url=self.API_BASE_URL, headers=self.headers, model=self.model, messages=[{"role": "user", "content": inp} for inp in inps], max_tokens=self.max_gen_toks, - temperature=0.0, + temperature=self._get_temperature(0.0), # stop=until, )) @@ -155,6 +156,10 @@ def sameuntil_chunks(xs, size): return re_ord.get_original(res) + def _get_temperature(self, temperature): + """Return a valid temperature value for this provider.""" + return temperature + def _model_call(self, inps): # Isn't used because we override _loglikelihood_tokens raise NotImplementedError() diff --git a/src/evaluator.py b/src/evaluator.py index f70e8012..44cd4dab 100644 --- a/src/evaluator.py +++ b/src/evaluator.py @@ -11,6 +11,7 @@ from model_prompt import MODEL_PROMPT_MAP from chatlm import ChatLM +from minimax_lm import MiniMaxLM, MINIMAX_MODELS import tasks as ta @positional_deprecated @@ -74,7 +75,9 @@ def simple_evaluate( if isinstance(model, str): if model_args is None: model_args = "" - if model[:3] != "gpt": + if model in MINIMAX_MODELS: + lm = MiniMaxLM(model) + elif model[:3] != "gpt": lm = lm_eval.models.get_model(model).create_from_arg_string( model_args, {"batch_size": batch_size, "max_batch_size": max_batch_size, "device": device} ) diff --git a/src/factscore_package/minimax_lm.py b/src/factscore_package/minimax_lm.py new file mode 100644 index 00000000..7e9d3554 --- /dev/null +++ b/src/factscore_package/minimax_lm.py @@ -0,0 +1,27 @@ +from .openai_lm import OpenAIModel +import os + + +class MiniMaxModel(OpenAIModel): + """MiniMax LLM for FActScore evaluation via OpenAI-compatible API.""" + + def __init__(self, model_name="MiniMax-M2.7", cache_file=None): + key = os.environ.get("MINIMAX_API_KEY", "") + super().__init__( + model_name=model_name, + cache_file=cache_file, + key=key, + api_base="https://api.minimax.io/v1", + ) + # MiniMax requires temperature in (0.0, 1.0] + self.temp = 0.7 + + def _generate(self, prompt, max_sequence_length=2048, max_output_length=128): + if self.add_n % self.save_interval == 0: + self.save_cache() + message = [{"role": "user", "content": prompt}] + response = self.call_ChatGPT( + message, model_name=self.model_name, temp=self.temp, max_len=max_sequence_length + ) + output = response.choices[0].message.content + return output, response diff --git a/src/factscore_package/openai_lm.py b/src/factscore_package/openai_lm.py index 12282a7e..63331b70 100644 --- a/src/factscore_package/openai_lm.py +++ b/src/factscore_package/openai_lm.py @@ -7,16 +7,17 @@ import numpy as np import logging -#os.environ["http_proxy"] = "http://localhost:27890" -#os.environ["https_proxy"] = "http://localhost:27890" class OpenAIModel(LM): - def __init__(self, model_name, cache_file=None, key=""): + def __init__(self, model_name, cache_file=None, key="", api_base=None): self.model_name = model_name self.temp = 0.7 self.save_interval = 100 - self.client = OpenAI(api_key=key.strip()) + client_kwargs = {"api_key": key.strip()} + if api_base: + client_kwargs["base_url"] = api_base + self.client = OpenAI(**client_kwargs) super().__init__(cache_file) def load_model(self): diff --git a/src/minimax_lm.py b/src/minimax_lm.py new file mode 100644 index 00000000..4b55c6f6 --- /dev/null +++ b/src/minimax_lm.py @@ -0,0 +1,34 @@ +from chatlm import ChatLM + + +# MiniMax models and their context window sizes +MINIMAX_MODELS = { + "MiniMax-M2.7": 204800, + "MiniMax-M2.7-highspeed": 204800, + "MiniMax-M2.5": 204800, + "MiniMax-M2.5-highspeed": 204800, +} + + +class MiniMaxLM(ChatLM): + """Language model class for MiniMax's OpenAI-compatible API. + + MiniMax provides an OpenAI-compatible chat completions endpoint at + https://api.minimax.io/v1/chat/completions. This class configures + ChatLM to use MiniMax instead of OpenAI. + + Environment variable: MINIMAX_API_KEY + """ + + API_BASE_URL = "https://api.minimax.io/v1/chat/completions" + API_KEY_ENV = "MINIMAX_API_KEY" + + @property + def max_length(self): + return MINIMAX_MODELS.get(self.model, 204800) + + def _get_temperature(self, temperature): + """MiniMax requires temperature in (0.0, 1.0].""" + if temperature <= 0.0: + return 0.01 + return min(temperature, 1.0) diff --git a/tests/test_minimax.py b/tests/test_minimax.py new file mode 100644 index 00000000..896d237c --- /dev/null +++ b/tests/test_minimax.py @@ -0,0 +1,366 @@ +"""Tests for MiniMax LLM provider integration in PIXIU.""" +import os +import sys +import json +import asyncio +import unittest +from unittest.mock import patch, MagicMock, AsyncMock +from types import ModuleType + +# Add src to path +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src")) + +# Mock lm_eval and its submodules (project-specific submodule dependency) +_lm_eval_mock = MagicMock() +_lm_eval_mock.base.BaseLM = type("BaseLM", (), { + "__init__": lambda self: None, + "cache_hook": MagicMock(), +}) +sys.modules["lm_eval"] = _lm_eval_mock +sys.modules["lm_eval.base"] = _lm_eval_mock.base +sys.modules["lm_eval.utils"] = _lm_eval_mock.utils +sys.modules["lm_eval.metrics"] = _lm_eval_mock.metrics +sys.modules["lm_eval.models"] = _lm_eval_mock.models +sys.modules["lm_eval.tasks"] = _lm_eval_mock.tasks + + +class TestMiniMaxLMConfig(unittest.TestCase): + """Test MiniMaxLM class configuration and attributes.""" + + def test_minimax_models_dict(self): + from minimax_lm import MINIMAX_MODELS + self.assertIn("MiniMax-M2.7", MINIMAX_MODELS) + self.assertIn("MiniMax-M2.7-highspeed", MINIMAX_MODELS) + self.assertIn("MiniMax-M2.5", MINIMAX_MODELS) + self.assertIn("MiniMax-M2.5-highspeed", MINIMAX_MODELS) + + def test_minimax_context_windows(self): + from minimax_lm import MINIMAX_MODELS + for model, ctx_len in MINIMAX_MODELS.items(): + self.assertEqual(ctx_len, 204800, f"{model} should have 204K context") + + def test_minimax_api_base_url(self): + from minimax_lm import MiniMaxLM + self.assertEqual( + MiniMaxLM.API_BASE_URL, + "https://api.minimax.io/v1/chat/completions", + ) + + def test_minimax_api_key_env(self): + from minimax_lm import MiniMaxLM + self.assertEqual(MiniMaxLM.API_KEY_ENV, "MINIMAX_API_KEY") + + @patch.dict(os.environ, {"MINIMAX_API_KEY": "test-key-123"}) + @patch("transformers.GPT2TokenizerFast.from_pretrained") + def test_minimax_lm_init(self, mock_tokenizer): + mock_tokenizer.return_value = MagicMock() + from minimax_lm import MiniMaxLM + lm = MiniMaxLM("MiniMax-M2.7") + self.assertEqual(lm.model, "MiniMax-M2.7") + self.assertIn("Bearer test-key-123", lm.headers["Authorization"]) + + @patch.dict(os.environ, {"MINIMAX_API_KEY": "test-key"}) + @patch("transformers.GPT2TokenizerFast.from_pretrained") + def test_minimax_max_length(self, mock_tokenizer): + mock_tokenizer.return_value = MagicMock() + from minimax_lm import MiniMaxLM + lm = MiniMaxLM("MiniMax-M2.7") + self.assertEqual(lm.max_length, 204800) + + @patch.dict(os.environ, {"MINIMAX_API_KEY": "test-key"}) + @patch("transformers.GPT2TokenizerFast.from_pretrained") + def test_minimax_max_length_highspeed(self, mock_tokenizer): + mock_tokenizer.return_value = MagicMock() + from minimax_lm import MiniMaxLM + lm = MiniMaxLM("MiniMax-M2.7-highspeed") + self.assertEqual(lm.max_length, 204800) + + @patch.dict(os.environ, {"MINIMAX_API_KEY": "test-key"}) + @patch("transformers.GPT2TokenizerFast.from_pretrained") + def test_minimax_temperature_clamping_zero(self, mock_tokenizer): + mock_tokenizer.return_value = MagicMock() + from minimax_lm import MiniMaxLM + lm = MiniMaxLM("MiniMax-M2.7") + self.assertGreater(lm._get_temperature(0.0), 0.0) + + @patch.dict(os.environ, {"MINIMAX_API_KEY": "test-key"}) + @patch("transformers.GPT2TokenizerFast.from_pretrained") + def test_minimax_temperature_clamping_negative(self, mock_tokenizer): + mock_tokenizer.return_value = MagicMock() + from minimax_lm import MiniMaxLM + lm = MiniMaxLM("MiniMax-M2.7") + self.assertGreater(lm._get_temperature(-0.5), 0.0) + + @patch.dict(os.environ, {"MINIMAX_API_KEY": "test-key"}) + @patch("transformers.GPT2TokenizerFast.from_pretrained") + def test_minimax_temperature_clamping_high(self, mock_tokenizer): + mock_tokenizer.return_value = MagicMock() + from minimax_lm import MiniMaxLM + lm = MiniMaxLM("MiniMax-M2.7") + self.assertLessEqual(lm._get_temperature(1.5), 1.0) + + @patch.dict(os.environ, {"MINIMAX_API_KEY": "test-key"}) + @patch("transformers.GPT2TokenizerFast.from_pretrained") + def test_minimax_temperature_valid(self, mock_tokenizer): + mock_tokenizer.return_value = MagicMock() + from minimax_lm import MiniMaxLM + lm = MiniMaxLM("MiniMax-M2.7") + self.assertEqual(lm._get_temperature(0.5), 0.5) + + @patch.dict(os.environ, {"MINIMAX_API_KEY": "test-key"}) + @patch("transformers.GPT2TokenizerFast.from_pretrained") + def test_minimax_temperature_boundary(self, mock_tokenizer): + mock_tokenizer.return_value = MagicMock() + from minimax_lm import MiniMaxLM + lm = MiniMaxLM("MiniMax-M2.7") + self.assertEqual(lm._get_temperature(1.0), 1.0) + + @patch("transformers.GPT2TokenizerFast.from_pretrained") + def test_missing_api_key_raises(self, mock_tokenizer): + mock_tokenizer.return_value = MagicMock() + from minimax_lm import MiniMaxLM + env = os.environ.copy() + env.pop("MINIMAX_API_KEY", None) + with patch.dict(os.environ, env, clear=True): + with self.assertRaises(KeyError): + MiniMaxLM("MiniMax-M2.7") + + +class TestChatLMConfig(unittest.TestCase): + """Test ChatLM base class configuration.""" + + def test_chatl_api_base_url(self): + from chatlm import ChatLM + self.assertEqual( + ChatLM.API_BASE_URL, + "https://api.openai.com/v1/chat/completions", + ) + + def test_chatl_api_key_env(self): + from chatlm import ChatLM + self.assertEqual(ChatLM.API_KEY_ENV, "OPENAI_API_SECRET_KEY") + + @patch.dict(os.environ, {"OPENAI_API_SECRET_KEY": "sk-test"}) + @patch("transformers.GPT2TokenizerFast.from_pretrained") + def test_chatl_default_temperature(self, mock_tokenizer): + mock_tokenizer.return_value = MagicMock() + from chatlm import ChatLM + lm = ChatLM("gpt-4") + self.assertEqual(lm._get_temperature(0.0), 0.0) + self.assertEqual(lm._get_temperature(0.5), 0.5) + + @patch.dict(os.environ, {"OPENAI_API_SECRET_KEY": "sk-test"}) + @patch("transformers.GPT2TokenizerFast.from_pretrained") + def test_chatl_inherits_base_lm(self, mock_tokenizer): + mock_tokenizer.return_value = MagicMock() + from chatlm import ChatLM + lm = ChatLM("gpt-4") + self.assertEqual(lm.model, "gpt-4") + + def test_minimax_is_subclass_of_chatlm(self): + from chatlm import ChatLM + from minimax_lm import MiniMaxLM + self.assertTrue(issubclass(MiniMaxLM, ChatLM)) + + +class TestEvaluatorRouting(unittest.TestCase): + """Test model routing in evaluator.py.""" + + def test_minimax_models_in_evaluator(self): + from minimax_lm import MINIMAX_MODELS + self.assertIn("MiniMax-M2.7", MINIMAX_MODELS) + self.assertIn("MiniMax-M2.7-highspeed", MINIMAX_MODELS) + self.assertIn("MiniMax-M2.5", MINIMAX_MODELS) + self.assertIn("MiniMax-M2.5-highspeed", MINIMAX_MODELS) + + def test_evaluator_has_minimax_models(self): + from minimax_lm import MINIMAX_MODELS + self.assertTrue(len(MINIMAX_MODELS) >= 4) + + @patch.dict(os.environ, {"MINIMAX_API_KEY": "test-key"}) + @patch("transformers.GPT2TokenizerFast.from_pretrained") + def test_minimax_model_creates_minimax_lm(self, mock_tokenizer): + mock_tokenizer.return_value = MagicMock() + from minimax_lm import MiniMaxLM, MINIMAX_MODELS + for model_name in MINIMAX_MODELS: + lm = MiniMaxLM(model_name) + self.assertIsInstance(lm, MiniMaxLM) + self.assertEqual(lm.model, model_name) + + @patch.dict(os.environ, {"OPENAI_API_SECRET_KEY": "sk-test"}) + @patch("transformers.GPT2TokenizerFast.from_pretrained") + def test_gpt_model_creates_chatlm(self, mock_tokenizer): + mock_tokenizer.return_value = MagicMock() + from chatlm import ChatLM + from minimax_lm import MiniMaxLM + lm = ChatLM("gpt-4") + self.assertIsInstance(lm, ChatLM) + self.assertNotIsInstance(lm, MiniMaxLM) + + +class TestMiniMaxFActScore(unittest.TestCase): + """Test MiniMax integration in FActScore package.""" + + def test_minimax_lm_import(self): + try: + from factscore_package.minimax_lm import MiniMaxModel + except ImportError: + self.fail("Should be able to import MiniMaxModel from factscore_package") + + @patch.dict(os.environ, {"MINIMAX_API_KEY": "test-key-factscore"}) + @patch("factscore_package.lm.LM.load_cache", return_value={}) + def test_minimax_model_init(self, mock_cache): + from factscore_package.minimax_lm import MiniMaxModel + model = MiniMaxModel("MiniMax-M2.7", cache_file="/tmp/test_cache") + self.assertEqual(model.model_name, "MiniMax-M2.7") + self.assertEqual(model.temp, 0.7) + + @patch.dict(os.environ, {"MINIMAX_API_KEY": "test-key-factscore"}) + @patch("factscore_package.lm.LM.load_cache", return_value={}) + def test_minimax_model_default_model(self, mock_cache): + from factscore_package.minimax_lm import MiniMaxModel + model = MiniMaxModel(cache_file="/tmp/test_cache") + self.assertEqual(model.model_name, "MiniMax-M2.7") + + @patch.dict(os.environ, {"MINIMAX_API_KEY": "test-api-key"}) + @patch("factscore_package.lm.LM.load_cache", return_value={}) + def test_minimax_model_api_base(self, mock_cache): + from factscore_package.minimax_lm import MiniMaxModel + model = MiniMaxModel("MiniMax-M2.7", cache_file="/tmp/test_cache") + self.assertEqual(model.client.base_url.host, "api.minimax.io") + + @patch("factscore_package.lm.LM.load_cache", return_value={}) + def test_openai_model_accepts_api_base(self, mock_cache): + from factscore_package.openai_lm import OpenAIModel + model = OpenAIModel( + "test-model", + cache_file="/tmp/test_cache", + key="test-key", + api_base="https://custom.api.com/v1", + ) + self.assertEqual(model.client.base_url.host, "custom.api.com") + + @patch.dict(os.environ, {"MINIMAX_API_KEY": "test-key"}) + @patch("factscore_package.lm.LM.load_cache", return_value={}) + def test_minimax_model_inherits_openai(self, mock_cache): + from factscore_package.openai_lm import OpenAIModel + from factscore_package.minimax_lm import MiniMaxModel + model = MiniMaxModel(cache_file="/tmp/test_cache") + self.assertIsInstance(model, OpenAIModel) + + +class TestOaCompletion(unittest.TestCase): + """Test the oa_completion async function.""" + + @patch("httpx.AsyncClient") + def test_oa_completion_uses_provided_url(self, mock_client_cls): + from chatlm import oa_completion + mock_client = AsyncMock() + mock_response = MagicMock() + mock_response.json.return_value = { + "choices": [{"message": {"content": "test response"}}] + } + mock_client.post.return_value = mock_response + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=None) + mock_client_cls.return_value = mock_client + + result = asyncio.run(oa_completion( + url="https://api.minimax.io/v1/chat/completions", + headers={"Authorization": "Bearer test"}, + model="MiniMax-M2.7", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + temperature=0.5, + )) + self.assertEqual(result, ["test response"]) + call_args = mock_client.post.call_args + self.assertEqual(call_args.kwargs["url"], "https://api.minimax.io/v1/chat/completions") + + @patch("httpx.AsyncClient") + def test_oa_completion_sends_model_name(self, mock_client_cls): + from chatlm import oa_completion + mock_client = AsyncMock() + mock_response = MagicMock() + mock_response.json.return_value = { + "choices": [{"message": {"content": "ok"}}] + } + mock_client.post.return_value = mock_response + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=None) + mock_client_cls.return_value = mock_client + + asyncio.run(oa_completion( + url="https://api.minimax.io/v1/chat/completions", + headers={"Authorization": "Bearer test"}, + model="MiniMax-M2.7-highspeed", + messages=[{"role": "user", "content": "test"}], + max_tokens=50, + temperature=0.01, + )) + call_args = mock_client.post.call_args + sent_json = call_args.kwargs["json"] + self.assertEqual(sent_json["model"], "MiniMax-M2.7-highspeed") + self.assertEqual(sent_json["temperature"], 0.01) + + +class TestMiniMaxIntegration(unittest.TestCase): + """Integration tests (require MINIMAX_API_KEY).""" + + @unittest.skipUnless( + os.environ.get("MINIMAX_API_KEY"), + "MINIMAX_API_KEY not set, skipping integration test", + ) + @patch("transformers.GPT2TokenizerFast.from_pretrained") + def test_minimax_greedy_until_live(self, mock_tokenizer): + """Test MiniMaxLM.greedy_until with live API.""" + mock_tok = MagicMock() + mock_tok.encode.return_value = [1, 2, 3] + mock_tok.decode.return_value = "test" + mock_tok.eos_token_id = 2 + mock_tokenizer.return_value = mock_tok + + from minimax_lm import MiniMaxLM + from chatlm import oa_completion + lm = MiniMaxLM("MiniMax-M2.7") + result = asyncio.run(oa_completion( + url=MiniMaxLM.API_BASE_URL, + headers=lm.headers, + model="MiniMax-M2.7", + messages=[{"role": "user", "content": "What is 1+1? Answer briefly."}], + max_tokens=32, + temperature=lm._get_temperature(0.0), + )) + self.assertEqual(len(result), 1) + self.assertIsInstance(result[0], str) + self.assertTrue(len(result[0]) > 0) + + @unittest.skipUnless( + os.environ.get("MINIMAX_API_KEY"), + "MINIMAX_API_KEY not set, skipping integration test", + ) + @patch("factscore_package.lm.LM.load_cache", return_value={}) + def test_minimax_factscore_live(self, mock_cache): + """Test MiniMaxModel._generate with live API.""" + from factscore_package.minimax_lm import MiniMaxModel + model = MiniMaxModel("MiniMax-M2.7", cache_file="/tmp/test_cache") + output, response = model._generate("What is 2+2?", max_output_length=32) + self.assertIsInstance(output, str) + self.assertTrue(len(output) > 0) + + @unittest.skipUnless( + os.environ.get("MINIMAX_API_KEY"), + "MINIMAX_API_KEY not set, skipping integration test", + ) + @patch("factscore_package.lm.LM.load_cache", return_value={}) + def test_minimax_highspeed_model_live(self, mock_cache): + """Test MiniMax-M2.7-highspeed model.""" + from factscore_package.minimax_lm import MiniMaxModel + model = MiniMaxModel("MiniMax-M2.7-highspeed", cache_file="/tmp/test_cache") + output, response = model._generate("Say hello", max_output_length=16) + self.assertIsInstance(output, str) + self.assertTrue(len(output) > 0) + + +if __name__ == "__main__": + unittest.main()