From 8360a9409742abe601e58c54e4d1b2bedccf7bfd Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Thu, 19 Nov 2026 16:36:03 +0800 Subject: [PATCH 1/2] feat(benchmark): add Qwen-Image-Bench with Q-Judger --- docs/en/benchmarks/qwen_image_bench.md | 165 ++++++++++++++ docs/en/get_started/supported_dataset/aigc.md | 2 + docs/zh/benchmarks/qwen_image_bench.md | 160 +++++++++++++ docs/zh/get_started/supported_dataset/aigc.md | 2 + evalscope/api/judge/contracts.py | 4 + evalscope/api/judge/executor.py | 4 +- evalscope/api/judge/types.py | 3 + evalscope/benchmarks/_index.json | 1 + .../benchmarks/_meta/qwen_image_bench.json | 104 +++++++++ .../benchmarks/qwen_image_bench/__init__.py | 1 + .../benchmarks/qwen_image_bench/checklists.py | 132 +++++++++++ .../qwen_image_bench_adapter.py | 211 ++++++++++++++++++ .../benchmarks/qwen_image_bench/utils.py | 106 +++++++++ evalscope/metrics/semantics/catalog.py | 5 + tests/api/judge/test_migrated_adapters.py | 76 +++++++ 15 files changed, 975 insertions(+), 1 deletion(-) create mode 100644 docs/en/benchmarks/qwen_image_bench.md create mode 100644 docs/zh/benchmarks/qwen_image_bench.md create mode 100644 evalscope/benchmarks/_meta/qwen_image_bench.json create mode 100644 evalscope/benchmarks/qwen_image_bench/__init__.py create mode 100644 evalscope/benchmarks/qwen_image_bench/checklists.py create mode 100644 evalscope/benchmarks/qwen_image_bench/qwen_image_bench_adapter.py create mode 100644 evalscope/benchmarks/qwen_image_bench/utils.py diff --git a/docs/en/benchmarks/qwen_image_bench.md b/docs/en/benchmarks/qwen_image_bench.md new file mode 100644 index 000000000..abc2c2a49 --- /dev/null +++ b/docs/en/benchmarks/qwen_image_bench.md @@ -0,0 +1,165 @@ +# Qwen-Image-Bench + + +## Overview + +Qwen-Image-Bench evaluates text-to-image models on professional creative tasks using the official Q-Judger. + +## Task Description + +- **Task Type**: Text-to-image generation and evaluation +- **Input**: A Chinese or English creative prompt +- **Output**: An image evaluated against applicable rubrics +- **Domain**: Image quality, aesthetics, alignment, real-world fidelity, and creative generation + +## Key Features + +- 1,000 bilingual prompts with 56 facets grouped into five dimensions. +- Supports both image generation and evaluation of existing images via a local JSONL manifest. + +## Evaluation Notes + +- Set `judge.models` to an OpenAI-compatible vision endpoint serving the official `Qwen/Qwen-Image-Bench` + weights. Serve with thinking enabled and reasoning separated from the final JSON. Use temperature 0, + top-k 1, top-p 1, repetition penalty 1.05, seed 42, and a 4,096-token generation limit for official settings. +- Set `extra_params.language` to `cn` (default) or `en`. Each run reports scores for the selected language. +- For existing images, use a local JSONL with `ID`, `dims_en`, `prompt` (or the selected official + `prompt_cn`/`prompt_en` field), and an absolute `image_path`. Set `model_task='image_generation'` + and leave `model` unset to score these images. +- Grades 0, 1, and 2 map to 0, 60, and 100. Inapplicable facets and failed requests are excluded from the + corresponding averages. Each image is averaged across its scored dimensions before averaging images. +- The overall score, dimensions, and subdimensions report their respective valid image counts. + +This configuration scores existing images against the English prompts. + +```python +from evalscope import TaskConfig, run_task + +run_task(TaskConfig( + model_id='my-image-model', + model_task='image_generation', + datasets=['qwen_image_bench'], + dataset_args={'qwen_image_bench': { + 'dataset_id': '/absolute/path/to/manifest.jsonl', + 'extra_params': {'language': 'en'}, + }}, + judge={'models': [{ + 'model_id': 'Qwen/Qwen-Image-Bench', + 'api_url': 'http://127.0.0.1:8000/v1', + 'generation_config': { + 'temperature': 0, + 'top_p': 1, + 'seed': 42, + 'max_tokens': 4096, + 'extra_body': { + 'top_k': 1, + 'repetition_penalty': 1.05, + 'chat_template_kwargs': {'enable_thinking': True}, + }, + }, + }]}, +)) +``` + + +## Properties + +| Property | Value | +|----------|-------| +| **Benchmark Name** | `qwen_image_bench` | +| **Dataset ID** | [Qwen/Qwen-Image-Bench](https://modelscope.cn/datasets/Qwen/Qwen-Image-Bench/summary) | +| **Paper** | [Paper](https://arxiv.org/abs/2605.28091) | +| **Tags** | `TextToImage` | +| **Metrics** | `qwen_image_bench_score` | +| **Default Shots** | 0-shot | +| **Evaluation Split** | `test` | + + +## Data Statistics + +| Metric | Value | +|--------|-------| +| Total Samples | 1,000 | +| Prompt Length (Mean) | 83.05 chars | +| Prompt Length (Min/Max) | 25 / 317 chars | + +## Sample Example + +**Subset**: `default` + +```json +{ + "input": [ + { + "id": "b8c6bf0d", + "content": "一只拆开的机械手表放在黑色工作台上,制表师用镊子夹起齿轮,特写镊子夹起齿轮,银色与暗灰色调,零部件结构精密,金属边缘细节清晰。" + } + ], + "target": "", + "id": 0, + "group_id": 0, + "metadata": { + "id": 1, + "prompt": "一只拆开的机械手表放在黑色工作台上,制表师用镊子夹起齿轮,特写镊子夹起齿轮,银色与暗灰色调,零部件结构精密,金属边缘细节清晰。", + "dimensions": [ + "Quality", + "Aesthetics", + "Alignment" + ], + "image_path": "", + "category": "", + "tags": [] + } +} +``` + +## Prompt Template + +*No prompt template defined.* + +## Extra Parameters + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `language` | `str` | `cn` | Language of the generation prompts (cn or en). Choices: ['cn', 'en'] | + +## Usage + +### Using CLI + +```bash +evalscope eval \ + --model YOUR_MODEL \ + --model-task image_generation \ + --eval-type text2image \ + --api-url http://127.0.0.1:18131/v1 \ + --api-key EMPTY_TOKEN \ + --datasets qwen_image_bench \ + --judge '{"strategy":"llm","models":[{"model_id":"YOUR_JUDGE_MODEL"}]}' \ + --limit 10 # Remove this line for formal evaluation +``` + +### Using Python + +```python +from evalscope import run_task +from evalscope.config import TaskConfig + +task_cfg = TaskConfig( + model='YOUR_MODEL', + eval_type='text2image', + model_task='image_generation', + api_url='http://127.0.0.1:18131/v1', + api_key='EMPTY_TOKEN', + datasets=['qwen_image_bench'], + judge={'strategy': 'llm', 'models': [{'model_id': 'YOUR_JUDGE_MODEL'}]}, + dataset_args={ + 'qwen_image_bench': { + # extra_params: {} # uses default extra parameters + } + }, + limit=10, # Remove this line for formal evaluation +) + +run_task(task_cfg=task_cfg) +``` diff --git a/docs/en/get_started/supported_dataset/aigc.md b/docs/en/get_started/supported_dataset/aigc.md index c269edd0e..36faffcb4 100644 --- a/docs/en/get_started/supported_dataset/aigc.md +++ b/docs/en/get_started/supported_dataset/aigc.md @@ -11,6 +11,7 @@ Below is the list of supported AIGC benchmarks. Click on a benchmark name for de | `general_t2i` | [General-T2I](../../benchmarks/general_t2i.md) | `Custom`, `TextToImage` | | `hpdv2` | [HPD-v2](../../benchmarks/hpdv2.md) | `TextToImage` | | `longtext_bench` | [LongText-Bench](../../benchmarks/longtext_bench.md) | `TextToImage` | +| `qwen_image_bench` | [Qwen-Image-Bench](../../benchmarks/qwen_image_bench.md) | `TextToImage` | | `tifa160` | [TIFA-160](../../benchmarks/tifa160.md) | `TextToImage` | :::{toctree} @@ -24,5 +25,6 @@ Below is the list of supported AIGC benchmarks. Click on a benchmark name for de ../../benchmarks/general_t2i.md ../../benchmarks/hpdv2.md ../../benchmarks/longtext_bench.md +../../benchmarks/qwen_image_bench.md ../../benchmarks/tifa160.md ::: diff --git a/docs/zh/benchmarks/qwen_image_bench.md b/docs/zh/benchmarks/qwen_image_bench.md new file mode 100644 index 000000000..a2229df05 --- /dev/null +++ b/docs/zh/benchmarks/qwen_image_bench.md @@ -0,0 +1,160 @@ +# Qwen-Image-Bench + + +## 概述 + +Qwen-Image-Bench 使用官方 Q-Judger 评估文本到图像模型在专业创意任务上的表现。 + +## 任务描述 + +- **任务类型**:文本到图像生成与评估 +- **输入**:中文或英文创意提示词 +- **输出**:根据适用评分标准进行评估的图像 +- **领域**:图像质量、美学、对齐度、真实世界保真度及创意生成 + +## 主要特点 + +- 包含 1,000 条双语提示词,涵盖归入五个维度的 56 个细分项。 +- 支持通过本地 JSONL 清单进行图像生成以及现有图像的评估。 + +## 评估说明 + +- 将 `judge.models` 设置为提供官方 `Qwen/Qwen-Image-Bench` 权重的 OpenAI 兼容视觉端点。启用思考模式,并将推理过程与最终 JSON 分离。官方设置下使用温度 0、top-k 1、top-p 1、重复惩罚 1.05、种子 42 以及 4,096 token 的生成限制。 +- 将 `extra_params.language` 设置为 `cn`(默认)或 `en`。每次运行报告所选语言的分数。 +- 对于现有图像,请使用包含 `ID`、`dims_en`、`prompt`(或选定的官方 `prompt_cn`/`prompt_en` 字段)以及绝对路径 `image_path` 的本地 JSONL 文件。设置 `model_task='image_generation'` 并保持 `model` 未设置以对这些图像进行评分。 +- 等级 0、1 和 2 分别映射为 0、60 和 100 分。不适用的细分项和失败的请求将从相应的平均值中排除。每张图像先在其得分维度上取平均,然后再对图像进行平均。 +- 总体分数、各维度及各子维度均报告其各自的有效图像数量。 + +此配置针对英文提示词对现有图像进行评分。 + +```python +from evalscope import TaskConfig, run_task + +run_task(TaskConfig( + model_id='my-image-model', + model_task='image_generation', + datasets=['qwen_image_bench'], + dataset_args={'qwen_image_bench': { + 'dataset_id': '/absolute/path/to/manifest.jsonl', + 'extra_params': {'language': 'en'}, + }}, + judge={'models': [{ + 'model_id': 'Qwen/Qwen-Image-Bench', + 'api_url': 'http://127.0.0.1:8000/v1', + 'generation_config': { + 'temperature': 0, + 'top_p': 1, + 'seed': 42, + 'max_tokens': 4096, + 'extra_body': { + 'top_k': 1, + 'repetition_penalty': 1.05, + 'chat_template_kwargs': {'enable_thinking': True}, + }, + }, + }]}, +)) +``` + + +## 属性 + +| 属性 | 值 | +|----------|-------| +| **基准测试名称** | `qwen_image_bench` | +| **数据集ID** | [Qwen/Qwen-Image-Bench](https://modelscope.cn/datasets/Qwen/Qwen-Image-Bench/summary) | +| **论文** | [Paper](https://arxiv.org/abs/2605.28091) | +| **标签** | `TextToImage` | +| **指标** | `qwen_image_bench_score` | +| **默认示例数** | 0-shot | +| **评估划分** | `test` | + + +## 数据统计 + +| 指标 | 值 | +|--------|-------| +| 总样本数 | 1,000 | +| 提示词长度(均值) | 83.05 字符 | +| 提示词长度(最小/最大) | 25 / 317 字符 | + +## 样例示例 + +**子集**: `default` + +```json +{ + "input": [ + { + "id": "b8c6bf0d", + "content": "一只拆开的机械手表放在黑色工作台上,制表师用镊子夹起齿轮,特写镊子夹起齿轮,银色与暗灰色调,零部件结构精密,金属边缘细节清晰。" + } + ], + "target": "", + "id": 0, + "group_id": 0, + "metadata": { + "id": 1, + "prompt": "一只拆开的机械手表放在黑色工作台上,制表师用镊子夹起齿轮,特写镊子夹起齿轮,银色与暗灰色调,零部件结构精密,金属边缘细节清晰。", + "dimensions": [ + "Quality", + "Aesthetics", + "Alignment" + ], + "image_path": "", + "category": "", + "tags": [] + } +} +``` + +## 提示模板 + +*未定义提示模板。* + +## 额外参数 + +| 参数 | 类型 | 默认值 | 描述 | +|-----------|------|---------|-------------| +| `language` | `str` | `cn` | 生成提示词的语言(cn 或 en)。选项:['cn', 'en'] | + +## 使用方法 + +### 使用 CLI + +```bash +evalscope eval \ + --model YOUR_MODEL \ + --model-task image_generation \ + --eval-type text2image \ + --api-url http://127.0.0.1:18131/v1 \ + --api-key EMPTY_TOKEN \ + --datasets qwen_image_bench \ + --judge '{"strategy":"llm","models":[{"model_id":"YOUR_JUDGE_MODEL"}]}' \ + --limit 10 # 正式评估时请删除此行 +``` + +### 使用 Python + +```python +from evalscope import run_task +from evalscope.config import TaskConfig + +task_cfg = TaskConfig( + model='YOUR_MODEL', + eval_type='text2image', + model_task='image_generation', + api_url='http://127.0.0.1:18131/v1', + api_key='EMPTY_TOKEN', + datasets=['qwen_image_bench'], + judge={'strategy': 'llm', 'models': [{'model_id': 'YOUR_JUDGE_MODEL'}]}, + dataset_args={ + 'qwen_image_bench': { + # extra_params: {} # uses default extra parameters + } + }, + limit=10, # 正式评估时请删除此行 +) + +run_task(task_cfg=task_cfg) +``` diff --git a/docs/zh/get_started/supported_dataset/aigc.md b/docs/zh/get_started/supported_dataset/aigc.md index 2798b1d5e..3c3ecb689 100644 --- a/docs/zh/get_started/supported_dataset/aigc.md +++ b/docs/zh/get_started/supported_dataset/aigc.md @@ -11,6 +11,7 @@ | `general_t2i` | [General-T2I](../../benchmarks/general_t2i.md) | `Custom`, `TextToImage` | | `hpdv2` | [HPD-v2](../../benchmarks/hpdv2.md) | `TextToImage` | | `longtext_bench` | [LongText-Bench](../../benchmarks/longtext_bench.md) | `TextToImage` | +| `qwen_image_bench` | [Qwen-Image-Bench](../../benchmarks/qwen_image_bench.md) | `TextToImage` | | `tifa160` | [TIFA-160](../../benchmarks/tifa160.md) | `TextToImage` | :::{toctree} @@ -24,5 +25,6 @@ ../../benchmarks/general_t2i.md ../../benchmarks/hpdv2.md ../../benchmarks/longtext_bench.md +../../benchmarks/qwen_image_bench.md ../../benchmarks/tifa160.md ::: diff --git a/evalscope/api/judge/contracts.py b/evalscope/api/judge/contracts.py index d46e9b4e2..8ee18fcab 100644 --- a/evalscope/api/judge/contracts.py +++ b/evalscope/api/judge/contracts.py @@ -37,6 +37,7 @@ class OutputContract(BaseModel): model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) schema_model: Type[BaseModel] + allow_json_fences: bool = False def instruction(self) -> str: """The format requirement appended to the benchmark's own prompt.""" @@ -56,6 +57,9 @@ def parse(self, response: str) -> ParseResult: if not response or not response.strip(): return ParseResult.failure('empty response') + if self.allow_json_fences: + response = response.strip().removeprefix('```json\n').removeprefix('```\n').removesuffix('```').strip() + try: data = json.loads( response, diff --git a/evalscope/api/judge/executor.py b/evalscope/api/judge/executor.py index a90c03324..30b277374 100644 --- a/evalscope/api/judge/executor.py +++ b/evalscope/api/judge/executor.py @@ -74,6 +74,8 @@ def _run_observation( for case in pending: verdict = self._run_case(definition, context, judge, repeat_id, case, completed, review) if verdict is None: + if not case.required: + continue observation.status = ScoreStatus.INVALID_SESSION observation.error = f'case {case.case_id} produced no usable verdict' return observation @@ -274,7 +276,7 @@ def _aggregate(self, review: JudgeReview) -> None: if tie_broken: review.metadata['tie_broken_by_primary'] = True review.status = ScoreStatus.DEGRADED - elif len(review.valid_observations) != expected or unresolved_ties or repeat_ties: + elif len(review.valid_observations) != expected or review.failure_counts or unresolved_ties or repeat_ties: review.status = ScoreStatus.DEGRADED def _aggregate_rule_fallback(self, review: JudgeReview) -> None: diff --git a/evalscope/api/judge/types.py b/evalscope/api/judge/types.py index 7979e0e03..52ebc8b3a 100644 --- a/evalscope/api/judge/types.py +++ b/evalscope/api/judge/types.py @@ -45,6 +45,9 @@ class JudgeCase(BaseModel): output_contract: OutputContract """Declares the verdict shape and is the only thing allowed to parse the response.""" + required: bool = True + """Allow other independent cases to contribute if this case fails.""" + metadata: Dict[str, Any] = Field(default_factory=dict) """Adapter-defined data carried through the request callback and onto the verdict.""" diff --git a/evalscope/benchmarks/_index.json b/evalscope/benchmarks/_index.json index 407760a17..739c51888 100644 --- a/evalscope/benchmarks/_index.json +++ b/evalscope/benchmarks/_index.json @@ -220,6 +220,7 @@ "process_bench": "evalscope.benchmarks.process_bench.process_bench_adapter", "pubmedqa": "evalscope.benchmarks.pumed_qa.pubmed_qa_adapter", "qasc": "evalscope.benchmarks.qasc.qasc_adapter", + "qwen_image_bench": "evalscope.benchmarks.qwen_image_bench.qwen_image_bench_adapter", "race": "evalscope.benchmarks.race.race_adapter", "real_world_qa": "evalscope.benchmarks.real_world_qa.real_world_qa_adapter", "ref_adv_s": "evalscope.benchmarks.ref_adv_s.ref_adv_s_adapter", diff --git a/evalscope/benchmarks/_meta/qwen_image_bench.json b/evalscope/benchmarks/_meta/qwen_image_bench.json new file mode 100644 index 000000000..78554646e --- /dev/null +++ b/evalscope/benchmarks/_meta/qwen_image_bench.json @@ -0,0 +1,104 @@ +{ + "meta": { + "pretty_name": "Qwen-Image-Bench", + "dataset_id": "Qwen/Qwen-Image-Bench", + "paper_url": "https://arxiv.org/abs/2605.28091", + "tags": [ + "TextToImage" + ], + "metrics": [ + "qwen_image_bench_score" + ], + "few_shot_num": 0, + "eval_split": "test", + "train_split": "", + "subset_list": [ + "default" + ], + "description": "\n## Overview\n\nQwen-Image-Bench evaluates text-to-image models on professional creative tasks using the official Q-Judger.\n\n## Task Description\n\n- **Task Type**: Text-to-image generation and evaluation\n- **Input**: A Chinese or English creative prompt\n- **Output**: An image evaluated against applicable rubrics\n- **Domain**: Image quality, aesthetics, alignment, real-world fidelity, and creative generation\n\n## Key Features\n\n- 1,000 bilingual prompts with 56 facets grouped into five dimensions.\n- Supports both image generation and evaluation of existing images via a local JSONL manifest.\n\n## Evaluation Notes\n\n- Set `judge.models` to an OpenAI-compatible vision endpoint serving the official `Qwen/Qwen-Image-Bench`\n weights. Serve with thinking enabled and reasoning separated from the final JSON. Use temperature 0,\n top-k 1, top-p 1, repetition penalty 1.05, seed 42, and a 4,096-token generation limit for official settings.\n- Set `extra_params.language` to `cn` (default) or `en`. Each run reports scores for the selected language.\n- For existing images, use a local JSONL with `ID`, `dims_en`, `prompt` (or the selected official\n `prompt_cn`/`prompt_en` field), and an absolute `image_path`. Set `model_task='image_generation'`\n and leave `model` unset to score these images.\n- Grades 0, 1, and 2 map to 0, 60, and 100. Inapplicable facets and failed requests are excluded from the\n corresponding averages. Each image is averaged across its scored dimensions before averaging images.\n- The overall score, dimensions, and subdimensions report their respective valid image counts.\n\nThis configuration scores existing images against the English prompts.\n\n```python\nfrom evalscope import TaskConfig, run_task\n\nrun_task(TaskConfig(\n model_id='my-image-model',\n model_task='image_generation',\n datasets=['qwen_image_bench'],\n dataset_args={'qwen_image_bench': {\n 'dataset_id': '/absolute/path/to/manifest.jsonl',\n 'extra_params': {'language': 'en'},\n }},\n judge={'models': [{\n 'model_id': 'Qwen/Qwen-Image-Bench',\n 'api_url': 'http://127.0.0.1:8000/v1',\n 'generation_config': {\n 'temperature': 0,\n 'top_p': 1,\n 'seed': 42,\n 'max_tokens': 4096,\n 'extra_body': {\n 'top_k': 1,\n 'repetition_penalty': 1.05,\n 'chat_template_kwargs': {'enable_thinking': True},\n },\n },\n }]},\n))\n```\n", + "prompt_template": "", + "system_prompt": "", + "few_shot_prompt_template": "", + "aggregation": "mean", + "extra_params": { + "language": { + "type": "str", + "description": "Language of the generation prompts (cn or en).", + "value": "cn", + "choices": [ + "cn", + "en" + ] + } + }, + "sandbox_config": {}, + "category": "aigc", + "primary_metric": { + "name": "qwen_image_bench_score", + "aggregation": "mean", + "dimensions": { + "scope": "overall" + } + }, + "requires_judge": true, + "eval_type": "text2image", + "requires_local_dataset": false + }, + "statistics": { + "total_samples": 1000, + "subset_stats": [ + { + "name": "default", + "sample_count": 1000, + "prompt_length_mean": 83.05, + "prompt_length_min": 25, + "prompt_length_max": 317, + "prompt_length_std": 42.81, + "target_length_mean": null + } + ], + "prompt_length": { + "mean": 83.05, + "min": 25, + "max": 317, + "std": 42.81 + }, + "target_length_mean": null, + "computed_at": "2026-11-19T16:34:27.702465" + }, + "sample_example": { + "data": { + "input": [ + { + "id": "b8c6bf0d", + "content": "一只拆开的机械手表放在黑色工作台上,制表师用镊子夹起齿轮,特写镊子夹起齿轮,银色与暗灰色调,零部件结构精密,金属边缘细节清晰。" + } + ], + "target": "", + "id": 0, + "group_id": 0, + "metadata": { + "id": 1, + "prompt": "一只拆开的机械手表放在黑色工作台上,制表师用镊子夹起齿轮,特写镊子夹起齿轮,银色与暗灰色调,零部件结构精密,金属边缘细节清晰。", + "dimensions": [ + "Quality", + "Aesthetics", + "Alignment" + ], + "image_path": "", + "category": "", + "tags": [] + } + }, + "subset": "default", + "truncated": false + }, + "readme": { + "en": "# Qwen-Image-Bench\n\n\n## Overview\n\nQwen-Image-Bench evaluates text-to-image models on professional creative tasks using the official Q-Judger.\n\n## Task Description\n\n- **Task Type**: Text-to-image generation and evaluation\n- **Input**: A Chinese or English creative prompt\n- **Output**: An image evaluated against applicable rubrics\n- **Domain**: Image quality, aesthetics, alignment, real-world fidelity, and creative generation\n\n## Key Features\n\n- 1,000 bilingual prompts with 56 facets grouped into five dimensions.\n- Supports both image generation and evaluation of existing images via a local JSONL manifest.\n\n## Evaluation Notes\n\n- Set `judge.models` to an OpenAI-compatible vision endpoint serving the official `Qwen/Qwen-Image-Bench`\n weights. Serve with thinking enabled and reasoning separated from the final JSON. Use temperature 0,\n top-k 1, top-p 1, repetition penalty 1.05, seed 42, and a 4,096-token generation limit for official settings.\n- Set `extra_params.language` to `cn` (default) or `en`. Each run reports scores for the selected language.\n- For existing images, use a local JSONL with `ID`, `dims_en`, `prompt` (or the selected official\n `prompt_cn`/`prompt_en` field), and an absolute `image_path`. Set `model_task='image_generation'`\n and leave `model` unset to score these images.\n- Grades 0, 1, and 2 map to 0, 60, and 100. Inapplicable facets and failed requests are excluded from the\n corresponding averages. Each image is averaged across its scored dimensions before averaging images.\n- The overall score, dimensions, and subdimensions report their respective valid image counts.\n\nThis configuration scores existing images against the English prompts.\n\n```python\nfrom evalscope import TaskConfig, run_task\n\nrun_task(TaskConfig(\n model_id='my-image-model',\n model_task='image_generation',\n datasets=['qwen_image_bench'],\n dataset_args={'qwen_image_bench': {\n 'dataset_id': '/absolute/path/to/manifest.jsonl',\n 'extra_params': {'language': 'en'},\n }},\n judge={'models': [{\n 'model_id': 'Qwen/Qwen-Image-Bench',\n 'api_url': 'http://127.0.0.1:8000/v1',\n 'generation_config': {\n 'temperature': 0,\n 'top_p': 1,\n 'seed': 42,\n 'max_tokens': 4096,\n 'extra_body': {\n 'top_k': 1,\n 'repetition_penalty': 1.05,\n 'chat_template_kwargs': {'enable_thinking': True},\n },\n },\n }]},\n))\n```\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `qwen_image_bench` |\n| **Dataset ID** | [Qwen/Qwen-Image-Bench](https://modelscope.cn/datasets/Qwen/Qwen-Image-Bench/summary) |\n| **Paper** | [Paper](https://arxiv.org/abs/2605.28091) |\n| **Tags** | `TextToImage` |\n| **Metrics** | `qwen_image_bench_score` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 1,000 |\n| Prompt Length (Mean) | 83.05 chars |\n| Prompt Length (Min/Max) | 25 / 317 chars |\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"b8c6bf0d\",\n \"content\": \"一只拆开的机械手表放在黑色工作台上,制表师用镊子夹起齿轮,特写镊子夹起齿轮,银色与暗灰色调,零部件结构精密,金属边缘细节清晰。\"\n }\n ],\n \"target\": \"\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"id\": 1,\n \"prompt\": \"一只拆开的机械手表放在黑色工作台上,制表师用镊子夹起齿轮,特写镊子夹起齿轮,银色与暗灰色调,零部件结构精密,金属边缘细节清晰。\",\n \"dimensions\": [\n \"Quality\",\n \"Aesthetics\",\n \"Alignment\"\n ],\n \"image_path\": \"\",\n \"category\": \"\",\n \"tags\": []\n }\n}\n```\n\n## Prompt Template\n\n*No prompt template defined.*\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `language` | `str` | `cn` | Language of the generation prompts (cn or en). Choices: ['cn', 'en'] |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --model-task image_generation \\\n --eval-type text2image \\\n --api-url http://127.0.0.1:18131/v1 \\\n --api-key EMPTY_TOKEN \\\n --datasets qwen_image_bench \\\n --judge '{\"strategy\":\"llm\",\"models\":[{\"model_id\":\"YOUR_JUDGE_MODEL\"}]}' \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n eval_type='text2image',\n model_task='image_generation',\n api_url='http://127.0.0.1:18131/v1',\n api_key='EMPTY_TOKEN',\n datasets=['qwen_image_bench'],\n judge={'strategy': 'llm', 'models': [{'model_id': 'YOUR_JUDGE_MODEL'}]},\n dataset_args={\n 'qwen_image_bench': {\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# Qwen-Image-Bench\n\n\n## 概述\n\nQwen-Image-Bench 使用官方 Q-Judger 评估文本到图像模型在专业创意任务上的表现。\n\n## 任务描述\n\n- **任务类型**:文本到图像生成与评估\n- **输入**:中文或英文创意提示词\n- **输出**:根据适用评分标准进行评估的图像\n- **领域**:图像质量、美学、对齐度、真实世界保真度及创意生成\n\n## 主要特点\n\n- 包含 1,000 条双语提示词,涵盖归入五个维度的 56 个细分项。\n- 支持通过本地 JSONL 清单进行图像生成以及现有图像的评估。\n\n## 评估说明\n\n- 将 `judge.models` 设置为提供官方 `Qwen/Qwen-Image-Bench` 权重的 OpenAI 兼容视觉端点。启用思考模式,并将推理过程与最终 JSON 分离。官方设置下使用温度 0、top-k 1、top-p 1、重复惩罚 1.05、种子 42 以及 4,096 token 的生成限制。\n- 将 `extra_params.language` 设置为 `cn`(默认)或 `en`。每次运行报告所选语言的分数。\n- 对于现有图像,请使用包含 `ID`、`dims_en`、`prompt`(或选定的官方 `prompt_cn`/`prompt_en` 字段)以及绝对路径 `image_path` 的本地 JSONL 文件。设置 `model_task='image_generation'` 并保持 `model` 未设置以对这些图像进行评分。\n- 等级 0、1 和 2 分别映射为 0、60 和 100 分。不适用的细分项和失败的请求将从相应的平均值中排除。每张图像先在其得分维度上取平均,然后再对图像进行平均。\n- 总体分数、各维度及各子维度均报告其各自的有效图像数量。\n\n此配置针对英文提示词对现有图像进行评分。\n\n```python\nfrom evalscope import TaskConfig, run_task\n\nrun_task(TaskConfig(\n model_id='my-image-model',\n model_task='image_generation',\n datasets=['qwen_image_bench'],\n dataset_args={'qwen_image_bench': {\n 'dataset_id': '/absolute/path/to/manifest.jsonl',\n 'extra_params': {'language': 'en'},\n }},\n judge={'models': [{\n 'model_id': 'Qwen/Qwen-Image-Bench',\n 'api_url': 'http://127.0.0.1:8000/v1',\n 'generation_config': {\n 'temperature': 0,\n 'top_p': 1,\n 'seed': 42,\n 'max_tokens': 4096,\n 'extra_body': {\n 'top_k': 1,\n 'repetition_penalty': 1.05,\n 'chat_template_kwargs': {'enable_thinking': True},\n },\n },\n }]},\n))\n```\n\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `qwen_image_bench` |\n| **数据集ID** | [Qwen/Qwen-Image-Bench](https://modelscope.cn/datasets/Qwen/Qwen-Image-Bench/summary) |\n| **论文** | [Paper](https://arxiv.org/abs/2605.28091) |\n| **标签** | `TextToImage` |\n| **指标** | `qwen_image_bench_score` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 1,000 |\n| 提示词长度(均值) | 83.05 字符 |\n| 提示词长度(最小/最大) | 25 / 317 字符 |\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"b8c6bf0d\",\n \"content\": \"一只拆开的机械手表放在黑色工作台上,制表师用镊子夹起齿轮,特写镊子夹起齿轮,银色与暗灰色调,零部件结构精密,金属边缘细节清晰。\"\n }\n ],\n \"target\": \"\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"id\": 1,\n \"prompt\": \"一只拆开的机械手表放在黑色工作台上,制表师用镊子夹起齿轮,特写镊子夹起齿轮,银色与暗灰色调,零部件结构精密,金属边缘细节清晰。\",\n \"dimensions\": [\n \"Quality\",\n \"Aesthetics\",\n \"Alignment\"\n ],\n \"image_path\": \"\",\n \"category\": \"\",\n \"tags\": []\n }\n}\n```\n\n## 提示模板\n\n*未定义提示模板。*\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `language` | `str` | `cn` | 生成提示词的语言(cn 或 en)。选项:['cn', 'en'] |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --model-task image_generation \\\n --eval-type text2image \\\n --api-url http://127.0.0.1:18131/v1 \\\n --api-key EMPTY_TOKEN \\\n --datasets qwen_image_bench \\\n --judge '{\"strategy\":\"llm\",\"models\":[{\"model_id\":\"YOUR_JUDGE_MODEL\"}]}' \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n eval_type='text2image',\n model_task='image_generation',\n api_url='http://127.0.0.1:18131/v1',\n api_key='EMPTY_TOKEN',\n datasets=['qwen_image_bench'],\n judge={'strategy': 'llm', 'models': [{'model_id': 'YOUR_JUDGE_MODEL'}]},\n dataset_args={\n 'qwen_image_bench': {\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "8fe00d7143344b96994fff0e927045de", + "needs_translation": false + }, + "updated_at": "2026-11-19T16:34:28.688163", + "translation_updated_at": "2026-11-19T16:34:30" +} diff --git a/evalscope/benchmarks/qwen_image_bench/__init__.py b/evalscope/benchmarks/qwen_image_bench/__init__.py new file mode 100644 index 000000000..cefeea27d --- /dev/null +++ b/evalscope/benchmarks/qwen_image_bench/__init__.py @@ -0,0 +1 @@ +"""Qwen-Image-Bench text-to-image evaluation.""" diff --git a/evalscope/benchmarks/qwen_image_bench/checklists.py b/evalscope/benchmarks/qwen_image_bench/checklists.py new file mode 100644 index 000000000..48bd17a1a --- /dev/null +++ b/evalscope/benchmarks/qwen_image_bench/checklists.py @@ -0,0 +1,132 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# Scoring prompts from QwenLM/Qwen-Image-Bench (Apache-2.0), revision 8ab1fb4. +# https://github.com/QwenLM/Qwen-Image-Bench/blob/8ab1fb4/checklists.py + +QUALITY_CHECKLIST = """## Realism +- Physical Logic: Does the image adhere to real-world physical laws (e.g., gravity, reflection, shadow direction, object stability)? +- Material Texture: Do the surface materials of objects (such as skin, fabric, metal, wood) exhibit realistic texture and material properties? +## Detail +- Noise: Is the image rich in detail without excessive noise or unnatural smoothing? +- Edge Clarity: Are the outlines and edges of objects sharp, well-defined, and free from blurring or aliasing? +- Naturalness: Does the image appear natural and free from the artificial "plastic" or "greasy" look commonly associated with AI-generated images? +## Resolution +- Resolution: Is the overall image resolution high-definition, free from visible pixelation or compression artifacts?""" + + +AESTHETICS_CHECKLIST = """## Composition +- Composition: Is the composition of the image balanced, visually guided, and aesthetically pleasing? +## Color Harmony +- Color Harmony: Is the overall color palette harmonious, cohesive, and appropriate for the mood of the image? +## Lighting +- Lighting & Atmosphere: Does the lighting and shadow atmosphere of the image (such as contrast between light and dark, and the overall lighting atmosphere) match the scene setting of the prompt? +## Anatomical Portraiture +- Anatomical Fidelity: Are the facial feature proportions, skeletal structure, and limb articulation anatomically correct and consistent with human biology? Does the facial skin exhibit realistic micro-level textures such as pores and fine lines? +## Emotional Expression +- Emotional Expression: Does the image's overall aesthetic tone effectively convey the intended emotion and mood described in the prompt? +## Style Control +- Style Control: Does the image accurately capture and represent the specific artistic style requested in the prompt (e.g., Van Gogh's brushwork, Cyberpunk aesthetic)?""" + +ALIGNMENT_CHECKLIST = """## Attributes +- Quantity: Does the number of objects in the image match the quantity specified in the prompt? +- Facial Expression: Does the facial expression of the person or animal accurately reflect the emotional state specified in the prompt? +- Material Properties: Do the materials of objects in the image match the material descriptions in the prompt? +- Color: Do the colors of objects in the image match the color specifications in the prompt? +- Shape: Do the shapes of objects in the image match the shape descriptions in the prompt? +- Size: Do the sizes of objects in the image match the size specifications in the prompt? +## Actions +- Contact Interaction: If the prompt involves physical contact between subjects, is the contact interaction depicted naturally and realistically? +- Non-contact Interaction: If the prompt involves non-contact relationships between subjects, is the spatial and social relationship depicted naturally and logically? +- Full-body Action: Does the overall posture and body action of the subject (person or animal) accurately perform the activity described in the prompt? +## Layout +- 2D Space: Are the relative positions of objects on the 2D plane (e.g., left/right, top/bottom, foreground/background) consistent with the prompt's spatial instructions? +- 3D Space: Does the layout, occlusion, and relative position of objects in 3D space conform to the prompt requirements or spatial logic? +## Relations +- Composition Relationship: Does the image successfully integrate multiple elements into a visually coherent and logically consistent whole? +- Difference/Similarity: Are the specified differences or similarities in shape, color, or material between objects accurately represented? +- Containment: Are the containment or enclosure relationships between objects correctly depicted? +## Scene +- Real-world Scene: Does the scene type and environmental setting (e.g., office, forest, street) match the location described in the prompt? +- Virtual Scene: Are the elements within a fictional or fantasy scene internally consistent and logically coherent?""" + +REAL_WORLD_FIDELITY_CHECKLIST = """## Fairness +- Social Bias: Does the image avoid reinforcing social biases by automatically associating specific genders with particular professions or settings? +- Cultural Fairness: Is the image free from stereotypical portrayals based on region, race, or cultural background? +## Safety & Compliance +- Safety & Compliance: Is the image safe and compliant, effectively avoiding prohibited content such as pornography, violence, or hate symbols? +## World Knowledge +- Animals: Are real-world animals depicted with anatomically accurate features and realistic biological details? +- Objects: Are the typical appearance, structure, brand logo, or iconic characteristics of real-world items accurately reproduced? +- Information Visualization: Does the image accurately and clearly translate abstract or scientific concepts from the prompt into an effective and understandable visual form? +- Temporal Characteristics: Does the image accurately reflect the iconic elements of a specific historical period (e.g., technology, clothing, architecture, lifestyle of that era)? +- Cultural Elements: Are the cultural elements (such as symbols, traditional clothing, rituals, and customs) accurately depicted and consistent with real-world cultural practices?""" + + +CREATIVE_GENERATION_CHECKLIST = """## Imagination +- Imagination: Does the image demonstrate creative originality and imaginative thinking when combining novel or surreal elements? +## Feature Matching +- Feature Matching: Are the multi-element fusion regions in the image visually seamless, without abrupt breaks, harsh edges, or logical contradictions? +## Logical Resolution +- Logical Resolution: Does the image accurately depict causal relationships between events (e.g., breaking glass → shards flying, rain → wet surfaces)? +## Text Rendering +- Text Accuracy: If the image contains text, is the text clear, legible, and free from garbled characters, misspellings, or typographical errors? +- Text Layout: Is the text layout (e.g., centering, alignment, line spacing, margins) in the image visually appealing and professionally structured? +- Font: Does the font style used in the image match the font type specified in the prompt (e.g., SimSun, Heiti, handwritten, serif)? +- Cross-lingual Generation: Does the image correctly follow the translation instructions in the prompt, producing accurate text in the target language? +## Design Applications +- Graphic Design: Does the graphic design (e.g., advertisement, poster) exhibit a clear information hierarchy, effective visual guidance, and professional layout? +- Product Design: Does the product design in the image demonstrate reasonable industrial design logic (e.g., ergonomic grip, logical interface placement, structural integrity)? +- Spatial Design: Does the interior or architectural space conform to the principles of perspective, proportion, and building design standards? +- Fashion Styling: Does the clothing cut and silhouette match the style described in the prompt (e.g., Hanfu, cyberpunk, haute couture)? Does the makeup style (e.g., smoky eyes, nude makeup, theatrical look) suit the occasion and character setting? +- Game Design: Do the game props and UI elements have practical in-game usability (e.g., icon recognizability, interactive affordances, clear feedback cues)? +- Art Design: Does the image successfully demonstrate the specific artistic design style required by the prompt (e.g., unique brushstrokes, distinctive color scheme, coherent artistic language)? +## Visual Storytelling +- Cinematic Style: Does the image reproduce the signature visual language of the specific director referenced in the prompt (e.g., Wes Anderson's symmetrical composition, Wong Kar-wai's warm color palette)? +- Camera / Lens Style: Does the image reflect the characteristic imaging effects of the specific photographic equipment or lens referenced in the prompt (e.g., film grain, bokeh, digital sharpening)? +- Storyboard Creation: Does the image's scene composition follow the panel layout requirements outlined in the prompt (e.g., three-panel, four-panel, split-screen)? +- Shot Sizes: Does the image meet the framing and shot size requirements specified in the prompt (e.g., close-up, medium shot, wide shot)? +- Composition: Does the image follow the specific composition rules required by the prompt (e.g., rule of thirds, golden ratio, leading lines)? +- Angles: Does the camera angle comply with the prompt's specification (e.g., bird's-eye view, low angle, Dutch angle)? +- Comic Creation: Does the image conform to the comic style required by the prompt (e.g., American comics, Japanese manga, European BD)?""" + + +DIM_TO_CHECKLIST = { + 'Quality': QUALITY_CHECKLIST, + 'Aesthetics': AESTHETICS_CHECKLIST, + 'Alignment': ALIGNMENT_CHECKLIST, + 'Real-world Fidelity': REAL_WORLD_FIDELITY_CHECKLIST, + 'Creative Generation': CREATIVE_GENERATION_CHECKLIST, +} + +SYSTEM_PROMPT = ( + 'You are an expert evaluator for text-to-image (T2I) generation quality. ' + 'Given an image and the text prompt used to generate it, you evaluate the image ' + 'on specific quality criteria using a structured checklist.' +) + +USER_PROMPT_TEMPLATE = """\ +# Text Prompt Used to Generate the Image +{prompt} + +# Generated Image + + +# Evaluation Dimension +{level1_dim} + +# Scoring Rules +- **0 (Fail)**: Clear defect present. Would noticeably reduce image quality. +- **1 (Pass)**: No defect. Meets baseline expectations. +- **2 (Excel)**: Exceptionally executed. Only when concrete excellence is observable. +- **N/A**: This criterion does not apply to this image/prompt. + +# Evaluation Checklist +{format_checklist} + +# Output Format +Respond with a valid JSON object only (no markdown code blocks): +{{ + "{{level2_dim}}": {{ + "{{level3_dim}}": {{"score": 0|1|2}}, + "{{level3_dim}}": {{"score": "N/A"}} + }} +}}""" diff --git a/evalscope/benchmarks/qwen_image_bench/qwen_image_bench_adapter.py b/evalscope/benchmarks/qwen_image_bench/qwen_image_bench_adapter.py new file mode 100644 index 000000000..50d1c7bce --- /dev/null +++ b/evalscope/benchmarks/qwen_image_bench/qwen_image_bench_adapter.py @@ -0,0 +1,211 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from pathlib import Path +from typing import Any, Sequence + +from evalscope.api.benchmark import BenchmarkMeta, Text2ImageAdapter +from evalscope.api.dataset import DataLoader, Dataset, LocalDataLoader, Sample, download_dataset_file +from evalscope.api.judge import ( + CaseVerdict, + JudgeCase, + JudgeContext, + JudgeDefinition, + JudgeRequest, + JudgeReview, + Placement, +) +from evalscope.api.messages import ChatMessageSystem, ChatMessageUser, ContentImage, ContentText +from evalscope.api.metric import MetricSelector, Score +from evalscope.api.registry import register_benchmark +from evalscope.constants import FileConstants, HubType, ScoreStatus, ScoringPolicy, Tags + +from .checklists import DIM_TO_CHECKLIST, SYSTEM_PROMPT, USER_PROMPT_TEMPLATE +from .utils import GRADE_CONTRACT, SCORE_NAME, prepare_judge_image, reduce_grades + +DATASET_FILE = 'qwen_image_bench_hf_v0518.jsonl' + + +@register_benchmark( + BenchmarkMeta( + name='qwen_image_bench', + pretty_name='Qwen-Image-Bench', + dataset_id='Qwen/Qwen-Image-Bench', + hf_dataset_id='Qwen/Qwen-Image-Bench', + hf_dataset_revision='d2493deb153b020cf169c7e3f57d15e4dd697038', + paper_url='https://arxiv.org/abs/2605.28091', + description=""" +## Overview + +Qwen-Image-Bench evaluates text-to-image models on professional creative tasks using the official Q-Judger. + +## Task Description + +- **Task Type**: Text-to-image generation and evaluation +- **Input**: A Chinese or English creative prompt +- **Output**: An image evaluated against applicable rubrics +- **Domain**: Image quality, aesthetics, alignment, real-world fidelity, and creative generation + +## Key Features + +- 1,000 bilingual prompts with 56 facets grouped into five dimensions. +- Supports both image generation and evaluation of existing images via a local JSONL manifest. + +## Evaluation Notes + +- Set `judge.models` to an OpenAI-compatible vision endpoint serving the official `Qwen/Qwen-Image-Bench` + weights. Serve with thinking enabled and reasoning separated from the final JSON. Use temperature 0, + top-k 1, top-p 1, repetition penalty 1.05, seed 42, and a 4,096-token generation limit for official settings. +- Set `extra_params.language` to `cn` (default) or `en`. Each run reports scores for the selected language. +- For existing images, use a local JSONL with `ID`, `dims_en`, `prompt` (or the selected official + `prompt_cn`/`prompt_en` field), and an absolute `image_path`. Set `model_task='image_generation'` + and leave `model` unset to score these images. +- Grades 0, 1, and 2 map to 0, 60, and 100. Inapplicable facets and failed requests are excluded from the + corresponding averages. Each image is averaged across its scored dimensions before averaging images. +- The overall score, dimensions, and subdimensions report their respective valid image counts. + +This configuration scores existing images against the English prompts. + +```python +from evalscope import TaskConfig, run_task + +run_task(TaskConfig( + model_id='my-image-model', + model_task='image_generation', + datasets=['qwen_image_bench'], + dataset_args={'qwen_image_bench': { + 'dataset_id': '/absolute/path/to/manifest.jsonl', + 'extra_params': {'language': 'en'}, + }}, + judge={'models': [{ + 'model_id': 'Qwen/Qwen-Image-Bench', + 'api_url': 'http://127.0.0.1:8000/v1', + 'generation_config': { + 'temperature': 0, + 'top_p': 1, + 'seed': 42, + 'max_tokens': 4096, + 'extra_body': { + 'top_k': 1, + 'repetition_penalty': 1.05, + 'chat_template_kwargs': {'enable_thinking': True}, + }, + }, + }]}, +)) +``` +""", + tags=[Tags.TEXT_TO_IMAGE], + metric_list=[SCORE_NAME], + primary_metric=MetricSelector(name=SCORE_NAME, aggregation='mean', dimensions={'scope': 'overall'}), + subset_list=['default'], + few_shot_num=0, + train_split=None, + eval_split='test', + evaluation_version='v1.0', + extra_params={ + 'language': { + 'type': 'str', + 'description': 'Language of the generation prompts (cn or en).', + 'value': 'cn', + 'choices': ['cn', 'en'], + }, + }, + ) +) +class QwenImageBenchAdapter(Text2ImageAdapter): + scoring_policy = ScoringPolicy.JUDGE_ONLY + doc_eval_type = 'text2image' + + def load_subset(self, subset: str, data_loader: type[DataLoader]) -> Dataset: + """Use a single dataset file, leaving the released example images on the hub.""" + if self.dataset_hub == HubType.LOCAL: + return super().load_subset(subset, data_loader) + path = download_dataset_file( + self.dataset_id, + DATASET_FILE, + data_source=self.dataset_hub, + revision=self.dataset_revision, + force_redownload=self.force_redownload, + cache_dir=self.dataset_dir, + ) + return LocalDataLoader( + data_id_or_path=path, + split=self.eval_split, + subset=subset, + sample_fields=self.record_to_sample, + filter_func=self.sample_filter, + limit=self.limit, + repeats=self.repeats, + shuffle=self.shuffle, + seed=self.seed, + ).load() + + def record_to_sample(self, record: dict[str, Any]) -> Sample: + """Select the prompt language and retain only the fields needed to score the image.""" + prompt_field = f'prompt_{self.extra_params.get("language", "cn")}' + prompt = record['prompt'] if 'prompt' in record else record[prompt_field] + dimensions = dict.fromkeys( + item.split('/', 1)[0].strip() for item in record['dims_en'].split(';') if item.strip() + ) + if not dimensions or dimensions.keys() - DIM_TO_CHECKLIST.keys(): + raise ValueError(f'Invalid Qwen-Image-Bench dimensions for ID {record["ID"]}: {list(dimensions)}') + return super().record_to_sample( + { + 'id': record['ID'], + 'prompt': prompt, + 'dimensions': list(dimensions), + 'image_path': record.get('image_path', ''), + } + ) + + def judge_definition(self, context: JudgeContext) -> JudgeDefinition: + """Send the official checklist for each applicable dimension.""" + metadata = context.task_state.metadata or {} + image_path = metadata.get(FileConstants.IMAGE_PATH, '') + artifact_dir = None + if self._task_config is not None: + artifact_dir = Path(self._task_config.work_dir).resolve() / 'judge_images' / self.name + try: + judge_image = prepare_judge_image(image_path, artifact_dir) + except (OSError, ValueError) as error: + reason = f'Cannot load the generated image: {error}' + score = Score(prediction=image_path, status=ScoreStatus.EXCLUDED, explanation=reason) + return JudgeDefinition.skip(score, reason=reason) + + cases = [ + JudgeCase( + case_id=dimension, output_contract=GRADE_CONTRACT, required=False, metadata={'dimension': dimension} + ) + for dimension in metadata['dimensions'] + ] + + def request( + case: JudgeCase, placement: Placement, completed: Sequence[CaseVerdict], judge_context: JudgeContext + ) -> JudgeRequest: + dimension = case.metadata['dimension'] + before_image, after_image = USER_PROMPT_TEMPLATE.split('', 1) + before_image = before_image.format(prompt=metadata['prompt']) + after_image = after_image.format(level1_dim=dimension, format_checklist=DIM_TO_CHECKLIST[dimension]) + return JudgeRequest( + messages=[ + ChatMessageSystem(content=SYSTEM_PROMPT), + ChatMessageUser( + content=[ + ContentText(text=before_image), + ContentImage(image=judge_image), + ContentText(text=after_image), + ] + ), + ] + ) + + def finalize(score: Score, review: JudgeReview, judge_context: JudgeContext) -> Score: + score.prediction = score.extracted_prediction = image_path + if score.value: + components = {key.partition(':')[2] or 'overall': value for key, value in score.value.items()} + score.value.clear() + self._record_metric_result(score, SCORE_NAME, components) + return score + + return JudgeDefinition.workflow( + cases=cases, request=request, reduce=reduce_grades, main_score_name=SCORE_NAME, finalize=finalize + ) diff --git a/evalscope/benchmarks/qwen_image_bench/utils.py b/evalscope/benchmarks/qwen_image_bench/utils.py new file mode 100644 index 000000000..300b59667 --- /dev/null +++ b/evalscope/benchmarks/qwen_image_bench/utils.py @@ -0,0 +1,106 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from hashlib import sha256 +from io import BytesIO +from pathlib import Path +from statistics import mean +from tempfile import NamedTemporaryFile +from typing import Literal, Sequence + +from PIL import Image +from pydantic import BaseModel, RootModel + +from evalscope.api.judge import CaseVerdict, JudgeContext, OutputContract, ReducedVerdict +from evalscope.utils.io_utils import PIL_to_base64 +from evalscope.utils.uri_utils import file_as_data + +from .checklists import DIM_TO_CHECKLIST + +SCORE_NAME = 'qwen_image_bench_score' +SCORE_MAP = {0: 0.0, 1: 60.0, 2: 100.0} + + +def prepare_judge_image(image_path: str, artifact_dir: Path | None) -> str: + """Apply the official preprocessing and keep one PNG for all judge requests on this image.""" + image_bytes, _ = file_as_data(image_path) + with Image.open(BytesIO(image_bytes)) as source: + image = source.convert('RGB') + if max(image.size) > 1024: + image = image.resize((1024, 1024), Image.Resampling.LANCZOS) + if artifact_dir is None: + return PIL_to_base64(image, format='PNG', add_header=True) + + artifact_dir.mkdir(parents=True, exist_ok=True) + target = artifact_dir / f'{sha256(image_bytes).hexdigest()}.png' + if not target.is_file(): + # Publish only complete PNGs when repeated images are reviewed concurrently. + with NamedTemporaryFile(dir=artifact_dir, delete=False) as output: + pending = Path(output.name) + try: + image.save(pending, format='PNG') + pending.replace(target) + finally: + pending.unlink(missing_ok=True) + return str(target) + + +class FacetGrade(BaseModel): + score: Literal[0, 1, 2, 'N/A'] + + +class DimensionGrade(RootModel[dict[str, FacetGrade | dict[str, FacetGrade]]]): + """The official nested rubric verdict and the flat variant accepted by the upstream scorer.""" + + +GRADE_CONTRACT = OutputContract(schema_model=DimensionGrade, allow_json_fences=True) + + +def _facet_parents(checklist: str) -> dict[str, str]: + parents = {} + for line in checklist.splitlines(): + if line.startswith('## '): + parent = line[3:] + elif line.startswith('- '): + facet = line[2:].split(':', 1)[0] + parents[facet] = parent + return parents + + +FACET_PARENTS = {dimension: _facet_parents(checklist) for dimension, checklist in DIM_TO_CHECKLIST.items()} + + +def _subdimension_scores(dimension: str, grade: DimensionGrade) -> dict[str, float]: + """Repair known facet placement before aggregating non-N/A grades.""" + grouped: dict[str, dict[str, FacetGrade]] = {} + parents = FACET_PARENTS[dimension] + for label, value in grade.root.items(): + facets = {label: value} if isinstance(value, FacetGrade) else value + for facet, facet_grade in facets.items(): + if dimension == 'Creative Generation' and facet == 'Feature Mapping': + facet = 'Feature Matching' + grouped.setdefault(parents.get(facet, label), {})[facet] = facet_grade + + scores = {} + for subdimension, facets in grouped.items(): + applicable = [SCORE_MAP[facet.score] for facet in facets.values() if facet.score in SCORE_MAP] + if applicable: + scores[subdimension] = mean(applicable) + return scores + + +def reduce_grades(verdicts: Sequence[CaseVerdict], context: JudgeContext) -> ReducedVerdict: + """Average subdimensions within dimensions and scored dimensions within the image.""" + values = {} + dimension_scores = [] + for verdict in verdicts: + dimension = verdict.metadata['dimension'] + subdimensions = _subdimension_scores(dimension, verdict.value) + if not subdimensions: + continue + dimension_score = mean(subdimensions.values()) + dimension_scores.append(dimension_score) + values[f'{SCORE_NAME}:{dimension}'] = dimension_score + for label, score in subdimensions.items(): + values[f'{SCORE_NAME}:{dimension}/{label}'] = score + if dimension_scores: + values[SCORE_NAME] = mean(dimension_scores) + return ReducedVerdict(value=values) diff --git a/evalscope/metrics/semantics/catalog.py b/evalscope/metrics/semantics/catalog.py index bbd524957..9464e5128 100644 --- a/evalscope/metrics/semantics/catalog.py +++ b/evalscope/metrics/semantics/catalog.py @@ -387,6 +387,11 @@ metric_name='Quality Score', value_range=ValueRange(min=0, max=5), ), + 'qwen_image_bench_score': MetricEntry( + baseline='quality.judge_score.unbounded', + metric_name='Qwen-Image-Bench Score', + value_range=ValueRange(min=0, max=100), + ), # MASK: 1 - P(lie) under pressure, and the share of pressured statements that match the belief. 'honesty_score': MetricEntry(baseline='quality.score.ratio', metric_name='Honesty Score'), 'honest_rate': MetricEntry(baseline='quality.score.ratio', metric_name='Honest Rate'), diff --git a/tests/api/judge/test_migrated_adapters.py b/tests/api/judge/test_migrated_adapters.py index af801d3a1..87e14664e 100644 --- a/tests/api/judge/test_migrated_adapters.py +++ b/tests/api/judge/test_migrated_adapters.py @@ -1,9 +1,11 @@ """Smoke tests for migrated Native judge adapters.""" import json +from pathlib import Path from typing import Any, List import pytest +from PIL import Image from evalscope.api.dataset import Sample from evalscope.api.evaluator import TaskState @@ -27,6 +29,80 @@ def generate(self, messages): raise RuntimeError('judge transport unavailable') +def test_qwen_image_bench_uses_chinese_prompts_without_task_config() -> None: + adapter = get_benchmark('qwen_image_bench') + sample = adapter.record_to_sample({ + 'ID': 1, + 'prompt_cn': '红色立方体', + 'prompt_en': 'a red cube', + 'dims_en': 'Quality / Realism / Physical Logic', + }) + assert sample.metadata['prompt'] == '红色立方体' + + +@pytest.mark.parametrize( + ('replies', 'expected_status', 'expected'), + [ + (['{"Physical Logic":{"score":0},"Detail":{"Noise":{"score":"N/A"}}}', + '```json\n{"Attributes":{"Color":{"score":2}}}\n```'], ScoreStatus.SUCCESS, 50.0), + (['not JSON', '{"Attributes":{"Color":{"score":1}}}'], ScoreStatus.DEGRADED, 60.0), + (['not JSON', '[ERROR] judge transport unavailable'], ScoreStatus.EXCLUDED, None), + (['{"Realism":{"Physical Logic":{"score":"N/A"}}}', '{"Attributes":{"Color":{"score":"N/A"}}}'], + ScoreStatus.EXCLUDED, None), + ], +) +def test_qwen_image_bench_contract( + tmp_path: Path, replies: list[str], expected_status: ScoreStatus, expected: float | None +) -> None: + image = tmp_path / 'sample.png' + Image.new('RGB', (8, 12)).save(image) + manifest = tmp_path / 'dataset.jsonl' + manifest.write_text(json.dumps({ + 'ID': 1, + 'prompt_cn': '红色立方体', + 'prompt_en': 'a red cube', + 'dims_en': 'Quality / Detail / Noise; Alignment / Attributes / Color; Quality / Realism / Physical Logic', + 'image_path': str(image), + }) + '\n', encoding='utf-8') + adapter = get_benchmark('qwen_image_bench', TaskConfig( + datasets=['qwen_image_bench'], + dataset_args={'qwen_image_bench': {'dataset_id': str(manifest), 'extra_params': {'language': 'en'}}}, + judge={'models': [{'model_id': 'Qwen/Qwen-Image-Bench'}]}, + work_dir=str(tmp_path / 'evaluation'), + )) + judge = ScriptedJudge(replies) + adapter.llm_judge = judge + sample = adapter.load_dataset()['default'][0] + assert sample.metadata['prompt'] == 'a red cube' + task = TaskState(model='m', sample=sample, output=ModelOutput.from_content('m', ''), completed=True) + + sample_score = adapter.calculate_metrics(task) + score = sample_score.score + assert score.status is expected_status + assert len(judge.calls) == 2 + assert score.prediction == str(image) + artifacts = list((tmp_path / 'evaluation' / 'judge_images').rglob('*.png')) + assert len(artifacts) == 1 + attempts = score.metadata['judge_attempts'] + assert [attempt['messages'][1]['content'][1]['image'] for attempt in attempts] == [str(artifacts[0])] * 2 + if expected is None: + assert score.value == {} + assert adapter.aggregate_scores([sample_score]) == [] + else: + assert score.main_value == expected + by_component = { + item.dimensions.get('component'): item for item in adapter.aggregate_scores([sample_score]) + } + assert by_component[None].score == expected and by_component[None].num == 1 + assert by_component['alignment_attributes'].score == score.value['qwen_image_bench_score:alignment'] + if expected_status is ScoreStatus.DEGRADED: + assert 'quality' not in by_component + assert score.judge_summary.failures == {'parse_error': 1} + else: + assert by_component['quality_realism'].score == 0.0 + assert 'quality_detail' not in by_component + + @pytest.mark.parametrize(('judge', 'expected'), [ (ScriptedJudge(['{"explanation": "The condition is preserved.", "criteria_met": true}']), 1.0), (ScriptedJudge(['{"explanation": "The condition is omitted.", "criteria_met": false}']), 0.0), From e84ed8c02625fc9389193e1966e9715dc8285715 Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Thu, 19 Nov 2026 17:52:50 +0800 Subject: [PATCH 2/2] docs(benchmark): shorten the Qwen-Image-Bench description --- docs/en/benchmarks/qwen_image_bench.md | 61 ++++--------------- docs/zh/benchmarks/qwen_image_bench.md | 61 +++++-------------- .../benchmarks/_meta/qwen_image_bench.json | 16 ++--- .../qwen_image_bench_adapter.py | 59 ++++-------------- 4 files changed, 45 insertions(+), 152 deletions(-) diff --git a/docs/en/benchmarks/qwen_image_bench.md b/docs/en/benchmarks/qwen_image_bench.md index abc2c2a49..8df789fe8 100644 --- a/docs/en/benchmarks/qwen_image_bench.md +++ b/docs/en/benchmarks/qwen_image_bench.md @@ -3,63 +3,26 @@ ## Overview -Qwen-Image-Bench evaluates text-to-image models on professional creative tasks using the official Q-Judger. +Qwen-Image-Bench measures how well text-to-image models handle creative tasks. ## Task Description -- **Task Type**: Text-to-image generation and evaluation -- **Input**: A Chinese or English creative prompt -- **Output**: An image evaluated against applicable rubrics -- **Domain**: Image quality, aesthetics, alignment, real-world fidelity, and creative generation +- **Task Type**: Text-to-image generation +- **Input**: A Chinese or English prompt +- **Output**: A generated image +- **Domain**: Image quality, aesthetics, prompt alignment, real-world fidelity, and creativity ## Key Features -- 1,000 bilingual prompts with 56 facets grouped into five dimensions. -- Supports both image generation and evaluation of existing images via a local JSONL manifest. +- 1,000 bilingual prompts. Set `extra_params.language` to `cn` (default) or `en`. +- Score existing images with a local JSONL (`ID`, `prompt`, `dims_en`, and absolute `image_path`). + Set `model_task='image_generation'` and omit `model`. ## Evaluation Notes -- Set `judge.models` to an OpenAI-compatible vision endpoint serving the official `Qwen/Qwen-Image-Bench` - weights. Serve with thinking enabled and reasoning separated from the final JSON. Use temperature 0, - top-k 1, top-p 1, repetition penalty 1.05, seed 42, and a 4,096-token generation limit for official settings. -- Set `extra_params.language` to `cn` (default) or `en`. Each run reports scores for the selected language. -- For existing images, use a local JSONL with `ID`, `dims_en`, `prompt` (or the selected official - `prompt_cn`/`prompt_en` field), and an absolute `image_path`. Set `model_task='image_generation'` - and leave `model` unset to score these images. -- Grades 0, 1, and 2 map to 0, 60, and 100. Inapplicable facets and failed requests are excluded from the - corresponding averages. Each image is averaged across its scored dimensions before averaging images. -- The overall score, dimensions, and subdimensions report their respective valid image counts. - -This configuration scores existing images against the English prompts. - -```python -from evalscope import TaskConfig, run_task - -run_task(TaskConfig( - model_id='my-image-model', - model_task='image_generation', - datasets=['qwen_image_bench'], - dataset_args={'qwen_image_bench': { - 'dataset_id': '/absolute/path/to/manifest.jsonl', - 'extra_params': {'language': 'en'}, - }}, - judge={'models': [{ - 'model_id': 'Qwen/Qwen-Image-Bench', - 'api_url': 'http://127.0.0.1:8000/v1', - 'generation_config': { - 'temperature': 0, - 'top_p': 1, - 'seed': 42, - 'max_tokens': 4096, - 'extra_body': { - 'top_k': 1, - 'repetition_penalty': 1.05, - 'chat_template_kwargs': {'enable_thinking': True}, - }, - }, - }]}, -)) -``` +- Configure `judge.models` with an OpenAI-compatible endpoint serving the official Q-Judger + (`Qwen/Qwen-Image-Bench`). See the [official inference settings](https://github.com/QwenLM/Qwen-Image-Bench#inference-parameters). +- Scores range from 0 to 100. Higher is better. The report shows overall and breakdown scores with their valid image counts. ## Properties @@ -91,7 +54,7 @@ run_task(TaskConfig( { "input": [ { - "id": "b8c6bf0d", + "id": "58f90ec0", "content": "一只拆开的机械手表放在黑色工作台上,制表师用镊子夹起齿轮,特写镊子夹起齿轮,银色与暗灰色调,零部件结构精密,金属边缘细节清晰。" } ], diff --git a/docs/zh/benchmarks/qwen_image_bench.md b/docs/zh/benchmarks/qwen_image_bench.md index a2229df05..c438c17f6 100644 --- a/docs/zh/benchmarks/qwen_image_bench.md +++ b/docs/zh/benchmarks/qwen_image_bench.md @@ -3,58 +3,25 @@ ## 概述 -Qwen-Image-Bench 使用官方 Q-Judger 评估文本到图像模型在专业创意任务上的表现。 +Qwen-Image-Bench 用于衡量文本到图像模型在处理创意任务时的表现。 ## 任务描述 -- **任务类型**:文本到图像生成与评估 -- **输入**:中文或英文创意提示词 -- **输出**:根据适用评分标准进行评估的图像 -- **领域**:图像质量、美学、对齐度、真实世界保真度及创意生成 +- **任务类型**:文本到图像生成 +- **输入**:中文或英文提示词 +- **输出**:生成的图像 +- **领域**:图像质量、美学、提示词对齐度、真实世界保真度和创造力 ## 主要特点 -- 包含 1,000 条双语提示词,涵盖归入五个维度的 56 个细分项。 -- 支持通过本地 JSONL 清单进行图像生成以及现有图像的评估。 +- 包含 1,000 条双语提示词。将 `extra_params.language` 设置为 `cn`(默认)或 `en`。 +- 使用本地 JSONL 文件对现有图像进行评分(需包含 `ID`、`prompt`、`dims_en` 以及绝对路径的 `image_path`)。 + 设置 `model_task='image_generation'` 并省略 `model` 参数。 ## 评估说明 -- 将 `judge.models` 设置为提供官方 `Qwen/Qwen-Image-Bench` 权重的 OpenAI 兼容视觉端点。启用思考模式,并将推理过程与最终 JSON 分离。官方设置下使用温度 0、top-k 1、top-p 1、重复惩罚 1.05、种子 42 以及 4,096 token 的生成限制。 -- 将 `extra_params.language` 设置为 `cn`(默认)或 `en`。每次运行报告所选语言的分数。 -- 对于现有图像,请使用包含 `ID`、`dims_en`、`prompt`(或选定的官方 `prompt_cn`/`prompt_en` 字段)以及绝对路径 `image_path` 的本地 JSONL 文件。设置 `model_task='image_generation'` 并保持 `model` 未设置以对这些图像进行评分。 -- 等级 0、1 和 2 分别映射为 0、60 和 100 分。不适用的细分项和失败的请求将从相应的平均值中排除。每张图像先在其得分维度上取平均,然后再对图像进行平均。 -- 总体分数、各维度及各子维度均报告其各自的有效图像数量。 - -此配置针对英文提示词对现有图像进行评分。 - -```python -from evalscope import TaskConfig, run_task - -run_task(TaskConfig( - model_id='my-image-model', - model_task='image_generation', - datasets=['qwen_image_bench'], - dataset_args={'qwen_image_bench': { - 'dataset_id': '/absolute/path/to/manifest.jsonl', - 'extra_params': {'language': 'en'}, - }}, - judge={'models': [{ - 'model_id': 'Qwen/Qwen-Image-Bench', - 'api_url': 'http://127.0.0.1:8000/v1', - 'generation_config': { - 'temperature': 0, - 'top_p': 1, - 'seed': 42, - 'max_tokens': 4096, - 'extra_body': { - 'top_k': 1, - 'repetition_penalty': 1.05, - 'chat_template_kwargs': {'enable_thinking': True}, - }, - }, - }]}, -)) -``` +- 配置 `judge.models`,指向提供官方 Q-Judger (`Qwen/Qwen-Image-Bench`) 服务的 OpenAI 兼容端点。请参阅[官方推理设置](https://github.com/QwenLM/Qwen-Image-Bench#inference-parameters)。 +- 分数范围为 0 到 100,分数越高越好。报告将显示总体得分及细分得分,并列出有效图像的数量。 ## 属性 @@ -75,8 +42,8 @@ run_task(TaskConfig( | 指标 | 值 | |--------|-------| | 总样本数 | 1,000 | -| 提示词长度(均值) | 83.05 字符 | -| 提示词长度(最小/最大) | 25 / 317 字符 | +| 提示词长度(平均值) | 83.05 字符 | +| 提示词长度(最小/最大值) | 25 / 317 字符 | ## 样例示例 @@ -86,7 +53,7 @@ run_task(TaskConfig( { "input": [ { - "id": "b8c6bf0d", + "id": "58f90ec0", "content": "一只拆开的机械手表放在黑色工作台上,制表师用镊子夹起齿轮,特写镊子夹起齿轮,银色与暗灰色调,零部件结构精密,金属边缘细节清晰。" } ], @@ -116,7 +83,7 @@ run_task(TaskConfig( | 参数 | 类型 | 默认值 | 描述 | |-----------|------|---------|-------------| -| `language` | `str` | `cn` | 生成提示词的语言(cn 或 en)。选项:['cn', 'en'] | +| `language` | `str` | `cn` | 生成提示词的语言(cn 或 en)。可选值:['cn', 'en'] | ## 使用方法 diff --git a/evalscope/benchmarks/_meta/qwen_image_bench.json b/evalscope/benchmarks/_meta/qwen_image_bench.json index 78554646e..3df278f26 100644 --- a/evalscope/benchmarks/_meta/qwen_image_bench.json +++ b/evalscope/benchmarks/_meta/qwen_image_bench.json @@ -15,7 +15,7 @@ "subset_list": [ "default" ], - "description": "\n## Overview\n\nQwen-Image-Bench evaluates text-to-image models on professional creative tasks using the official Q-Judger.\n\n## Task Description\n\n- **Task Type**: Text-to-image generation and evaluation\n- **Input**: A Chinese or English creative prompt\n- **Output**: An image evaluated against applicable rubrics\n- **Domain**: Image quality, aesthetics, alignment, real-world fidelity, and creative generation\n\n## Key Features\n\n- 1,000 bilingual prompts with 56 facets grouped into five dimensions.\n- Supports both image generation and evaluation of existing images via a local JSONL manifest.\n\n## Evaluation Notes\n\n- Set `judge.models` to an OpenAI-compatible vision endpoint serving the official `Qwen/Qwen-Image-Bench`\n weights. Serve with thinking enabled and reasoning separated from the final JSON. Use temperature 0,\n top-k 1, top-p 1, repetition penalty 1.05, seed 42, and a 4,096-token generation limit for official settings.\n- Set `extra_params.language` to `cn` (default) or `en`. Each run reports scores for the selected language.\n- For existing images, use a local JSONL with `ID`, `dims_en`, `prompt` (or the selected official\n `prompt_cn`/`prompt_en` field), and an absolute `image_path`. Set `model_task='image_generation'`\n and leave `model` unset to score these images.\n- Grades 0, 1, and 2 map to 0, 60, and 100. Inapplicable facets and failed requests are excluded from the\n corresponding averages. Each image is averaged across its scored dimensions before averaging images.\n- The overall score, dimensions, and subdimensions report their respective valid image counts.\n\nThis configuration scores existing images against the English prompts.\n\n```python\nfrom evalscope import TaskConfig, run_task\n\nrun_task(TaskConfig(\n model_id='my-image-model',\n model_task='image_generation',\n datasets=['qwen_image_bench'],\n dataset_args={'qwen_image_bench': {\n 'dataset_id': '/absolute/path/to/manifest.jsonl',\n 'extra_params': {'language': 'en'},\n }},\n judge={'models': [{\n 'model_id': 'Qwen/Qwen-Image-Bench',\n 'api_url': 'http://127.0.0.1:8000/v1',\n 'generation_config': {\n 'temperature': 0,\n 'top_p': 1,\n 'seed': 42,\n 'max_tokens': 4096,\n 'extra_body': {\n 'top_k': 1,\n 'repetition_penalty': 1.05,\n 'chat_template_kwargs': {'enable_thinking': True},\n },\n },\n }]},\n))\n```\n", + "description": "\n## Overview\n\nQwen-Image-Bench measures how well text-to-image models handle creative tasks.\n\n## Task Description\n\n- **Task Type**: Text-to-image generation\n- **Input**: A Chinese or English prompt\n- **Output**: A generated image\n- **Domain**: Image quality, aesthetics, prompt alignment, real-world fidelity, and creativity\n\n## Key Features\n\n- 1,000 bilingual prompts. Set `extra_params.language` to `cn` (default) or `en`.\n- Score existing images with a local JSONL (`ID`, `prompt`, `dims_en`, and absolute `image_path`).\n Set `model_task='image_generation'` and omit `model`.\n\n## Evaluation Notes\n\n- Configure `judge.models` with an OpenAI-compatible endpoint serving the official Q-Judger\n (`Qwen/Qwen-Image-Bench`). See the [official inference settings](https://github.com/QwenLM/Qwen-Image-Bench#inference-parameters).\n- Scores range from 0 to 100. Higher is better. The report shows overall and breakdown scores with their valid image counts.\n", "prompt_template": "", "system_prompt": "", "few_shot_prompt_template": "", @@ -64,13 +64,13 @@ "std": 42.81 }, "target_length_mean": null, - "computed_at": "2026-11-19T16:34:27.702465" + "computed_at": "2026-11-19T17:51:26.667375" }, "sample_example": { "data": { "input": [ { - "id": "b8c6bf0d", + "id": "58f90ec0", "content": "一只拆开的机械手表放在黑色工作台上,制表师用镊子夹起齿轮,特写镊子夹起齿轮,银色与暗灰色调,零部件结构精密,金属边缘细节清晰。" } ], @@ -94,11 +94,11 @@ "truncated": false }, "readme": { - "en": "# Qwen-Image-Bench\n\n\n## Overview\n\nQwen-Image-Bench evaluates text-to-image models on professional creative tasks using the official Q-Judger.\n\n## Task Description\n\n- **Task Type**: Text-to-image generation and evaluation\n- **Input**: A Chinese or English creative prompt\n- **Output**: An image evaluated against applicable rubrics\n- **Domain**: Image quality, aesthetics, alignment, real-world fidelity, and creative generation\n\n## Key Features\n\n- 1,000 bilingual prompts with 56 facets grouped into five dimensions.\n- Supports both image generation and evaluation of existing images via a local JSONL manifest.\n\n## Evaluation Notes\n\n- Set `judge.models` to an OpenAI-compatible vision endpoint serving the official `Qwen/Qwen-Image-Bench`\n weights. Serve with thinking enabled and reasoning separated from the final JSON. Use temperature 0,\n top-k 1, top-p 1, repetition penalty 1.05, seed 42, and a 4,096-token generation limit for official settings.\n- Set `extra_params.language` to `cn` (default) or `en`. Each run reports scores for the selected language.\n- For existing images, use a local JSONL with `ID`, `dims_en`, `prompt` (or the selected official\n `prompt_cn`/`prompt_en` field), and an absolute `image_path`. Set `model_task='image_generation'`\n and leave `model` unset to score these images.\n- Grades 0, 1, and 2 map to 0, 60, and 100. Inapplicable facets and failed requests are excluded from the\n corresponding averages. Each image is averaged across its scored dimensions before averaging images.\n- The overall score, dimensions, and subdimensions report their respective valid image counts.\n\nThis configuration scores existing images against the English prompts.\n\n```python\nfrom evalscope import TaskConfig, run_task\n\nrun_task(TaskConfig(\n model_id='my-image-model',\n model_task='image_generation',\n datasets=['qwen_image_bench'],\n dataset_args={'qwen_image_bench': {\n 'dataset_id': '/absolute/path/to/manifest.jsonl',\n 'extra_params': {'language': 'en'},\n }},\n judge={'models': [{\n 'model_id': 'Qwen/Qwen-Image-Bench',\n 'api_url': 'http://127.0.0.1:8000/v1',\n 'generation_config': {\n 'temperature': 0,\n 'top_p': 1,\n 'seed': 42,\n 'max_tokens': 4096,\n 'extra_body': {\n 'top_k': 1,\n 'repetition_penalty': 1.05,\n 'chat_template_kwargs': {'enable_thinking': True},\n },\n },\n }]},\n))\n```\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `qwen_image_bench` |\n| **Dataset ID** | [Qwen/Qwen-Image-Bench](https://modelscope.cn/datasets/Qwen/Qwen-Image-Bench/summary) |\n| **Paper** | [Paper](https://arxiv.org/abs/2605.28091) |\n| **Tags** | `TextToImage` |\n| **Metrics** | `qwen_image_bench_score` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 1,000 |\n| Prompt Length (Mean) | 83.05 chars |\n| Prompt Length (Min/Max) | 25 / 317 chars |\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"b8c6bf0d\",\n \"content\": \"一只拆开的机械手表放在黑色工作台上,制表师用镊子夹起齿轮,特写镊子夹起齿轮,银色与暗灰色调,零部件结构精密,金属边缘细节清晰。\"\n }\n ],\n \"target\": \"\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"id\": 1,\n \"prompt\": \"一只拆开的机械手表放在黑色工作台上,制表师用镊子夹起齿轮,特写镊子夹起齿轮,银色与暗灰色调,零部件结构精密,金属边缘细节清晰。\",\n \"dimensions\": [\n \"Quality\",\n \"Aesthetics\",\n \"Alignment\"\n ],\n \"image_path\": \"\",\n \"category\": \"\",\n \"tags\": []\n }\n}\n```\n\n## Prompt Template\n\n*No prompt template defined.*\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `language` | `str` | `cn` | Language of the generation prompts (cn or en). Choices: ['cn', 'en'] |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --model-task image_generation \\\n --eval-type text2image \\\n --api-url http://127.0.0.1:18131/v1 \\\n --api-key EMPTY_TOKEN \\\n --datasets qwen_image_bench \\\n --judge '{\"strategy\":\"llm\",\"models\":[{\"model_id\":\"YOUR_JUDGE_MODEL\"}]}' \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n eval_type='text2image',\n model_task='image_generation',\n api_url='http://127.0.0.1:18131/v1',\n api_key='EMPTY_TOKEN',\n datasets=['qwen_image_bench'],\n judge={'strategy': 'llm', 'models': [{'model_id': 'YOUR_JUDGE_MODEL'}]},\n dataset_args={\n 'qwen_image_bench': {\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", - "zh": "# Qwen-Image-Bench\n\n\n## 概述\n\nQwen-Image-Bench 使用官方 Q-Judger 评估文本到图像模型在专业创意任务上的表现。\n\n## 任务描述\n\n- **任务类型**:文本到图像生成与评估\n- **输入**:中文或英文创意提示词\n- **输出**:根据适用评分标准进行评估的图像\n- **领域**:图像质量、美学、对齐度、真实世界保真度及创意生成\n\n## 主要特点\n\n- 包含 1,000 条双语提示词,涵盖归入五个维度的 56 个细分项。\n- 支持通过本地 JSONL 清单进行图像生成以及现有图像的评估。\n\n## 评估说明\n\n- 将 `judge.models` 设置为提供官方 `Qwen/Qwen-Image-Bench` 权重的 OpenAI 兼容视觉端点。启用思考模式,并将推理过程与最终 JSON 分离。官方设置下使用温度 0、top-k 1、top-p 1、重复惩罚 1.05、种子 42 以及 4,096 token 的生成限制。\n- 将 `extra_params.language` 设置为 `cn`(默认)或 `en`。每次运行报告所选语言的分数。\n- 对于现有图像,请使用包含 `ID`、`dims_en`、`prompt`(或选定的官方 `prompt_cn`/`prompt_en` 字段)以及绝对路径 `image_path` 的本地 JSONL 文件。设置 `model_task='image_generation'` 并保持 `model` 未设置以对这些图像进行评分。\n- 等级 0、1 和 2 分别映射为 0、60 和 100 分。不适用的细分项和失败的请求将从相应的平均值中排除。每张图像先在其得分维度上取平均,然后再对图像进行平均。\n- 总体分数、各维度及各子维度均报告其各自的有效图像数量。\n\n此配置针对英文提示词对现有图像进行评分。\n\n```python\nfrom evalscope import TaskConfig, run_task\n\nrun_task(TaskConfig(\n model_id='my-image-model',\n model_task='image_generation',\n datasets=['qwen_image_bench'],\n dataset_args={'qwen_image_bench': {\n 'dataset_id': '/absolute/path/to/manifest.jsonl',\n 'extra_params': {'language': 'en'},\n }},\n judge={'models': [{\n 'model_id': 'Qwen/Qwen-Image-Bench',\n 'api_url': 'http://127.0.0.1:8000/v1',\n 'generation_config': {\n 'temperature': 0,\n 'top_p': 1,\n 'seed': 42,\n 'max_tokens': 4096,\n 'extra_body': {\n 'top_k': 1,\n 'repetition_penalty': 1.05,\n 'chat_template_kwargs': {'enable_thinking': True},\n },\n },\n }]},\n))\n```\n\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `qwen_image_bench` |\n| **数据集ID** | [Qwen/Qwen-Image-Bench](https://modelscope.cn/datasets/Qwen/Qwen-Image-Bench/summary) |\n| **论文** | [Paper](https://arxiv.org/abs/2605.28091) |\n| **标签** | `TextToImage` |\n| **指标** | `qwen_image_bench_score` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 1,000 |\n| 提示词长度(均值) | 83.05 字符 |\n| 提示词长度(最小/最大) | 25 / 317 字符 |\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"b8c6bf0d\",\n \"content\": \"一只拆开的机械手表放在黑色工作台上,制表师用镊子夹起齿轮,特写镊子夹起齿轮,银色与暗灰色调,零部件结构精密,金属边缘细节清晰。\"\n }\n ],\n \"target\": \"\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"id\": 1,\n \"prompt\": \"一只拆开的机械手表放在黑色工作台上,制表师用镊子夹起齿轮,特写镊子夹起齿轮,银色与暗灰色调,零部件结构精密,金属边缘细节清晰。\",\n \"dimensions\": [\n \"Quality\",\n \"Aesthetics\",\n \"Alignment\"\n ],\n \"image_path\": \"\",\n \"category\": \"\",\n \"tags\": []\n }\n}\n```\n\n## 提示模板\n\n*未定义提示模板。*\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `language` | `str` | `cn` | 生成提示词的语言(cn 或 en)。选项:['cn', 'en'] |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --model-task image_generation \\\n --eval-type text2image \\\n --api-url http://127.0.0.1:18131/v1 \\\n --api-key EMPTY_TOKEN \\\n --datasets qwen_image_bench \\\n --judge '{\"strategy\":\"llm\",\"models\":[{\"model_id\":\"YOUR_JUDGE_MODEL\"}]}' \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n eval_type='text2image',\n model_task='image_generation',\n api_url='http://127.0.0.1:18131/v1',\n api_key='EMPTY_TOKEN',\n datasets=['qwen_image_bench'],\n judge={'strategy': 'llm', 'models': [{'model_id': 'YOUR_JUDGE_MODEL'}]},\n dataset_args={\n 'qwen_image_bench': {\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", - "content_hash": "8fe00d7143344b96994fff0e927045de", + "en": "# Qwen-Image-Bench\n\n\n## Overview\n\nQwen-Image-Bench measures how well text-to-image models handle creative tasks.\n\n## Task Description\n\n- **Task Type**: Text-to-image generation\n- **Input**: A Chinese or English prompt\n- **Output**: A generated image\n- **Domain**: Image quality, aesthetics, prompt alignment, real-world fidelity, and creativity\n\n## Key Features\n\n- 1,000 bilingual prompts. Set `extra_params.language` to `cn` (default) or `en`.\n- Score existing images with a local JSONL (`ID`, `prompt`, `dims_en`, and absolute `image_path`).\n Set `model_task='image_generation'` and omit `model`.\n\n## Evaluation Notes\n\n- Configure `judge.models` with an OpenAI-compatible endpoint serving the official Q-Judger\n (`Qwen/Qwen-Image-Bench`). See the [official inference settings](https://github.com/QwenLM/Qwen-Image-Bench#inference-parameters).\n- Scores range from 0 to 100. Higher is better. The report shows overall and breakdown scores with their valid image counts.\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `qwen_image_bench` |\n| **Dataset ID** | [Qwen/Qwen-Image-Bench](https://modelscope.cn/datasets/Qwen/Qwen-Image-Bench/summary) |\n| **Paper** | [Paper](https://arxiv.org/abs/2605.28091) |\n| **Tags** | `TextToImage` |\n| **Metrics** | `qwen_image_bench_score` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 1,000 |\n| Prompt Length (Mean) | 83.05 chars |\n| Prompt Length (Min/Max) | 25 / 317 chars |\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"58f90ec0\",\n \"content\": \"一只拆开的机械手表放在黑色工作台上,制表师用镊子夹起齿轮,特写镊子夹起齿轮,银色与暗灰色调,零部件结构精密,金属边缘细节清晰。\"\n }\n ],\n \"target\": \"\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"id\": 1,\n \"prompt\": \"一只拆开的机械手表放在黑色工作台上,制表师用镊子夹起齿轮,特写镊子夹起齿轮,银色与暗灰色调,零部件结构精密,金属边缘细节清晰。\",\n \"dimensions\": [\n \"Quality\",\n \"Aesthetics\",\n \"Alignment\"\n ],\n \"image_path\": \"\",\n \"category\": \"\",\n \"tags\": []\n }\n}\n```\n\n## Prompt Template\n\n*No prompt template defined.*\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `language` | `str` | `cn` | Language of the generation prompts (cn or en). Choices: ['cn', 'en'] |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --model-task image_generation \\\n --eval-type text2image \\\n --api-url http://127.0.0.1:18131/v1 \\\n --api-key EMPTY_TOKEN \\\n --datasets qwen_image_bench \\\n --judge '{\"strategy\":\"llm\",\"models\":[{\"model_id\":\"YOUR_JUDGE_MODEL\"}]}' \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n eval_type='text2image',\n model_task='image_generation',\n api_url='http://127.0.0.1:18131/v1',\n api_key='EMPTY_TOKEN',\n datasets=['qwen_image_bench'],\n judge={'strategy': 'llm', 'models': [{'model_id': 'YOUR_JUDGE_MODEL'}]},\n dataset_args={\n 'qwen_image_bench': {\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# Qwen-Image-Bench\n\n\n## 概述\n\nQwen-Image-Bench 用于衡量文本到图像模型在处理创意任务时的表现。\n\n## 任务描述\n\n- **任务类型**:文本到图像生成\n- **输入**:中文或英文提示词\n- **输出**:生成的图像\n- **领域**:图像质量、美学、提示词对齐度、真实世界保真度和创造力\n\n## 主要特点\n\n- 包含 1,000 条双语提示词。将 `extra_params.language` 设置为 `cn`(默认)或 `en`。\n- 使用本地 JSONL 文件对现有图像进行评分(需包含 `ID`、`prompt`、`dims_en` 以及绝对路径的 `image_path`)。\n 设置 `model_task='image_generation'` 并省略 `model` 参数。\n\n## 评估说明\n\n- 配置 `judge.models`,指向提供官方 Q-Judger (`Qwen/Qwen-Image-Bench`) 服务的 OpenAI 兼容端点。请参阅[官方推理设置](https://github.com/QwenLM/Qwen-Image-Bench#inference-parameters)。\n- 分数范围为 0 到 100,分数越高越好。报告将显示总体得分及细分得分,并列出有效图像的数量。\n\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `qwen_image_bench` |\n| **数据集ID** | [Qwen/Qwen-Image-Bench](https://modelscope.cn/datasets/Qwen/Qwen-Image-Bench/summary) |\n| **论文** | [Paper](https://arxiv.org/abs/2605.28091) |\n| **标签** | `TextToImage` |\n| **指标** | `qwen_image_bench_score` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 1,000 |\n| 提示词长度(平均值) | 83.05 字符 |\n| 提示词长度(最小/最大值) | 25 / 317 字符 |\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"58f90ec0\",\n \"content\": \"一只拆开的机械手表放在黑色工作台上,制表师用镊子夹起齿轮,特写镊子夹起齿轮,银色与暗灰色调,零部件结构精密,金属边缘细节清晰。\"\n }\n ],\n \"target\": \"\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"id\": 1,\n \"prompt\": \"一只拆开的机械手表放在黑色工作台上,制表师用镊子夹起齿轮,特写镊子夹起齿轮,银色与暗灰色调,零部件结构精密,金属边缘细节清晰。\",\n \"dimensions\": [\n \"Quality\",\n \"Aesthetics\",\n \"Alignment\"\n ],\n \"image_path\": \"\",\n \"category\": \"\",\n \"tags\": []\n }\n}\n```\n\n## 提示模板\n\n*未定义提示模板。*\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `language` | `str` | `cn` | 生成提示词的语言(cn 或 en)。可选值:['cn', 'en'] |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --model-task image_generation \\\n --eval-type text2image \\\n --api-url http://127.0.0.1:18131/v1 \\\n --api-key EMPTY_TOKEN \\\n --datasets qwen_image_bench \\\n --judge '{\"strategy\":\"llm\",\"models\":[{\"model_id\":\"YOUR_JUDGE_MODEL\"}]}' \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n eval_type='text2image',\n model_task='image_generation',\n api_url='http://127.0.0.1:18131/v1',\n api_key='EMPTY_TOKEN',\n datasets=['qwen_image_bench'],\n judge={'strategy': 'llm', 'models': [{'model_id': 'YOUR_JUDGE_MODEL'}]},\n dataset_args={\n 'qwen_image_bench': {\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "e98d26f3e4694460163c62fb09a94057", "needs_translation": false }, - "updated_at": "2026-11-19T16:34:28.688163", - "translation_updated_at": "2026-11-19T16:34:30" + "updated_at": "2026-11-19T17:51:27.307218", + "translation_updated_at": "2026-11-19T17:51:29" } diff --git a/evalscope/benchmarks/qwen_image_bench/qwen_image_bench_adapter.py b/evalscope/benchmarks/qwen_image_bench/qwen_image_bench_adapter.py index 50d1c7bce..b8c7ead94 100644 --- a/evalscope/benchmarks/qwen_image_bench/qwen_image_bench_adapter.py +++ b/evalscope/benchmarks/qwen_image_bench/qwen_image_bench_adapter.py @@ -35,63 +35,26 @@ description=""" ## Overview -Qwen-Image-Bench evaluates text-to-image models on professional creative tasks using the official Q-Judger. +Qwen-Image-Bench measures how well text-to-image models handle creative tasks. ## Task Description -- **Task Type**: Text-to-image generation and evaluation -- **Input**: A Chinese or English creative prompt -- **Output**: An image evaluated against applicable rubrics -- **Domain**: Image quality, aesthetics, alignment, real-world fidelity, and creative generation +- **Task Type**: Text-to-image generation +- **Input**: A Chinese or English prompt +- **Output**: A generated image +- **Domain**: Image quality, aesthetics, prompt alignment, real-world fidelity, and creativity ## Key Features -- 1,000 bilingual prompts with 56 facets grouped into five dimensions. -- Supports both image generation and evaluation of existing images via a local JSONL manifest. +- 1,000 bilingual prompts. Set `extra_params.language` to `cn` (default) or `en`. +- Score existing images with a local JSONL (`ID`, `prompt`, `dims_en`, and absolute `image_path`). + Set `model_task='image_generation'` and omit `model`. ## Evaluation Notes -- Set `judge.models` to an OpenAI-compatible vision endpoint serving the official `Qwen/Qwen-Image-Bench` - weights. Serve with thinking enabled and reasoning separated from the final JSON. Use temperature 0, - top-k 1, top-p 1, repetition penalty 1.05, seed 42, and a 4,096-token generation limit for official settings. -- Set `extra_params.language` to `cn` (default) or `en`. Each run reports scores for the selected language. -- For existing images, use a local JSONL with `ID`, `dims_en`, `prompt` (or the selected official - `prompt_cn`/`prompt_en` field), and an absolute `image_path`. Set `model_task='image_generation'` - and leave `model` unset to score these images. -- Grades 0, 1, and 2 map to 0, 60, and 100. Inapplicable facets and failed requests are excluded from the - corresponding averages. Each image is averaged across its scored dimensions before averaging images. -- The overall score, dimensions, and subdimensions report their respective valid image counts. - -This configuration scores existing images against the English prompts. - -```python -from evalscope import TaskConfig, run_task - -run_task(TaskConfig( - model_id='my-image-model', - model_task='image_generation', - datasets=['qwen_image_bench'], - dataset_args={'qwen_image_bench': { - 'dataset_id': '/absolute/path/to/manifest.jsonl', - 'extra_params': {'language': 'en'}, - }}, - judge={'models': [{ - 'model_id': 'Qwen/Qwen-Image-Bench', - 'api_url': 'http://127.0.0.1:8000/v1', - 'generation_config': { - 'temperature': 0, - 'top_p': 1, - 'seed': 42, - 'max_tokens': 4096, - 'extra_body': { - 'top_k': 1, - 'repetition_penalty': 1.05, - 'chat_template_kwargs': {'enable_thinking': True}, - }, - }, - }]}, -)) -``` +- Configure `judge.models` with an OpenAI-compatible endpoint serving the official Q-Judger + (`Qwen/Qwen-Image-Bench`). See the [official inference settings](https://github.com/QwenLM/Qwen-Image-Bench#inference-parameters). +- Scores range from 0 to 100. Higher is better. The report shows overall and breakdown scores with their valid image counts. """, tags=[Tags.TEXT_TO_IMAGE], metric_list=[SCORE_NAME],