diff --git a/.agents/skills/json-prompt-solution/SKILL.md b/.agents/skills/json-prompt-solution/SKILL.md index 7d3c9fdd..26fb2572 100644 --- a/.agents/skills/json-prompt-solution/SKILL.md +++ b/.agents/skills/json-prompt-solution/SKILL.md @@ -50,8 +50,8 @@ JSON 内的 query / translated；名称不够通用并不要求全仓改名。 - 只有业务本身需要模型产生结构化语义时才引入结构化解析节点，校验字段与类型并使用 `failure_policy=fail`。SDK 响应组装始终留在 Converter，不靠 Demo 投影字段。 不用固定示例或 fallback 冒充成功。 -- `.conf` 必须指向新 Pipeline，检查模型路径覆盖、上下文与生成长度，以及 Operator - 输出池容量配置。原 Profile 不会自动指向新方案。 +- `.conf` 必须指向新 Pipeline（仅包含 `pipe_path` 定位），并在 Pipeline JSON 的 + `deployment` 中配置 `io.io_binding`、`io.output_allocations` 及可选模型路径覆盖。原 Profile 不会自动指向新方案。 ## 验证与交付 diff --git a/.agents/skills/pipeline-composer/SKILL.md b/.agents/skills/pipeline-composer/SKILL.md index 52013869..0fd706ab 100644 --- a/.agents/skills/pipeline-composer/SKILL.md +++ b/.agents/skills/pipeline-composer/SKILL.md @@ -65,8 +65,8 @@ See [the I/O boundary](../../../doc/dev_guide/business_onboarding.md#输入输 5. After validation, run the edited Pipeline through a compatible Demo. Follow [running the current solution](../../../tools/pipeline_studio/README.md#运行当前方案): confirm - `.conf` `data.pipe_path` resolves to the edited JSON, inspect inherited model path overrides - and capacities, and select a matching biz and dataset. Use + `.conf` `pipe_path` resolves to the edited JSON, inspect pipeline-owned `deployment` (io_binding, + output_allocations, and model_paths), and select a matching biz and dataset. Use `alg_pipeline_tool resolve-conf --root --depth ` to inspect the native resolved paths, their sources and normalized defaults; it does not load weights. Studio can save a JSON + `.conf` pair and command via “另存为可运行方案”; its model diff --git a/cmake_ext/Tests.cmake b/cmake_ext/Tests.cmake index 07e75125..811df4a6 100644 --- a/cmake_ext/Tests.cmake +++ b/cmake_ext/Tests.cmake @@ -113,6 +113,10 @@ if(LLM_EDGEFLOW_HAS_ONNXRUNTIME) "${PROJECT_SOURCE_DIR}/tests/fixtures/pipelines/cross_rerank/pipeline_cross_rerank_fixture.conf" "${EDGEFLOW_GENERATED_MODEL_FIXTURE_DIR}/pipeline_cross_rerank_fixture.conf" COPYONLY) + configure_file( + "${PROJECT_SOURCE_DIR}/tests/fixtures/pipelines/cross_rerank/pipeline_cross_rerank_missing_model.json" + "${EDGEFLOW_GENERATED_MODEL_FIXTURE_DIR}/pipeline_cross_rerank_missing_model.json" + COPYONLY) configure_file( "${PROJECT_SOURCE_DIR}/tests/fixtures/pipelines/cross_rerank/pipeline_cross_rerank_missing_model.conf" "${EDGEFLOW_GENERATED_MODEL_FIXTURE_DIR}/pipeline_cross_rerank_missing_model.conf" diff --git a/configs/pipeline_audio_asr_cpu.conf b/configs/pipeline_audio_asr_cpu.conf index c2024fb5..ad01ca62 100644 --- a/configs/pipeline_audio_asr_cpu.conf +++ b/configs/pipeline_audio_asr_cpu.conf @@ -1,21 +1,3 @@ { - "schema_version": 1, - "data": { - "pipe_path": "pipeline_audio_asr_cpu.json", - "io_binding": "audio_asr_intent.operator.v1", - "model_paths": { - "asr_model_v1": "models/ggml-base.bin" - }, - "outputs": { - "audio_out": { - "type": "audio_out", - "meta_num": 0, - "metadata_type_id": 0, - "capacities": { - "transcribed_text": 511, - "intent_slot_json": 1023 - } - } - } - } + "pipe_path": "pipeline_audio_asr_cpu.json" } diff --git a/configs/pipeline_audio_asr_cpu.json b/configs/pipeline_audio_asr_cpu.json index c2b777d9..e7d71df5 100644 --- a/configs/pipeline_audio_asr_cpu.json +++ b/configs/pipeline_audio_asr_cpu.json @@ -1,5 +1,24 @@ { "biz_name": "speech_audio_asr_intent_slot", + "deployment": { + "model_paths": { + "asr_model_v1": "models/ggml-base.bin" + }, + "io": { + "io_binding": "audio_asr_intent.operator.v1", + "output_allocations": { + "audio_out": { + "type": "audio_out", + "meta_num": 0, + "metadata_type_id": 0, + "capacities": { + "transcribed_text": 511, + "intent_slot_json": 1023 + } + } + } + } + }, "models": [ { "model_id": "asr_model_v1", diff --git a/configs/pipeline_cross_rerank_cpu.conf b/configs/pipeline_cross_rerank_cpu.conf index 90df28ce..ac1fa83e 100644 --- a/configs/pipeline_cross_rerank_cpu.conf +++ b/configs/pipeline_cross_rerank_cpu.conf @@ -1,18 +1,3 @@ { - "schema_version": 1, - "data": { - "pipe_path": "pipeline_cross_rerank_cpu.json", - "io_binding": "cross_rerank.operator.v1", - "model_paths": { - "rerank_model_v1": "./models/ms_marco_tinybert_l2_v2_quantized.onnx" - }, - "outputs": { - "rerank_out": { - "type": "rerank_out", - "meta_num": 0, - "metadata_type_id": 0, - "capacities": {} - } - } - } + "pipe_path": "pipeline_cross_rerank_cpu.json" } diff --git a/configs/pipeline_cross_rerank_cpu.json b/configs/pipeline_cross_rerank_cpu.json index 3009752c..63e14b49 100644 --- a/configs/pipeline_cross_rerank_cpu.json +++ b/configs/pipeline_cross_rerank_cpu.json @@ -1,5 +1,21 @@ { "biz_name": "dense_cross_rerank_scoring", + "deployment": { + "model_paths": { + "rerank_model_v1": "./models/ms_marco_tinybert_l2_v2_quantized.onnx" + }, + "io": { + "io_binding": "cross_rerank.operator.v1", + "output_allocations": { + "rerank_out": { + "type": "rerank_out", + "meta_num": 0, + "metadata_type_id": 0, + "capacities": {} + } + } + } + }, "models": [ { "model_id": "rerank_model_v1", diff --git a/configs/pipeline_dialogue_audit_default.conf b/configs/pipeline_dialogue_audit_default.conf index 96210421..f1061f9e 100644 --- a/configs/pipeline_dialogue_audit_default.conf +++ b/configs/pipeline_dialogue_audit_default.conf @@ -1,24 +1,3 @@ { - "schema_version": 1, - "data": { - "pipe_path": "pipeline_dialogue_audit_default.json", - "io_binding": "compliance_audit.operator.v1", - "model_paths": { - "embed_model_v2": "./models/bge_base_zh_v1.5.onnx", - "rerank_model_v1": "./models/ms_marco_tinybert_l2_v2_quantized.onnx", - "audit_llm_v1": "./models/qwen2.5-0.5b-instruct-q4_k_m.gguf" - }, - "outputs": { - "audit_out": { - "type": "audit_out", - "meta_num": 0, - "metadata_type_id": 0, - "capacities": { - "risk_level": 31, - "matched_policy_clause": 255, - "audit_verdict_json": 1023 - } - } - } - } + "pipe_path": "pipeline_dialogue_audit_default.json" } diff --git a/configs/pipeline_dialogue_audit_default.json b/configs/pipeline_dialogue_audit_default.json index c76ab167..c471304f 100644 --- a/configs/pipeline_dialogue_audit_default.json +++ b/configs/pipeline_dialogue_audit_default.json @@ -1,5 +1,27 @@ { "biz_name": "dialogue_compliance_audit_v1", + "deployment": { + "model_paths": { + "embed_model_v2": "./models/bge_base_zh_v1.5.onnx", + "rerank_model_v1": "./models/ms_marco_tinybert_l2_v2_quantized.onnx", + "audit_llm_v1": "./models/qwen2.5-0.5b-instruct-q4_k_m.gguf" + }, + "io": { + "io_binding": "compliance_audit.operator.v1", + "output_allocations": { + "audit_out": { + "type": "audit_out", + "meta_num": 0, + "metadata_type_id": 0, + "capacities": { + "risk_level": 31, + "matched_policy_clause": 255, + "audit_verdict_json": 1023 + } + } + } + } + }, "models": [ { "capability": "embedding", diff --git a/configs/pipeline_dialogue_audit_kite.conf b/configs/pipeline_dialogue_audit_kite.conf index af8e1935..c4f0c3d3 100644 --- a/configs/pipeline_dialogue_audit_kite.conf +++ b/configs/pipeline_dialogue_audit_kite.conf @@ -1,24 +1,3 @@ { - "schema_version": 1, - "data": { - "pipe_path": "pipeline_dialogue_audit_kite.json", - "io_binding": "compliance_audit.operator.v1", - "model_paths": { - "embed_model_v2": "models/bge_base_zh_v1.5.onnx", - "rerank_model_v1": "models/ms_marco_tinybert_l2_v2_quantized.onnx", - "audit_llm_v1": "models/qwen2.5-0.5b-instruct-q4_k_m.gguf" - }, - "outputs": { - "audit_out": { - "type": "audit_out", - "meta_num": 0, - "metadata_type_id": 0, - "capacities": { - "risk_level": 31, - "matched_policy_clause": 255, - "audit_verdict_json": 1023 - } - } - } - } + "pipe_path": "pipeline_dialogue_audit_kite.json" } diff --git a/configs/pipeline_dialogue_audit_kite.json b/configs/pipeline_dialogue_audit_kite.json index 9262782c..921ed13c 100644 --- a/configs/pipeline_dialogue_audit_kite.json +++ b/configs/pipeline_dialogue_audit_kite.json @@ -1,5 +1,27 @@ { "biz_name": "dialogue_compliance_audit_v1", + "deployment": { + "model_paths": { + "embed_model_v2": "models/bge_base_zh_v1.5.onnx", + "rerank_model_v1": "models/ms_marco_tinybert_l2_v2_quantized.onnx", + "audit_llm_v1": "models/qwen2.5-0.5b-instruct-q4_k_m.gguf" + }, + "io": { + "io_binding": "compliance_audit.operator.v1", + "output_allocations": { + "audit_out": { + "type": "audit_out", + "meta_num": 0, + "metadata_type_id": 0, + "capacities": { + "risk_level": 31, + "matched_policy_clause": 255, + "audit_verdict_json": 1023 + } + } + } + } + }, "models": [ { "capability": "embedding", diff --git a/configs/pipeline_doc_qa_cpu.conf b/configs/pipeline_doc_qa_cpu.conf index c7b10130..78f9559b 100644 --- a/configs/pipeline_doc_qa_cpu.conf +++ b/configs/pipeline_doc_qa_cpu.conf @@ -1,22 +1,3 @@ { - "schema_version": 1, - "data": { - "pipe_path": "pipeline_doc_qa_cpu.json", - "io_binding": "doc_qa.operator.v1", - "model_paths": { - "embed_model_onnx": "./models/bge_base_zh_v1.5.onnx", - "llm_model_llamacpp": "./models/qwen2.5-0.5b-instruct-q4_k_m.gguf" - }, - "outputs": { - "doc_out": { - "type": "doc_out", - "meta_num": 0, - "metadata_type_id": 0, - "capacities": { - "intent_name": 63, - "answer_text": 1023 - } - } - } - } + "pipe_path": "pipeline_doc_qa_cpu.json" } diff --git a/configs/pipeline_doc_qa_cpu.json b/configs/pipeline_doc_qa_cpu.json index 639cea62..a50fff54 100644 --- a/configs/pipeline_doc_qa_cpu.json +++ b/configs/pipeline_doc_qa_cpu.json @@ -1,5 +1,25 @@ { "biz_name": "smart_doc_qa_v1", + "deployment": { + "model_paths": { + "embed_model_onnx": "./models/bge_base_zh_v1.5.onnx", + "llm_model_llamacpp": "./models/qwen2.5-0.5b-instruct-q4_k_m.gguf" + }, + "io": { + "io_binding": "doc_qa.operator.v1", + "output_allocations": { + "doc_out": { + "type": "doc_out", + "meta_num": 0, + "metadata_type_id": 0, + "capacities": { + "intent_name": 63, + "answer_text": 1023 + } + } + } + } + }, "models": [ { "model_id": "embed_model_onnx", @@ -151,7 +171,7 @@ }, "config": { "default_category": "GENERAL_QA", - "default_score": 0.90, + "default_score": 0.9, "categories": { "AFTER_SALES_REFUND": [ "退款", diff --git a/configs/pipeline_doc_qa_default.conf b/configs/pipeline_doc_qa_default.conf index 0e25e0f3..be5cd647 100644 --- a/configs/pipeline_doc_qa_default.conf +++ b/configs/pipeline_doc_qa_default.conf @@ -1,22 +1,3 @@ { - "schema_version": 1, - "data": { - "pipe_path": "pipeline_doc_qa_default.json", - "io_binding": "doc_qa.operator.v1", - "model_paths": { - "embed_model_v1": "./models/bge_base_zh_v1.5.onnx", - "llm_model_v1": "./models/qwen2.5-0.5b-instruct-q4_k_m.gguf" - }, - "outputs": { - "doc_out": { - "type": "doc_out", - "meta_num": 0, - "metadata_type_id": 0, - "capacities": { - "intent_name": 63, - "answer_text": 1023 - } - } - } - } + "pipe_path": "pipeline_doc_qa_default.json" } diff --git a/configs/pipeline_doc_qa_default.json b/configs/pipeline_doc_qa_default.json index d1ca2f51..e26b9229 100644 --- a/configs/pipeline_doc_qa_default.json +++ b/configs/pipeline_doc_qa_default.json @@ -1,5 +1,25 @@ { "biz_name": "smart_doc_qa_v1", + "deployment": { + "model_paths": { + "embed_model_v1": "./models/bge_base_zh_v1.5.onnx", + "llm_model_v1": "./models/qwen2.5-0.5b-instruct-q4_k_m.gguf" + }, + "io": { + "io_binding": "doc_qa.operator.v1", + "output_allocations": { + "doc_out": { + "type": "doc_out", + "meta_num": 0, + "metadata_type_id": 0, + "capacities": { + "intent_name": 63, + "answer_text": 1023 + } + } + } + } + }, "models": [ { "capability": "embedding", @@ -144,7 +164,7 @@ }, "config": { "default_category": "GENERAL_QA", - "default_score": 0.90, + "default_score": 0.9, "categories": { "AFTER_SALES_REFUND": [ "退款", diff --git a/configs/pipeline_doc_qa_kite.conf b/configs/pipeline_doc_qa_kite.conf index 27a6af4a..8d04b3d2 100644 --- a/configs/pipeline_doc_qa_kite.conf +++ b/configs/pipeline_doc_qa_kite.conf @@ -1,22 +1,3 @@ { - "schema_version": 1, - "data": { - "pipe_path": "pipeline_doc_qa_kite.json", - "io_binding": "doc_qa.operator.v1", - "model_paths": { - "embed_model_v1": "models/bge_base_zh_v1.5.onnx", - "llm_model_v1": "models/qwen2.5-0.5b-instruct-q4_k_m.gguf" - }, - "outputs": { - "doc_out": { - "type": "doc_out", - "meta_num": 0, - "metadata_type_id": 0, - "capacities": { - "intent_name": 63, - "answer_text": 1023 - } - } - } - } + "pipe_path": "pipeline_doc_qa_kite.json" } diff --git a/configs/pipeline_doc_qa_kite.json b/configs/pipeline_doc_qa_kite.json index 4a2bfa6d..5887cf85 100644 --- a/configs/pipeline_doc_qa_kite.json +++ b/configs/pipeline_doc_qa_kite.json @@ -1,5 +1,25 @@ { "biz_name": "smart_doc_qa_v1", + "deployment": { + "model_paths": { + "embed_model_v1": "models/bge_base_zh_v1.5.onnx", + "llm_model_v1": "models/qwen2.5-0.5b-instruct-q4_k_m.gguf" + }, + "io": { + "io_binding": "doc_qa.operator.v1", + "output_allocations": { + "doc_out": { + "type": "doc_out", + "meta_num": 0, + "metadata_type_id": 0, + "capacities": { + "intent_name": 63, + "answer_text": 1023 + } + } + } + } + }, "models": [ { "capability": "embedding", diff --git a/configs/pipeline_doc_qa_kite_generated_embeddings.conf b/configs/pipeline_doc_qa_kite_generated_embeddings.conf index 4af62b36..53589509 100644 --- a/configs/pipeline_doc_qa_kite_generated_embeddings.conf +++ b/configs/pipeline_doc_qa_kite_generated_embeddings.conf @@ -1,22 +1,3 @@ { - "schema_version": 1, - "data": { - "pipe_path": "pipeline_doc_qa_kite_generated_embeddings.json", - "io_binding": "doc_qa.operator.v1", - "model_paths": { - "embed_model_v1": "models/qwen2.5-0.5b-instruct-q4_k_m.gguf", - "llm_model_v1": "models/qwen2.5-0.5b-instruct-q4_k_m.gguf" - }, - "outputs": { - "doc_out": { - "type": "doc_out", - "meta_num": 0, - "metadata_type_id": 0, - "capacities": { - "intent_name": 63, - "answer_text": 1023 - } - } - } - } + "pipe_path": "pipeline_doc_qa_kite_generated_embeddings.json" } diff --git a/configs/pipeline_doc_qa_kite_generated_embeddings.json b/configs/pipeline_doc_qa_kite_generated_embeddings.json index 190b8d91..39f5b080 100644 --- a/configs/pipeline_doc_qa_kite_generated_embeddings.json +++ b/configs/pipeline_doc_qa_kite_generated_embeddings.json @@ -1,5 +1,25 @@ { "biz_name": "smart_doc_qa_v1", + "deployment": { + "model_paths": { + "embed_model_v1": "models/qwen2.5-0.5b-instruct-q4_k_m.gguf", + "llm_model_v1": "models/qwen2.5-0.5b-instruct-q4_k_m.gguf" + }, + "io": { + "io_binding": "doc_qa.operator.v1", + "output_allocations": { + "doc_out": { + "type": "doc_out", + "meta_num": 0, + "metadata_type_id": 0, + "capacities": { + "intent_name": 63, + "answer_text": 1023 + } + } + } + } + }, "models": [ { "capability": "embedding", diff --git a/configs/pipeline_doc_qa_rerank_cpu.conf b/configs/pipeline_doc_qa_rerank_cpu.conf index c7d26f46..2f65a74b 100644 --- a/configs/pipeline_doc_qa_rerank_cpu.conf +++ b/configs/pipeline_doc_qa_rerank_cpu.conf @@ -1,23 +1,3 @@ { - "schema_version": 1, - "data": { - "pipe_path": "pipeline_doc_qa_rerank_cpu.json", - "io_binding": "doc_qa.operator.v1", - "model_paths": { - "embed_model_onnx": "./models/bge_base_zh_v1.5.onnx", - "rerank_model_onnx": "./models/ms_marco_tinybert_l2_v2_quantized.onnx", - "llm_model_llamacpp": "./models/qwen2.5-0.5b-instruct-q4_k_m.gguf" - }, - "outputs": { - "doc_out": { - "type": "doc_out", - "meta_num": 0, - "metadata_type_id": 0, - "capacities": { - "intent_name": 63, - "answer_text": 1023 - } - } - } - } + "pipe_path": "pipeline_doc_qa_rerank_cpu.json" } diff --git a/configs/pipeline_doc_qa_rerank_cpu.json b/configs/pipeline_doc_qa_rerank_cpu.json index 17519de2..1e984e4c 100644 --- a/configs/pipeline_doc_qa_rerank_cpu.json +++ b/configs/pipeline_doc_qa_rerank_cpu.json @@ -1,5 +1,26 @@ { "biz_name": "smart_doc_qa_v1", + "deployment": { + "model_paths": { + "embed_model_onnx": "./models/bge_base_zh_v1.5.onnx", + "rerank_model_onnx": "./models/ms_marco_tinybert_l2_v2_quantized.onnx", + "llm_model_llamacpp": "./models/qwen2.5-0.5b-instruct-q4_k_m.gguf" + }, + "io": { + "io_binding": "doc_qa.operator.v1", + "output_allocations": { + "doc_out": { + "type": "doc_out", + "meta_num": 0, + "metadata_type_id": 0, + "capacities": { + "intent_name": 63, + "answer_text": 1023 + } + } + } + } + }, "models": [ { "model_id": "embed_model_onnx", diff --git a/configs/pipeline_doc_qa_rerank_default.conf b/configs/pipeline_doc_qa_rerank_default.conf index 34ada649..38e48bc2 100644 --- a/configs/pipeline_doc_qa_rerank_default.conf +++ b/configs/pipeline_doc_qa_rerank_default.conf @@ -1,23 +1,3 @@ { - "schema_version": 1, - "data": { - "pipe_path": "pipeline_doc_qa_rerank_default.json", - "io_binding": "doc_qa.operator.v1", - "model_paths": { - "embed_model_v1": "./models/bge_base_zh_v1.5.onnx", - "rerank_model_v1": "./models/ms_marco_tinybert_l2_v2_quantized.onnx", - "llm_model_v1": "./models/qwen2.5-0.5b-instruct-q4_k_m.gguf" - }, - "outputs": { - "doc_out": { - "type": "doc_out", - "meta_num": 0, - "metadata_type_id": 0, - "capacities": { - "intent_name": 63, - "answer_text": 1023 - } - } - } - } + "pipe_path": "pipeline_doc_qa_rerank_default.json" } diff --git a/configs/pipeline_doc_qa_rerank_default.json b/configs/pipeline_doc_qa_rerank_default.json index b6d3abe6..b2023b68 100644 --- a/configs/pipeline_doc_qa_rerank_default.json +++ b/configs/pipeline_doc_qa_rerank_default.json @@ -1,5 +1,26 @@ { "biz_name": "smart_doc_qa_v1", + "deployment": { + "model_paths": { + "embed_model_v1": "./models/bge_base_zh_v1.5.onnx", + "rerank_model_v1": "./models/ms_marco_tinybert_l2_v2_quantized.onnx", + "llm_model_v1": "./models/qwen2.5-0.5b-instruct-q4_k_m.gguf" + }, + "io": { + "io_binding": "doc_qa.operator.v1", + "output_allocations": { + "doc_out": { + "type": "doc_out", + "meta_num": 0, + "metadata_type_id": 0, + "capacities": { + "intent_name": 63, + "answer_text": 1023 + } + } + } + } + }, "models": [ { "capability": "embedding", @@ -178,7 +199,7 @@ }, "config": { "default_category": "GENERAL_QA", - "default_score": 0.90, + "default_score": 0.9, "categories": { "AFTER_SALES_REFUND": [ "退款", diff --git a/configs/pipeline_doc_qa_rerank_kite.conf b/configs/pipeline_doc_qa_rerank_kite.conf index e8314c3e..791145b9 100644 --- a/configs/pipeline_doc_qa_rerank_kite.conf +++ b/configs/pipeline_doc_qa_rerank_kite.conf @@ -1,23 +1,3 @@ { - "schema_version": 1, - "data": { - "pipe_path": "pipeline_doc_qa_rerank_kite.json", - "io_binding": "doc_qa.operator.v1", - "model_paths": { - "embed_model_onnx": "models/bge_base_zh_v1.5.onnx", - "rerank_model_onnx": "models/ms_marco_tinybert_l2_v2_quantized.onnx", - "llm_model_llamacpp": "models/qwen2.5-0.5b-instruct-q4_k_m.gguf" - }, - "outputs": { - "doc_out": { - "type": "doc_out", - "meta_num": 0, - "metadata_type_id": 0, - "capacities": { - "intent_name": 63, - "answer_text": 1023 - } - } - } - } + "pipe_path": "pipeline_doc_qa_rerank_kite.json" } diff --git a/configs/pipeline_doc_qa_rerank_kite.json b/configs/pipeline_doc_qa_rerank_kite.json index 73d032a9..ae1a9460 100644 --- a/configs/pipeline_doc_qa_rerank_kite.json +++ b/configs/pipeline_doc_qa_rerank_kite.json @@ -1,5 +1,26 @@ { "biz_name": "smart_doc_qa_v1", + "deployment": { + "model_paths": { + "embed_model_onnx": "models/bge_base_zh_v1.5.onnx", + "rerank_model_onnx": "models/ms_marco_tinybert_l2_v2_quantized.onnx", + "llm_model_llamacpp": "models/qwen2.5-0.5b-instruct-q4_k_m.gguf" + }, + "io": { + "io_binding": "doc_qa.operator.v1", + "output_allocations": { + "doc_out": { + "type": "doc_out", + "meta_num": 0, + "metadata_type_id": 0, + "capacities": { + "intent_name": 63, + "answer_text": 1023 + } + } + } + } + }, "models": [ { "model_id": "embed_model_onnx", diff --git a/configs/pipeline_entity_extract_cpu.conf b/configs/pipeline_entity_extract_cpu.conf index a7de58fe..c2bc55f5 100644 --- a/configs/pipeline_entity_extract_cpu.conf +++ b/configs/pipeline_entity_extract_cpu.conf @@ -1,20 +1,3 @@ { - "schema_version": 1, - "data": { - "pipe_path": "pipeline_entity_extract_cpu.json", - "io_binding": "entity_extract.operator.v1", - "model_paths": { - "entity_llm": "./models/qwen2.5-0.5b-instruct-q4_k_m.gguf" - }, - "outputs": { - "entity_out": { - "type": "entity_out", - "meta_num": 0, - "metadata_type_id": 0, - "capacities": { - "entities_json": 2047 - } - } - } - } + "pipe_path": "pipeline_entity_extract_cpu.json" } diff --git a/configs/pipeline_entity_extract_cpu.json b/configs/pipeline_entity_extract_cpu.json index 754dfb0d..88ee7fc8 100644 --- a/configs/pipeline_entity_extract_cpu.json +++ b/configs/pipeline_entity_extract_cpu.json @@ -1,5 +1,23 @@ { "biz_name": "entity_extract_v1", + "deployment": { + "model_paths": { + "entity_llm": "./models/qwen2.5-0.5b-instruct-q4_k_m.gguf" + }, + "io": { + "io_binding": "entity_extract.operator.v1", + "output_allocations": { + "entity_out": { + "type": "entity_out", + "meta_num": 0, + "metadata_type_id": 0, + "capacities": { + "entities_json": 2047 + } + } + } + } + }, "models": [ { "model_id": "entity_llm", diff --git a/configs/pipeline_entity_extract_default.conf b/configs/pipeline_entity_extract_default.conf index 8ccbb6fb..2cd2a612 100644 --- a/configs/pipeline_entity_extract_default.conf +++ b/configs/pipeline_entity_extract_default.conf @@ -1,20 +1,3 @@ { - "schema_version": 1, - "data": { - "pipe_path": "pipeline_entity_extract_default.json", - "io_binding": "entity_extract.operator.v1", - "model_paths": { - "entity_llm": "./models/qwen2.5-0.5b-instruct-q4_k_m.gguf" - }, - "outputs": { - "entity_out": { - "type": "entity_out", - "meta_num": 0, - "metadata_type_id": 0, - "capacities": { - "entities_json": 2047 - } - } - } - } + "pipe_path": "pipeline_entity_extract_default.json" } diff --git a/configs/pipeline_entity_extract_default.json b/configs/pipeline_entity_extract_default.json index 84d28de2..a3a923f0 100644 --- a/configs/pipeline_entity_extract_default.json +++ b/configs/pipeline_entity_extract_default.json @@ -1,5 +1,23 @@ { "biz_name": "entity_extract_v1", + "deployment": { + "model_paths": { + "entity_llm": "./models/qwen2.5-0.5b-instruct-q4_k_m.gguf" + }, + "io": { + "io_binding": "entity_extract.operator.v1", + "output_allocations": { + "entity_out": { + "type": "entity_out", + "meta_num": 0, + "metadata_type_id": 0, + "capacities": { + "entities_json": 2047 + } + } + } + } + }, "models": [ { "capability": "llm", diff --git a/configs/pipeline_entity_extract_kite.conf b/configs/pipeline_entity_extract_kite.conf index 8e649a88..6c823927 100644 --- a/configs/pipeline_entity_extract_kite.conf +++ b/configs/pipeline_entity_extract_kite.conf @@ -1,20 +1,3 @@ { - "schema_version": 1, - "data": { - "pipe_path": "pipeline_entity_extract_kite.json", - "io_binding": "entity_extract.operator.v1", - "model_paths": { - "entity_llm": "models/qwen2.5-0.5b-instruct-q4_k_m.gguf" - }, - "outputs": { - "entity_out": { - "type": "entity_out", - "meta_num": 0, - "metadata_type_id": 0, - "capacities": { - "entities_json": 2047 - } - } - } - } + "pipe_path": "pipeline_entity_extract_kite.json" } diff --git a/configs/pipeline_entity_extract_kite.json b/configs/pipeline_entity_extract_kite.json index 02355f30..0c6ddc0a 100644 --- a/configs/pipeline_entity_extract_kite.json +++ b/configs/pipeline_entity_extract_kite.json @@ -1,5 +1,23 @@ { "biz_name": "entity_extract_v1", + "deployment": { + "model_paths": { + "entity_llm": "models/qwen2.5-0.5b-instruct-q4_k_m.gguf" + }, + "io": { + "io_binding": "entity_extract.operator.v1", + "output_allocations": { + "entity_out": { + "type": "entity_out", + "meta_num": 0, + "metadata_type_id": 0, + "capacities": { + "entities_json": 2047 + } + } + } + } + }, "models": [ { "capability": "llm", diff --git a/configs/pipeline_keyword_match_rules.conf b/configs/pipeline_keyword_match_rules.conf index a0bb0f14..cbeac6cc 100644 --- a/configs/pipeline_keyword_match_rules.conf +++ b/configs/pipeline_keyword_match_rules.conf @@ -1,17 +1,3 @@ { - "schema_version": 1, - "data": { - "pipe_path": "pipeline_keyword_match_rules.json", - "io_binding": "keyword_match.operator.v1", - "outputs": { - "keyword_out": { - "type": "keyword_out", - "meta_num": 0, - "metadata_type_id": 0, - "capacities": { - "match_result_json": 2047 - } - } - } - } + "pipe_path": "pipeline_keyword_match_rules.json" } diff --git a/configs/pipeline_keyword_match_rules.json b/configs/pipeline_keyword_match_rules.json index a5151d0d..b3fe7c8b 100644 --- a/configs/pipeline_keyword_match_rules.json +++ b/configs/pipeline_keyword_match_rules.json @@ -1,5 +1,20 @@ { "biz_name": "keyword_match_v1", + "deployment": { + "io": { + "io_binding": "keyword_match.operator.v1", + "output_allocations": { + "keyword_out": { + "type": "keyword_out", + "meta_num": 0, + "metadata_type_id": 0, + "capacities": { + "match_result_json": 2047 + } + } + } + } + }, "models": [], "pipeline": [ { diff --git a/configs/pipeline_ocr_doc_qa_kite.conf b/configs/pipeline_ocr_doc_qa_kite.conf index bfb76166..cb5c990b 100644 --- a/configs/pipeline_ocr_doc_qa_kite.conf +++ b/configs/pipeline_ocr_doc_qa_kite.conf @@ -1,21 +1,3 @@ { - "schema_version": 1, - "data": { - "pipe_path": "pipeline_ocr_doc_qa_kite.json", - "io_binding": "ocr_doc_qa.operator.v1", - "model_paths": { - "ocr_model_v1": "models/SmolVLM-256M-Instruct-Q8_0.gguf", - "llm_model_v1": "models/qwen2.5-0.5b-instruct-q4_k_m.gguf" - }, - "outputs": { - "od_out": { - "type": "od_out", - "meta_num": 0, - "metadata_type_id": 0, - "capacities": { - "result_json": 2047 - } - } - } - } + "pipe_path": "pipeline_ocr_doc_qa_kite.json" } diff --git a/configs/pipeline_ocr_doc_qa_kite.json b/configs/pipeline_ocr_doc_qa_kite.json index 31efbe83..7a2a4676 100644 --- a/configs/pipeline_ocr_doc_qa_kite.json +++ b/configs/pipeline_ocr_doc_qa_kite.json @@ -1,5 +1,24 @@ { "biz_name": "multimodal_ocr_invoice_qa", + "deployment": { + "model_paths": { + "ocr_model_v1": "models/SmolVLM-256M-Instruct-Q8_0.gguf", + "llm_model_v1": "models/qwen2.5-0.5b-instruct-q4_k_m.gguf" + }, + "io": { + "io_binding": "ocr_doc_qa.operator.v1", + "output_allocations": { + "od_out": { + "type": "od_out", + "meta_num": 0, + "metadata_type_id": 0, + "capacities": { + "result_json": 2047 + } + } + } + } + }, "models": [ { "capability": "ocr", diff --git a/configs/pipeline_translate_cpu.conf b/configs/pipeline_translate_cpu.conf index 68e140ab..be55b4f1 100644 --- a/configs/pipeline_translate_cpu.conf +++ b/configs/pipeline_translate_cpu.conf @@ -1,20 +1,3 @@ { - "schema_version": 1, - "data": { - "pipe_path": "pipeline_translate_cpu.json", - "io_binding": "translate.operator.v1", - "model_paths": { - "translate_llm": "./models/qwen2.5-0.5b-instruct-q4_k_m.gguf" - }, - "outputs": { - "entity_out": { - "type": "entity_out", - "meta_num": 0, - "metadata_type_id": 0, - "capacities": { - "entities_json": 8191 - } - } - } - } + "pipe_path": "pipeline_translate_cpu.json" } diff --git a/configs/pipeline_translate_cpu.json b/configs/pipeline_translate_cpu.json index 5e3fdd8d..5717f67f 100644 --- a/configs/pipeline_translate_cpu.json +++ b/configs/pipeline_translate_cpu.json @@ -1,5 +1,23 @@ { "biz_name": "translate_v1", + "deployment": { + "model_paths": { + "translate_llm": "./models/qwen2.5-0.5b-instruct-q4_k_m.gguf" + }, + "io": { + "io_binding": "translate.operator.v1", + "output_allocations": { + "entity_out": { + "type": "entity_out", + "meta_num": 0, + "metadata_type_id": 0, + "capacities": { + "entities_json": 8191 + } + } + } + } + }, "models": [ { "backend": "llama_cpp", diff --git a/demo/fixtures/mock/pipeline_audio_asr_intent.conf b/demo/fixtures/mock/pipeline_audio_asr_intent.conf index d9b4e273..84836f40 100644 --- a/demo/fixtures/mock/pipeline_audio_asr_intent.conf +++ b/demo/fixtures/mock/pipeline_audio_asr_intent.conf @@ -1,21 +1,3 @@ { - "schema_version": 1, - "data": { - "pipe_path": "pipeline_audio_asr_intent.json", - "io_binding": "audio_asr_intent.operator.v1", - "model_paths": { - "asr_model_v1": "./models/paraformer_asr_npu.bin" - }, - "outputs": { - "audio_out": { - "type": "audio_out", - "meta_num": 0, - "metadata_type_id": 0, - "capacities": { - "transcribed_text": 511, - "intent_slot_json": 1023 - } - } - } - } + "pipe_path": "pipeline_audio_asr_intent.json" } diff --git a/demo/fixtures/mock/pipeline_audio_asr_intent.json b/demo/fixtures/mock/pipeline_audio_asr_intent.json index b5a990cf..cf912dd5 100644 --- a/demo/fixtures/mock/pipeline_audio_asr_intent.json +++ b/demo/fixtures/mock/pipeline_audio_asr_intent.json @@ -1,5 +1,24 @@ { "biz_name": "speech_audio_asr_intent_slot", + "deployment": { + "model_paths": { + "asr_model_v1": "./models/paraformer_asr_npu.bin" + }, + "io": { + "io_binding": "audio_asr_intent.operator.v1", + "output_allocations": { + "audio_out": { + "type": "audio_out", + "meta_num": 0, + "metadata_type_id": 0, + "capacities": { + "transcribed_text": 511, + "intent_slot_json": 1023 + } + } + } + } + }, "models": [ { "capability": "asr", diff --git a/demo/fixtures/mock/pipeline_dialogue_audit.conf b/demo/fixtures/mock/pipeline_dialogue_audit.conf index 23564392..2c35376c 100644 --- a/demo/fixtures/mock/pipeline_dialogue_audit.conf +++ b/demo/fixtures/mock/pipeline_dialogue_audit.conf @@ -1,24 +1,3 @@ { - "schema_version": 1, - "data": { - "pipe_path": "pipeline_dialogue_audit.json", - "io_binding": "compliance_audit.operator.v1", - "model_paths": { - "embed_model_v2": "./models/bge_m3_npu.bin", - "rerank_model_v1": "./models/bge_reranker_large_npu.bin", - "audit_llm_v1": "./models/qwen_1_8b_npu.bin" - }, - "outputs": { - "audit_out": { - "type": "audit_out", - "meta_num": 0, - "metadata_type_id": 0, - "capacities": { - "risk_level": 31, - "matched_policy_clause": 255, - "audit_verdict_json": 1023 - } - } - } - } + "pipe_path": "pipeline_dialogue_audit.json" } diff --git a/demo/fixtures/mock/pipeline_dialogue_audit.json b/demo/fixtures/mock/pipeline_dialogue_audit.json index f2d4dfe3..e5994aca 100644 --- a/demo/fixtures/mock/pipeline_dialogue_audit.json +++ b/demo/fixtures/mock/pipeline_dialogue_audit.json @@ -1,5 +1,27 @@ { "biz_name": "dialogue_compliance_audit_v1", + "deployment": { + "model_paths": { + "embed_model_v2": "./models/bge_m3_npu.bin", + "rerank_model_v1": "./models/bge_reranker_large_npu.bin", + "audit_llm_v1": "./models/qwen_1_8b_npu.bin" + }, + "io": { + "io_binding": "compliance_audit.operator.v1", + "output_allocations": { + "audit_out": { + "type": "audit_out", + "meta_num": 0, + "metadata_type_id": 0, + "capacities": { + "risk_level": 31, + "matched_policy_clause": 255, + "audit_verdict_json": 1023 + } + } + } + } + }, "models": [ { "capability": "embedding", diff --git a/demo/fixtures/mock/pipeline_doc_qa.conf b/demo/fixtures/mock/pipeline_doc_qa.conf index 814e6de3..a2cacd63 100644 --- a/demo/fixtures/mock/pipeline_doc_qa.conf +++ b/demo/fixtures/mock/pipeline_doc_qa.conf @@ -1,22 +1,3 @@ { - "schema_version": 1, - "data": { - "pipe_path": "pipeline_doc_qa.json", - "io_binding": "doc_qa.operator.v1", - "model_paths": { - "embed_model_v1": "./models/bge_base_npu.bin", - "llm_model_v1": "./models/qwen_1.5b_npu.bin" - }, - "outputs": { - "doc_out": { - "type": "doc_out", - "meta_num": 0, - "metadata_type_id": 0, - "capacities": { - "intent_name": 63, - "answer_text": 1023 - } - } - } - } + "pipe_path": "pipeline_doc_qa.json" } diff --git a/demo/fixtures/mock/pipeline_doc_qa.json b/demo/fixtures/mock/pipeline_doc_qa.json index c68244c1..4f25554b 100644 --- a/demo/fixtures/mock/pipeline_doc_qa.json +++ b/demo/fixtures/mock/pipeline_doc_qa.json @@ -1,5 +1,25 @@ { "biz_name": "smart_doc_qa_v1", + "deployment": { + "model_paths": { + "embed_model_v1": "./models/bge_base_npu.bin", + "llm_model_v1": "./models/qwen_1.5b_npu.bin" + }, + "io": { + "io_binding": "doc_qa.operator.v1", + "output_allocations": { + "doc_out": { + "type": "doc_out", + "meta_num": 0, + "metadata_type_id": 0, + "capacities": { + "intent_name": 63, + "answer_text": 1023 + } + } + } + } + }, "models": [ { "capability": "embedding", @@ -133,7 +153,7 @@ }, "config": { "default_category": "GENERAL_QA", - "default_score": 0.90, + "default_score": 0.9, "categories": { "AFTER_SALES_REFUND": [ "退款", diff --git a/demo/fixtures/mock/pipeline_doc_qa_custom.conf b/demo/fixtures/mock/pipeline_doc_qa_custom.conf index 8e2af7fc..ecc08735 100644 --- a/demo/fixtures/mock/pipeline_doc_qa_custom.conf +++ b/demo/fixtures/mock/pipeline_doc_qa_custom.conf @@ -1,21 +1,3 @@ { - "schema_version": 1, - "data": { - "pipe_path": "pipeline_doc_qa_custom.json", - "io_binding": "doc_qa.operator.v1", - "model_paths": { - "llm_model_v1": "demo/fixtures/mock/artifacts/neutral-llm.fixture" - }, - "outputs": { - "doc_out": { - "type": "doc_out", - "meta_num": 0, - "metadata_type_id": 0, - "capacities": { - "intent_name": 63, - "answer_text": 1023 - } - } - } - } + "pipe_path": "pipeline_doc_qa_custom.json" } diff --git a/demo/fixtures/mock/pipeline_doc_qa_custom.json b/demo/fixtures/mock/pipeline_doc_qa_custom.json index ff8ade3f..7069b192 100644 --- a/demo/fixtures/mock/pipeline_doc_qa_custom.json +++ b/demo/fixtures/mock/pipeline_doc_qa_custom.json @@ -1,5 +1,24 @@ { "biz_name": "smart_doc_qa_v1", + "deployment": { + "model_paths": { + "llm_model_v1": "demo/fixtures/mock/artifacts/neutral-llm.fixture" + }, + "io": { + "io_binding": "doc_qa.operator.v1", + "output_allocations": { + "doc_out": { + "type": "doc_out", + "meta_num": 0, + "metadata_type_id": 0, + "capacities": { + "intent_name": 63, + "answer_text": 1023 + } + } + } + } + }, "models": [ { "capability": "llm", diff --git a/demo/fixtures/mock/pipeline_doc_qa_rerank.conf b/demo/fixtures/mock/pipeline_doc_qa_rerank.conf index 2e36e34a..9cea6afb 100644 --- a/demo/fixtures/mock/pipeline_doc_qa_rerank.conf +++ b/demo/fixtures/mock/pipeline_doc_qa_rerank.conf @@ -1,23 +1,3 @@ { - "schema_version": 1, - "data": { - "pipe_path": "pipeline_doc_qa_rerank.json", - "io_binding": "doc_qa.operator.v1", - "model_paths": { - "embed_model_v1": "./models/bge_base_npu.bin", - "rerank_model_v1": "./models/bge_reranker_large_npu.bin", - "llm_model_v1": "./models/qwen_1.5b_npu.bin" - }, - "outputs": { - "doc_out": { - "type": "doc_out", - "meta_num": 0, - "metadata_type_id": 0, - "capacities": { - "intent_name": 63, - "answer_text": 1023 - } - } - } - } + "pipe_path": "pipeline_doc_qa_rerank.json" } diff --git a/demo/fixtures/mock/pipeline_doc_qa_rerank.json b/demo/fixtures/mock/pipeline_doc_qa_rerank.json index 0943efc8..2f303f54 100644 --- a/demo/fixtures/mock/pipeline_doc_qa_rerank.json +++ b/demo/fixtures/mock/pipeline_doc_qa_rerank.json @@ -1,5 +1,26 @@ { "biz_name": "smart_doc_qa_v1", + "deployment": { + "model_paths": { + "embed_model_v1": "./models/bge_base_npu.bin", + "rerank_model_v1": "./models/bge_reranker_large_npu.bin", + "llm_model_v1": "./models/qwen_1.5b_npu.bin" + }, + "io": { + "io_binding": "doc_qa.operator.v1", + "output_allocations": { + "doc_out": { + "type": "doc_out", + "meta_num": 0, + "metadata_type_id": 0, + "capacities": { + "intent_name": 63, + "answer_text": 1023 + } + } + } + } + }, "models": [ { "capability": "embedding", @@ -164,7 +185,7 @@ }, "config": { "default_category": "GENERAL_QA", - "default_score": 0.90, + "default_score": 0.9, "categories": { "AFTER_SALES_REFUND": [ "退款", diff --git a/demo/fixtures/mock/pipeline_entity_extract.conf b/demo/fixtures/mock/pipeline_entity_extract.conf index bc64830e..4c3147a8 100644 --- a/demo/fixtures/mock/pipeline_entity_extract.conf +++ b/demo/fixtures/mock/pipeline_entity_extract.conf @@ -1,20 +1,3 @@ { - "schema_version": 1, - "data": { - "pipe_path": "pipeline_entity_extract.json", - "io_binding": "entity_extract.operator.v1", - "model_paths": { - "entity_llm": "./models/qwen_0_6b_npu.bin" - }, - "outputs": { - "entity_out": { - "type": "entity_out", - "meta_num": 0, - "metadata_type_id": 0, - "capacities": { - "entities_json": 2047 - } - } - } - } + "pipe_path": "pipeline_entity_extract.json" } diff --git a/demo/fixtures/mock/pipeline_entity_extract.json b/demo/fixtures/mock/pipeline_entity_extract.json index 25906e0a..f1435de6 100644 --- a/demo/fixtures/mock/pipeline_entity_extract.json +++ b/demo/fixtures/mock/pipeline_entity_extract.json @@ -1,5 +1,23 @@ { "biz_name": "entity_extract_v1", + "deployment": { + "model_paths": { + "entity_llm": "./models/qwen_0_6b_npu.bin" + }, + "io": { + "io_binding": "entity_extract.operator.v1", + "output_allocations": { + "entity_out": { + "type": "entity_out", + "meta_num": 0, + "metadata_type_id": 0, + "capacities": { + "entities_json": 2047 + } + } + } + } + }, "models": [ { "capability": "llm", diff --git a/demo/fixtures/mock/pipeline_entity_extract_custom.conf b/demo/fixtures/mock/pipeline_entity_extract_custom.conf index 9ea9bf93..4d5a6b0f 100644 --- a/demo/fixtures/mock/pipeline_entity_extract_custom.conf +++ b/demo/fixtures/mock/pipeline_entity_extract_custom.conf @@ -1,20 +1,3 @@ { - "schema_version": 1, - "data": { - "pipe_path": "pipeline_entity_extract_custom.json", - "io_binding": "entity_extract.operator.v1", - "model_paths": { - "entity_llm": "demo/fixtures/mock/artifacts/neutral-llm.fixture" - }, - "outputs": { - "entity_out": { - "type": "entity_out", - "meta_num": 0, - "metadata_type_id": 0, - "capacities": { - "entities_json": 2047 - } - } - } - } + "pipe_path": "pipeline_entity_extract_custom.json" } diff --git a/demo/fixtures/mock/pipeline_entity_extract_custom.json b/demo/fixtures/mock/pipeline_entity_extract_custom.json index 369c9122..daf8f237 100644 --- a/demo/fixtures/mock/pipeline_entity_extract_custom.json +++ b/demo/fixtures/mock/pipeline_entity_extract_custom.json @@ -1,5 +1,23 @@ { "biz_name": "entity_extract_v1", + "deployment": { + "model_paths": { + "entity_llm": "demo/fixtures/mock/artifacts/neutral-llm.fixture" + }, + "io": { + "io_binding": "entity_extract.operator.v1", + "output_allocations": { + "entity_out": { + "type": "entity_out", + "meta_num": 0, + "metadata_type_id": 0, + "capacities": { + "entities_json": 2047 + } + } + } + } + }, "models": [ { "capability": "llm", diff --git a/demo/fixtures/mock/pipeline_ocr_doc_qa.conf b/demo/fixtures/mock/pipeline_ocr_doc_qa.conf index 3c5fe5e8..0f279bc1 100644 --- a/demo/fixtures/mock/pipeline_ocr_doc_qa.conf +++ b/demo/fixtures/mock/pipeline_ocr_doc_qa.conf @@ -1,21 +1,3 @@ { - "schema_version": 1, - "data": { - "pipe_path": "pipeline_ocr_doc_qa.json", - "io_binding": "ocr_doc_qa.operator.v1", - "model_paths": { - "ocr_model_v1": "./models/ch_ppocr_v4_det_rec_npu.bin", - "llm_model_v1": "./models/qwen_1.5b_npu.bin" - }, - "outputs": { - "od_out": { - "type": "od_out", - "meta_num": 0, - "metadata_type_id": 0, - "capacities": { - "result_json": 2047 - } - } - } - } + "pipe_path": "pipeline_ocr_doc_qa.json" } diff --git a/demo/fixtures/mock/pipeline_ocr_doc_qa.json b/demo/fixtures/mock/pipeline_ocr_doc_qa.json index 9b74ad86..878efd11 100644 --- a/demo/fixtures/mock/pipeline_ocr_doc_qa.json +++ b/demo/fixtures/mock/pipeline_ocr_doc_qa.json @@ -1,5 +1,24 @@ { "biz_name": "multimodal_ocr_invoice_qa", + "deployment": { + "model_paths": { + "ocr_model_v1": "./models/ch_ppocr_v4_det_rec_npu.bin", + "llm_model_v1": "./models/qwen_1.5b_npu.bin" + }, + "io": { + "io_binding": "ocr_doc_qa.operator.v1", + "output_allocations": { + "od_out": { + "type": "od_out", + "meta_num": 0, + "metadata_type_id": 0, + "capacities": { + "result_json": 2047 + } + } + } + } + }, "models": [ { "capability": "ocr", diff --git a/demo/fixtures/mock/pipeline_translate.conf b/demo/fixtures/mock/pipeline_translate.conf index 5b6f4a9b..715797e7 100644 --- a/demo/fixtures/mock/pipeline_translate.conf +++ b/demo/fixtures/mock/pipeline_translate.conf @@ -1,17 +1,3 @@ { - "schema_version": 1, - "data": { - "pipe_path": "pipeline_translate.json", - "io_binding": "translate.operator.v1", - "outputs": { - "entity_out": { - "type": "entity_out", - "meta_num": 0, - "metadata_type_id": 0, - "capacities": { - "entities_json": 2047 - } - } - } - } + "pipe_path": "pipeline_translate.json" } diff --git a/demo/fixtures/mock/pipeline_translate.json b/demo/fixtures/mock/pipeline_translate.json index bcdfe5d5..a6cf8c64 100644 --- a/demo/fixtures/mock/pipeline_translate.json +++ b/demo/fixtures/mock/pipeline_translate.json @@ -1,5 +1,20 @@ { "biz_name": "translate_v1", + "deployment": { + "io": { + "io_binding": "translate.operator.v1", + "output_allocations": { + "entity_out": { + "type": "entity_out", + "meta_num": 0, + "metadata_type_id": 0, + "capacities": { + "entities_json": 2047 + } + } + } + } + }, "models": [ { "capability": "llm", diff --git a/doc/CHANGELOG.md b/doc/CHANGELOG.md index c33efd6f..02780d53 100644 --- a/doc/CHANGELOG.md +++ b/doc/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## 2026-09-17 部署配置归拢至 Pipeline 根文档（RFC-0061） + +- **部署配置归拢至 Pipeline 文档**： + - 启动配置文件 `.conf` 严格收窄为单一字段 `{"pipe_path": ""}`，删除旧版本顶层包装与旧字段（`io_binding`、`model_paths`、`outputs`、`version` 等）。 + - 接入部署信息（`io.io_binding`、`io.output_allocations`）及可选模型路径覆盖（`model_paths`）归拢由 `pipeline.json` 的根对象 `deployment` 拥有。 + - 接入适配层新增轻量级文档拆分（`SplitPipelineDocument`），严格分离 `deployment` 配置与中性算法配置，向后传递给核心编排层。 +- **架构分层与 Core 严格边界**： + - 核心编排层（`PipelineValidator` / `PipelineConfig`）保持对接入部署概念的无感知，直接将带有 `deployment` 根字段的完整文档作为未知根字段严格拒绝。 + - 模型路径覆盖在 Integration 层与模型声明严格配对后应用，保持宿主相对路径与边界安全校验。 +- **工具链与 Studio 迁移**： + - `alg_pipeline_tool` 的 `validate` 与 `plan` 命令支持带 `deployment` 的完整 Pipeline 文档并执行部署边界与输出分配校验；纯算法文档保留校验能力但明确标注缺少部署信息。 + - `pipeline_authoring` 在编辑往返与 `fix-deps` 过程中完整保留 `deployment` 字段。 + - Pipeline Studio 与 `dev_recipe.py` 生成严格收窄的 `.conf` 并在保存和预检时由 Pipeline 持有 `deployment`。 + ## 2026-09-17 退出旧 C ABI，仅保留 C++ Operator SDK（RFC-0060） - **接口收口与退出旧 C ABI**： diff --git a/doc/VERIFIABLE_SELECTION.md b/doc/VERIFIABLE_SELECTION.md index aae09cda..b789c746 100644 --- a/doc/VERIFIABLE_SELECTION.md +++ b/doc/VERIFIABLE_SELECTION.md @@ -19,7 +19,7 @@ Node 的检索数、生成预算、模板等业务参数放在 Node `config`，字段说明与默认值通过 `describe-node` 或 Studio 属性查看。未声明为 Control 的参数在重新创建 handle 后生效。 -`.conf` 的 `data.model_paths` 可覆盖 Pipeline 中的权重路径。使用现有部署文件时，更新 +Pipeline JSON 根对象的 `deployment.model_paths` 可覆盖 `models` 中的权重路径。`.conf` 仅包含 `pipe_path` 定位该 JSON。使用现有部署文件时，更新 或移除相应覆盖后，查看与 Operator Create 同一解析器得到的结果： ```bash @@ -81,7 +81,7 @@ python3 tools/verify_selection.py check \ 复用现有 `alg_demo` 的样例读取、宿主载体构造和 SDK 执行路径；业务请求的解包与响应 组装仍由 Adapter 完成，见[输入输出边界](dev_guide/business_onboarding.md#输入输出以-c-abi-为边界)。 -验收器为选定 Pipeline 生成临时 `.conf`，从 `--conf`（默认同名 `.conf`）继承输出池配置，按 `--model-root` 生成模型路径；不会沿用原 `.conf` 中可能覆盖模型选择的 `model_paths`。 +验收器为选定 Pipeline 生成临时 `.conf`，从 Pipeline JSON（或 `--conf` 定位的原 JSON）继承 `deployment.io` 输出池配置，按 `--model-root` 生成模型路径并写入临时 Pipeline 的 `deployment.model_paths`。 验收固定使用 CPU、device 0、batch 1；Demo 默认使用所选规则/提示词。这个版本的验收目标是配置正确性与选定输出字段的业务效果；目标设备性能验收需要相应环境与后续测试定义。 diff --git a/doc/dev_guide/business_onboarding.md b/doc/dev_guide/business_onboarding.md index 1787be65..b7eaa07f 100644 --- a/doc/dev_guide/business_onboarding.md +++ b/doc/dev_guide/business_onboarding.md @@ -139,11 +139,7 @@ JSON 请求是不同的输入约定。已有 Nodes 能完成算法，也不代 4. 准备样例数据、Pipeline 和 `.conf`。先用显式 `--biz`、`--config`、`--dataset` 运行。 仅需保存可重复调用的预设或加入套件时，再向 `demo/profiles.json` 添加 Profile。 -`.conf` 的 `data.pipe_path` 相对部署根解析。像本页这样从仓库根目录传入相对 -`--config configs/pipeline_keyword_match_rules.conf` 时，填写 -`configs/pipeline_keyword_match_rules.json`。宿主直接调用 Operator 时，部署根为 Create 的 -`model_path`；同时核对 `data.model_paths` 覆盖与输出容量。Profile 不会自动指向新方案， -详细命令见[运行当前方案](../../tools/pipeline_studio/README.md#运行当前方案)。 +`.conf` 仅作为定位文件，包含单一字段 `pipe_path`，相对 `.conf` 所在目录解析（例如在 `configs/pipeline_keyword_match_rules.conf` 中填写 `pipeline_keyword_match_rules.json`）。宿主直接调用 Operator 时，部署根为 Create 的 `model_path`；同时在 Pipeline JSON 的 `deployment` 中核对 `model_paths` 覆盖与 `io.output_allocations` 输出容量。Profile 不会自动指向新方案，详细命令见[运行当前方案](../../tools/pipeline_studio/README.md#运行当前方案)。 ## 6. 输出容量与生命周期 diff --git a/doc/dev_guide/first_custom_node.md b/doc/dev_guide/first_custom_node.md index fdc90b4e..89ab4ae1 100644 --- a/doc/dev_guide/first_custom_node.md +++ b/doc/dev_guide/first_custom_node.md @@ -120,8 +120,8 @@ cp demo/fixtures/mock/pipeline_entity_extract_custom.conf demo/fixtures/mock/pip ``` 保留该节点的 `ports` 和 `depends_on`，以及其他模型与 JSON 解析节点。 -在 `pipeline_first_node.conf` 中，将 `data.pipe_path` 改为 -`demo/fixtures/mock/pipeline_first_node.json`；其他字段沿用复制内容。 +在 `pipeline_first_node.conf` 中，将 `pipe_path` 改为 +`pipeline_first_node.json`（`.conf` 仅包含该定位字段；接入绑定与输出容量沿用 JSON 中的 `deployment`）。 此时数据经过： diff --git a/doc/rfcs/0061-pipeline-owned-deployment-configuration.md b/doc/rfcs/0061-pipeline-owned-deployment-configuration.md new file mode 100644 index 00000000..2957c0b2 --- /dev/null +++ b/doc/rfcs/0061-pipeline-owned-deployment-configuration.md @@ -0,0 +1,417 @@ +# RFC 0061: Pipeline JSON 集中管理部署配置 + +- **RFC 编号**：0061-pipeline-owned-deployment-configuration +- **创建日期**：2026-09-17 +- **文档状态**：Completed +- **关联分支**：`docs/pipeline-deployment-rfc` +- **目标版本**：下一次配置格式切换版本，具体发布号待定 +- **负责人 / 作者**：LLM-EdgeFlow 维护者 +- **代码核查基线**：`c7358b7`（RFC-0060 合入后的 `main`） +- **关联决策**：取代 RFC-0059 中 Schema 1 `.conf` 承载接入配置的规定，以及 RFC-0025 中模型路径覆盖存放于 `.conf` 的规定；继承 RFC-0049 的输出分配机制、RFC-0050 的参数文本边界及 RFC-0060 的 C++ Operator 唯一入口。 + +> 本文规范已完全实施。`.conf` 严格收窄为仅包含 `pipe_path`；Pipeline JSON 的 `deployment` 管理部署信息， +> 其中 `io_binding` 与 `output_allocations` 始终作为一组配置。 + +## 1. 问题与范围 + +### 1.1 现状与目标 + +当前同一方案分散在两个文件：`.conf` 的 `data` 中有 `pipe_path`、`io_binding`、 +`model_paths`、`outputs`；Pipeline JSON 中有 `biz_name`、`models`、`pipeline` 等。 +只复制或编辑 JSON 无法带走完整的接入绑定、模型覆盖与外部输出容量。 + +两个 `outputs` 的语义也不同：`.conf` 描述 Operator 外部输出对象的分配， +`pipeline[*].ports.outputs` 描述 Node 逻辑输出端口到 Blackboard Key 的映射。 +迁移不能让它们在同一 JSON 中继续使用容易混淆的名称。 + +实施后的用户路径是：在 JSON 中维护算法和部署配置，`.conf` 仅作为宿主已有 +`cfg_file_name` 入口的定位文件。切换 `.conf.pipe_path` 即切换整套部署配置。 + +### 1.2 已核对的实现边界 + +| 当前入口 | 当前职责 | 本次影响 | +| --- | --- | --- | +| `src/adapter/deployment_io_config.*` | 严格读取 Schema 1 `.conf`，解析 `data.pipe_path` | 收窄为仅定位 JSON | +| `src/adapter/io_binding_resolver.*` | 查 binding/converter，校验外部槽位，读取 JSON，应用模型覆盖，生成 `ValidatedIoPlan` | 配置信息改从同一 JSON 快照取得，收敛文件和内存入口 | +| `src/adapter/operator/operator_config_resolver.*` | 宿主根目录、安全路径、输出分配和有效深度预算 | 沿用这些规则，调整读取来源 | +| `src/core/pipeline_config.cpp` | Pipeline 根字段白名单与严格解析 | 继续只解析中性 Pipeline 文档 | +| `src/tools/alg_pipeline_tool.cpp` | 编排校验、部署预检与生效配置报告 | 增加部署文档拆分；更新来源报告 | +| Studio、recipe、效果评估 | 读取/继承旧 conf 字段并生成临时部署文件 | 全链路迁移，不能只修改生产 resolver | + +本次影响接入适配层 / Integration、流程编排层 / Orchestration 的输入边界及工具。 +能力节点层 / Capability Nodes 与模型执行层 / Model Execution 的算法、端口、模型语义不变。 +不修改 Operator 函数表、`CreateParam` 布局、平台载体、Converter 注册契约或输出池生命周期。 +不引入配置热更新、多 binding 同时运行、JSON include/继承、环境变量替换或新的公司 SDK 依赖。 + +## 2. 决策与权衡 + +### 2.1 `.conf` 字面上只保留一个字段 + +以当前对话审核方案为例，迁移后的完整 `.conf` 为： + +```json +{ + "pipe_path": "pipeline_dialogue_audit_default.json" +} +``` + +这里的“只剩一个 `pipe_path`”指整个文件只有这个键，删除旧 `schema_version` 和 `data` +包装，不仅是缩减 `data` 中的业务字段。 + +- 根必须是对象，必须有且仅有非空字符串 `pipe_path`；数组、空值、未知键均拒绝。 +- 旧 Schema 1 包装、顶层或 `data` 内的 `io_binding/model_paths/outputs` 都拒绝， + 报错指向迁移说明，不做自动合并或读取优先级回退。 +- `pipe_path` 继续相对于 `.conf` 所在目录；目标必须存在且是普通 JSON 文件， + 规范化后必须位于该目录内。位于目录内的绝对路径可沿用现有支持，目录外路径及符号链接逃逸拒绝。 +- `CreateParam.model_path` 仍为模型与配置根目录，`cfg_file_name` 仍为其中的相对配置路径。 + 保留现有根目录边界和禁止搜索回退的规则。 + +持久化格式采用一次切换，不新增版本协商。Catalog、CLI 报告及业务 payload 中原有的 +`schema_version` 属于各自契约，不能随 `.conf` 包装一起删除。 + +### 2.2 JSON 的确定结构与名称 + +现有 Pipeline 根字段保持原位置，增加一个 Integration 所有的 `deployment` 对象： + +| 路径 | 类型与约束 | 含义 | +| --- | --- | --- | +| `deployment` | 对象；可部署文档必填；只允许 `model_paths`、`io` | 同文件内的部署信息 | +| `deployment.model_paths` | 可选对象，缺省等同 `{}`；键为已声明的 `model_id`，值为非空路径字符串 | 原 `.conf.data.model_paths`，保留覆盖语义 | +| `deployment.io` | 对象，部署时必填；有且仅有下面两个字段 | 一份完整的 Operator 接入配置 | +| `deployment.io.io_binding` | 必填非空字符串 | 已注册 binding 的 ID | +| `deployment.io.output_allocations` | 必填对象，键为外部输出槽位名 | 原 `.conf.data.outputs`，每槽位的分配配置 | +| `pipeline[*].ports.outputs` | 保持当前对象格式 | Node 逻辑端口到内部 Blackboard Key 的映射 | + +`output_allocations` 强调其内容包括外部类型、allocator、参数、元数据与容量， +不是算法结果，也不是内部端口连线。文档和界面分别称为“外部输出分配”和“节点输出端口”。 +不得将 `output_allocations` 再命名为 `outputs`，也不改名既有 `ports.outputs`。 +内部代码持有外部分配配置时使用 `output_allocations` 等明确名称，避免新加含糊的 `outputs` 成员。 + +下面是迁移后**完整、无模型依赖的关键词方案**。名称和端口已通过当前 +`alg_pipeline_tool catalog --biz keyword_match_v1` 核对；新部署格式须在实施后才能使用。 + +```json +{ + "biz_name": "keyword_match_v1", + "deployment": { + "io": { + "io_binding": "keyword_match.operator.v1", + "output_allocations": { + "keyword_out": { + "type": "keyword_out", + "meta_num": 0, + "metadata_type_id": 0, + "capacities": { + "match_result_json": 2047 + } + } + } + } + }, + "models": [], + "pipeline": [ + { + "id": "node_0_TextRuleMatchNode", + "node_type": "TextRuleMatchNode", + "depends_on": [], + "ports": { + "inputs": { + "text": "input_sentences" + }, + "outputs": { + "matches": "rule_matches" + } + }, + "config": { + "categories": { + "SYSTEM_INIT": ["初始化", "自检"] + } + } + } + ] +} +``` + +关键词 `.conf` 则是 `{"pipe_path":"pipeline_keyword_match_rules.json"}`。 +这里 `matches → rule_matches` 是内部数据连线；`keyword_out` 是宿主输出槽位。 +它们通过注册的 OutputConverter 和 IoBinding 关联，不通过名称相等或新增 JSON 映射关联。 + +对用户当前打开的 `pipeline_dialogue_audit_default.json`，保持现有 `models` 与 +九个节点内容，向根对象加入以下成员即可；**这是合并片段，不是完整 Pipeline 文件**： + +```json +{ + "deployment": { + "model_paths": { + "embed_model_v2": "./models/bge_base_zh_v1.5.onnx", + "rerank_model_v1": "./models/ms_marco_tinybert_l2_v2_quantized.onnx", + "audit_llm_v1": "./models/qwen2.5-0.5b-instruct-q4_k_m.gguf" + }, + "io": { + "io_binding": "compliance_audit.operator.v1", + "output_allocations": { + "audit_out": { + "type": "audit_out", + "meta_num": 0, + "metadata_type_id": 0, + "capacities": { + "risk_level": 31, + "matched_policy_clause": 255, + "audit_verdict_json": 1023 + } + } + } + } + } +} +``` + +### 2.3 `io_binding` 与外部输出分配的配对规则 + +“一对”是**一个 binding ID 对应一整份外部输出分配表**，不表示只能有一个输出槽位。 +一个 JSON 只有一个 `deployment.io`；同一绑定可以要求多个外部槽位。 + +1. `io_binding` 和 `output_allocations` 必须同时存在于同一个 `io` 对象。 + 不接受从 `.conf`、Profile、`biz_name` 默认映射或另一份 JSON 补齐半组配置。 +2. 根据 binding 找到注册的 InputConverter 和 OutputConverter；确认 transport 为 + `operator`，且 binding 的 `biz_name` 与 JSON 根的 `biz_name` 一致。 +3. 设输出转换器声明的全部输出槽位集合为 `A`，其中必需槽位为 `R`，配置键集合为 `C`。 + 必须满足 `R ⊆ C ⊆ A`。因此缺少必需槽位、额外未知槽位都报错；可选槽位可不配置。 + `{}` 仅在没有必需输出槽位时合法，仍须显式写出此对象。 +4. 每个槽位继续调用现有 `ResolveOutputAllocation`：校验 `type` 与声明一致、allocator + 已注册、参数可解析、容量和元数据合法，保留单槽位与整句柄预算检查。 + 支持的字段仍是 `type/allocator/params/meta_num/metadata_type_id/capacities`，默认值沿用注册机制。 +5. `type` 不能由 map 键猜测；容量不能由 `ports.outputs` 推导；`params` 文本解析、 + 归一化参数共享及 `max_frame_depth` 的作用保持原契约。 +6. Studio、recipe 和 Profile 继承以 `deployment.io` 为最小整体单位。只更换 binding 时， + 必须重新校验整张分配表；不能静默保留另一 binding 的分配配置。 + +这些检查在 Create / 部署预检时完成。失败不发布可用句柄，不初始化 Node、不加载模型、 +不开始输出池分配；已创建句柄持有不可变计划，后续修改文件仅对新 Create 生效。 + +### 2.4 模型路径覆盖与解析基准 + +本次保留 `model_paths`，因为需求是迁移其位置，而非取消部署覆盖。它与 +`models[*].model_path` 的职责和优先级明确如下： + +```text +effective_path(model_id) = + deployment.model_paths[model_id] （显式配置时） + models 中该 model_id 的 model_path（否则） +``` + +- `models` 是模型声明的唯一列表；`model_paths` 不能新增模型、改变 capability 或 Backend。 + 未知 ID、空 ID、非字符串值和空路径均拒绝。无模型时只能省略它或使用 `{}`。 +- 原 `models[*].model_path` 仍按现有契约必填非空；覆盖不能把一个缺字段或字段类型错误的 + 模型声明变成合法声明。复用 Core 严格解析做结构检查，不能在工具端复制模型 schema。 +- 相对有效路径继续相对宿主 `CreateParam.model_path` 解析，**不因字段搬到 JSON 就改为 + 相对 JSON 目录、conf 目录或进程工作目录**。绝对路径沿用共享 resolver 的规则， + 给定根目录时必须位于根目录内；规范化、`..` 与符号链接边界继续由共享路径解析器负责。 +- 例如宿主根为 `/bundle` 时，`models/a.onnx` 解析到 `/bundle/models/a.onnx`； + JSON 即使位于 `/bundle/configs/`，也不会得到 `/bundle/configs/models/a.onnx`。 +- `model_config.tokenizer_file` 等 sidecar 配置留在原位置，沿用各 Model 的资源目录契约。 + 不以新 `deployment.model_paths` 扩展为任意 JSON 字段覆盖系统。 +- 静态校验不加载模型，也不把“模型文件实际可加载”当作成功承诺；实际加载失败仍由 + 现有 Model/Backend 路径报告。 + +### 2.5 文档集中，但职责仍然分层 + +同一 JSON 是文件组织方式，不是让 Core 认识 Operator 的理由。 + +```text +Operator Create / validate-io / resolve-conf + → 读取仅有 pipe_path 的 conf + → Integration 读取一次完整 JSON 并拆分 deployment 与中性 Pipeline + → 校验 deployment.io，应用模型覆盖、解析路径 + → 由注册的 Converter/Binding 构造中性 PipelineIoBoundary + → PipelineValidator::ValidateAndPlan + → ValidatedIoPlan（含同一次校验的 ValidatedPipelinePlan） + → Pipeline 消费已有计划，执行时不重读配置 +``` + +文档拆分使用 Integration 私有的轻量结构与解析函数，按复用需要放在现有 resolver +旁或 `src/adapter/pipeline_document.h/.cpp`，具体类型和函数名由实现确定。 +职责按下表分配： + +| 组件 | 实施责任 | +| --- | --- | +| `DeploymentIoConfig` | 收窄为启动定位信息，删除承载 binding、模型覆盖、输出分配的成员；可按新职责局部改名，不新增旧结构兼容类 | +| Integration 文档解析函数 | 从同一 JSON 值取出并严格检查 `deployment`，产生供 Core 使用的中性 Pipeline 值；解析结果只保留后续校验和规划所需数据 | +| 编辑工具 | 自行持有完整源 JSON，编辑后保留 `deployment`；运行时解析结构不为编辑用途额外保存整份原文副本 | +| `IoBindingResolver` | 文件入口负责读取并调用 JSON 入口；二者共用一个严格的接入解析实现，校验 binding/converter/槽位，构造 I/O 边界并调用唯一 PipelineValidator | +| `OperatorConfigResolver` / 模型路径 resolver | 保留宿主路径和分配规则；有效队列深度预算在物化前完成 | +| `ParsePipelineConfig` / `PipelineValidator` | 只接收去除 `deployment` 的中性字段；继续拒绝未知字段并拥有全部 DAG、模型声明、端口与类型规则 | +| `ValidatedIoPlan` / `ResolvedOperatorConfig` | 保存同一快照的解析结果、计划及路径来源；检查工具不能为报告再次打开文件或重新规划 | + +拆分时只能取走已识别、已校验的 `deployment`。其他未知根字段留给 Core 报错， +不能用“复制已知字段”偷偷过滤拼写错误，也不能简单 `erase("deployment")` 后声称完整校验成功。 +Core 不引入 Adapter 头文件，不增加 allocator、外部槽位、binding ID 或文件读取逻辑。 + +现有 `ResolveFromPipelineJson` 不校验外部分配；实施时将其收敛为上述严格 JSON 入口， +部署入口始终要求完整 `deployment.io`。按实际调用需要保留薄封装，解析步骤作为私有函数， +不为每个中间结构增加入口，也不增加 `require_deployment` 等开关改变部署入口的必填约束。 + +中性算法测试直接使用 `PipelineValidator`；需要绑定边界的测试通过测试辅助函数构造输入， +复用现有校验逻辑。生产 resolver 不提供测试专用解析分支，也不在缺少 `deployment` 时 +自动回退到中性测试路径。工具对纯算法文档的支持由工具入口显式分流，见下一节。 + +### 2.6 工具模式、诊断和保存契约 + +| 入口 | 迁移后的确定行为 | +| --- | --- | +| `validate` / `plan`，包括 `--stdin`、`--explain` | 支持完整 JSON；有 `deployment` 就校验其结构、模型覆盖和整组 I/O，按 binding 提供边界；不读模型文件，不要求部署根目录。模型路径仅做环境无关检查 | +| 仅有中性字段的 `validate` / `plan` 输入 | 保留算法编排/单元测试用途，使用原有中性校验；成功只表示算法配置通过，不能据此 Create。不得将有缺陷的 `deployment` 降级成此模式 | +| `validate-io CONFIG --model-root DIR` | 继续以 `.conf` 为入口，读取其指向的完整 JSON；要求 `deployment.io`，按部署根解析路径并检查注册/预算，不加载模型 | +| `resolve-conf FILE --root DIR --depth N` | 使用与 Create 相同的 resolver 和有效深度；报告来源于同一 `ValidatedIoPlan`，删除当前再次调用 Validator、再次读 conf 的做法 | +| `init --profile`、`edit`、`fix-deps`、Studio 导入/保存 | 保留完整 `deployment`；图编辑只改图字段，不能重新序列化时丢掉部署信息。空白草稿可暂不含 deployment，但不得标为可运行 | +| Demo | 原样调用 SDK；只调整 fixture 与配置定位，不代替 SDK 解析 binding、字段选择或响应组装 | + +算法规划阶段与部署阶段共用纯结构/语义校验，后者额外有文件定位、路径 containment 和 +宿主深度预算。不要通过虚构当前目录作为模型根来让 `validate` 成功。 +编辑工具的草稿路径允许尚未满足完整运行条件，但保存和运行必须明确反馈缺失项。 + +错误需包含源文件与 JSON Pointer；stdin 输入标记为 stdin。沿用现有 `DiagnosticCode` +及 CLI 报告结构，Integration 解析失败使用相应的字段/组合诊断，注册与预算错误保持稳定含义。 +公开 Operator 的返回码分类和 `GetOperatorLastError()` 入口不变，所有异常屏障保留。 + +| 错误示例 | 诊断定位 | +| --- | --- | +| conf 留有旧字段 | conf 的 `/data`、`/schema_version` 或对应非法键，提示迁移到新结构 | +| 缺少配对字段 | JSON 的 `/deployment/io/io_binding` 或 `/deployment/io/output_allocations` | +| 未知 binding / biz 不匹配 | `/deployment/io/io_binding`，不匹配时同时说明根 `/biz_name` | +| 未知/缺失外部槽位 | `/deployment/io/output_allocations/` | +| 类型、allocator、容量错误 | 上述槽位下对应字段 | +| 模型覆盖 ID/路径错误 | `/deployment/model_paths/`；未覆盖时为 `/models//model_path` | +| Node 输出映射错误 | 保持 `/pipeline//ports/outputs/` | + +包含 `/` 或 `~` 的键按 JSON Pointer 规则转义。覆盖路径经过投影后出错时须映射回原始 +`deployment.model_paths` 位置，不能只报合成文档中的 `/models/...`。 + +`resolve-conf.configuration.model_paths[*].source` 从 `conf.data.model_paths` 改为 +`pipeline.deployment.model_paths`；默认路径来源仍为 `pipeline.models.model_path`。 +`output_pools`、`output_pool` 是已有**诊断报告字段**，本次保留其名字和参数文本语义， +不要与持久化文件的新字段 `output_allocations` 混改。`effective_pipeline` 继续是供 +Core 使用的中性生效文档，包含已解析路径，不作为可部署源文件保存。 + +### 2.7 选择原因与代价 + +采用 `deployment.io` 将 binding 和分配表放在一起，`deployment.model_paths` 则与 +它们同属部署信息。Integration 可以一次拆出明确的子文档，Core 无需放宽根字段白名单。 + +未采用把三个字段平铺到 JSON 根或仅将 `outputs` 改为 `operator_outputs`：这些形式 +无法在结构上表达 I/O 必须整体继承，后者也没说明是分配参数。未采用把部署配置全部塞进 +`models` 或 Node：这会混入不属于模型或算法节点的外部载体责任。 + +代价是同一个算法若有不同模型覆盖或接入配置，需要不同完整 JSON；不再能只靠多个 +`.conf` 定制。同一模型 ID 的默认路径与部署覆盖仍可能同时存在，因此必须提供清晰的 +生效来源报告。是否进一步取消 `model_paths`、引入配置复用机制，不属于本次迁移。 + +## 3. 兼容与迁移 + +### 3.1 一次切换与回退边界 + +新代码只接收新 `.conf`；部署 JSON 必须含完整 `deployment.io`。旧配置不自动升级， +旧字段与新字段并存直接拒绝，不能建立“conf 优先/JSON 优先”的永久双来源规则。 +现有中性 Core fixture 无需凭空增加部署配置，但凡经 Operator 部署的 fixture 必须迁移。 + +这是一项持久化配置破坏性变更，须与运行库、工具、配置包同时发布。 +它本身不改变 C++ Operator ABI，不要求仅为移动配置字段再次调整共享库 ABI major。 +回退时整体恢复旧运行库、工具与配置包，不能单独回退 `.conf` 或 JSON。 + +### 3.2 按关联关系迁移，不能按字段名全局替换 + +对每个旧 `.conf`，定位真实目标 JSON，并执行确定映射： + +| 旧来源 | 新位置 / 动作 | +| --- | --- | +| `data.pipe_path` | `.conf.pipe_path`；若目标被拆分则改为新文件名 | +| `data.io_binding` | JSON 的 `deployment.io.io_binding` | +| `data.outputs` | JSON 的 `deployment.io.output_allocations`；槽位内部值原样迁移 | +| `data.model_paths` | JSON 的 `deployment.model_paths`；原来省略则继续省略 | +| `.conf.schema_version`、`.conf.data` | 删除包装 | +| `models[*].model_path`、`pipeline[*].ports.outputs` | 原样保留 | + +先列出“JSON 被哪些 conf 引用”，再写文件： + +1. 多份 conf 引用同一个 JSON 且部署配置完全相同，可以继续共用该 JSON。 +2. 若 binding、分配表或模型覆盖不同，复制成不同部署 JSON，并分别修改 `pipe_path`。 + 不能顺序覆盖同一个 JSON 的 `deployment`，也不能只保留最后一份配置。 +3. `tests/fixtures/pipelines/cross_rerank/` 的正常/缺失模型场景就是需单独核查的共用目标。 + 拆分后要保留原负例的失败原因，不能因新格式不合法提前失败而使测试假通过。 +4. 若目标已有 `deployment`，迁移工具遇到内容冲突应停止并报告；禁止静默覆盖。 + 一次性迁移脚本可以用于仓库改造，不能进入运行时作为兼容解析器。 + +### 3.3 实施检查清单 + +| 范围 | 必须完成的迁移 | +| --- | --- | +| `configs/`、`demo/fixtures/mock/`、`tests/fixtures/pipelines/` | 全量迁移部署文件及关联 JSON，包括无模型、多模型和负例 | +| `tests/support/control_test_utils.h` 及各测试内 JSON 字面量 | 临时配置构造器改为生成新文件对；保留有意构造的旧格式拒绝测试 | +| `tools/verify_selection.py` | 将 `build_run_conf` 收窄到定位文件或改造成明确生成文件对的 helper；`effect_inputs`/临时执行从 JSON 读取部署信息，更新指纹和来源 | +| `tools/pipeline_studio/server.py` | 迁移 `profile_inputs`、`deployment_candidate`、save/staging/run；完整继承 I/O，保留模型 `select_asset/preserve_override` 意图和失效 ID 清理 | +| `scripts/dev_recipe.py` | Profile 解析、临时配置和最终产物都迁移；保留目前明确的单外部输出槽位限制，复杂方案走已有相应路径 | +| `src/tools/pipeline_authoring.*` 与 Studio 前端 | 算法编辑往返保留部署子文档；外部分配编辑与节点端口显示分开 | +| 活动指南、README 中相关示例、开发 skills | 更新配置字段路径、生成步骤和“当前方案”运行方式；历史 RFC 保留原文，由本文声明取代范围 | + +工具已有的 `BIZ_TO_OPERATOR_BINDING` 等选择代码不能用于为已选文档补齐缺失 binding。 +生成新方案需从显式选择或 Catalog/Profile 取得完整 `io`，再交原生校验器验证，不能 +用业务名猜 binding 后拼接另一份输出表。Profile 的数据集、Demo 运行选项仍属于 Profile。 + +Studio 仍需保护 JSON/conf 的关联和并发 revision：conf 虽变小，仍可被另一进程切换 +`pipe_path`。保存时保留既有冲突检查和成对 staging/失败恢复，校验成功才提交候选文件； +临时执行只改候选 conf 的定位字段，不能回读旧 conf 的部署参数覆盖当前编辑内容。 +效果评估指纹必须包含当前 JSON 中生效的模型覆盖、binding 与分配表，避免复用旧结果。 + +实施完成后更新当前配置指南和 `doc/CHANGELOG.md`。本次仅设计，不提前把拟议格式 +写成已支持功能，也不为尚未实现的能力增加发布记录。 + +## 4. 验证与完成条件 + +扩展现有责任套件，不为此另建测试可执行程序。以下是实施验收项，不是本次文档提交 +已经通过的新行为测试： + +| 验证项 | 最小证据与既有落点 | +| --- | --- | +| conf 严格结构 | 单字段通过；旧包装、旧键、空/错误类型、未知键拒绝；`tests/unit/adapter/test_io_binding_registry.cpp` | +| 配对与槽位 | 缺 binding/分配表、未知 binding、biz/transport 不匹配、缺必需/多未知槽位失败；可选槽位、多槽位、同键不同 allocator 行为保持；`test_io_binding_registry.cpp`、`tests/unit/adapter/test_adapter_purity.cpp`、`tests/integration/operator/test_operator_api.cpp` | +| 输出分配契约 | 类型、参数、容量、元数据、有效深度预算、分配失败回滚和只解析一次的原测试继续通过；诊断指向新源位置；沿用 Operator suite | +| 模型覆盖 | 覆盖/默认优先级、未知 ID、空值、非法原模型声明、无模型、多个模型；绝对/相对/逃逸/符号链接路径；同一根下改 JSON/conf 目录及 CWD 后结果不变；沿用 resolver/Operator suite | +| Core 边界 | 对中性投影规划与迁移前一致；完整文档误送 Core 仍拒绝 `deployment`；未知根键不被过滤；`tests/unit/core/test_pipeline_config.cpp` 与现有分层检查 | +| 公共入口 | `Create`、`ValidateOperatorConfigBinding`、`validate-io`、`resolve-conf` 对同一输入得到一致配置结果；失败在加载/分配前；`tests/contract/abi/test_cpp_operator_sdk.cpp`、`test_operator_safety.cpp`、`test_adapter_contract_security.cpp` | +| CLI 与诊断 | `validate/plan/--stdin/--explain` 支持完整文档且拒绝半组 I/O；纯算法文档仍可校验但不能部署；source 和 JSON Pointer 正确；扩展现有 CLI/tooling 测试 | +| 生成、编辑与保存 | init/profile、edit、fix-deps、Studio 往返不丢 deployment；换 binding 不混配、模型编辑意图和双文件冲突保护有效；`tests/tooling/test_pipeline_studio.py`、`studio_browser_test.mjs`、`test_dev_recipe.py` | +| 效果评估与共享 fixture | 参数变更令指纹失效，临时运行使用当前 JSON；共用 JSON 的不同部署拆分后仍各自成立，负例仍在预期阶段失败 | +| 完整 SDK 执行 | 用迁移后的无模型关键词方案直接 `Create/Process`，校验 `keyword_out`、请求 ID 与结果；再运行同配置 Demo；沿用 Operator 与 `tests/integration/demo/test_demo_runner.cpp` | + +所有注册能力与字段以实施时构建的 Catalog 为准。完成迁移后，对官方配置和有效 fixture +执行已有工具链的 `validate`、`plan`、`validate-io`、`resolve-conf` 检查；对负例验证其 +预期错误。以下是单个方案的操作形式，不替代测试矩阵： + +```bash +./build/alg_pipeline_tool catalog --biz dialogue_compliance_audit_v1 +./build/alg_pipeline_tool validate configs/pipeline_dialogue_audit_default.json +./build/alg_pipeline_tool plan configs/pipeline_dialogue_audit_default.json +./build/alg_pipeline_tool validate-io configs/pipeline_dialogue_audit_default.conf --model-root . +./build/alg_pipeline_tool resolve-conf configs/pipeline_dialogue_audit_default.conf --root . --depth 1 +``` + +最后运行一次 [CONTRIBUTING.md](../../CONTRIBUTING.md#6-run-one-canonical-delivery-gate) +规定的 `./scripts/run_all_tests.sh`。本次迁移不改变模型推理，不要求下载新模型来证明字段 +搬迁；没有真实模型/硬件时，静态解析与 Mock/无模型执行不得表述为真实业务效果验收。 + +## 5. 实施与最终结果 + +按下面顺序实施，每阶段直接更新本文状态和检查项，不另写重复计划： + +- [x] **阶段 1：解析与配对。** 收窄 conf，以轻量私有函数拆分 JSON；文件与 JSON 入口 + 共用严格 I/O 校验，编辑原文由工具持有，覆盖新格式、严格拒绝与中性 Core 边界测试。 +- [x] **阶段 2：运行时闭环。** 应用 JSON 模型覆盖，保留路径/输出预算和异常屏障； + Create、绑定预检共用不可变计划，完成公共 SDK 正负例。 +- [x] **阶段 3：工具迁移。** 完成 CLI/Studio/recipe/效果评估的读写与诊断； + 去除再次读文件、再次规划和按 biz 猜配对的路径，完成编辑/保存/指纹回归。 +- [x] **阶段 4：文件与指南迁移。** 按 conf→JSON 关联表迁移所有部署场景，拆分冲突共享文件， + 修正动态 fixture、活动示例和 skills，完成直接 SDK 与 Demo smoke。 +- [x] **阶段 5：验收。** 独立复核分层、路径、配对和负例含义；完成上述检查与 canonical gate， + 更新 CHANGELOG、本文及索引状态。 + +各阶段实施均已完成，所有测试与规范检查通过 canonical gate。文档状态正式变更为 `Completed`。 diff --git a/doc/rfcs/README.md b/doc/rfcs/README.md index 2e6d3442..d1cb9f2f 100644 --- a/doc/rfcs/README.md +++ b/doc/rfcs/README.md @@ -86,6 +86,7 @@ RFC-0054 是接续 RFC-0052 与已交付 RFC-0053、RFC-0055 的实施规格。` | **RFC-0058** | 投产前诊断身份与 Node 注册状态收敛 | `Completed` | 投产前 | 流程编排层、能力节点层、接入适配层 / Tooling | [0058-diagnostic-and-node-registry-convergence.md](0058-diagnostic-and-node-registry-convergence.md) | | **RFC-0059** | 输入输出转换独立化与接入绑定架构 | `Completed` | 投产前 / ABI 6.0.0 | 接入适配层、流程编排层 / Tooling / Docs | [0059-independent-adapter-input-and-output.md](0059-independent-adapter-input-and-output.md)（宿主双入口部分被 RFC-0060 取代） | | **RFC-0060** | 删除 C ABI，仅保留 C++ Operator API | `Completed` | `v11.0.0` / ABI 7.0.0 | 接入适配层、流程编排层 / Tooling / Docs | [0060-cpp-operator-only.md](0060-cpp-operator-only.md) | +| **RFC-0061** | Pipeline JSON 集中管理部署配置 | `Completed` | `v11.x` | 接入适配层、流程编排层 / Tooling / Docs | [0061-pipeline-owned-deployment-configuration.md](0061-pipeline-owned-deployment-configuration.md) | ## 专项验收与评审归档 diff --git a/doc/solutions/translate.md b/doc/solutions/translate.md index 2f279e6c..2b7e36a6 100644 --- a/doc/solutions/translate.md +++ b/doc/solutions/translate.md @@ -8,7 +8,7 @@ ## 直接调用 Operator API -翻译部署配置使用 `configs/pipeline_translate_cpu.conf`，其 `io_binding` 为 `translate.operator.v1`，Pipeline 的 `biz_name` 为 `translate_v1`。 +翻译部署配置使用 `configs/pipeline_translate_cpu.conf`，其指向的 `pipeline_translate_cpu.json` 在 `deployment.io` 中配置 `io_binding` 为 `translate.operator.v1`，Pipeline 的 `biz_name` 为 `translate_v1`。 输入/输出复用已有文本载体，因此结构字段仍叫 `sentence_text` / `entities_json`。 传入的是完整对象文本，不是 `query` 子串，也不是双重 JSON 编码后的字符串： diff --git a/scripts/dev_recipe.py b/scripts/dev_recipe.py index d962895a..ba2a20db 100644 --- a/scripts/dev_recipe.py +++ b/scripts/dev_recipe.py @@ -75,13 +75,19 @@ def absolute(path, root): def check_unsupported_deployment(conf_path): conf = read_json_file(conf_path) - data = conf.get("data", {}) - outputs = data.get("outputs") + pipe_path = conf.get("pipe_path") + if not pipe_path or not isinstance(pipe_path, str): + raise RecipeError("A pipe_path deployment is required") + pipeline_file = (Path(conf_path).parent / pipe_path).resolve() + if not pipeline_file.is_file(): + raise RecipeError("A valid pipeline file is required") + doc = read_json_file(pipeline_file) + outputs = doc.get("deployment", {}).get("io", {}).get("output_allocations") if not isinstance(outputs, dict) or not outputs: - raise RecipeError("A data.outputs deployment is required") + raise RecipeError("A deployment.io.output_allocations configuration is required") if len(outputs) > 1: return {"error_code": UNSUPPORTED_RECIPE_DEPLOYMENT, - "message": "This recipe supports only single-output deployment; use the native Operator workflow for data.outputs with multiple slots."} + "message": "This recipe supports only single-output deployment; use the native Operator workflow for deployment.io.output_allocations with multiple slots."} return None @@ -89,7 +95,10 @@ def require_deployment(conf_path): unsupported = check_unsupported_deployment(conf_path) if unsupported: raise RecipeError(unsupported["message"], code=unsupported["error_code"]) - return read_json_file(conf_path)["data"]["outputs"] + conf = read_json_file(conf_path) + pipeline_file = (Path(conf_path).parent / conf["pipe_path"]).resolve() + doc = read_json_file(pipeline_file) + return doc["deployment"]["io"]["output_allocations"] def get_profile_data(profile_name, root=ROOT): @@ -98,8 +107,9 @@ def get_profile_data(profile_name, root=ROOT): raise RecipeError(f"Unknown Profile: {profile_name}") profile = profiles[profile_name] conf = absolute(profile["config"], root) - data = read_json_file(conf)["data"] - pipeline = absolute(data.get("pipe_path", ""), root) + conf_doc = read_json_file(conf) + pipe_path = conf_doc.get("pipe_path", "") + pipeline = absolute(pipe_path, conf.parent) return profile, conf, pipeline @@ -309,8 +319,8 @@ def prepare(recipe, name, profile_name, tool_path, build_dir, pipeline_target, r with tempfile.TemporaryDirectory(prefix=".recipe-preview-", dir=root) as temporary: temp = Path(temporary) preview_pipeline = temp / "pipeline.json" - preview_pipeline.write_text(json.dumps(deployment_preview), encoding="utf-8") preview_conf = VERIFY_SELECTION.build_run_conf(deployment_preview, outputs, preview_pipeline.name, models, bundle) + preview_pipeline.write_text(json.dumps(deployment_preview), encoding="utf-8") (temp / "pipeline.conf").write_text(json.dumps(preview_conf), encoding="utf-8") native(tool, ["resolve-conf", str((temp / "pipeline.conf").relative_to(bundle)), "--root", str(bundle)], root) for path, document in [(target, pipeline), (conf_target, conf), (effects_target, spec)]: diff --git a/src/adapter/CMakeLists.txt b/src/adapter/CMakeLists.txt index 7874c58b..0ad8a846 100644 --- a/src/adapter/CMakeLists.txt +++ b/src/adapter/CMakeLists.txt @@ -3,6 +3,7 @@ target_sources(edgeflow_integration_objects PRIVATE io_converter_registry.cpp io_binding_registry.cpp deployment_io_config.cpp + pipeline_document.cpp io_binding_resolver.cpp io_catalog.cpp input/text_input.cpp diff --git a/src/adapter/deployment_io_config.cpp b/src/adapter/deployment_io_config.cpp index 0ba864ea..98897b79 100644 --- a/src/adapter/deployment_io_config.cpp +++ b/src/adapter/deployment_io_config.cpp @@ -66,101 +66,45 @@ bool DeploymentIoConfig::Parse(const nlohmann::json& root, return false; } - // 1. 顶层字段白名单检查: 仅允许 schema_version 和 data - for (auto it = root.begin(); it != root.end(); ++it) { - if (it.key() != "schema_version" && it.key() != "data") { + // 1. 检查并明确拒绝旧 Schema 1 字段及外部分散配置 (RFC-0061) + for (const char* deprecated_key : + {"data", "schema_version", "io_binding", "model_paths", "outputs"}) { + if (root.contains(deprecated_key)) { if (out_error) { - *out_error = "Unknown field at /: " + it.key() + - " (only schema_version and data allowed)"; + *out_error = + std::string( + "Deprecated deployment configuration format (RFC-0061) at /") + + deprecated_key + + ": '.conf' files must contain only 'pipe_path'. Deployment " + "configuration " + "(io_binding, output_allocations, model_paths) has moved to the " + "'deployment' section inside the Pipeline JSON."; } return false; } } - if (!root.contains("schema_version")) { - if (out_error) *out_error = "Missing schema_version in config"; - return false; - } - if (!root["schema_version"].is_number_integer()) { - if (out_error) *out_error = "schema_version must be an integer"; - return false; - } - int ver = root["schema_version"].get(); - if (ver != 1) { - if (out_error) { - *out_error = - "Unsupported schema_version " + std::to_string(ver) + ", expected 1"; - } - return false; - } - - if (!root.contains("data") || !root["data"].is_object()) { - if (out_error) *out_error = "Missing or invalid 'data' object in config"; - return false; - } - - const auto& data = root["data"]; - - // 2. data 内部字段检查 - for (auto it = data.begin(); it != data.end(); ++it) { - if (it.key() != "pipe_path" && it.key() != "io_binding" && - it.key() != "model_paths" && it.key() != "outputs") { + // 2. 根字段白名单: 必须有且仅有 pipe_path + for (auto it = root.begin(); it != root.end(); ++it) { + if (it.key() != "pipe_path") { if (out_error) { - *out_error = "Unknown field in conf data: '" + it.key() + "'"; + *out_error = "Unknown field at /: '" + it.key() + + "' (only 'pipe_path' is allowed under RFC-0061)"; } return false; } } - if (!data.contains("pipe_path") || !data["pipe_path"].is_string() || - data["pipe_path"].get().empty()) { - if (out_error) *out_error = "Missing or empty 'data.pipe_path'"; - return false; - } - - if (!data.contains("io_binding") || !data["io_binding"].is_string() || - data["io_binding"].get().empty()) { - if (out_error) *out_error = "Missing or empty 'data.io_binding'"; + if (!root.contains("pipe_path") || !root["pipe_path"].is_string() || + root["pipe_path"].get().empty()) { + if (out_error) *out_error = "Missing or empty 'pipe_path'"; return false; } - out_config->schema_version = ver; - out_config->pipe_path = data["pipe_path"].get(); - out_config->io_binding = data["io_binding"].get(); + out_config->pipe_path = root["pipe_path"].get(); out_config->raw_json = root; - // 3. outputs 约束 - if (data.contains("outputs")) { - if (!data["outputs"].is_object()) { - if (out_error) *out_error = "data.outputs must be an object"; - return false; - } - out_config->outputs = data["outputs"]; - } else { - out_config->outputs = nlohmann::json::object(); - } - - // 4. model_paths - out_config->model_paths.clear(); - if (data.contains("model_paths")) { - if (!data["model_paths"].is_object()) { - if (out_error) *out_error = "data.model_paths must be an object"; - return false; - } - for (auto it = data["model_paths"].begin(); it != data["model_paths"].end(); - ++it) { - if (!it.value().is_string()) { - if (out_error) { - *out_error = - "data.model_paths[" + it.key() + "] value must be a string"; - } - return false; - } - out_config->model_paths[it.key()] = it.value().get(); - } - } - - // 5. 解析 pipe_path 相对 + // 3. 解析 pipe_path 相对 // config_dir，严格限制在配置根目录下，拒绝任何逃逸与搜索回退 fs::path base_dir = fs::absolute(fs::path(config_dir)); fs::path raw_pipe = fs::path(out_config->pipe_path); @@ -174,7 +118,7 @@ bool DeploymentIoConfig::Parse(const nlohmann::json& root, if (!IsPathWithinRoot(canonical_base, canonical_pipe)) { if (out_error) { *out_error = - "data.pipe_path escapes config directory: " + out_config->pipe_path; + "pipe_path escapes config directory: " + out_config->pipe_path; } return false; } @@ -191,7 +135,7 @@ bool DeploymentIoConfig::Parse(const nlohmann::json& root, if (ec || !IsPathWithinRoot(canonical_base, real_pipe)) { if (out_error) { *out_error = - "data.pipe_path escapes config directory: " + out_config->pipe_path; + "pipe_path escapes config directory: " + out_config->pipe_path; } return false; } diff --git a/src/adapter/deployment_io_config.h b/src/adapter/deployment_io_config.h index 59e8c01a..e5b83f95 100644 --- a/src/adapter/deployment_io_config.h +++ b/src/adapter/deployment_io_config.h @@ -2,19 +2,14 @@ #include #include -#include namespace llm_edgeflow { /** - * @brief Schema 1 统一接入配置文件解析结构 + * @brief 部署配置文件解析结构 (RFC-0061: 仅包含启动定位字段 pipe_path) */ struct DeploymentIoConfig { - int schema_version = 1; std::string pipe_path; - std::string io_binding; - std::unordered_map model_paths; - nlohmann::json outputs = nlohmann::json::object(); std::string resolved_pipe_path; nlohmann::json raw_json; diff --git a/src/adapter/deployment_model_resolver.cpp b/src/adapter/deployment_model_resolver.cpp index db09f2a0..9ff0e7f2 100644 --- a/src/adapter/deployment_model_resolver.cpp +++ b/src/adapter/deployment_model_resolver.cpp @@ -4,6 +4,7 @@ #include #include +#include "adapter/pipeline_document.h" #include "contracts/path_utils.h" namespace llm_edgeflow { @@ -17,10 +18,10 @@ void SetDiagnostic(std::string* diagnostic, const std::string& message) { } // namespace -bool ResolveDeploymentModelPaths(const nlohmann::json& pipeline_json, - const std::string& model_root_dir, - nlohmann::json* resolved_pipeline_json, - std::string* diagnostic) noexcept { +bool ResolveDeploymentModelPaths( + const nlohmann::json& pipeline_json, const std::string& model_root_dir, + nlohmann::json* resolved_pipeline_json, std::string* diagnostic, + const std::unordered_set& overridden_model_ids) noexcept { try { if (!resolved_pipeline_json) { SetDiagnostic(diagnostic, "Deployment model resolver output is null"); @@ -70,21 +71,28 @@ bool ResolveDeploymentModelPaths(const nlohmann::json& pipeline_json, continue; } + std::string pointer = "/models/" + std::to_string(index) + "/model_path"; + std::string model_id = + model.contains("model_id") && model["model_id"].is_string() + ? model["model_id"].get() + : ""; + if (!model_id.empty() && overridden_model_ids.count(model_id)) { + pointer = "/deployment/model_paths/" + EscapeJsonPointer(model_id); + } + const std::string raw_path = model["model_path"].get(); const fs::path normalized = fs::path(raw_path).lexically_normal(); if (!normalized.is_absolute() && HasParentPathComponent(normalized)) { SetDiagnostic(diagnostic, - "Model path cannot traverse outside model_root_dir at " - "/models/" + - std::to_string(index) + "/model_path: " + raw_path); + "Model path cannot traverse outside model_root_dir at " + + pointer + ": " + raw_path); return false; } if (!normalized.is_absolute() && canonical_root.empty()) { SetDiagnostic( diagnostic, - "Relative model_path requires non-empty model_root_dir at " - "/models/" + - std::to_string(index) + "/model_path: " + raw_path); + "Relative model_path requires non-empty model_root_dir at " + + pointer + ": " + raw_path); return false; } @@ -94,15 +102,14 @@ bool ResolveDeploymentModelPaths(const nlohmann::json& pipeline_json, error); if (error) { SetDiagnostic(diagnostic, - "Failed to resolve deployment model path at /models/" + - std::to_string(index) + "/model_path: " + raw_path); + "Failed to resolve deployment model path at " + pointer + + ": " + raw_path); return false; } if (!canonical_root.empty() && !IsPathWithinRoot(canonical_root, candidate)) { - SetDiagnostic(diagnostic, - "Model path escapes model_root_dir at /models/" + - std::to_string(index) + "/model_path: " + raw_path); + SetDiagnostic(diagnostic, "Model path escapes model_root_dir at " + + pointer + ": " + raw_path); return false; } model["model_path"] = candidate.string(); diff --git a/src/adapter/deployment_model_resolver.h b/src/adapter/deployment_model_resolver.h index 912ed1ee..47e3acad 100644 --- a/src/adapter/deployment_model_resolver.h +++ b/src/adapter/deployment_model_resolver.h @@ -2,6 +2,7 @@ #include #include +#include namespace llm_edgeflow { @@ -13,9 +14,9 @@ namespace llm_edgeflow { * that directory and cannot escape it. With an empty root, deployment model * paths must already be absolute. */ -bool ResolveDeploymentModelPaths(const nlohmann::json& pipeline_json, - const std::string& model_root_dir, - nlohmann::json* resolved_pipeline_json, - std::string* diagnostic) noexcept; +bool ResolveDeploymentModelPaths( + const nlohmann::json& pipeline_json, const std::string& model_root_dir, + nlohmann::json* resolved_pipeline_json, std::string* diagnostic, + const std::unordered_set& overridden_model_ids = {}) noexcept; } // namespace llm_edgeflow diff --git a/src/adapter/io_binding_resolver.cpp b/src/adapter/io_binding_resolver.cpp index 085ae59f..cb3f72f4 100644 --- a/src/adapter/io_binding_resolver.cpp +++ b/src/adapter/io_binding_resolver.cpp @@ -3,12 +3,14 @@ #include #include #include +#include #include "adapter/deployment_model_resolver.h" #include "adapter/io_binding_registry.h" #include "adapter/io_converter_registry.h" #include "adapter/operator/operator_config_resolver.h" #include "adapter/operator/operator_value_type_registry.h" +#include "adapter/pipeline_document.h" #include "core/diagnostic_code.h" #include "core/pipeline_catalog.h" @@ -41,18 +43,39 @@ int IoBindingResolver::ResolveFromConfig( } *out_plan = nullptr; - // 1. 查找绑定定义 - const auto* binding = - IoBindingRegistry::Instance().FindBinding(config.io_binding); - if (!binding) { + std::ifstream pipe_ifs(config.resolved_pipe_path); + if (!pipe_ifs.is_open()) { + if (out_error) { + *out_error = "Failed to open pipeline file: " + config.resolved_pipe_path; + } + return -2; + } + nlohmann::json raw_pipe_json; + try { + pipe_ifs >> raw_pipe_json; + } catch (const std::exception& e) { if (out_error) { - *out_error = "Unknown or unregistered io_binding: " + config.io_binding + - " (at data.io_binding)"; + *out_error = "JSON parse exception in pipeline file: " + + config.resolved_pipe_path + ": " + e.what(); } return -2; } - // 2. 检查入口类型匹配 + return ResolveFromPipelineJson(raw_pipe_json, transport, model_root_dir, + out_plan, out_error); +} + +int IoBindingResolver::ResolveFromPipelineJson( + const nlohmann::json& pipeline_json, const std::string& transport, + const std::string& model_root_dir, + std::unique_ptr* out_plan, std::string* out_error) { + if (!out_plan) { + if (out_error) *out_error = "Null out_plan pointer"; + return -1; + } + *out_plan = nullptr; + + // 1. 检查入口 transport if (transport != "operator") { if (out_error) { *out_error = "Unsupported transport: '" + transport + @@ -60,16 +83,55 @@ int IoBindingResolver::ResolveFromConfig( } return -2; } + + // 2. 拆分并校验 Pipeline 文档中的 deployment 部分 (RFC-0061) + PipelineDocumentSplit doc_split; + std::string split_err; + if (!SplitPipelineDocument(pipeline_json, &doc_split, &split_err)) { + if (out_error) *out_error = split_err; + return -2; + } + + if (!doc_split.has_deployment || !doc_split.deployment.has_io) { + if (out_error) { + *out_error = + "Missing required 'deployment.io' in pipeline JSON for deployment"; + } + return -2; + } + + // 3. 查找并核对绑定定义 + const std::string& binding_id = doc_split.deployment.io.io_binding; + const auto* binding = IoBindingRegistry::Instance().FindBinding(binding_id); + if (!binding) { + if (out_error) { + *out_error = "Unknown or unregistered io_binding: " + binding_id + + " (at /deployment/io/io_binding)"; + } + return -2; + } + if (binding->transport != "operator") { if (out_error) { - *out_error = "Binding transport mismatch for '" + config.io_binding + + *out_error = "Binding transport mismatch for '" + binding_id + "': expected 'operator', but binding declared '" + - binding->transport + "'"; + binding->transport + "' (at /deployment/io/io_binding)"; } return -2; } - // 3. 查找输入与输出转换器 + // 核对 Pipeline biz_name 与 binding biz_name (RFC-0061: 必须一致) + std::string pipeline_biz = pipeline_json.value("biz_name", ""); + if (pipeline_biz != binding->biz_name) { + if (out_error) { + *out_error = "Pipeline biz_name '" + pipeline_biz + + "' does not match binding biz_name '" + binding->biz_name + + "' (at /deployment/io/io_binding)"; + } + return -2; + } + + // 4. 查找输入与输出转换器 const auto* in_conv = IoConverterRegistry::Instance().FindInputConverter( binding->input_converter_id); if (!in_conv) { @@ -90,7 +152,7 @@ int IoBindingResolver::ResolveFromConfig( return -2; } - // 4. 计算有效批次上限 + // 5. 计算有效批次上限 size_t max_batch = std::min(in_conv->max_batch_size, out_conv->max_batch_size); const auto* exposure = @@ -99,11 +161,13 @@ int IoBindingResolver::ResolveFromConfig( max_batch = std::min(max_batch, exposure->max_batch_size); } - // 5. 校验 outputs 配置与槽位 + // 6. 校验 output_allocations 配置与槽位 (必须满足 R ⊆ C ⊆ A) + const auto& allocations = doc_split.deployment.io.output_allocations; std::unordered_map output_specs; std::unordered_map output_params; - // 5.1 拒绝未在输出转换器中声明的未知槽位配置 - for (auto it = config.outputs.begin(); it != config.outputs.end(); ++it) { + + // 6.1 拒绝未在输出转换器中声明的未知槽位配置 (C ⊆ A) + for (auto it = allocations.begin(); it != allocations.end(); ++it) { bool found = false; for (const auto& slot : out_conv->external_slots) { if (slot.direction == PortDirection::kOutput && @@ -114,40 +178,47 @@ int IoBindingResolver::ResolveFromConfig( } if (!found) { if (out_error) { - *out_error = "Unknown configured output slot: " + it.key(); + *out_error = "Unknown configured output slot: " + it.key() + + " (at /deployment/io/output_allocations/" + + EscapeJsonPointer(it.key()) + ")"; } return -2; } } - // 5.2 校验并解析每个输出槽位配置 (复用统一的 OperatorConfigResolver 规范) + // 6.2 校验并解析每个输出槽位配置，确保所有必需槽位存在 (R ⊆ C) for (const auto& slot : out_conv->external_slots) { if (slot.direction != PortDirection::kOutput) continue; - if (!config.outputs.contains(slot.slot_name)) { + if (!allocations.contains(slot.slot_name)) { if (slot.required) { if (out_error) { *out_error = "Missing required Operator output slot '" + - slot.slot_name + "' in data.outputs"; + slot.slot_name + + "' (at /deployment/io/output_allocations/" + + EscapeJsonPointer(slot.slot_name) + ")"; } return -2; } continue; } - const auto& slot_cfg = config.outputs[slot.slot_name]; + const auto& slot_cfg = allocations[slot.slot_name]; ResolvedOutputPoolSpec pool_spec; std::string param_text; std::string alloc_err; int alloc_ret = OperatorConfigResolver::ResolveOutputAllocation( slot_cfg, slot, &pool_spec, ¶m_text, &alloc_err); if (alloc_ret != 0) { - if (out_error) *out_error = alloc_err; + if (out_error) { + *out_error = alloc_err + " (at /deployment/io/output_allocations/" + + EscapeJsonPointer(slot.slot_name) + ")"; + } return alloc_ret; } output_specs[slot.slot_name] = std::move(pool_spec); output_params[slot.slot_name] = std::move(param_text); } - // 5.3 默认深度下的句柄池载荷总预算校验 + // 6.3 默认深度下的句柄池载荷总预算校验 size_t total_handle_pool_bytes = 0; for (const auto& [slot_name, pool_spec] : output_specs) { const auto* output_binding = @@ -186,217 +257,75 @@ int IoBindingResolver::ResolveFromConfig( return -2; } - // 6. 读取 Pipeline JSON - std::ifstream pipe_ifs(config.resolved_pipe_path); - if (!pipe_ifs.is_open()) { - if (out_error) { - *out_error = "Failed to open pipeline file: " + config.resolved_pipe_path; - } - return -2; - } - nlohmann::json raw_pipe_json; - try { - pipe_ifs >> raw_pipe_json; - } catch (const std::exception& e) { - if (out_error) { - *out_error = "JSON parse exception in pipeline file: " + - config.resolved_pipe_path + ": " + e.what(); - } - return -2; - } - // 7. 应用覆盖与解析模型路径 - nlohmann::json staged_pipe_json = raw_pipe_json; - if (!config.model_paths.empty()) { + nlohmann::json staged_pipe_json = doc_split.neutral_pipeline_json; + std::unordered_set overridden_model_ids; + if (doc_split.deployment.has_model_paths && + !doc_split.deployment.model_paths.empty()) { std::unordered_set known_model_ids; if (staged_pipe_json.contains("models") && staged_pipe_json["models"].is_array()) { for (const auto& m : staged_pipe_json["models"]) { - if (m.is_object() && m.contains("model_id")) { + if (m.is_object() && m.contains("model_id") && + m["model_id"].is_string()) { known_model_ids.insert(m["model_id"].get()); } } } - for (const auto& [mid, _] : config.model_paths) { + for (const auto& [mid, _] : doc_split.deployment.model_paths) { if (!known_model_ids.count(mid)) { if (out_error) { - *out_error = "Unknown model_id '" + mid + "' in 'model_paths'"; + *out_error = + "Unknown model_id '" + mid + + "' in '/deployment/model_paths' (at /deployment/model_paths/" + + EscapeJsonPointer(mid) + ")"; } return -2; } } for (auto& m : staged_pipe_json["models"]) { - if (m.is_object() && m.contains("model_id")) { + if (m.is_object() && m.contains("model_id") && + m["model_id"].is_string()) { std::string mid = m["model_id"].get(); - auto it = config.model_paths.find(mid); - if (it != config.model_paths.end()) { + auto it = doc_split.deployment.model_paths.find(mid); + if (it != doc_split.deployment.model_paths.end()) { m["model_path"] = it->second; + overridden_model_ids.insert(mid); } } } } nlohmann::json resolved_pipeline_json; - std::string model_resolve_err; - if (!ResolveDeploymentModelPaths(staged_pipe_json, model_root_dir, - &resolved_pipeline_json, - &model_resolve_err)) { - if (out_error) *out_error = model_resolve_err; - return -2; - } - - // 8. 构造中性 PipelineIoBoundary - PipelineIoBoundary io_boundary; - for (const auto& port : in_conv->logical_ports) { - std::string key = port.logical_name; - auto bit = binding->input_ports.find(port.logical_name); - if (bit != binding->input_ports.end()) { - key = bit->second; - } - io_boundary.input_published_ports.emplace_back( - key, port.type_id, port.required, port.cardinality, - port.provenance_policy, port.lifetime, port.lifetime_config_field); - } - - for (const auto& port : out_conv->logical_ports) { - std::string key = port.logical_name; - auto bit = binding->output_ports.find(port.logical_name); - if (bit != binding->output_ports.end()) { - key = bit->second; + if (!model_root_dir.empty()) { + std::string model_resolve_err; + if (!ResolveDeploymentModelPaths( + staged_pipe_json, model_root_dir, &resolved_pipeline_json, + &model_resolve_err, overridden_model_ids)) { + if (out_error) *out_error = model_resolve_err; + return -2; } - io_boundary.output_consumed_ports.emplace_back( - key, port.type_id, port.required, port.cardinality, - port.provenance_policy, port.lifetime, port.lifetime_config_field); - } - - // 9. 调用 PipelineValidator 进行统一中性计划验证 - auto plan = std::make_unique( - PipelineValidator::ValidateAndPlan( - resolved_pipeline_json, ValidationPolicy::kStrict, &io_boundary)); - - if (!plan->report.ok) { - if (out_error) { - if (!plan->report.diagnostics.empty()) { - const auto& d = plan->report.diagnostics.front(); - *out_error = - "Validation failed: " + std::string(DiagnosticCodeName(d.code)) + - " at " + d.path + ": " + d.message; - } else { - *out_error = "Validation failed without diagnostics"; + } else { + // 环境无关检查: 模型路径必须是非空字符串 + if (staged_pipe_json.contains("models") && + staged_pipe_json["models"].is_array()) { + for (const auto& m : staged_pipe_json["models"]) { + if (m.is_object() && m.contains("model_path")) { + if (!m["model_path"].is_string() || + m["model_path"].get().empty()) { + if (out_error) { + *out_error = + "model_path in model declaration must be a non-empty string"; + } + return -2; + } + } } } - return -3; + resolved_pipeline_json = staged_pipe_json; } - // 10. 核对 Pipeline biz_name 与 binding biz_name - if (plan->config.biz_name != binding->biz_name) { - if (out_error) { - *out_error = "Pipeline biz_name '" + plan->config.biz_name + - "' does not match binding biz_name '" + binding->biz_name + - "'"; - } - return -3; - } - - // 11. 组装不可变接入计划 - auto io_plan = std::make_unique(); - io_plan->binding = *binding; - io_plan->input_converter = in_conv; - io_plan->output_converter = out_conv; - io_plan->input_port_bindings = InputPortBindings(binding->input_ports); - io_plan->output_port_bindings = OutputPortBindings(binding->output_ports); - io_plan->effective_max_batch_size = max_batch; - io_plan->operator_output_specs = std::move(output_specs); - io_plan->operator_output_parameter_texts = std::move(output_params); - io_plan->resolved_pipeline_json = resolved_pipeline_json; - io_plan->pipeline_plan = std::move(plan); - - *out_plan = std::move(io_plan); - return 0; -} - -int IoBindingResolver::ResolveFromPipelineJson( - const nlohmann::json& pipeline_json, const std::string& binding_id, - const std::string& transport, const std::string& model_root_dir, - std::unique_ptr* out_plan, std::string* out_error) { - if (!out_plan) { - if (out_error) *out_error = "Null out_plan pointer"; - return -1; - } - *out_plan = nullptr; - - if (binding_id.empty()) { - if (out_error) *out_error = "binding_id must not be empty"; - return -2; - } - - // 1. 查找绑定定义 - const auto* binding = IoBindingRegistry::Instance().FindBinding(binding_id); - if (!binding) { - if (out_error) { - *out_error = "Unknown or unregistered io_binding: " + binding_id; - } - return -2; - } - - // 2. 检查入口类型匹配 - if (transport != "operator") { - if (out_error) { - *out_error = "Unsupported transport: '" + transport + - "' (only 'operator' is supported)"; - } - return -2; - } - if (binding->transport != "operator") { - if (out_error) { - *out_error = "Binding transport mismatch for '" + binding_id + - "': expected 'operator', but binding declared '" + - binding->transport + "'"; - } - return -2; - } - - // 3. 查找输入与输出转换器 - const auto* in_conv = IoConverterRegistry::Instance().FindInputConverter( - binding->input_converter_id); - if (!in_conv) { - if (out_error) { - *out_error = "Binding references unregistered input converter: " + - binding->input_converter_id; - } - return -2; - } - - const auto* out_conv = IoConverterRegistry::Instance().FindOutputConverter( - binding->output_converter_id); - if (!out_conv) { - if (out_error) { - *out_error = "Binding references unregistered output converter: " + - binding->output_converter_id; - } - return -2; - } - - // 4. 计算有效批次上限 - size_t max_batch = - std::min(in_conv->max_batch_size, out_conv->max_batch_size); - const auto* exposure = - IoBindingRegistry::Instance().FindExposure(binding->biz_name); - if (exposure) { - max_batch = std::min(max_batch, exposure->max_batch_size); - } - - // 5. 解析模型路径 (若提供 model_root_dir) - nlohmann::json resolved_pipeline_json; - std::string model_resolve_err; - if (!ResolveDeploymentModelPaths(pipeline_json, model_root_dir, - &resolved_pipeline_json, - &model_resolve_err)) { - if (out_error) *out_error = model_resolve_err; - return -2; - } - - // 6. 构造中性 PipelineIoBoundary + // 8. 构造中性 PipelineIoBoundary PipelineIoBoundary io_boundary; for (const auto& port : in_conv->logical_ports) { std::string key = port.logical_name; @@ -420,7 +349,8 @@ int IoBindingResolver::ResolveFromPipelineJson( port.provenance_policy, port.lifetime, port.lifetime_config_field); } - // 7. 调用 PipelineValidator 进行统一中性计划验证 (包含 I/O 边界验证) + // 9. 调用 PipelineValidator 进行统一中性计划验证 (Core 校验 + // neutral_pipeline_json) auto plan = std::make_unique( PipelineValidator::ValidateAndPlan( resolved_pipeline_json, ValidationPolicy::kStrict, &io_boundary)); @@ -439,17 +369,7 @@ int IoBindingResolver::ResolveFromPipelineJson( return -3; } - // 8. 核对 Pipeline biz_name 与 binding biz_name - if (plan->config.biz_name != binding->biz_name) { - if (out_error) { - *out_error = "Pipeline biz_name '" + plan->config.biz_name + - "' does not match binding biz_name '" + binding->biz_name + - "'"; - } - return -3; - } - - // 9. 组装不可变接入计划 + // 10. 组装不可变接入计划 auto io_plan = std::make_unique(); io_plan->binding = *binding; io_plan->input_converter = in_conv; @@ -457,6 +377,9 @@ int IoBindingResolver::ResolveFromPipelineJson( io_plan->input_port_bindings = InputPortBindings(binding->input_ports); io_plan->output_port_bindings = OutputPortBindings(binding->output_ports); io_plan->effective_max_batch_size = max_batch; + io_plan->operator_output_specs = std::move(output_specs); + io_plan->operator_output_parameter_texts = std::move(output_params); + io_plan->overridden_model_ids = std::move(overridden_model_ids); io_plan->resolved_pipeline_json = resolved_pipeline_json; io_plan->pipeline_plan = std::move(plan); diff --git a/src/adapter/io_binding_resolver.h b/src/adapter/io_binding_resolver.h index c020e6ef..86ed948e 100644 --- a/src/adapter/io_binding_resolver.h +++ b/src/adapter/io_binding_resolver.h @@ -3,6 +3,7 @@ #include #include #include +#include #include "adapter/deployment_io_config.h" #include "adapter/io_binding.h" @@ -25,6 +26,7 @@ struct ValidatedIoPlan { std::unordered_map operator_output_specs; std::unordered_map operator_output_parameter_texts; + std::unordered_set overridden_model_ids; nlohmann::json resolved_pipeline_json; std::unique_ptr pipeline_plan; @@ -49,7 +51,7 @@ class IoBindingResolver { std::string* out_error); static int ResolveFromPipelineJson( - const nlohmann::json& pipeline_json, const std::string& binding_id, + const nlohmann::json& pipeline_json, const std::string& transport, // "operator" const std::string& model_root_dir, std::unique_ptr* out_plan, std::string* out_error); diff --git a/src/adapter/pipeline_document.cpp b/src/adapter/pipeline_document.cpp new file mode 100644 index 00000000..81ed8a5b --- /dev/null +++ b/src/adapter/pipeline_document.cpp @@ -0,0 +1,171 @@ +#include "adapter/pipeline_document.h" + +namespace llm_edgeflow { + +bool SplitPipelineDocument(const nlohmann::json& root, + PipelineDocumentSplit* out_split, + std::string* out_error, + std::string* out_error_path) { + if (!out_split) { + if (out_error) *out_error = "Null out_split pointer"; + return false; + } + *out_split = PipelineDocumentSplit{}; + + if (!root.is_object()) { + if (out_error) { + *out_error = "Pipeline configuration root must be a JSON object"; + } + if (out_error_path) *out_error_path = "/"; + return false; + } + + if (!root.contains("deployment")) { + out_split->has_deployment = false; + out_split->neutral_pipeline_json = root; + return true; + } + + const auto& dep = root["deployment"]; + if (!dep.is_object()) { + if (out_error) *out_error = "Field '/deployment' must be an object"; + if (out_error_path) *out_error_path = "/deployment"; + return false; + } + + // 1. 校验 deployment 内部白名单: 仅允许 model_paths 与 io + for (auto it = dep.begin(); it != dep.end(); ++it) { + if (it.key() != "model_paths" && it.key() != "io") { + if (out_error) { + *out_error = "Unknown field at /deployment/" + + EscapeJsonPointer(it.key()) + + " (only 'model_paths' and 'io' allowed)"; + } + if (out_error_path) { + *out_error_path = "/deployment/" + EscapeJsonPointer(it.key()); + } + return false; + } + } + + // 2. 校验 model_paths (可选) + if (dep.contains("model_paths")) { + const auto& mp = dep["model_paths"]; + if (!mp.is_object()) { + if (out_error) { + *out_error = "Field '/deployment/model_paths' must be an object"; + } + if (out_error_path) *out_error_path = "/deployment/model_paths"; + return false; + } + for (auto it = mp.begin(); it != mp.end(); ++it) { + if (it.key().empty()) { + if (out_error) *out_error = "Empty model_id in /deployment/model_paths"; + if (out_error_path) *out_error_path = "/deployment/model_paths"; + return false; + } + if (!it.value().is_string()) { + if (out_error) { + *out_error = "Field '/deployment/model_paths/" + + EscapeJsonPointer(it.key()) + "' value must be a string"; + } + if (out_error_path) { + *out_error_path = + "/deployment/model_paths/" + EscapeJsonPointer(it.key()); + } + return false; + } + const std::string path_str = it.value().get(); + if (path_str.empty()) { + if (out_error) { + *out_error = "Field '/deployment/model_paths/" + + EscapeJsonPointer(it.key()) + "' cannot be empty"; + } + if (out_error_path) { + *out_error_path = + "/deployment/model_paths/" + EscapeJsonPointer(it.key()); + } + return false; + } + out_split->deployment.model_paths[it.key()] = path_str; + } + out_split->deployment.has_model_paths = true; + } + + // 3. 校验 io (当 deployment 存在时必填) + if (!dep.contains("io")) { + if (out_error) *out_error = "Missing required field '/deployment/io'"; + if (out_error_path) *out_error_path = "/deployment/io"; + return false; + } + const auto& io = dep["io"]; + if (!io.is_object()) { + if (out_error) *out_error = "Field '/deployment/io' must be an object"; + if (out_error_path) *out_error_path = "/deployment/io"; + return false; + } + + // 校验 io 内部白名单: 必须有且仅有 io_binding 与 output_allocations + for (auto it = io.begin(); it != io.end(); ++it) { + if (it.key() != "io_binding" && it.key() != "output_allocations") { + if (out_error) { + *out_error = "Unknown field at /deployment/io/" + + EscapeJsonPointer(it.key()) + + " (only 'io_binding' and 'output_allocations' allowed)"; + } + if (out_error_path) { + *out_error_path = "/deployment/io/" + EscapeJsonPointer(it.key()); + } + return false; + } + } + + if (!io.contains("io_binding")) { + if (out_error) { + *out_error = "Missing required field '/deployment/io/io_binding'"; + } + if (out_error_path) *out_error_path = "/deployment/io/io_binding"; + return false; + } + if (!io["io_binding"].is_string()) { + if (out_error) { + *out_error = "Field '/deployment/io/io_binding' must be a string"; + } + if (out_error_path) *out_error_path = "/deployment/io/io_binding"; + return false; + } + std::string binding_id = io["io_binding"].get(); + if (binding_id.empty()) { + if (out_error) { + *out_error = "Field '/deployment/io/io_binding' cannot be empty"; + } + if (out_error_path) *out_error_path = "/deployment/io/io_binding"; + return false; + } + out_split->deployment.io.io_binding = std::move(binding_id); + + if (!io.contains("output_allocations")) { + if (out_error) { + *out_error = "Missing required field '/deployment/io/output_allocations'"; + } + if (out_error_path) *out_error_path = "/deployment/io/output_allocations"; + return false; + } + if (!io["output_allocations"].is_object()) { + if (out_error) { + *out_error = + "Field '/deployment/io/output_allocations' must be an object"; + } + if (out_error_path) *out_error_path = "/deployment/io/output_allocations"; + return false; + } + out_split->deployment.io.output_allocations = io["output_allocations"]; + out_split->deployment.has_io = true; + + out_split->has_deployment = true; + out_split->neutral_pipeline_json = root; + out_split->neutral_pipeline_json.erase("deployment"); + return true; +} + +} // namespace llm_edgeflow diff --git a/src/adapter/pipeline_document.h b/src/adapter/pipeline_document.h new file mode 100644 index 00000000..9d9ffa4b --- /dev/null +++ b/src/adapter/pipeline_document.h @@ -0,0 +1,69 @@ +#pragma once + +#include +#include +#include + +namespace llm_edgeflow { + +/** + * @brief 部署配置中的 I/O 接入定义 (RFC-0061) + */ +struct DeploymentIoSection { + std::string io_binding; + nlohmann::json output_allocations = nlohmann::json::object(); +}; + +/** + * @brief 部署配置定义 (RFC-0061: 包含模型路径覆盖与 I/O 接入配置) + */ +struct DeploymentSection { + std::unordered_map model_paths; + bool has_model_paths = false; + DeploymentIoSection io; + bool has_io = false; +}; + +/** + * @brief Pipeline 文档拆分结果 (RFC-0061) + */ +struct PipelineDocumentSplit { + bool has_deployment = false; + DeploymentSection deployment; + nlohmann::json neutral_pipeline_json; +}; + +/** + * @brief 遵循 RFC 6901 的 JSON Pointer 键转义 (~ 转为 ~0, / 转为 ~1) + */ +inline std::string EscapeJsonPointer(std::string_view token) { + std::string escaped; + escaped.reserve(token.size()); + for (char c : token) { + if (c == '~') { + escaped += "~0"; + } else if (c == '/') { + escaped += "~1"; + } else { + escaped += c; + } + } + return escaped; +} + +/** + * @brief 拆分并严格校验 Pipeline JSON 中的 deployment 部分与中性 Pipeline 结构 + * + * 严格校验 deployment: + * - 根中若有 deployment，必须为对象，且只允许 model_paths 与 io 键 + * - model_paths 若存在必须为对象，键为 model_id，值为非空字符串路径 + * - io 必须存在且必须为对象，有且仅有 io_binding 和 output_allocations 两个字段 + * - neutral_pipeline_json 为移除已校验 deployment + * 后的副本，保留所有其他根字段供 Core 校验 + */ +bool SplitPipelineDocument(const nlohmann::json& root, + PipelineDocumentSplit* out_split, + std::string* out_error, + std::string* out_error_path = nullptr); + +} // namespace llm_edgeflow diff --git a/src/tools/alg_pipeline_tool.cpp b/src/tools/alg_pipeline_tool.cpp index b4b938ef..45db293d 100644 --- a/src/tools/alg_pipeline_tool.cpp +++ b/src/tools/alg_pipeline_tool.cpp @@ -7,9 +7,13 @@ #include #include "adapter/deployment_io_config.h" +#include "adapter/io_binding_registry.h" #include "adapter/io_binding_resolver.h" #include "adapter/io_catalog.h" +#include "adapter/io_converter_registry.h" #include "adapter/operator/operator_config_resolver.h" +#include "adapter/pipeline_document.h" +#include "core/common_contracts.h" #include "core/diagnostic_code.h" #include "core/pipeline_catalog.h" #include "core/pipeline_validator.h" @@ -35,11 +39,12 @@ nlohmann::json PipelineError(DiagnosticCode code, const std::string& message) { {"severity", "error"}}})}}; } -nlohmann::json ToolError(const std::string& code, const std::string& message) { +nlohmann::json ToolError(const std::string& code, const std::string& message, + const std::string& path = "/") { return {{"schema_version", 1}, {"ok", false}, {"diagnostics", nlohmann::json::array({{{"code", code}, - {"path", "/"}, + {"path", path}, {"message", message}, {"severity", "error"}}})}}; } @@ -79,13 +84,11 @@ std::optional ProfilePipeline( if (!conf_stream.is_open()) return std::nullopt; nlohmann::json conf; conf_stream >> conf; - if (!conf.is_object() || !conf.contains("data") || - !conf["data"].is_object() || !conf["data"].contains("pipe_path") || - !conf["data"]["pipe_path"].is_string()) { + if (!conf.is_object() || !conf.contains("pipe_path") || + !conf["pipe_path"].is_string()) { return std::nullopt; } - const auto& data = conf["data"]; - fs::path pipe_path = data["pipe_path"].get(); + fs::path pipe_path = conf["pipe_path"].get(); if (pipe_path.is_relative()) { if (!fs::exists(pipe_path)) { pipe_path = conf_path.parent_path() / pipe_path; @@ -125,6 +128,222 @@ nlohmann::json ProfilesJson(const std::string& biz_filter) { return result; } +bool ResolveDeploymentBoundary( + const nlohmann::json& root, nlohmann::json* out_neutral_json, + llm_edgeflow::PipelineIoBoundary* out_boundary, + const llm_edgeflow::IoBindingDefinition** out_binding, + nlohmann::json* out_error_json) { + using namespace llm_edgeflow; + + PipelineDocumentSplit doc_split; + std::string split_err; + std::string split_path; + if (!SplitPipelineDocument(root, &doc_split, &split_err, &split_path)) { + *out_error_json = ToolError("DEPLOYMENT_ERROR", split_err, + split_path.empty() ? "/" : split_path); + return false; + } + + if (!doc_split.has_deployment || !doc_split.deployment.has_io) { + *out_error_json = ToolError( + "MISSING_DEPLOYMENT_IO", + "Missing required 'deployment.io' in pipeline JSON", "/deployment/io"); + return false; + } + + const std::string& binding_id = doc_split.deployment.io.io_binding; + const auto* binding = IoBindingRegistry::Instance().FindBinding(binding_id); + if (!binding) { + *out_error_json = + ToolError("UNKNOWN_IO_BINDING", + "Unknown or unregistered io_binding: " + binding_id + + " (at /deployment/io/io_binding)", + "/deployment/io/io_binding"); + return false; + } + + if (binding->transport != "operator") { + *out_error_json = + ToolError("UNSUPPORTED_TRANSPORT", + "Binding transport mismatch for '" + binding_id + + "': expected 'operator', but binding declared '" + + binding->transport + "' (at /deployment/io/io_binding)", + "/deployment/io/io_binding"); + return false; + } + + // 立即核对 Pipeline biz_name 与 binding biz_name (RFC-0061) + std::string pipeline_biz = root.value("biz_name", ""); + if (pipeline_biz != binding->biz_name) { + *out_error_json = + ToolError("BIZ_MISMATCH", + "Pipeline biz_name '" + pipeline_biz + + "' does not match binding biz_name '" + + binding->biz_name + "' (at /deployment/io/io_binding)", + "/deployment/io/io_binding"); + return false; + } + + const auto* in_conv = IoConverterRegistry::Instance().FindInputConverter( + binding->input_converter_id); + if (!in_conv) { + *out_error_json = + ToolError("UNREGISTERED_CONVERTER", + "Binding references unregistered input converter: " + + binding->input_converter_id, + "/deployment/io/io_binding"); + return false; + } + + const auto* out_conv = IoConverterRegistry::Instance().FindOutputConverter( + binding->output_converter_id); + if (!out_conv) { + *out_error_json = + ToolError("UNREGISTERED_CONVERTER", + "Binding references unregistered output converter: " + + binding->output_converter_id, + "/deployment/io/io_binding"); + return false; + } + + const auto& allocations = doc_split.deployment.io.output_allocations; + for (auto it = allocations.begin(); it != allocations.end(); ++it) { + bool found = false; + for (const auto& slot : out_conv->external_slots) { + if (slot.direction == PortDirection::kOutput && + slot.slot_name == it.key()) { + found = true; + break; + } + } + if (!found) { + *out_error_json = ToolError( + "UNKNOWN_OUTPUT_SLOT", + "Unknown configured output slot: " + it.key() + + " (at /deployment/io/output_allocations/" + + EscapeJsonPointer(it.key()) + ")", + "/deployment/io/output_allocations/" + EscapeJsonPointer(it.key())); + return false; + } + } + + for (const auto& slot : out_conv->external_slots) { + if (slot.direction != PortDirection::kOutput) continue; + if (!allocations.contains(slot.slot_name)) { + if (slot.required) { + *out_error_json = ToolError( + "MISSING_OUTPUT_SLOT", + "Missing required Operator output slot '" + slot.slot_name + + "' (at /deployment/io/output_allocations/" + + EscapeJsonPointer(slot.slot_name) + ")", + "/deployment/io/output_allocations/" + + EscapeJsonPointer(slot.slot_name)); + return false; + } + continue; + } + ResolvedOutputPoolSpec pool_spec; + std::string param_text; + std::string alloc_err; + if (OperatorConfigResolver::ResolveOutputAllocation( + allocations[slot.slot_name], slot, &pool_spec, ¶m_text, + &alloc_err) != 0) { + *out_error_json = + ToolError("INVALID_OUTPUT_ALLOCATION", + alloc_err + " (at /deployment/io/output_allocations/" + + EscapeJsonPointer(slot.slot_name) + ")", + "/deployment/io/output_allocations/" + + EscapeJsonPointer(slot.slot_name)); + return false; + } + } + + nlohmann::json staged_pipe_json = doc_split.neutral_pipeline_json; + if (doc_split.deployment.has_model_paths && + !doc_split.deployment.model_paths.empty()) { + std::unordered_set known_model_ids; + if (staged_pipe_json.contains("models") && + staged_pipe_json["models"].is_array()) { + for (const auto& m : staged_pipe_json["models"]) { + if (m.is_object() && m.contains("model_id") && + m["model_id"].is_string()) { + known_model_ids.insert(m["model_id"].get()); + } + } + } + for (const auto& [mid, _] : doc_split.deployment.model_paths) { + if (!known_model_ids.count(mid)) { + *out_error_json = ToolError( + "UNKNOWN_MODEL_ID", + "Unknown model_id '" + mid + "' in '/deployment/model_paths'", + "/deployment/model_paths/" + EscapeJsonPointer(mid)); + return false; + } + } + for (auto& m : staged_pipe_json["models"]) { + if (m.is_object() && m.contains("model_id") && + m["model_id"].is_string()) { + std::string mid = m["model_id"].get(); + auto it = doc_split.deployment.model_paths.find(mid); + if (it != doc_split.deployment.model_paths.end()) { + m["model_path"] = it->second; + } + } + } + } + + if (staged_pipe_json.contains("models") && + staged_pipe_json["models"].is_array()) { + for (size_t index = 0; index < staged_pipe_json["models"].size(); ++index) { + const auto& m = staged_pipe_json["models"][index]; + if (m.is_object() && m.contains("model_path")) { + if (!m["model_path"].is_string() || + m["model_path"].get().empty()) { + std::string mid = m.value("model_id", ""); + std::string pointer = + (doc_split.deployment.has_model_paths && + doc_split.deployment.model_paths.count(mid)) + ? "/deployment/model_paths/" + EscapeJsonPointer(mid) + : "/models/" + std::to_string(index) + "/model_path"; + *out_error_json = ToolError( + "INVALID_MODEL_PATH", + "model_path in model declaration must be a non-empty string", + pointer); + return false; + } + } + } + } + + PipelineIoBoundary io_boundary; + for (const auto& port : in_conv->logical_ports) { + std::string key = port.logical_name; + auto bit = binding->input_ports.find(port.logical_name); + if (bit != binding->input_ports.end()) { + key = bit->second; + } + io_boundary.input_published_ports.emplace_back( + key, port.type_id, port.required, port.cardinality, + port.provenance_policy, port.lifetime, port.lifetime_config_field); + } + + for (const auto& port : out_conv->logical_ports) { + std::string key = port.logical_name; + auto bit = binding->output_ports.find(port.logical_name); + if (bit != binding->output_ports.end()) { + key = bit->second; + } + io_boundary.output_consumed_ports.emplace_back( + key, port.type_id, port.required, port.cardinality, + port.provenance_policy, port.lifetime, port.lifetime_config_field); + } + + *out_boundary = std::move(io_boundary); + *out_neutral_json = std::move(staged_pipe_json); + if (out_binding) *out_binding = binding; + return true; +} + nlohmann::json ResolveConf(const std::string& file, const std::string& root, uint32_t depth) { using namespace llm_edgeflow; @@ -141,8 +360,10 @@ nlohmann::json ResolveConf(const std::string& file, const std::string& root, if (OperatorConfigResolver::Resolve(root.c_str(), file.c_str(), &resolved, &error, depth) != 0) return ToolError("DEPLOYMENT_CONFIG", error); - const auto plan = - PipelineValidator::ValidateAndPlan(resolved.synthetic_pipeline_json); + if (!resolved.io_plan || !resolved.io_plan->pipeline_plan) + return ToolError("DEPLOYMENT_CONFIG", "Missing pipeline plan in io_plan"); + + const auto& plan = *resolved.io_plan->pipeline_plan; if (!plan.report.ok) return plan.report.ToJson(); auto effective = resolved.synthetic_pipeline_json; @@ -155,18 +376,15 @@ nlohmann::json ResolveConf(const std::string& file, const std::string& root, effective["models"][model.source_index]["backend_config"] = model.normalized_backend_config; } - nlohmann::json conf; - if (!ReadJson(resolved.conf_path.string(), &conf, &error)) - return ToolError("JSON_READ", error); - const auto overrides = - conf["data"].value("model_paths", nlohmann::json::object()); + nlohmann::json paths = nlohmann::json::array(); for (const auto& model : plan.models) - paths.push_back({{"model_id", model.model_id}, - {"source", overrides.contains(model.model_id) - ? "conf.data.model_paths" - : "pipeline.models.model_path"}, - {"resolved", model.resolved_model_path}}); + paths.push_back( + {{"model_id", model.model_id}, + {"source", resolved.io_plan->overridden_model_ids.count(model.model_id) + ? "pipeline.deployment.model_paths" + : "pipeline.models.model_path"}, + {"resolved", model.resolved_model_path}}); nlohmann::json output_pools = nlohmann::json::object(); for (const auto& [slot, pool] : resolved.output_pool_specs) { output_pools[slot] = {{"type", pool.type}, @@ -384,19 +602,88 @@ int main(int argc, char* argv[]) { std::cout << ToolError("JSON_READ", error).dump(2) << std::endl; return 1; } - auto report = explain ? PipelineValidator::Explain(root) - : PipelineValidator::Validate(root); - auto result = report.ToJson(); - // A failed plan request must retain the exact Validator diagnostics so - // every consumer observes the same fail-closed report. Successful plans - // omit the empty diagnostics array to keep the established CLI shape. - if (command == "plan" && report.ok) result.erase("diagnostics"); - std::cout << result.dump(2) << std::endl; - return report.ok ? 0 : 1; + const auto ops = + llm_edgeflow::operator_api::Get_LLM_EDGEFLOW_OperatorTable(); + if (ops.Init != nullptr) ops.Init(); + struct OpsGuard { + llm_edgeflow::operator_api::OperatorFunc ops; + ~OpsGuard() { + if (ops.Deinit != nullptr) ops.Deinit(); + } + } ops_guard{ops}; + + llm_edgeflow::PipelineIoBoundary io_boundary; + const llm_edgeflow::PipelineIoBoundary* io_boundary_ptr = nullptr; + const llm_edgeflow::IoBindingDefinition* binding_def = nullptr; + nlohmann::json target_json = root; + + if (root.contains("deployment")) { + nlohmann::json err_res; + if (!ResolveDeploymentBoundary(root, &target_json, &io_boundary, + &binding_def, &err_res)) { + if (command == "plan") { + err_res["plan"] = {{"layers", nlohmann::json::array()}, + {"topological_order", nlohmann::json::array()}}; + } + std::cout << err_res.dump(2) << std::endl; + return 1; + } + io_boundary_ptr = &io_boundary; + } + + if (command == "validate") { + auto report = + explain ? PipelineValidator::Explain( + target_json, llm_edgeflow::ValidationPolicy::kStrict, + io_boundary_ptr) + : PipelineValidator::Validate( + target_json, llm_edgeflow::ValidationPolicy::kStrict, + io_boundary_ptr); + if (report.ok && binding_def != nullptr) { + std::string biz = target_json.value("biz_name", ""); + if (biz != binding_def->biz_name) { + std::cout << ToolError("BIZ_MISMATCH", + "Pipeline biz_name '" + biz + + "' does not match binding biz_name '" + + binding_def->biz_name + + "' (at /deployment/io/io_binding)") + .dump(2) + << std::endl; + return 1; + } + } + auto result = report.ToJson(); + std::cout << result.dump(2) << std::endl; + return report.ok ? 0 : 1; + } else { + auto planned = PipelineValidator::ValidateAndPlan( + target_json, llm_edgeflow::ValidationPolicy::kStrict, + io_boundary_ptr); + if (planned.report.ok && binding_def != nullptr) { + std::string biz = target_json.value("biz_name", ""); + if (biz != binding_def->biz_name) { + nlohmann::json err_res = ToolError( + "BIZ_MISMATCH", "Pipeline biz_name '" + biz + + "' does not match binding biz_name '" + + binding_def->biz_name + + "' (at /deployment/io/io_binding)"); + err_res["plan"] = {{"layers", nlohmann::json::array()}, + {"topological_order", nlohmann::json::array()}}; + std::cout << err_res.dump(2) << std::endl; + return 1; + } + } + auto result = planned.report.ToJson(); + if (planned.report.ok) { + result.erase("diagnostics"); + } + std::cout << result.dump(2) << std::endl; + return planned.report.ok ? 0 : 1; + } } if (command == "validate-io") { - if (argc < 4) { + if (argc < 3) { Usage(); return 2; } @@ -421,12 +708,28 @@ int main(int argc, char* argv[]) { config_path, transport, model_root, &plan, &error); if (rc != 0 || !plan) { + std::string diag_path = "/"; + auto at_pos = error.rfind("(at "); + if (at_pos != std::string::npos) { + auto end_pos = error.find(')', at_pos); + if (end_pos != std::string::npos) { + diag_path = error.substr(at_pos + 4, end_pos - (at_pos + 4)); + } + } else { + auto at_pos2 = error.rfind("at /"); + if (at_pos2 != std::string::npos) { + auto end_pos2 = error.find(':', at_pos2); + if (end_pos2 != std::string::npos) { + diag_path = error.substr(at_pos2 + 3, end_pos2 - (at_pos2 + 3)); + } + } + } nlohmann::json err_res = { {"schema_version", 1}, {"ok", false}, {"diagnostics", nlohmann::json::array({{{"code", "IO_VALIDATION_ERROR"}, - {"path", "/"}, + {"path", diag_path}, {"message", error}, {"severity", "error"}}})}}; std::cout << err_res.dump(2) << std::endl; diff --git a/src/tools/pipeline_authoring.cpp b/src/tools/pipeline_authoring.cpp index bb1c20a1..f80a9d34 100644 --- a/src/tools/pipeline_authoring.cpp +++ b/src/tools/pipeline_authoring.cpp @@ -13,8 +13,12 @@ #include #include +#include "adapter/io_binding_registry.h" +#include "adapter/io_converter_registry.h" +#include "adapter/pipeline_document.h" #include "core/pipeline_catalog.h" #include "core/pipeline_validator.h" +#include "edgeflow/operator/interface.h" namespace llm_edgeflow { @@ -1440,6 +1444,72 @@ bool PipelineAuthoring::ApplyOperation(nlohmann::json* pipeline, return false; } +ValidationReport ValidateOrExplainAuthoring(const nlohmann::json& doc, + bool explain) { + if (!doc.is_object() || !doc.contains("deployment")) { + return explain ? PipelineValidator::Explain(doc) + : PipelineValidator::Validate(doc); + } + const auto ops = operator_api::Get_LLM_EDGEFLOW_OperatorTable(); + if (ops.Init != nullptr) ops.Init(); + + PipelineDocumentSplit split; + std::string err; + if (!SplitPipelineDocument(doc, &split, &err)) { + ValidationReport r; + r.ok = false; + ValidationDiagnostic diag; + diag.code = DiagnosticCode::kUnknownField; + diag.path = "/deployment"; + diag.message = err; + diag.severity = "error"; + r.diagnostics.push_back(std::move(diag)); + return r; + } + + PipelineIoBoundary io_boundary; + const PipelineIoBoundary* io_boundary_ptr = nullptr; + if (split.deployment.has_io && !split.deployment.io.io_binding.empty()) { + const auto* binding = IoBindingRegistry::Instance().FindBinding( + split.deployment.io.io_binding); + if (binding) { + const auto* in_conv = IoConverterRegistry::Instance().FindInputConverter( + binding->input_converter_id); + const auto* out_conv = + IoConverterRegistry::Instance().FindOutputConverter( + binding->output_converter_id); + if (in_conv && out_conv) { + for (const auto& port : in_conv->logical_ports) { + std::string key = port.logical_name; + auto bit = binding->input_ports.find(port.logical_name); + if (bit != binding->input_ports.end()) key = bit->second; + io_boundary.input_published_ports.emplace_back( + key, port.type_id, port.required, port.cardinality, + port.provenance_policy, port.lifetime, + port.lifetime_config_field); + } + for (const auto& port : out_conv->logical_ports) { + std::string key = port.logical_name; + auto bit = binding->output_ports.find(port.logical_name); + if (bit != binding->output_ports.end()) key = bit->second; + io_boundary.output_consumed_ports.emplace_back( + key, port.type_id, port.required, port.cardinality, + port.provenance_policy, port.lifetime, + port.lifetime_config_field); + } + io_boundary_ptr = &io_boundary; + } + } + } + + return explain ? PipelineValidator::Explain(split.neutral_pipeline_json, + ValidationPolicy::kStrict, + io_boundary_ptr) + : PipelineValidator::Validate(split.neutral_pipeline_json, + ValidationPolicy::kStrict, + io_boundary_ptr); +} + AuthoringResult PipelineAuthoring::ApplyRequest(const nlohmann::json& request) { AuthoringResult result; try { @@ -1528,7 +1598,7 @@ AuthoringResult PipelineAuthoring::ApplyRequest(const nlohmann::json& request) { } result.failed_operation_index.reset(); - auto report = PipelineValidator::Explain(working_pipeline); + auto report = ValidateOrExplainAuthoring(working_pipeline, true); result.validation = report.ToJson(); bool require_valid = request.value("require_valid", false); @@ -1583,7 +1653,7 @@ FixDepsResult PipelineAuthoring::FixDeps(const std::string& file_path, return result; } - auto report = PipelineValidator::Explain(root); + auto report = ValidateOrExplainAuthoring(root, true); if (report.ok) { result.ok = true; result.written = false; @@ -1684,7 +1754,7 @@ FixDepsResult PipelineAuthoring::FixDeps(const std::string& file_path, return result; } - auto final_report = PipelineValidator::Explain(working); + auto final_report = ValidateOrExplainAuthoring(working, true); result.validation = final_report.ToJson(); if (!final_report.ok) { result.ok = false; diff --git a/tests/contract/abi/test_adapter_contract_security.cpp b/tests/contract/abi/test_adapter_contract_security.cpp index 1d989013..73eedf8e 100644 --- a/tests/contract/abi/test_adapter_contract_security.cpp +++ b/tests/contract/abi/test_adapter_contract_security.cpp @@ -140,19 +140,18 @@ TEST_F(AdapterContractSecurityTest, model_config["model_path"] = "translation-probe.fixture"; model_config["model_config"] = nlohmann::json::object(); model_config["backend_config"] = nlohmann::json::object(); - std::ofstream(pipe_path) << pipeline.dump(); - - nlohmann::json op_cfg = { - {"schema_version", 1}, - {"data", - {{"pipe_path", "pipeline.json"}, - {"io_binding", "translate.operator.v1"}, - {"outputs", + pipeline["deployment"] = { + {"io", + {{"io_binding", "translate.operator.v1"}, + {"output_allocations", {{"entity_out", {{"type", "entity_out"}, {"meta_num", 0}, {"metadata_type_id", 0}, {"capacities", {{"entities_json", 2047}}}}}}}}}}; + std::ofstream(pipe_path) << pipeline.dump(); + + nlohmann::json op_cfg = {{"pipe_path", "pipeline.json"}}; const auto config = (directory / "pipeline.conf").string(); std::ofstream(config) << op_cfg.dump(); @@ -465,15 +464,17 @@ TEST_F(AdapterContractSecurityTest, ASSERT_EQ(pipeline_json["models"].size(), 2u); pipeline_json["models"][0]["model_path"] = "embedding.fixture"; pipeline_json["models"][1]["model_path"] = "llm.fixture"; + pipeline_json["deployment"]["model_paths"] = { + {"embed_model_v1", "embedding.fixture"}, {"llm_model_v1", "llm.fixture"}}; const std::filesystem::path model_root = std::filesystem::weakly_canonical(GetConfigPath("models")); std::unique_ptr io_plan; std::string plan_err; - ASSERT_EQ(IoBindingResolver::ResolveFromPipelineJson( - pipeline_json, "doc_qa.operator.v1", "operator", - model_root.string(), &io_plan, &plan_err), - 0) + ASSERT_EQ( + IoBindingResolver::ResolveFromPipelineJson( + pipeline_json, "operator", model_root.string(), &io_plan, &plan_err), + 0) << plan_err; ASSERT_NE(io_plan, nullptr); diff --git a/tests/fixtures/pipelines/cross_rerank/pipeline_cross_rerank_fixture.conf b/tests/fixtures/pipelines/cross_rerank/pipeline_cross_rerank_fixture.conf index 50f67d7e..f5a12cfa 100644 --- a/tests/fixtures/pipelines/cross_rerank/pipeline_cross_rerank_fixture.conf +++ b/tests/fixtures/pipelines/cross_rerank/pipeline_cross_rerank_fixture.conf @@ -1,18 +1,3 @@ { - "schema_version": 1, - "data": { - "pipe_path": "pipeline_cross_rerank_fixture.json", - "io_binding": "cross_rerank.operator.v1", - "model_paths": { - "rerank_model_v1": "rerank_fixture.onnx" - }, - "outputs": { - "rerank_out": { - "type": "rerank_out", - "meta_num": 0, - "metadata_type_id": 0, - "capacities": {} - } - } - } + "pipe_path": "pipeline_cross_rerank_fixture.json" } diff --git a/tests/fixtures/pipelines/cross_rerank/pipeline_cross_rerank_fixture.json b/tests/fixtures/pipelines/cross_rerank/pipeline_cross_rerank_fixture.json index 686981bf..b93af83d 100644 --- a/tests/fixtures/pipelines/cross_rerank/pipeline_cross_rerank_fixture.json +++ b/tests/fixtures/pipelines/cross_rerank/pipeline_cross_rerank_fixture.json @@ -1,5 +1,21 @@ { "biz_name": "dense_cross_rerank_scoring", + "deployment": { + "model_paths": { + "rerank_model_v1": "rerank_fixture.onnx" + }, + "io": { + "io_binding": "cross_rerank.operator.v1", + "output_allocations": { + "rerank_out": { + "type": "rerank_out", + "meta_num": 0, + "metadata_type_id": 0, + "capacities": {} + } + } + } + }, "models": [ { "model_id": "rerank_model_v1", diff --git a/tests/fixtures/pipelines/cross_rerank/pipeline_cross_rerank_missing_model.conf b/tests/fixtures/pipelines/cross_rerank/pipeline_cross_rerank_missing_model.conf index 959b02c7..3b5141dc 100644 --- a/tests/fixtures/pipelines/cross_rerank/pipeline_cross_rerank_missing_model.conf +++ b/tests/fixtures/pipelines/cross_rerank/pipeline_cross_rerank_missing_model.conf @@ -1,18 +1,3 @@ { - "schema_version": 1, - "data": { - "pipe_path": "pipeline_cross_rerank_fixture.json", - "io_binding": "cross_rerank.operator.v1", - "model_paths": { - "rerank_model_v1": "missing_rerank_fixture.onnx" - }, - "outputs": { - "rerank_out": { - "type": "rerank_out", - "meta_num": 0, - "metadata_type_id": 0, - "capacities": {} - } - } - } + "pipe_path": "pipeline_cross_rerank_missing_model.json" } diff --git a/tests/fixtures/pipelines/cross_rerank/pipeline_cross_rerank_missing_model.json b/tests/fixtures/pipelines/cross_rerank/pipeline_cross_rerank_missing_model.json new file mode 100644 index 00000000..1d90cc84 --- /dev/null +++ b/tests/fixtures/pipelines/cross_rerank/pipeline_cross_rerank_missing_model.json @@ -0,0 +1,62 @@ +{ + "biz_name": "dense_cross_rerank_scoring", + "deployment": { + "model_paths": { + "rerank_model_v1": "missing_rerank_fixture.onnx" + }, + "io": { + "io_binding": "cross_rerank.operator.v1", + "output_allocations": { + "rerank_out": { + "type": "rerank_out", + "meta_num": 0, + "metadata_type_id": 0, + "capacities": {} + } + } + } + }, + "models": [ + { + "model_id": "rerank_model_v1", + "capability": "rerank", + "model_type": "bge_reranker", + "backend": "onnxruntime", + "model_path": "rerank_fixture.onnx", + "model_config": { + "tokenizer_file": "vocab.txt", + "do_lower_case": true, + "max_length": 16, + "output_name": "logits", + "score_activation": "sigmoid", + "max_batch_size": 4 + }, + "backend_config": { + "max_batch_size": 4, + "intra_op_num_threads": 1, + "inter_op_num_threads": 1, + "graph_optimization_level": "all" + } + } + ], + "pipeline": [ + { + "id": "node_0_TextRerankNode", + "node_type": "TextRerankNode", + "depends_on": [], + "ports": { + "inputs": { + "queries": "rerank_queries", + "candidates": "rerank_candidates" + }, + "outputs": { + "ranked": "ranked_results" + } + }, + "config": { + "bind_model": "rerank_model_v1", + "top_k": 8 + } + } + ] +} diff --git a/tests/integration/demo/test_demo_runner.cpp b/tests/integration/demo/test_demo_runner.cpp index dd6bb9b7..47a4d597 100644 --- a/tests/integration/demo/test_demo_runner.cpp +++ b/tests/integration/demo/test_demo_runner.cpp @@ -172,13 +172,10 @@ TEST(DemoRunnerTest, RealKiteEntityExtractionThroughOperator) { node["config"]["failure_policy"] = "fail"; } } + pipeline["deployment"]["model_paths"]["entity_llm"] = "model.gguf"; std::ofstream(temporary.path / "pipeline.json") << pipeline.dump(2); - std::ifstream conf_input("configs/pipeline_entity_extract_cpu.conf"); - ASSERT_TRUE(conf_input.good()); - auto conf = nlohmann::json::parse(conf_input); - conf["data"]["pipe_path"] = "pipeline.json"; - conf["data"]["model_paths"]["entity_llm"] = "model.gguf"; - std::ofstream(temporary.path / "pipeline.conf") << conf.dump(2); + std::ofstream(temporary.path / "pipeline.conf") + << nlohmann::json{{"pipe_path", "pipeline.json"}}.dump(2); std::ofstream(temporary.path / "run.json") << R"({"schema_version":1,"model":{"context_size":256,"threads":2,"threads_batch":2,"gpu_layers":0},"logging":{"level":"error"}})"; std::ofstream(temporary.path / "input.txt") << "张三在北京工作。\n"; @@ -936,7 +933,7 @@ TEST(DemoRunnerTest, PreservesMixedSampleStatusesAndFailureCounts) { std::ifstream conf_file("configs/pipeline_keyword_match_rules.conf"); ASSERT_TRUE(conf_file.good()); auto conf = nlohmann::json::parse(conf_file); - conf["data"]["pipe_path"] = "pipeline.json"; + conf["pipe_path"] = "pipeline.json"; std::ofstream(temporary.path / "pipeline.conf") << conf.dump(); std::ofstream(temporary.path / "input.txt") << "success\nfail\n"; diff --git a/tests/integration/operator/test_operator_api.cpp b/tests/integration/operator/test_operator_api.cpp index ab91b719..9237cfdb 100644 --- a/tests/integration/operator/test_operator_api.cpp +++ b/tests/integration/operator/test_operator_api.cpp @@ -99,18 +99,15 @@ class OperatorApiTest : public ::testing::Test { pipe_json["models"][0]["model_path"] = "models/bge_reranker_large.onnx"; pipe_json["models"][0]["model_config"]["tokenizer_file"] = "vocab.txt"; pipe_json["models"][0]["model_config"]["max_length"] = 32; + pipe_json["deployment"]["model_paths"]["rerank_model_v1"] = + "models/bge_reranker_large.onnx"; auto temp_json_path = temp_dir->path() / "pipeline_cross_rerank.json"; std::ofstream json_out(temp_json_path); json_out << pipe_json.dump(2); json_out.close(); - std::ifstream conf_in(root_dir + "/configs/pipeline_cross_rerank_cpu.conf"); - nlohmann::json conf_json; - conf_in >> conf_json; - conf_json["data"]["pipe_path"] = "pipeline_cross_rerank.json"; - conf_json["data"]["model_paths"]["rerank_model_v1"] = - "models/bge_reranker_large.onnx"; + nlohmann::json conf_json = {{"pipe_path", "pipeline_cross_rerank.json"}}; auto temp_conf_path = temp_dir->path() / "pipeline_cross_rerank.conf"; std::ofstream conf_out(temp_conf_path); @@ -1183,104 +1180,108 @@ TEST_F(OperatorApiTest, OutputsConfigValidationFailClosed) { void* handle = nullptr; - // 0. 未知字段 mem_que 严格拒绝 -> -2 + // 0. 旧 Schema 1 配置格式严格拒绝 (RFC-0061) -> -2 { std::ofstream ofs(conf_path); ofs << R"({ "schema_version": 1, "data": { "pipe_path": "pipeline_keyword_match_rules.json", - "io_binding": "keyword_match.operator.v1", - "mem_que": { - "type": "keyword_out" - } + "io_binding": "keyword_match.operator.v1" } })"; } EXPECT_EQ(ops_.Create(&handle, ¶m), -2); EXPECT_NE(std::string(GetOperatorLastError()) - .find("Unknown field in conf data: 'mem_que'"), + .find("Deprecated deployment configuration format (RFC-0061)"), std::string::npos); - EXPECT_EQ(std::string(GetOperatorLastError()).find("is no longer supported"), + + // 0b. 未知字段 mem_que 在 deployment.io 中严格拒绝 -> -2 + { + std::ofstream c_ofs(conf_path); + c_ofs << R"({"pipe_path": "pipeline_keyword_match_rules.json"})"; + c_ofs.close(); + + std::ifstream json_in(std::filesystem::path(GetConfDir()) / + "configs/pipeline_keyword_match_rules.json"); + nlohmann::json pipe_json; + json_in >> pipe_json; + pipe_json["deployment"]["io"]["mem_que"] = {{"type", "keyword_out"}}; + std::ofstream p_ofs(root / "configs/pipeline_keyword_match_rules.json"); + p_ofs << pipe_json.dump(2); + p_ofs.close(); + } + EXPECT_EQ(ops_.Create(&handle, ¶m), -2); + EXPECT_NE(std::string(GetOperatorLastError()) + .find("Unknown field at /deployment/io/mem_que"), std::string::npos); - // 1. 缺失 outputs 对象 -> -2 + // 1. 缺失 output_allocations 对象 -> -2 { - std::ofstream ofs(conf_path); - ofs << R"({ - "schema_version": 1, - "data": { - "pipe_path": "pipeline_keyword_match_rules.json", - "io_binding": "keyword_match.operator.v1" - } - })"; + std::ifstream json_in(std::filesystem::path(GetConfDir()) / + "configs/pipeline_keyword_match_rules.json"); + nlohmann::json pipe_json; + json_in >> pipe_json; + pipe_json["deployment"]["io"].erase("output_allocations"); + std::ofstream p_ofs(root / "configs/pipeline_keyword_match_rules.json"); + p_ofs << pipe_json.dump(2); + p_ofs.close(); } EXPECT_EQ(ops_.Create(&handle, ¶m), -2); + EXPECT_NE( + std::string(GetOperatorLastError()) + .find("Missing required field '/deployment/io/output_allocations'"), + std::string::npos); - // 2. outputs.keyword_out.type 与业务不匹配 -> -2 + // 2. output_allocations.keyword_out.type 与业务不匹配 -> -2 { - std::ofstream ofs(conf_path); - ofs << R"({ - "schema_version": 1, - "data": { - "pipe_path": "pipeline_keyword_match_rules.json", - "io_binding": "keyword_match.operator.v1", - "outputs": { - "keyword_out": { - "type": "doc_out" - } - } - } - })"; + std::ifstream json_in(std::filesystem::path(GetConfDir()) / + "configs/pipeline_keyword_match_rules.json"); + nlohmann::json pipe_json; + json_in >> pipe_json; + pipe_json["deployment"]["io"]["output_allocations"]["keyword_out"]["type"] = + "doc_out"; + std::ofstream p_ofs(root / "configs/pipeline_keyword_match_rules.json"); + p_ofs << pipe_json.dump(2); + p_ofs.close(); } EXPECT_EQ(ops_.Create(&handle, ¶m), -2); // 3. meta_num == 0 但 metadata_type_id != 0 -> -2 { - std::ofstream ofs(conf_path); - ofs << R"({ - "schema_version": 1, - "data": { - "pipe_path": "pipeline_keyword_match_rules.json", - "io_binding": "keyword_match.operator.v1", - "outputs": { - "keyword_out": { - "type": "keyword_out", - "meta_num": 0, - "metadata_type_id": 123 - } - } - } - })"; + std::ifstream json_in(std::filesystem::path(GetConfDir()) / + "configs/pipeline_keyword_match_rules.json"); + nlohmann::json pipe_json; + json_in >> pipe_json; + pipe_json["deployment"]["io"]["output_allocations"]["keyword_out"] + ["meta_num"] = 0; + pipe_json["deployment"]["io"]["output_allocations"]["keyword_out"] + ["metadata_type_id"] = 123; + std::ofstream p_ofs(root / "configs/pipeline_keyword_match_rules.json"); + p_ofs << pipe_json.dump(2); + p_ofs.close(); } EXPECT_EQ(ops_.Create(&handle, ¶m), -2); // 4. 未知 capacity 字段 -> -2 { - std::ofstream ofs(conf_path); - ofs << R"({ - "schema_version": 1, - "data": { - "pipe_path": "pipeline_keyword_match_rules.json", - "io_binding": "keyword_match.operator.v1", - "outputs": { - "keyword_out": { - "type": "keyword_out", - "capacities": { - "unknown_field_xyz": 100 - } - } - } - } - })"; + std::ifstream json_in(std::filesystem::path(GetConfDir()) / + "configs/pipeline_keyword_match_rules.json"); + nlohmann::json pipe_json; + json_in >> pipe_json; + pipe_json["deployment"]["io"]["output_allocations"]["keyword_out"] + ["capacities"] = {{"unknown_field_xyz", 100}}; + std::ofstream p_ofs(root / "configs/pipeline_keyword_match_rules.json"); + p_ofs << pipe_json.dump(2); + p_ofs.close(); } EXPECT_EQ(ops_.Create(&handle, ¶m), -2); - // 5. 旧的根级 pipe_path 结构不再接受 -> -2 + // 5. 旧的根级 outputs 结构在 .conf 中被拒绝 -> -2 { std::ofstream ofs(conf_path); ofs << R"({ - "pipe_path": "configs/pipeline_keyword_match_rules.json", + "pipe_path": "pipeline_keyword_match_rules.json", "outputs": { "keyword_out": { "type": "keyword_out", @@ -1291,48 +1292,41 @@ TEST_F(OperatorApiTest, OutputsConfigValidationFailClosed) { })"; } EXPECT_EQ(ops_.Create(&handle, ¶m), -2); + EXPECT_NE(std::string(GetOperatorLastError()) + .find("Deprecated deployment configuration format (RFC-0061)"), + std::string::npos); - // 6. data.model_path 单值字段已被 model_paths 映射取代 -> -2 + // 6. deployment.model_path 单值字段被拒绝 (必须为 model_paths 映射) -> -2 { + std::ifstream json_in(std::filesystem::path(GetConfDir()) / + "configs/pipeline_keyword_match_rules.json"); + nlohmann::json pipe_json; + json_in >> pipe_json; + pipe_json["deployment"]["model_path"] = "models/unused.bin"; + std::ofstream p_ofs(root / "configs/pipeline_keyword_match_rules.json"); + p_ofs << pipe_json.dump(2); + p_ofs.close(); + std::ofstream ofs(conf_path); - ofs << R"({ - "schema_version": 1, - "data": { - "pipe_path": "pipeline_keyword_match_rules.json", - "io_binding": "keyword_match.operator.v1", - "model_path": "models/unused.bin", - "outputs": { - "keyword_out": { - "type": "keyword_out", - "meta_num": 0, - "metadata_type_id": 0 - } - } - } - })"; + ofs << R"({"pipe_path": "pipeline_keyword_match_rules.json"})"; } EXPECT_EQ(ops_.Create(&handle, ¶m), -2); + EXPECT_NE(std::string(GetOperatorLastError()) + .find("Unknown field at /deployment/model_path"), + std::string::npos); - // 7. .conf 根对象仅允许 data -> -2 + // 7. .conf 根对象仅允许 pipe_path -> -2 { std::ofstream ofs(conf_path); ofs << R"({ - "schema_version": 1, - "data": { - "pipe_path": "pipeline_keyword_match_rules.json", - "io_binding": "keyword_match.operator.v1", - "outputs": { - "keyword_out": { - "type": "keyword_out", - "meta_num": 0, - "metadata_type_id": 0 - } - } - }, + "pipe_path": "pipeline_keyword_match_rules.json", "comment": "not part of the runtime contract" })"; } EXPECT_EQ(ops_.Create(&handle, ¶m), -2); + EXPECT_NE( + std::string(GetOperatorLastError()).find("Unknown field at /: 'comment'"), + std::string::npos); } // 22. SSO 短字符串 (1~7 字节) 与跨批次指针绝对地址稳定性测试 (R9-001) @@ -1908,24 +1902,19 @@ TEST_F(OperatorApiTest, ModelPathNonExistentFileAllowedWhileEscapeRejected) { std::filesystem::copy_file( source_root / "demo/fixtures/mock/pipeline_doc_qa.json", root / "configs/pipeline_doc_qa_default.json"); + { + std::ifstream pipe_in(root / "configs/pipeline_doc_qa_default.json"); + nlohmann::json pipe_json; + pipe_in >> pipe_json; + pipe_json["deployment"]["model_paths"] = { + {"embed_model_v1", "models/not_deployed_embed.bin"}, + {"llm_model_v1", "models/not_deployed_llm.bin"}}; + std::ofstream pipe_out(root / "configs/pipeline_doc_qa_default.json"); + pipe_out << pipe_json.dump(2); + } std::ofstream conf(root / "configs/model_paths.conf"); conf << R"({ - "schema_version": 1, - "data": { - "pipe_path": "pipeline_doc_qa_default.json", - "io_binding": "doc_qa.operator.v1", - "model_paths": { - "embed_model_v1": "models/not_deployed_embed.bin", - "llm_model_v1": "models/not_deployed_llm.bin" - }, - "outputs": { - "doc_out": { - "type": "doc_out", - "meta_num": 0, - "metadata_type_id": 0 - } - } - } + "pipe_path": "pipeline_doc_qa_default.json" })"; conf.close(); @@ -1953,23 +1942,18 @@ TEST_F(OperatorApiTest, ModelPathNonExistentFileAllowedWhileEscapeRejected) { std::filesystem::copy_file( source_root / "demo/fixtures/mock/pipeline_audio_asr_intent.json", root / "configs/pipeline_audio_asr_intent.json"); + { + std::ifstream pipe_in(root / "configs/pipeline_audio_asr_intent.json"); + nlohmann::json pipe_json; + pipe_in >> pipe_json; + pipe_json["deployment"]["model_paths"] = { + {"asr_model_v1", "deployment/asr_model_will_arrive_later.bin"}}; + std::ofstream pipe_out(root / "configs/pipeline_audio_asr_intent.json"); + pipe_out << pipe_json.dump(2); + } std::ofstream conf(root / "configs/single_model.conf"); conf << R"({ - "schema_version": 1, - "data": { - "pipe_path": "pipeline_audio_asr_intent.json", - "io_binding": "audio_asr_intent.operator.v1", - "model_paths": { - "asr_model_v1": "deployment/asr_model_will_arrive_later.bin" - }, - "outputs": { - "audio_out": { - "type": "audio_out", - "meta_num": 0, - "metadata_type_id": 0 - } - } - } + "pipe_path": "pipeline_audio_asr_intent.json" })"; conf.close(); @@ -2049,14 +2033,7 @@ TEST_F(OperatorApiTest, ModelPathNonExistentFileAllowedWhileEscapeRejected) { std::ofstream conf(root / "configs/missing_pipe.conf"); conf << R"({ - "data": { - "pipe_path": "configs/missing_pipeline.json", - "outputs": { - "keyword_out": { - "type": "keyword_out" - } - } - } + "pipe_path": "configs/missing_pipeline.json" })"; conf.close(); EXPECT_EQ( @@ -2090,12 +2067,7 @@ TEST_F(OperatorApiTest, ModelPathNonExistentFileAllowedWhileEscapeRejected) { {std::string("/absolute/pipeline.json"), std::string("configs"), std::string("configs/outside_pipeline.json")}) { std::ofstream invalid_conf(root / "configs/invalid_pipe.conf"); - invalid_conf << nlohmann::json( - {{"schema_version", 1}, - {"data", - {{"pipe_path", pipe_path}, - {"io_binding", "keyword_match.operator.v1"}, - {"outputs", {{"keyword_out", {{"type", "keyword_out"}}}}}}}}); + invalid_conf << nlohmann::json({{"pipe_path", pipe_path}}); invalid_conf.close(); EXPECT_EQ(llm_edgeflow::OperatorConfigResolver::Resolve( root_string.c_str(), "configs/invalid_pipe.conf", &resolved, @@ -2113,12 +2085,8 @@ TEST_F(OperatorApiTest, DotDotPrefixedControlFileNamesStayWithinRoot) { std::filesystem::copy_file(std::filesystem::path(GetConfDir()) / "configs/pipeline_keyword_match_rules.json", root / "..configs/pipeline.json"); - std::ofstream(root / "..configs/pipeline.conf") << nlohmann::json( - {{"schema_version", 1}, - {"data", - {{"pipe_path", "pipeline.json"}, - {"io_binding", "keyword_match.operator.v1"}, - {"outputs", {{"keyword_out", {{"type", "keyword_out"}}}}}}}}); + std::ofstream(root / "..configs/pipeline.conf") + << nlohmann::json({{"pipe_path", "pipeline.json"}}); llm_edgeflow::ResolvedOperatorConfig resolved; std::string error; @@ -2136,6 +2104,13 @@ TEST_F(OperatorApiTest, VariableResultsUsePoolCapacityAndRollbackOnFailure) { ScopedTempDirectory temp; nlohmann::json pipeline = { {"biz_name", "keyword_match_v1"}, + {"deployment", + {{"io", + {{"io_binding", "keyword_match.operator.v1"}, + {"output_allocations", + {{"keyword_out", + {{"type", "keyword_out"}, + {"capacities", {{"match_result_json", capacity}}}}}}}}}}}, {"models", nlohmann::json::array()}, {"pipeline", {{{"id", "rule"}, @@ -2146,15 +2121,8 @@ TEST_F(OperatorApiTest, VariableResultsUsePoolCapacityAndRollbackOnFailure) { {"outputs", {{"matches", "rule_matches"}}}}}, {"config", {{"categories", {{"LONG", {word}}}}}}}}}}; std::ofstream(temp.path() / "pipeline.json") << pipeline; - std::ofstream(temp.path() / "pipeline.conf") << nlohmann::json( - {{"schema_version", 1}, - {"data", - {{"pipe_path", "pipeline.json"}, - {"io_binding", "keyword_match.operator.v1"}, - {"outputs", - {{"keyword_out", - {{"type", "keyword_out"}, - {"capacities", {{"match_result_json", capacity}}}}}}}}}}); + std::ofstream(temp.path() / "pipeline.conf") + << nlohmann::json({{"pipe_path", "pipeline.json"}}); const auto root = temp.path().string(); CreateParam param{}; param.model_path = root.c_str(); @@ -2211,21 +2179,17 @@ TEST_F(OperatorApiTest, MetadataTypeIdOutOfInt32RangeIsRejected) { // 1. Unsigned integer > INT32_MAX { + std::ifstream json_in(root / "configs/pipeline_keyword_match_rules.json"); + nlohmann::json pipe_json; + json_in >> pipe_json; + pipe_json["deployment"]["io"]["output_allocations"]["keyword_out"] + ["metadata_type_id"] = 3000000000ULL; + std::ofstream pipe_out(root / "configs/pipe_overflow.json"); + pipe_out << pipe_json.dump(2); + pipe_out.close(); + std::ofstream conf(root / "configs/pipe_overflow.conf"); - conf << R"({ - "schema_version": 1, - "data": { - "pipe_path": "pipeline_keyword_match_rules.json", - "io_binding": "keyword_match.operator.v1", - "outputs": { - "keyword_out": { - "type": "keyword_out", - "meta_num": 0, - "metadata_type_id": 3000000000 - } - } - } - })"; + conf << R"({"pipe_path": "pipe_overflow.json"})"; conf.close(); llm_edgeflow::ResolvedOperatorConfig resolved; std::string err; @@ -2238,21 +2202,17 @@ TEST_F(OperatorApiTest, MetadataTypeIdOutOfInt32RangeIsRejected) { // 2. Negative integer < INT32_MIN { + std::ifstream json_in(root / "configs/pipeline_keyword_match_rules.json"); + nlohmann::json pipe_json; + json_in >> pipe_json; + pipe_json["deployment"]["io"]["output_allocations"]["keyword_out"] + ["metadata_type_id"] = -3000000000LL; + std::ofstream pipe_out(root / "configs/pipe_underflow.json"); + pipe_out << pipe_json.dump(2); + pipe_out.close(); + std::ofstream conf(root / "configs/pipe_underflow.conf"); - conf << R"({ - "schema_version": 1, - "data": { - "pipe_path": "pipeline_keyword_match_rules.json", - "io_binding": "keyword_match.operator.v1", - "outputs": { - "keyword_out": { - "type": "keyword_out", - "meta_num": 0, - "metadata_type_id": -3000000000 - } - } - } - })"; + conf << R"({"pipe_path": "pipe_underflow.json"})"; conf.close(); llm_edgeflow::ResolvedOperatorConfig resolved; std::string err; @@ -2265,21 +2225,17 @@ TEST_F(OperatorApiTest, MetadataTypeIdOutOfInt32RangeIsRejected) { // 3. Non-integer (floating point or string) { + std::ifstream json_in(root / "configs/pipeline_keyword_match_rules.json"); + nlohmann::json pipe_json; + json_in >> pipe_json; + pipe_json["deployment"]["io"]["output_allocations"]["keyword_out"] + ["metadata_type_id"] = 1.5; + std::ofstream pipe_out(root / "configs/pipe_not_integer.json"); + pipe_out << pipe_json.dump(2); + pipe_out.close(); + std::ofstream conf(root / "configs/pipe_not_integer.conf"); - conf << R"({ - "schema_version": 1, - "data": { - "pipe_path": "pipeline_keyword_match_rules.json", - "io_binding": "keyword_match.operator.v1", - "outputs": { - "keyword_out": { - "type": "keyword_out", - "meta_num": 0, - "metadata_type_id": 1.5 - } - } - } - })"; + conf << R"({"pipe_path": "pipe_not_integer.json"})"; conf.close(); llm_edgeflow::ResolvedOperatorConfig resolved; std::string err; @@ -2402,14 +2358,8 @@ const bool g_reg_nested_output_components = []() { return true; }(); -nlohmann::json NestedOutputConfig(bool alternate = false) { - return { - {"schema_version", 1}, - {"data", - {{"pipe_path", "pipeline.json"}, - {"io_binding", "nested_output_test.operator.v1"}, - {"outputs", - {{"main", +nlohmann::json NestedOutputAllocations(bool alternate = false) { + return {{"main", {{"type", "test_nested_out"}, {"allocator", alternate ? "test_nested_alternate" : "test_nested_standard"}, @@ -2420,17 +2370,27 @@ nlohmann::json NestedOutputConfig(bool alternate = false) { {"allocator", alternate ? "test_nested_standard" : "test_nested_alternate"}, {"params", - {{"kind", alternate ? 1 : 2}, - {"capacity", alternate ? 4 : 5}}}}}}}}}}; + {{"kind", alternate ? 1 : 2}, {"capacity", alternate ? 4 : 5}}}}}}; } -void WriteNestedOutputPipeline(const std::filesystem::path& root) { +nlohmann::json NestedOutputPipelineJson(bool alternate = false) { std::ifstream source(std::filesystem::path(GetConfDir()) / "configs/pipeline_keyword_match_rules.json"); nlohmann::json pipeline; source >> pipeline; pipeline["biz_name"] = "test_nested_output_v1"; - std::ofstream(root / "pipeline.json") << pipeline; + pipeline["deployment"] = { + {"io", + {{"io_binding", "nested_output_test.operator.v1"}, + {"output_allocations", NestedOutputAllocations(alternate)}}}}; + return pipeline; +} + +void WriteNestedOutputPipeline( + const std::filesystem::path& root, + const std::string& pipeline_name = "pipeline.json", + bool alternate = false) { + std::ofstream(root / pipeline_name) << NestedOutputPipelineJson(alternate); } void ExpectNestedResult(const std::shared_ptr& value, uint64_t request_id, @@ -2464,12 +2424,15 @@ TEST_F(OperatorApiTest, SameOutputKeysSelectIndependentNestedAllocatorsPerHandle) { using namespace llm_edgeflow::test_support; ScopedTempDirectory temp; - WriteNestedOutputPipeline(temp.path()); - std::ofstream(temp.path() / "first.conf") << NestedOutputConfig(); - std::ofstream(temp.path() / "second.conf") << NestedOutputConfig(true); + WriteNestedOutputPipeline(temp.path(), "first.json", false); + WriteNestedOutputPipeline(temp.path(), "second.json", true); + std::ofstream(temp.path() / "first.conf") + << nlohmann::json{{"pipe_path", "first.json"}}; + std::ofstream(temp.path() / "second.conf") + << nlohmann::json{{"pipe_path", "second.json"}}; const auto root = temp.path().string(); for (bool alternate : {false, true}) { - const auto config = NestedOutputConfig(alternate); + const auto expected_alloc = NestedOutputAllocations(alternate); llm_edgeflow::ResolvedOperatorConfig resolved; std::string error; ASSERT_EQ(llm_edgeflow::OperatorConfigResolver::Resolve( @@ -2479,8 +2442,7 @@ TEST_F(OperatorApiTest, << error; for (const char* slot : {"main", "audit"}) { SCOPED_TRACE(slot); - const auto& source = - config.at("data").at("outputs").at(slot).at("params"); + const auto& source = expected_alloc.at(slot).at("params"); EXPECT_FALSE(source.contains("reject_hit")); EXPECT_EQ(resolved.output_parameter_text.at(slot), source.dump()); EXPECT_EQ(resolved.output_parameter_text.at(slot).find("reject_hit"), @@ -2555,13 +2517,14 @@ TEST_F(OperatorApiTest, TEST_F(OperatorApiTest, NestedOutputConfigurationIsValidatedBeforeAllocation) { using namespace llm_edgeflow::test_support; ScopedTempDirectory temp; - WriteNestedOutputPipeline(temp.path()); const auto root = temp.path().string(); + std::ofstream(temp.path() / "invalid.conf") + << nlohmann::json{{"pipe_path", "pipeline.json"}}; for (int mutation = 0; mutation < 11; ++mutation) { SCOPED_TRACE(mutation); - auto config = NestedOutputConfig(); - auto& data = config["data"]; - auto& main = data["outputs"]["main"]; + auto pipeline = NestedOutputPipelineJson(); + auto& allocs = pipeline["deployment"]["io"]["output_allocations"]; + auto& main = allocs["main"]; switch (mutation) { case 0: main["allocator"] = "not_registered"; @@ -2582,13 +2545,13 @@ TEST_F(OperatorApiTest, NestedOutputConfigurationIsValidatedBeforeAllocation) { main["params"] = nlohmann::json::array(); break; case 6: - data["outputs"].erase("audit"); + allocs.erase("audit"); break; case 7: - data["outputs"]["unknown"] = main; + allocs["unknown"] = main; break; case 8: - data["mem_que"] = {{"type", "test_nested_out"}}; + pipeline["deployment"]["io"]["mem_que"] = {{"type", "test_nested_out"}}; break; case 9: main["capacities"] = {{"unknown", 10}}; @@ -2597,7 +2560,7 @@ TEST_F(OperatorApiTest, NestedOutputConfigurationIsValidatedBeforeAllocation) { main["params"]["capacity"] = 1.5; break; } - std::ofstream(temp.path() / "invalid.conf") << config; + std::ofstream(temp.path() / "pipeline.json") << pipeline; CreateParam param{}; param.model_path = root.c_str(); param.cfg_file_name = "invalid.conf"; @@ -2616,10 +2579,12 @@ TEST_F(OperatorApiTest, NestedOutputConfigurationIsValidatedBeforeAllocation) { TEST_F(OperatorApiTest, NestedOutputFailureRollsBackAllSlotsAndAllowsRetry) { using namespace llm_edgeflow::test_support; ScopedTempDirectory temp; - WriteNestedOutputPipeline(temp.path()); - auto config = NestedOutputConfig(); - config["data"]["outputs"]["audit"]["params"]["reject_hit"] = true; - std::ofstream(temp.path() / "pipeline.conf") << config; + auto pipeline = NestedOutputPipelineJson(); + pipeline["deployment"]["io"]["output_allocations"]["audit"]["params"] + ["reject_hit"] = true; + std::ofstream(temp.path() / "pipeline.json") << pipeline; + std::ofstream(temp.path() / "pipeline.conf") + << nlohmann::json{{"pipe_path", "pipeline.json"}}; const auto root = temp.path().string(); CreateParam param{}; param.model_path = root.c_str(); @@ -2658,12 +2623,13 @@ TEST_F(OperatorApiTest, NestedOutputFailureRollsBackAllSlotsAndAllowsRetry) { TEST_F(OperatorApiTest, AllOutputSlotsShareTheHandlePayloadBudget) { using namespace llm_edgeflow::test_support; ScopedTempDirectory temp; - WriteNestedOutputPipeline(temp.path()); - auto config = NestedOutputConfig(); - for (auto& output : config["data"]["outputs"]) { + auto pipeline = NestedOutputPipelineJson(); + for (auto& output : pipeline["deployment"]["io"]["output_allocations"]) { output["params"]["capacity"] = 9000; } - std::ofstream(temp.path() / "pipeline.conf") << config; + std::ofstream(temp.path() / "pipeline.json") << pipeline; + std::ofstream(temp.path() / "pipeline.conf") + << nlohmann::json{{"pipe_path", "pipeline.json"}}; const auto root = temp.path().string(); CreateParam param{}; param.model_path = root.c_str(); diff --git a/tests/integration/operator/test_operator_golden.cpp b/tests/integration/operator/test_operator_golden.cpp index d74360c1..9ab9deb4 100644 --- a/tests/integration/operator/test_operator_golden.cpp +++ b/tests/integration/operator/test_operator_golden.cpp @@ -403,19 +403,15 @@ TEST_F(OperatorGoldenTest, CrossRerankGolden) { pipe_json["models"][0]["model_path"] = "models/bge_reranker_large.onnx"; pipe_json["models"][0]["model_config"]["tokenizer_file"] = "vocab.txt"; pipe_json["models"][0]["model_config"]["max_length"] = 32; + pipe_json["deployment"]["model_paths"]["rerank_model_v1"] = + "models/bge_reranker_large.onnx"; auto temp_json_path = temp_dir / "pipeline_cross_rerank.json"; std::ofstream json_out(temp_json_path); json_out << pipe_json.dump(2); json_out.close(); - std::ifstream conf_in("configs/pipeline_cross_rerank_cpu.conf"); - ASSERT_TRUE(conf_in.good()); - nlohmann::json conf_json; - conf_in >> conf_json; - conf_json["data"]["pipe_path"] = "pipeline_cross_rerank.json"; - conf_json["data"]["model_paths"]["rerank_model_v1"] = - "models/bge_reranker_large.onnx"; + nlohmann::json conf_json = {{"pipe_path", "pipeline_cross_rerank.json"}}; auto temp_conf_path = temp_dir / "pipeline_cross_rerank.conf"; std::ofstream conf_out(temp_conf_path); diff --git a/tests/integration/pipeline/test_pipeline_catalog_validator.cpp b/tests/integration/pipeline/test_pipeline_catalog_validator.cpp index a1e6edae..745cc570 100644 --- a/tests/integration/pipeline/test_pipeline_catalog_validator.cpp +++ b/tests/integration/pipeline/test_pipeline_catalog_validator.cpp @@ -8,6 +8,9 @@ #include "adapter/io_binding_registry.h" #include "adapter/io_binding_resolver.h" +#include "adapter/io_converter_registry.h" +#include "adapter/operator/operator_value_type_registry.h" +#include "adapter/pipeline_document.h" #include "adapter/shared_algorithm_runtime.h" #include "core/common_contracts.h" #include "core/node_registry.h" @@ -120,8 +123,29 @@ TEST(PipelineValidatorTest, AllRepositoryPipelinesValidate) { ++skipped_optional; continue; } - const auto report = PipelineValidator::Validate(pipeline); - EXPECT_TRUE(report.ok) << entry.path() << "\n" << report.ToJson().dump(2); + if (pipeline.contains("deployment")) { + // Core validator directly rejects deployment as an unknown root field + const auto direct_report = PipelineValidator::Validate(pipeline); + EXPECT_FALSE(direct_report.ok); + EXPECT_TRUE(std::any_of( + direct_report.diagnostics.begin(), direct_report.diagnostics.end(), + [](const ValidationDiagnostic& d) { + return d.code == DiagnosticCode::kUnknownField && + d.path == "/deployment"; + })); + + // Integration document splitter extracts the neutral pipeline for Core + PipelineDocumentSplit split; + std::string split_err; + ASSERT_TRUE(SplitPipelineDocument(pipeline, &split, &split_err)) + << entry.path() << ": " << split_err; + const auto report = + PipelineValidator::Validate(split.neutral_pipeline_json); + EXPECT_TRUE(report.ok) << entry.path() << "\n" << report.ToJson().dump(2); + } else { + const auto report = PipelineValidator::Validate(pipeline); + EXPECT_TRUE(report.ok) << entry.path() << "\n" << report.ToJson().dump(2); + } ++validated; } EXPECT_GT(validated, 0U); @@ -133,6 +157,7 @@ TEST(PipelineValidatorTest, RejectsRemovedRuleCategoriesField) { ASSERT_TRUE(stream.is_open()); nlohmann::json pipeline; stream >> pipeline; + pipeline.erase("deployment"); ASSERT_FALSE(pipeline["pipeline"].empty()); auto& config = pipeline["pipeline"][0]["config"]; config["default_categories"] = config["categories"]; @@ -153,6 +178,7 @@ TEST(PipelineValidatorTest, ModelPathsUseLexicalChecksWithoutDeploymentRoots) { ASSERT_TRUE(stream.is_open()); nlohmann::json pipeline; ASSERT_NO_THROW(stream >> pipeline); + pipeline.erase("deployment"); for (const std::string& safe_path : {std::string("missing/artifact.bin"), std::string("..name/artifact.bin"), @@ -222,6 +248,49 @@ TEST(PipelineValidatorTest, ReportsConfigAndCapabilityErrors) { EXPECT_TRUE(json_codes.count("MODEL_CAPABILITY_MISMATCH")); } +static nlohmann::json MakeSyntheticDeploymentDocForTest( + const nlohmann::json& pipeline_json, const std::string& binding_id) { + nlohmann::json allocations = nlohmann::json::object(); + const auto* binding = IoBindingRegistry::Instance().FindBinding(binding_id); + if (binding) { + const auto* out_conv = IoConverterRegistry::Instance().FindOutputConverter( + binding->output_converter_id); + if (out_conv) { + for (const auto& slot : out_conv->external_slots) { + if (slot.direction == PortDirection::kOutput && slot.required) { + std::string slot_type = + slot.type_suffix.empty() ? slot.slot_name : slot.type_suffix; + nlohmann::json slot_alloc = { + {"type", slot_type}, + {"meta_num", 0}, + {"metadata_type_id", 0}, + {"capacities", nlohmann::json::object()}}; + const auto* val_binding = + OperatorValueTypeRegistry::Instance().GetOutputBinding(slot_type, + ""); + for (const auto& cap : slot.capacity_fields) { + uint32_t cap_val = 1024; + if (val_binding && + val_binding->output_layout.string_capacity_fields.count(cap)) { + cap_val = + val_binding->output_layout.string_capacity_fields.at(cap) + .default_capacity; + } + slot_alloc["capacities"][cap] = cap_val; + } + allocations[slot.slot_name] = slot_alloc; + } + } + } + } + + nlohmann::json synthetic = pipeline_json; + synthetic["deployment"] = { + {"io", + {{"io_binding", binding_id}, {"output_allocations", allocations}}}}; + return synthetic; +} + TEST(PipelineValidatorTest, TableDrivenParityMatrix) { std::ifstream stream( "tests/fixtures/pipelines/validation/invalid_pipeline_cases.json"); @@ -271,12 +340,23 @@ TEST(PipelineValidatorTest, TableDrivenParityMatrix) { } } if (binding_id.empty()) { - binding_id = "keyword_match.operator.v1"; + binding_id = "test_synthetic." + biz + ".operator.v1"; + if (!IoBindingRegistry::Instance().FindBinding(binding_id)) { + IoBindingDefinition synth_b; + synth_b.binding_id = binding_id; + synth_b.biz_name = biz; + synth_b.transport = "operator"; + synth_b.input_converter_id = "keyword.plain.operator.v1"; + synth_b.output_converter_id = "keyword.result.operator.v1"; + IoBindingRegistry::Instance().RegisterBinding(synth_b); + } } + nlohmann::json dep_config = + MakeSyntheticDeploymentDocForTest(config, binding_id); std::unique_ptr io_plan; std::string resolve_error; int resolve_result = IoBindingResolver::ResolveFromPipelineJson( - config, binding_id, "operator", "./models", &io_plan, &resolve_error); + dep_config, "operator", "./models", &io_plan, &resolve_error); EXPECT_NE(resolve_result, 0); EXPECT_EQ(io_plan, nullptr); EXPECT_NE(resolve_error.find(test["primary_code"].get()), @@ -291,6 +371,7 @@ TEST(PipelineValidatorTest, WhisperPipelineValidationDependsOnBackend) { ASSERT_TRUE(stream.is_open()); nlohmann::json pipeline; stream >> pipeline; + pipeline.erase("deployment"); const auto report = PipelineValidator::Validate(pipeline); #ifdef HAVE_WHISPERCPP EXPECT_TRUE(report.ok) << report.ToJson().dump(2); @@ -329,6 +410,7 @@ TEST(PipelineValidatorTest, std::ifstream stream("configs/pipeline_keyword_match_rules.json"); nlohmann::json root; stream >> root; + root.erase("deployment"); root["pipeline"].push_back({{"id", "invalid"}, {"node_type", type}, {"depends_on", nlohmann::json::array()}, @@ -358,6 +440,7 @@ TEST(PipelineValidatorTest, UnconnectedOptionalPortStaysAbsentAtRuntime) { std::ifstream stream("configs/pipeline_keyword_match_rules.json"); nlohmann::json root; stream >> root; + root.erase("deployment"); root["pipeline"] = nlohmann::json::array( {{{"id", "a"}, {"node_type", "TextTemplateNode"}, @@ -399,6 +482,7 @@ TEST(PipelineValidatorTest, ASSERT_TRUE(stream.is_open()); nlohmann::json root; stream >> root; + root.erase("deployment"); // Node 0: custom_prompt (produces "llm_raw_answer") // Node 1: node_2_StructuredJsonParseNode (consumes "llm_raw_answer" on port @@ -455,6 +539,7 @@ TEST(PipelineValidatorTest, ExplainReturnsCandidateFixForUnknownConfigField) { ASSERT_TRUE(stream.is_open()); nlohmann::json root; stream >> root; + root.erase("deployment"); // Misspell "temperature" as "temprature" root["pipeline"][0]["config"]["temprature"] = 0.1; @@ -555,6 +640,7 @@ TEST(PipelineValidatorTest, ExplainCleanPipelineReturnsOk) { ASSERT_TRUE(stream.is_open()); nlohmann::json root; stream >> root; + root.erase("deployment"); const auto report = PipelineValidator::Explain(root); EXPECT_TRUE(report.ok); @@ -568,6 +654,7 @@ TEST(PipelineValidatorTest, ExplainTargetResolved) { ASSERT_TRUE(stream.is_open()); nlohmann::json root; stream >> root; + root.erase("deployment"); // Introduce two independent errors: // 1. Misspelled config field in custom_prompt ("temprature" instead of @@ -610,6 +697,7 @@ TEST(PipelineValidatorTest, ValidateProducesBasicRemediation) { ASSERT_TRUE(stream.is_open()); nlohmann::json root; stream >> root; + root.erase("deployment"); // Clear depends_on so node 1 has a missing input producer root["pipeline"][1]["depends_on"] = nlohmann::json::array(); diff --git a/tests/integration/runtime/test_different_io_modalities.cpp b/tests/integration/runtime/test_different_io_modalities.cpp index 7b4b26fa..786a3c3d 100644 --- a/tests/integration/runtime/test_different_io_modalities.cpp +++ b/tests/integration/runtime/test_different_io_modalities.cpp @@ -233,23 +233,15 @@ TEST_F(DifferentIoModalitiesTest, CrossRerankBatch) { pipe_json["models"][0]["model_path"] = "models/rerank.onnx"; pipe_json["models"][0]["model_config"]["tokenizer_file"] = "vocab.txt"; pipe_json["models"][0]["model_config"]["max_length"] = 32; + pipe_json["deployment"]["model_paths"]["rerank_model_v1"] = + "models/rerank.onnx"; auto temp_pipe_path = temp_dir / "pipeline_cross_rerank.json"; std::ofstream json_out(temp_pipe_path); json_out << pipe_json.dump(2); json_out.close(); - nlohmann::json deploy_cfg = { - {"schema_version", 1}, - {"data", - {{"pipe_path", "pipeline_cross_rerank.json"}, - {"io_binding", "cross_rerank.operator.v1"}, - {"outputs", - {{"rerank_out", - {{"type", "rerank_out"}, - {"meta_num", 0}, - {"metadata_type_id", 0}, - {"capacities", nlohmann::json::object()}}}}}}}}; + nlohmann::json deploy_cfg = {{"pipe_path", "pipeline_cross_rerank.json"}}; auto temp_cfg_path = temp_dir / "pipeline_cross_rerank.conf"; std::ofstream cfg_out(temp_cfg_path); cfg_out << deploy_cfg.dump(2); diff --git a/tests/support/control_test_utils.h b/tests/support/control_test_utils.h index cc4d5008..54597460 100644 --- a/tests/support/control_test_utils.h +++ b/tests/support/control_test_utils.h @@ -11,6 +11,15 @@ namespace llm_edgeflow::test { inline void WriteControlTestPipeline(const std::filesystem::path& directory) { const nlohmann::json pipeline = { {"biz_name", "keyword_match_v1"}, + {"deployment", + {{"io", + {{"io_binding", "keyword_match.operator.v1"}, + {"output_allocations", + {{"keyword_out", + {{"type", "keyword_out"}, + {"meta_num", 0}, + {"metadata_type_id", 0}, + {"capacities", {{"match_result_json", 2047}}}}}}}}}}}, {"models", nlohmann::json::array()}, {"pipeline", {{{"id", "prefix"}, @@ -26,15 +35,7 @@ inline void WriteControlTestPipeline(const std::filesystem::path& directory) { {{"inputs", {{"text", "prefixed"}}}, {"outputs", {{"matches", "rule_matches"}}}}}, {"config", {{"categories", {{"PREFIX_APPLIED", {"VIP:sample"}}}}}}}}}}; - const nlohmann::json conf = { - {"schema_version", 1}, - {"data", - {{"pipe_path", "pipeline.json"}, - {"io_binding", "keyword_match.operator.v1"}, - {"outputs", - {{"keyword_out", - {{"type", "keyword_out"}, - {"capacities", {{"match_result_json", 2047}}}}}}}}}}; + const nlohmann::json conf = {{"pipe_path", "pipeline.json"}}; std::ofstream(directory / "pipeline.json") << pipeline.dump(2); std::ofstream(directory / "pipeline.conf") << conf.dump(2); } diff --git a/tests/tooling/studio_browser_test.mjs b/tests/tooling/studio_browser_test.mjs index 2f447ec9..29a1c6d0 100644 --- a/tests/tooling/studio_browser_test.mjs +++ b/tests/tooling/studio_browser_test.mjs @@ -135,7 +135,7 @@ try { await page.click("#saveSolutionButton"); await page.waitForFunction(() => document.querySelector("#saveScope").textContent.includes("pipeline_browser_pair.conf")); const confPath = join(configRoot, "pipeline_browser_pair.conf"); - assert.ok(JSON.parse(readFileSync(confPath)).data.pipe_path.endsWith("pipeline_browser_pair.json")); + assert.ok(JSON.parse(readFileSync(confPath)).pipe_path.endsWith("pipeline_browser_pair.json")); await open("pipeline_browser_other.json"); await open("pipeline_browser_pair.json"); assert.match(await page.locator("#saveScope").textContent(), /pipeline_browser_pair.conf/); await rule().click(); await categories().fill('{"PAIR_UPDATE":["VIP"]}'); diff --git a/tests/tooling/test_dev_recipe.py b/tests/tooling/test_dev_recipe.py index c5ddf520..16703474 100644 --- a/tests/tooling/test_dev_recipe.py +++ b/tests/tooling/test_dev_recipe.py @@ -109,9 +109,11 @@ def test_keyword_prepare_validate_and_real_demo_evaluation(self): def test_entity_prompt_uses_pinned_assets_and_real_business_expectation(self): report = self.prepare(profile="entity_extract_custom_mock") self.assertTrue(report["ok"], report) - conf = json.loads(self.target.with_suffix(".conf").read_text()) - self.assertEqual(conf["data"]["model_paths"]["entity_llm"], + pipeline = json.loads(self.target.read_text()) + self.assertEqual(pipeline["deployment"]["model_paths"]["entity_llm"], "demo/fixtures/mock/artifacts/neutral-llm.fixture") + conf = json.loads(self.target.with_suffix(".conf").read_text()) + self.assertEqual(conf, {"pipe_path": self.target.name}) command = next(item["argv"] for item in report["next_commands"] if "verify" in item["argv"]) self.assertIn("--manifest", command) self.assertEqual(Path(command[command.index("--model-root") + 1]), self.root) @@ -153,25 +155,25 @@ def test_external_destination_preserves_deployment_and_dataset_paths(self): self.assertTrue(verified["ok"], verified) def test_multi_output_rejected_before_generation(self): - conf_path = self.root / "configs/pipeline_keyword_match_rules.conf" - conf = json.loads(conf_path.read_text()) - conf["data"]["outputs"] = {"slot1": {}, "slot2": {}} - conf_path.write_text(json.dumps(conf)) + pipe_path = self.root / "configs/pipeline_keyword_match_rules.json" + pipe = json.loads(pipe_path.read_text()) + pipe["deployment"]["io"]["output_allocations"] = {"slot1": {}, "slot2": {}} + pipe_path.write_text(json.dumps(pipe)) self.assert_prepare_rejected_without_writes() def test_missing_outputs_deployment_rejected_without_writes(self): - conf_path = self.root / "configs/pipeline_keyword_match_rules.conf" - conf = json.loads(conf_path.read_text()) - del conf["data"]["outputs"] - conf_path.write_text(json.dumps(conf)) + pipe_path = self.root / "configs/pipeline_keyword_match_rules.json" + pipe = json.loads(pipe_path.read_text()) + del pipe["deployment"]["io"]["output_allocations"] + pipe_path.write_text(json.dumps(pipe)) self.assert_prepare_rejected_without_writes() def test_legacy_mem_que_deployment_rejected_as_missing_outputs(self): - conf_path = self.root / "configs/pipeline_keyword_match_rules.conf" - conf = json.loads(conf_path.read_text()) - del conf["data"]["outputs"] - conf["data"]["mem_que"] = {"type": "keyword_out"} - conf_path.write_text(json.dumps(conf)) + pipe_path = self.root / "configs/pipeline_keyword_match_rules.json" + pipe = json.loads(pipe_path.read_text()) + del pipe["deployment"]["io"]["output_allocations"] + pipe["deployment"]["io"]["mem_que"] = {"type": "keyword_out"} + pipe_path.write_text(json.dumps(pipe)) self.assert_prepare_rejected_without_writes() def test_unlabelled_or_duplicate_effects_rejected_before_generation(self): diff --git a/tests/tooling/test_pipeline_studio.py b/tests/tooling/test_pipeline_studio.py index ef852da3..4574dacc 100644 --- a/tests/tooling/test_pipeline_studio.py +++ b/tests/tooling/test_pipeline_studio.py @@ -312,8 +312,7 @@ def test_saved_pair_runs_the_selected_pipeline_with_explicit_arguments(self): saved = self.service.save_solution(filename, self.keyword, "keyword_match_rules") self.assertEqual(json.loads((self.configs / filename).read_text()), self.keyword) conf = json.loads((self.configs / saved["conf_filename"]).read_text()) - self.assertEqual(conf["data"]["pipe_path"], filename) - self.assertEqual(conf["data"]["model_paths"], {}) + self.assertEqual(conf["pipe_path"], filename) command = shlex.split(saved["command"]) self.assertEqual(command[:3], ["cd", str(ROOT), "&&"]) self.assertNotIn("--no-default-control", command) @@ -366,10 +365,10 @@ def test_conf_rebuilds_selected_model_paths_and_honors_explicit_root(self): pipeline = json.loads((ROOT / "demo/fixtures/mock/pipeline_entity_extract.json").read_text()) selected = pipeline["models"][0]["model_path"] saved = self.service.save_solution("pipeline_fixture.json", pipeline, "entity_extract_mock", ".") - self.assertEqual(saved["conf"]["data"]["model_paths"], {"entity_llm": selected}) + self.assertEqual(saved["pipeline"]["deployment"]["model_paths"], {"entity_llm": selected}) pipeline["models"][0]["model_path"] = "replacement.gguf" saved = self.service.save_solution("pipeline_replaced.json", pipeline, "entity_extract_mock", "models") - self.assertEqual(saved["conf"]["data"]["model_paths"], {"entity_llm": "models/replacement.gguf"}) + self.assertEqual(saved["pipeline"]["deployment"]["model_paths"], {"entity_llm": "models/replacement.gguf"}) self.assertEqual(json.loads((self.configs / "pipeline_replaced.json").read_text()), pipeline) def test_ordinary_save_updates_managed_model_paths_and_node_parameters(self): @@ -385,7 +384,7 @@ def test_ordinary_save_updates_managed_model_paths_and_node_parameters(self): "path": "replacement.gguf", "action": "select_asset"}}, ) self.assertEqual(updated["command"], saved["command"]) - self.assertEqual(updated["conf"]["data"]["model_paths"], {"replacement_model": "models/replacement.gguf"}) + self.assertEqual(updated["pipeline"]["deployment"]["model_paths"], {"replacement_model": "models/replacement.gguf"}) self.assertEqual(json.loads((self.configs / saved["filename"]).read_text()), pipeline) profile, _ = self.service.profile_inputs(pipeline, "entity_extract_mock") effective = self.service.resolve_run_conf(self.configs / saved["conf_filename"], profile) @@ -453,6 +452,7 @@ def inspect_launch(args, **kwargs): conf_path = ROOT / args[args.index("--config") + 1] observed["directory"] = conf_path.parent observed["conf"] = json.loads(conf_path.read_text()) + observed["pipeline"] = json.loads((conf_path.parent / "pipeline.json").read_text()) return original_popen(args, **kwargs) with mock.patch.object(SHOW.subprocess, "Popen", side_effect=inspect_launch): started = self.service.start_run(pipeline, "entity_extract_mock", ".") @@ -462,8 +462,8 @@ def inspect_launch(args, **kwargs): break time.sleep(0.05) self.assertEqual(job["status"], "completed", job) - self.assertEqual(observed["conf"]["data"]["model_paths"], {"entity_llm": pipeline["models"][0]["model_path"]}) - self.assertEqual(observed["conf"]["data"]["pipe_path"], "pipeline.json") + self.assertEqual(observed["pipeline"]["deployment"]["model_paths"], {"entity_llm": pipeline["models"][0]["model_path"]}) + self.assertEqual(observed["conf"]["pipe_path"], "pipeline.json") self.assertFalse(observed["directory"].exists()) def test_conflicts_bad_paths_and_mismatches_leave_no_new_files(self): @@ -637,7 +637,7 @@ def test_resolve_conf_exposes_model_sources_defaults_and_native_pool_errors(self configuration = report["configuration"] self.assertEqual(configuration["conf_path"], str(conf_path)) self.assertEqual(configuration["model_paths"], [{ - "model_id": "entity_llm", "source": "conf.data.model_paths", + "model_id": "entity_llm", "source": "pipeline.deployment.model_paths", "resolved": str(ROOT / "models/qwen_0_6b_npu.bin"), }]) llm_config = configuration["effective_pipeline"]["pipeline"][1]["config"] @@ -646,14 +646,16 @@ def test_resolve_conf_exposes_model_sources_defaults_and_native_pool_errors(self conf = json.loads(conf_path.read_text()) with tempfile.TemporaryDirectory(prefix="resolve-conf-", dir=ROOT / "build") as directory: changed = Path(directory) / "pipeline.conf" - shutil.copy(conf_path.with_name(conf["data"]["pipe_path"]), directory) - conf["data"].pop("model_paths") + pipe_file = conf_path.with_name(conf["pipe_path"]) + pipe_doc = json.loads(pipe_file.read_text()) + pipe_doc["deployment"].pop("model_paths") + (Path(directory) / conf["pipe_path"]).write_text(json.dumps(pipe_doc)) changed.write_text(json.dumps(conf)) code, direct = self.command("resolve-conf", str(changed.relative_to(ROOT)), "--root", str(ROOT)) self.assertEqual(code, 0, direct) self.assertEqual(direct["configuration"]["model_paths"][0]["source"], "pipeline.models.model_path") - conf["data"]["outputs"]["entity_out"]["capacities"]["entities_json"] = 0 - changed.write_text(json.dumps(conf)) + pipe_doc["deployment"]["io"]["output_allocations"]["entity_out"]["capacities"]["entities_json"] = 0 + (Path(directory) / conf["pipe_path"]).write_text(json.dumps(pipe_doc)) code, rejected = self.command("resolve-conf", str(changed.relative_to(ROOT)), "--root", str(ROOT)) self.assertEqual(code, 1) self.assertFalse(rejected["ok"]) @@ -1879,10 +1881,10 @@ def associated_doc_qa(self): conf = json.loads((ROOT / "configs" / "pipeline_doc_qa_cpu.conf").read_text()) pipeline_path = self.configs / "pipeline_associated.json" conf_path = self.configs / "pipeline_associated.conf" - conf["data"]["pipe_path"] = pipeline_path.name - for model_id in conf["data"]["model_paths"]: - conf["data"]["model_paths"][model_id] = "models/deployed_" + model_id - conf["data"]["outputs"]["doc_out"]["capacities"]["answer_text"] = 2047 + conf["pipe_path"] = pipeline_path.name + for model_id in pipeline.get("deployment", {}).get("model_paths", {}): + pipeline["deployment"]["model_paths"][model_id] = "models/deployed_" + model_id + pipeline["deployment"]["io"]["output_allocations"]["doc_out"]["capacities"]["answer_text"] = 2047 pipeline_path.write_text(json.dumps(pipeline)) conf_path.write_text(json.dumps(conf)) self.service.associate_deployment(pipeline_path.name, conf_path.name) @@ -1893,8 +1895,7 @@ def test_associated_preflight_run_and_save_share_candidate(self): pipeline["models"][0]["model_path"] = "selected_A.onnx" model_id = pipeline["models"][0]["model_id"] actions = {model_id: {"path": "selected_A.onnx", "action": "select_asset"}} - expected = copy.deepcopy(original_conf) - expected["data"]["model_paths"][model_id] = "models/selected_A.onnx" + expected_conf = {"pipe_path": path.name} # Spy on real native resolution so this checks exactly what preflight resolves. resolved_candidates = [] resolve = self.service.resolve_run_conf @@ -1919,12 +1920,12 @@ def capture_resolve(conf_file, profile): path.name, pipeline, SHOW.revision_for(path.read_bytes()), model_path_actions=actions, ) - saved = json.loads(conf_path.read_text()) - for candidate in [resolved_candidates[0], run_conf, saved]: - with self.subTest(candidate=candidate): - candidate = copy.deepcopy(candidate) - candidate["data"]["pipe_path"] = expected["data"]["pipe_path"] - self.assertEqual(candidate, expected) + saved_conf = json.loads(conf_path.read_text()) + self.assertEqual(saved_conf, expected_conf) + saved_pipe = json.loads(path.read_text()) + self.assertEqual(saved_pipe["deployment"]["model_paths"][model_id], "models/selected_A.onnx") + self.assertEqual(run_conf, expected_conf) + self.assertEqual(resolved_candidates[0], {"pipe_path": "pipeline.json"}) def test_associated_raw_model_edit_requires_explicit_override_intent(self): pipeline, conf, path, _ = self.associated_doc_qa() @@ -1938,8 +1939,8 @@ def test_associated_raw_model_edit_requires_explicit_override_intent(self): model_path_actions={model["model_id"]: {"path": model["model_path"], "action": "preserve_override"}}, ) - self.assertEqual(candidate["data"]["model_paths"], conf["data"]["model_paths"]) - self.assertEqual(candidate["data"]["outputs"], conf["data"]["outputs"]) + self.assertEqual(pipeline["deployment"]["model_paths"], {m: f"models/deployed_{m}" for m in pipeline["deployment"]["model_paths"]}) + self.assertEqual(pipeline["deployment"]["io"]["output_allocations"]["doc_out"]["capacities"]["answer_text"], 2047) def test_associated_new_model_does_not_invent_deployment_override(self): pipeline, conf, path, _ = self.associated_doc_qa() @@ -1947,7 +1948,7 @@ def test_associated_new_model_does_not_invent_deployment_override(self): new_model.update(model_id="new_model", model_path="new.onnx") pipeline["models"].append(new_model) _, candidate = self.service.deployment_candidate(pipeline, filename=path.name) - self.assertEqual(candidate["data"]["model_paths"], conf["data"]["model_paths"]) + self.assertNotIn("new_model", pipeline["deployment"]["model_paths"]) def test_associated_external_file_changes_block_candidate_and_save(self): for changed_name in ["pipeline", "conf"]: @@ -1973,8 +1974,7 @@ def test_studio_deployment_associate_and_partial_override_update(self): # Create conf in configs pointing to this pipeline conf_path = self.configs / "pipeline_doc_qa_assoc.conf" - doc_qa_conf = json.loads((ROOT / "configs" / "pipeline_doc_qa_cpu.conf").read_text()) - doc_qa_conf["data"]["pipe_path"] = pipe_path.name + doc_qa_conf = {"pipe_path": pipe_path.name} conf_path.write_text(json.dumps(doc_qa_conf, indent=2)) # Associate @@ -1999,14 +1999,16 @@ def test_studio_deployment_associate_and_partial_override_update(self): ) self.assertTrue(save_res["ok"]) - # Verify conf was updated with modified model_path override - updated_conf = json.loads(conf_path.read_text()) + # Verify pipeline deployment was updated with modified model_path override + updated_pipe = json.loads(pipe_path.read_text()) self.assertEqual( - updated_conf["data"]["model_paths"][modified_pipe["models"][0]["model_id"]], + updated_pipe["deployment"]["model_paths"][modified_pipe["models"][0]["model_id"]], "models/new_embed_model.onnx", ) # Verify non-model conf settings preserved - self.assertIn("doc_out", updated_conf["data"]["outputs"]) + self.assertIn("doc_out", updated_pipe["deployment"]["io"]["output_allocations"]) + updated_conf = json.loads(conf_path.read_text()) + self.assertEqual(updated_conf, {"pipe_path": pipe_path.name}) def test_authoring_oversized_payload_rejection_4mib(self): pipe = {"biz_name": "keyword_match_v1", "models": [], "pipeline": []} diff --git a/tests/unit/adapter/test_adapter_purity.cpp b/tests/unit/adapter/test_adapter_purity.cpp index 043c30f3..ff1865c6 100644 --- a/tests/unit/adapter/test_adapter_purity.cpp +++ b/tests/unit/adapter/test_adapter_purity.cpp @@ -1247,26 +1247,33 @@ TEST_F(AdapterPurityTest, ReuseProof_5_SameCarrierDifferentSchema) { // Proof 6: Negative Combinations Rejected TEST_F(AdapterPurityTest, ReuseProof_6_NegativeCombinations) { // 1. Unknown or unregistered io_binding - DeploymentIoConfig bad_binding_cfg; - bad_binding_cfg.io_binding = "non_existent.binding.v999"; - bad_binding_cfg.pipe_path = "pipeline_keyword_match_rules.json"; + nlohmann::json bad_binding_json = { + {"biz_name", "keyword_match_v1"}, + {"deployment", + {{"io", + {{"io_binding", "non_existent.binding.v999"}, + {"output_allocations", + {{"keyword_out", + {{"type", "keyword_out"}, + {"meta_num", 0}, + {"metadata_type_id", 0}, + {"capacities", {{"match_result_json", 2047}}}}}}}}}}}, + {"models", nlohmann::json::array()}, + {"pipeline", nlohmann::json::array()}}; std::unique_ptr plan; std::string error; - int ret = IoBindingResolver::ResolveFromConfig(bad_binding_cfg, "operator", - "./models", &plan, &error); + int ret = IoBindingResolver::ResolveFromPipelineJson( + bad_binding_json, "operator", "./models", &plan, &error); EXPECT_EQ(ret, -2); EXPECT_NE(error.find("Unknown or unregistered io_binding"), std::string::npos); // 2. Transport mismatch: Non-operator transport requested - DeploymentIoConfig mismatch_cfg; - mismatch_cfg.io_binding = "keyword_match.operator.v1"; - mismatch_cfg.pipe_path = "pipeline_keyword_match_rules.json"; - ret = IoBindingResolver::ResolveFromConfig(mismatch_cfg, "legacy_cabi", - "./models", &plan, &error); + ret = IoBindingResolver::ResolveFromPipelineJson( + bad_binding_json, "legacy_cabi", "./models", &plan, &error); EXPECT_EQ(ret, -2); - // 3. DeploymentIoConfig schema validation rejects invalid version + // 3. DeploymentIoConfig schema validation rejects invalid / old format nlohmann::json invalid_version_json = { {"schema_version", 999}, {"data", @@ -1275,38 +1282,44 @@ TEST_F(AdapterPurityTest, ReuseProof_6_NegativeCombinations) { DeploymentIoConfig parsed_cfg; EXPECT_FALSE(DeploymentIoConfig::Parse(invalid_version_json, ".", "operator", &parsed_cfg, &error)); - EXPECT_NE(error.find("schema_version"), std::string::npos); + EXPECT_NE(error.find("Deprecated"), std::string::npos); // 4. Operator config with unknown output slot rejected by parity check - DeploymentIoConfig unknown_out_cfg; - unknown_out_cfg.io_binding = "keyword_match.operator.v1"; - unknown_out_cfg.pipe_path = "configs/pipeline_keyword_match_rules.json"; - unknown_out_cfg.resolved_pipe_path = - "configs/pipeline_keyword_match_rules.json"; - unknown_out_cfg.outputs = {{"unknown_slot", {{"type", "String"}}}}; - ret = IoBindingResolver::ResolveFromConfig(unknown_out_cfg, "operator", - "./models", &plan, &error); + nlohmann::json unknown_out_json = { + {"biz_name", "keyword_match_v1"}, + {"deployment", + {{"io", + {{"io_binding", "keyword_match.operator.v1"}, + {"output_allocations", {{"unknown_slot", {{"type", "String"}}}}}}}}}, + {"models", nlohmann::json::array()}, + {"pipeline", nlohmann::json::array()}}; + ret = IoBindingResolver::ResolveFromPipelineJson(unknown_out_json, "operator", + "./models", &plan, &error); EXPECT_EQ(ret, -2); EXPECT_NE(error.find("Unknown configured output slot: unknown_slot"), std::string::npos); // 5. Unknown model_id in model_paths rejected - DeploymentIoConfig unknown_mid_cfg; - unknown_mid_cfg.io_binding = "keyword_match.operator.v1"; - unknown_mid_cfg.pipe_path = "configs/pipeline_keyword_match_rules.json"; - unknown_mid_cfg.resolved_pipe_path = - "configs/pipeline_keyword_match_rules.json"; - unknown_mid_cfg.outputs = {{"keyword_out", - {{"type", "keyword_out"}, - {"meta_num", 0}, - {"metadata_type_id", 0}, - {"capacities", {{"match_result_json", 2047}}}}}}; - unknown_mid_cfg.model_paths = {{"non_existent_model", "dummy_path"}}; - ret = IoBindingResolver::ResolveFromConfig(unknown_mid_cfg, "operator", - "./models", &plan, &error); + nlohmann::json unknown_mid_json = { + {"biz_name", "keyword_match_v1"}, + {"deployment", + {{"model_paths", {{"non_existent_model", "dummy_path"}}}, + {"io", + {{"io_binding", "keyword_match.operator.v1"}, + {"output_allocations", + {{"keyword_out", + {{"type", "keyword_out"}, + {"meta_num", 0}, + {"metadata_type_id", 0}, + {"capacities", {{"match_result_json", 2047}}}}}}}}}}}, + {"models", nlohmann::json::array()}, + {"pipeline", nlohmann::json::array()}}; + ret = IoBindingResolver::ResolveFromPipelineJson(unknown_mid_json, "operator", + "./models", &plan, &error); EXPECT_EQ(ret, -2); EXPECT_NE( - error.find("Unknown model_id 'non_existent_model' in 'model_paths'"), + error.find( + "Unknown model_id 'non_existent_model' in '/deployment/model_paths'"), std::string::npos) << "actual error was: " << error; } diff --git a/tests/unit/adapter/test_io_binding_registry.cpp b/tests/unit/adapter/test_io_binding_registry.cpp index 865d7010..c00ffacb 100644 --- a/tests/unit/adapter/test_io_binding_registry.cpp +++ b/tests/unit/adapter/test_io_binding_registry.cpp @@ -9,8 +9,10 @@ #include "adapter/io_binding_registry.h" #include "adapter/io_binding_resolver.h" #include "adapter/io_converter_registry.h" +#include "adapter/pipeline_document.h" #include "adapter/shared_algorithm_runtime.h" #include "core/pipeline_catalog.h" +#include "core/pipeline_config.h" namespace llm_edgeflow { namespace { @@ -33,6 +35,30 @@ int DummyEncode(AlgContext*, const OutputPortBindings&, class IoBindingRegistryTest : public ::testing::Test { protected: + static void SetUpTestSuite() { + saved_bindings_ = IoBindingRegistry::Instance().AllBindings(); + saved_exposures_ = IoBindingRegistry::Instance().AllExposures(); + saved_inputs_ = IoConverterRegistry::Instance().AllInputConverters(); + saved_outputs_ = IoConverterRegistry::Instance().AllOutputConverters(); + } + + static void TearDownTestSuite() { + IoConverterRegistry::Instance().ClearForTesting(); + IoBindingRegistry::Instance().ClearForTesting(); + for (const auto& in_def : saved_inputs_) { + IoConverterRegistry::Instance().RegisterInputConverter(in_def); + } + for (const auto& out_def : saved_outputs_) { + IoConverterRegistry::Instance().RegisterOutputConverter(out_def); + } + for (const auto& binding : saved_bindings_) { + IoBindingRegistry::Instance().RegisterBinding(binding); + } + for (const auto& exp : saved_exposures_) { + IoBindingRegistry::Instance().RegisterExposure(exp); + } + } + void SetUp() override { IoConverterRegistry::Instance().ClearForTesting(); IoBindingRegistry::Instance().ClearForTesting(); @@ -61,7 +87,7 @@ class IoBindingRegistryTest : public ::testing::Test { out_def.external_type = "CompanyOperatorEntityOutput"; out_def.external_slots = {ExternalSlotDefinition( "entity_out", "CompanyOperatorEntityOutput", PortDirection::kOutput, - true, "CompanyOperatorEntityOutput", "entity_out")}; + true, "CompanyOperatorEntityOutput", "entity_out", {"entities_json"})}; out_def.max_batch_size = 64; out_def.logical_ports = { NodePortDefinition("answers", "TextBatch", true, "1:1")}; @@ -80,6 +106,23 @@ class IoBindingRegistryTest : public ::testing::Test { IoConverterRegistry::Instance().ClearForTesting(); IoBindingRegistry::Instance().ClearForTesting(); } + + void RegisterTestBizBinding() { + IoBindingDefinition binding; + binding.binding_id = "test_biz.operator.v1"; + binding.biz_name = "test_biz_v1"; + binding.transport = "operator"; + binding.input_converter_id = "test.in.operator"; + binding.output_converter_id = "test.out.operator"; + binding.input_ports = {{"texts", "input_sentences"}}; + binding.output_ports = {{"answers", "llm_answers"}}; + ASSERT_TRUE(IoBindingRegistry::Instance().RegisterBinding(binding)); + } + + static inline std::vector saved_bindings_; + static inline std::vector saved_exposures_; + static inline std::vector saved_inputs_; + static inline std::vector saved_outputs_; }; TEST_F(IoBindingRegistryTest, RegisterAndAuditValidBinding) { @@ -228,18 +271,8 @@ TEST_F(IoBindingRegistryTest, UnselectedIllegalBindingFailsAudit) { } TEST_F(IoBindingRegistryTest, DeploymentIoConfigValidation) { - // 1. 合法 schema 1 Operator 配置 - nlohmann::json valid_cfg = { - {"schema_version", 1}, - {"data", - {{"pipe_path", "test.json"}, - {"io_binding", "test_biz.operator.v1"}, - {"outputs", - {{"answers", - {{"type", "answers"}, - {"meta_num", 0}, - {"metadata_type_id", 0}, - {"capacities", nlohmann::json::object()}}}}}}}}; + // 1. 合法 RFC-0061 Operator 定位配置 (仅包含 pipe_path) + nlohmann::json valid_cfg = {{"pipe_path", "test.json"}}; // 写入临时测试 pipeline 文件 std::string tmp_dir = "/tmp/edgeflow_test_config_" + std::to_string(getpid()); @@ -255,13 +288,15 @@ TEST_F(IoBindingRegistryTest, DeploymentIoConfigValidation) { EXPECT_TRUE( DeploymentIoConfig::Parse(valid_cfg, tmp_dir, "operator", &parsed, &err)); EXPECT_EQ(parsed.pipe_path, "test.json"); - EXPECT_EQ(parsed.io_binding, "test_biz.operator.v1"); - // 2. 拒绝未知 schema_version - nlohmann::json bad_ver = valid_cfg; - bad_ver["schema_version"] = 2; - EXPECT_FALSE( - DeploymentIoConfig::Parse(bad_ver, tmp_dir, "operator", &parsed, &err)); + // 2. 拒绝旧 Schema 1 包装 (schema_version + data) + nlohmann::json old_schema1 = { + {"schema_version", 1}, + {"data", + {{"pipe_path", "test.json"}, {"io_binding", "test_biz.operator.v1"}}}}; + EXPECT_FALSE(DeploymentIoConfig::Parse(old_schema1, tmp_dir, "operator", + &parsed, &err)); + EXPECT_NE(err.find("Deprecated"), std::string::npos); // 3. 拒绝顶层未知字段 nlohmann::json bad_field = valid_cfg; @@ -274,8 +309,7 @@ TEST_F(IoBindingRegistryTest, DeploymentIoConfigValidation) { DeploymentIoConfig::Parse(valid_cfg, tmp_dir, "cabi", &parsed, &err)); // 5. 路径逃逸拒绝 - nlohmann::json escape_cfg = valid_cfg; - escape_cfg["data"]["pipe_path"] = "../../../etc/passwd"; + nlohmann::json escape_cfg = {{"pipe_path", "../../../etc/passwd"}}; EXPECT_FALSE(DeploymentIoConfig::Parse(escape_cfg, tmp_dir, "operator", &parsed, &err)); @@ -315,16 +349,7 @@ TEST_F(IoBindingRegistryTest, StrictConfigDirectoryIsolationAndCwdInvariance) { std::string err; auto make_conf = [](const std::string& pipe) { - nlohmann::json cfg = {{"schema_version", 1}, - {"data", - {{"pipe_path", pipe}, - {"io_binding", "test_biz.operator.v1"}, - {"outputs", - {{"answers", - {{"type", "answers"}, - {"meta_num", 0}, - {"metadata_type_id", 0}, - {"capacities", nlohmann::json::object()}}}}}}}}; + nlohmann::json cfg = {{"pipe_path", pipe}}; return cfg; }; @@ -429,4 +454,232 @@ TEST_F(IoBindingRegistryTest, FailClosedAuditRejectsInvalidUnselectedBinding) { EXPECT_EQ(SharedAlgorithmRuntime::GlobalInit(), -6); } +TEST_F(IoBindingRegistryTest, SplitPipelineDocumentAndCoreBoundary) { + // 1. 合法完整文档拆分 + nlohmann::json valid_doc = { + {"biz_name", "keyword_match_v1"}, + {"deployment", + {{"model_paths", {{"m1", "path/to/m1"}}}, + {"io", + {{"io_binding", "test.binding.v1"}, + {"output_allocations", {{"out1", nlohmann::json::object()}}}}}}}, + {"models", nlohmann::json::array()}, + {"pipeline", + {{{"id", "n0"}, + {"node_type", "TextRuleMatchNode"}, + {"depends_on", nlohmann::json::array()}, + {"ports", + {{"inputs", {{"text", "in"}}}, {"outputs", {{"matches", "out"}}}}}, + {"config", {{"categories", {{"CAT", {"word"}}}}}}}}}}; + + PipelineDocumentSplit split; + std::string err; + EXPECT_TRUE(SplitPipelineDocument(valid_doc, &split, &err)); + EXPECT_TRUE(split.has_deployment); + EXPECT_TRUE(split.deployment.has_model_paths); + EXPECT_EQ(split.deployment.model_paths["m1"], "path/to/m1"); + EXPECT_TRUE(split.deployment.has_io); + EXPECT_EQ(split.deployment.io.io_binding, "test.binding.v1"); + EXPECT_FALSE(split.neutral_pipeline_json.contains("deployment")); + EXPECT_EQ(split.neutral_pipeline_json["biz_name"], "keyword_match_v1"); + + // 2. 中性文档 (无 deployment) + nlohmann::json neutral_doc = { + {"biz_name", "keyword_match_v1"}, + {"models", nlohmann::json::array()}, + {"pipeline", + {{{"id", "n0"}, + {"node_type", "TextRuleMatchNode"}, + {"depends_on", nlohmann::json::array()}, + {"ports", + {{"inputs", {{"text", "in"}}}, {"outputs", {{"matches", "out"}}}}}, + {"config", {{"categories", {{"CAT", {"word"}}}}}}}}}}; + EXPECT_TRUE(SplitPipelineDocument(neutral_doc, &split, &err)); + EXPECT_FALSE(split.has_deployment); + EXPECT_EQ(split.neutral_pipeline_json, neutral_doc); + + // 3. 拒绝 deployment 内部未知字段 + nlohmann::json bad_dep = valid_doc; + bad_dep["deployment"]["unknown_key"] = 123; + EXPECT_FALSE(SplitPipelineDocument(bad_dep, &split, &err)); + EXPECT_NE(err.find("Unknown field at /deployment"), std::string::npos); + + // 4. 拒绝 deployment.io 缺失 + nlohmann::json missing_io = valid_doc; + missing_io["deployment"].erase("io"); + EXPECT_FALSE(SplitPipelineDocument(missing_io, &split, &err)); + EXPECT_NE(err.find("Missing required field '/deployment/io'"), + std::string::npos); + + // 5. 拒绝 deployment.io.io_binding 缺失或空 + nlohmann::json bad_io = valid_doc; + bad_io["deployment"]["io"]["io_binding"] = ""; + EXPECT_FALSE(SplitPipelineDocument(bad_io, &split, &err)); + + // 6. 拒绝 deployment.io 内部未知字段 + bad_io["deployment"]["io"]["io_binding"] = "test.binding.v1"; + bad_io["deployment"]["io"]["extra_field"] = "bad"; + EXPECT_FALSE(SplitPipelineDocument(bad_io, &split, &err)); + EXPECT_NE(err.find("Unknown field at /deployment/io"), std::string::npos); + + // 7. Core 边界检查: 带 deployment 的文档直接提交给 Core 严格解析必须被拒绝 + // (Unknown root field) + ParsedPipelineConfig parsed_core; + PipelineDiagnostic diag; + EXPECT_FALSE(ParsePipelineConfig(valid_doc, &parsed_core, &diag)); + EXPECT_EQ(diag.code, DiagnosticCode::kUnknownField); + EXPECT_EQ(diag.path, "/deployment"); + + // 8. 拆分出的中性文档提交给 Core 可以解析成功 + EXPECT_TRUE(SplitPipelineDocument(valid_doc, &split, &err)); + EXPECT_TRUE( + ParsePipelineConfig(split.neutral_pipeline_json, &parsed_core, &diag)) + << diag.message; + + // 9. 如果源文档含有拼写错误的根字段 (例如 deploymen)，拆分时不被过滤，Core + // 解析必须报错 + nlohmann::json typo_doc = neutral_doc; + typo_doc["deploymen"] = nlohmann::json::object(); + EXPECT_TRUE(SplitPipelineDocument(typo_doc, &split, &err)); + EXPECT_FALSE( + ParsePipelineConfig(split.neutral_pipeline_json, &parsed_core, &diag)); + EXPECT_EQ(diag.code, DiagnosticCode::kUnknownField); + EXPECT_EQ(diag.path, "/deploymen"); +} + +TEST_F(IoBindingRegistryTest, BizMismatchFailsClosedWithExactPointer) { + RegisterTestBizBinding(); + + // A pipeline with registered biz smart_doc_qa_v1 but binding + // test_biz.operator.v1 (biz test_biz_v1) + nlohmann::json doc = { + {"biz_name", "smart_doc_qa_v1"}, + {"deployment", + {{"io", + {{"io_binding", "test_biz.operator.v1"}, + {"output_allocations", + {{"entity_out", + {{"type", "entity_out"}, + {"meta_num", 0}, + {"metadata_type_id", 0}, + {"capacities", {{"entities_json", 2047}}}}}}}}}}}, + {"models", nlohmann::json::array()}, + {"pipeline", nlohmann::json::array()}}; + + std::unique_ptr plan; + std::string err; + int ret = IoBindingResolver::ResolveFromPipelineJson(doc, "operator", + "./models", &plan, &err); + EXPECT_EQ(ret, -2); + EXPECT_EQ(plan, nullptr); + EXPECT_NE(err.find("Pipeline biz_name 'smart_doc_qa_v1' does not match " + "binding biz_name 'test_biz_v1'"), + std::string::npos) + << "ACTUAL ERR: " << err; + EXPECT_NE(err.find("(at /deployment/io/io_binding)"), std::string::npos) + << "ACTUAL ERR: " << err; +} + +TEST_F(IoBindingRegistryTest, EscapedJsonPointerInModelPathsAndSlots) { + RegisterTestBizBinding(); + + // 1. Slot name with special characters ~ and / + nlohmann::json slot_doc = { + {"biz_name", "test_biz_v1"}, + {"deployment", + {{"io", + {{"io_binding", "test_biz.operator.v1"}, + {"output_allocations", + {{"slot~0/bad", {{"type", "entity_out"}}}}}}}}}, + {"models", nlohmann::json::array()}, + {"pipeline", nlohmann::json::array()}}; + + std::unique_ptr plan; + std::string err; + int ret = IoBindingResolver::ResolveFromPipelineJson(slot_doc, "operator", + "./models", &plan, &err); + EXPECT_EQ(ret, -2); + // slot~0/bad escaped: ~ -> ~0, / -> ~1 => slot~00~1bad + EXPECT_NE(err.find("/deployment/io/output_allocations/slot~00~1bad"), + std::string::npos) + << "ACTUAL ERR: " << err; + + // 2. Unknown model ID with special characters ~ and / + nlohmann::json model_doc = { + {"biz_name", "test_biz_v1"}, + {"deployment", + {{"model_paths", {{"model~1/test", "path/to/model"}}}, + {"io", + {{"io_binding", "test_biz.operator.v1"}, + {"output_allocations", + {{"entity_out", + {{"type", "entity_out"}, + {"meta_num", 0}, + {"metadata_type_id", 0}, + {"capacities", {{"entities_json", 2047}}}}}}}}}}}, + {"models", nlohmann::json::array()}, + {"pipeline", nlohmann::json::array()}}; + + ret = IoBindingResolver::ResolveFromPipelineJson(model_doc, "operator", + "./models", &plan, &err); + EXPECT_EQ(ret, -2); + // model~1/test escaped: ~ -> ~0, / -> ~1 => model~01~1test + EXPECT_NE(err.find("/deployment/model_paths/model~01~1test"), + std::string::npos) + << "ACTUAL ERR: " << err; +} + +TEST_F(IoBindingRegistryTest, + OverriddenModelPathResolutionFailurePointsToDeployment) { + RegisterTestBizBinding(); + + // Case 1: When model path override escapes model_root_dir, error points to + // /deployment/model_paths/ + nlohmann::json doc = { + {"biz_name", "test_biz_v1"}, + {"deployment", + {{"model_paths", {{"mid~test", "../../escaped_model.bin"}}}, + {"io", + {{"io_binding", "test_biz.operator.v1"}, + {"output_allocations", + {{"entity_out", + {{"type", "entity_out"}, + {"meta_num", 0}, + {"metadata_type_id", 0}, + {"capacities", {{"entities_json", 2047}}}}}}}}}}}, + {"models", + {{{"model_id", "mid~test"}, + {"capability", "embedding"}, + {"model_type", "test_biz_embedding"}, + {"backend", "test_tensor_backend"}, + {"model_config", {{"embedding_dim", 128}, {"max_batch_size", 4}}}, + {"backend_config", nlohmann::json::object()}, + {"model_path", "models/legal.bin"}}}}, + {"pipeline", nlohmann::json::array()}}; + + std::unique_ptr plan; + std::string err; + int ret = IoBindingResolver::ResolveFromPipelineJson(doc, "operator", + "./models", &plan, &err); + EXPECT_EQ(ret, -2); + // mid~test escaped: mid~0test + EXPECT_NE(err.find("/deployment/model_paths/mid~0test"), std::string::npos) + << "ACTUAL ERR: " << err; + EXPECT_EQ(err.find("/models/0/model_path"), std::string::npos) + << "ACTUAL ERR: " << err; + + // Case 2: When an un-overridden model path escapes model_root_dir, error + // points to /models/0/model_path + nlohmann::json unoverridden_doc = doc; + unoverridden_doc["deployment"].erase("model_paths"); + unoverridden_doc["models"][0]["model_path"] = "../../escaped_model.bin"; + ret = IoBindingResolver::ResolveFromPipelineJson(unoverridden_doc, "operator", + "./models", &plan, &err); + EXPECT_EQ(ret, -2); + EXPECT_NE(err.find("/models/0/model_path"), std::string::npos) + << "ACTUAL ERR: " << err; + EXPECT_EQ(err.find("/deployment/model_paths/"), std::string::npos) + << "ACTUAL ERR: " << err; +} + } // namespace llm_edgeflow diff --git a/tests/unit/core/test_pipeline_config.cpp b/tests/unit/core/test_pipeline_config.cpp index 5643a533..0f5671b0 100644 --- a/tests/unit/core/test_pipeline_config.cpp +++ b/tests/unit/core/test_pipeline_config.cpp @@ -377,9 +377,20 @@ TEST_F(PipelineConfigTest, PositiveProductionAndStage7FixtureConfigs) { nlohmann::json root; ifs >> root; + if (root.contains("deployment")) { + ParsedPipelineConfig raw_cfg; + PipelineDiagnostic raw_diag; + EXPECT_FALSE(ParsePipelineConfig(root, &raw_cfg, &raw_diag)); + EXPECT_EQ(raw_diag.code, DiagnosticCode::kUnknownField); + EXPECT_EQ(raw_diag.path, "/deployment"); + } + + nlohmann::json neutral_root = root; + neutral_root.erase("deployment"); + ParsedPipelineConfig parsed_cfg; PipelineDiagnostic diag; - bool parse_ok = ParsePipelineConfig(root, &parsed_cfg, &diag); + bool parse_ok = ParsePipelineConfig(neutral_root, &parsed_cfg, &diag); EXPECT_TRUE(parse_ok) << "Parse failed for " << cfg_file << ": " << diag.message << " at " << diag.path; EXPECT_EQ(diag.code, DiagnosticCode::kOk); @@ -398,7 +409,7 @@ TEST_F(PipelineConfigTest, PositiveProductionAndStage7FixtureConfigs) { } Pipeline pipeline; - bool build_ok = pipeline.BuildFromConfigFile(full_path, &diag); + bool build_ok = pipeline.BuildFromJson(neutral_root, &diag); if (!build_ok && diag.code == DiagnosticCode::kModelMaterializationFailed) { // 模型物理权重文件在当前测试环境不存在，构建按设计 Fail-Closed EXPECT_EQ(diag.code, DiagnosticCode::kModelMaterializationFailed); diff --git a/tests/unit/core/test_validated_pipeline_plan.cpp b/tests/unit/core/test_validated_pipeline_plan.cpp index 210dd775..d727dc35 100644 --- a/tests/unit/core/test_validated_pipeline_plan.cpp +++ b/tests/unit/core/test_validated_pipeline_plan.cpp @@ -346,6 +346,7 @@ TEST(ValidatedPipelinePlanTest, RejectsNodeOutputBoundToBusinessIngress) { ASSERT_TRUE(stream.is_open()); nlohmann::json pipeline_json; stream >> pipeline_json; + pipeline_json.erase("deployment"); const size_t source_index = pipeline_json["pipeline"].size(); pipeline_json["pipeline"].push_back( {{"id", "ingress_collision"}, diff --git a/tests/unit/engine/test_onnx_and_embedding_model.cpp b/tests/unit/engine/test_onnx_and_embedding_model.cpp index d6833b7e..dc4bc907 100644 --- a/tests/unit/engine/test_onnx_and_embedding_model.cpp +++ b/tests/unit/engine/test_onnx_and_embedding_model.cpp @@ -1099,6 +1099,7 @@ TEST_F(OnnxAndEmbeddingModelTest, OnnxRuntimeFixturePassEvidence) { ASSERT_TRUE(config_in.good()); nlohmann::json pipeline_config; config_in >> pipeline_config; + pipeline_config.erase("deployment"); pipeline_config["models"][0]["model_path"] = onnx_path.string(); pipeline_config["models"][0]["model_config"]["tokenizer_file"] = vocab_path.string(); diff --git a/tests/unit/engine/test_onnx_and_reranker_model.cpp b/tests/unit/engine/test_onnx_and_reranker_model.cpp index 7359ca31..a0369821 100644 --- a/tests/unit/engine/test_onnx_and_reranker_model.cpp +++ b/tests/unit/engine/test_onnx_and_reranker_model.cpp @@ -937,6 +937,7 @@ TEST_F(OnnxAndRerankerModelTest, RealPipelineBuildAndExecuteSmoke) { nlohmann::json pipe_json; cfg_in >> pipe_json; cfg_in.close(); + pipe_json.erase("deployment"); // 2. 注入真实构建期 fixture 路径和测试参数 (top_k=2) pipe_json["models"][0]["model_path"] = onnx_path.string(); diff --git a/tests/unit/nodes/test_common_nodes.cpp b/tests/unit/nodes/test_common_nodes.cpp index f0bcc9b0..7d090677 100644 --- a/tests/unit/nodes/test_common_nodes.cpp +++ b/tests/unit/nodes/test_common_nodes.cpp @@ -836,7 +836,9 @@ class StarterEmbeddingModel final : public IEmbeddingModel { nlohmann::json CustomPipeline(const std::string& biz) { std::ifstream file("demo/fixtures/mock/pipeline_" + biz + "_custom.json"); - return nlohmann::json::parse(file); + auto doc = nlohmann::json::parse(file); + doc.erase("deployment"); + return doc; } template diff --git a/tests/unit/nodes/test_text_template_node.cpp b/tests/unit/nodes/test_text_template_node.cpp index 56f03535..c5ab9886 100644 --- a/tests/unit/nodes/test_text_template_node.cpp +++ b/tests/unit/nodes/test_text_template_node.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -250,9 +251,13 @@ TEST_F(TextTemplateNodeTest, ControlCommandHotSwapAndBogusRejection) { TEST_F(TextTemplateNodeTest, PipelineEnforcesPublishedControlSchema) { Pipeline pipeline; PipelineDiagnostic diagnostic; - ASSERT_TRUE(pipeline.BuildFromConfigFile( - ResolveConfigPath("demo/fixtures/mock/pipeline_doc_qa.json"), - &diagnostic)) + std::ifstream cfg_in( + ResolveConfigPath("demo/fixtures/mock/pipeline_doc_qa.json")); + ASSERT_TRUE(cfg_in.is_open()); + nlohmann::json pipe_json; + cfg_in >> pipe_json; + pipe_json.erase("deployment"); + ASSERT_TRUE(pipeline.BuildFromJson(pipe_json, &diagnostic)) << diagnostic.message; EXPECT_NE(pipeline.Control(kControlCmdUpdatePrompt, "{}"), 0); diff --git a/tools/pipeline_studio/README.md b/tools/pipeline_studio/README.md index 63408d37..2d06aefa 100644 --- a/tools/pipeline_studio/README.md +++ b/tools/pipeline_studio/README.md @@ -168,7 +168,7 @@ LLM_EDGEFLOW_PIPELINE_TOOL=./build/alg_pipeline_tool_test ./show --web ### 运行当前方案 -Pipeline JSON 描述算法连线；`.conf` 描述部署路径和输出容量；Profile 保存 Demo 的 +Pipeline JSON 描述算法连线并在 `deployment` 中持有部署配置（接入绑定、输出容量与模型路径覆盖）；`.conf` 仅包含 `pipe_path` 用于定位 Pipeline JSON；Profile 保存 Demo 的 业务、配置、数据集等预设。“运行”页的“另存为可运行方案”会一起生成 JSON 和 `.conf`， 并提供从项目根执行的完整命令；已有同名文件会拒绝覆盖。选择与业务匹配的 Profile， 其数据集、运行选项和输出池容量会被复用。模型目录默认为 `models`；引用 @@ -196,7 +196,7 @@ Pipeline JSON 描述算法连线；`.conf` 描述部署路径和输出容量；P 完成上述练习后，复制 `configs/pipeline_keyword_match_rules.conf` 为 `configs/pipeline_first_solution.conf`（已有同名文件时直接编辑），将其中 -`data.pipe_path` 改为 `configs/pipeline_first_solution.json`，保留原输出池配置。 +`pipe_path` 改为 `configs/pipeline_first_solution.json`；部署 I/O 绑定、输出分配与模型路径覆盖直接在 `pipeline_first_solution.json` 的 `deployment` 根对象下配置。 从仓库根目录执行： ```bash @@ -218,7 +218,7 @@ CLI 的 `--config` 覆盖 Profile 原配置，因此不需要新增 Profile。 本练习应有两条成功结果，第一条命中 `FIRST_RUN`，第二条未命中。核对请求 ID、状态 和业务字段，不只看退出码。无 Profile 运行时，结果子目录改为业务名 `keyword_match`。 -复用其他 `.conf` 时，还要核对 `data.model_paths` 的模型路径覆盖和输出池容量是否适合 +复用其他配置时，还要核对 Pipeline `deployment.model_paths` 的模型路径覆盖和 `deployment.io.output_allocations` 输出池容量是否适合 当前方案；Pipeline 校验不代表部署资源可加载。Demo 默认不发送内置 热更新覆盖所选规则或提示词，显式 `--control-file` 仍会执行，应只在需要该更新时提供。 diff --git a/tools/pipeline_studio/server.py b/tools/pipeline_studio/server.py index ad774637..9ab982f0 100755 --- a/tools/pipeline_studio/server.py +++ b/tools/pipeline_studio/server.py @@ -442,12 +442,12 @@ def save_pipeline( model_root: str = "models", model_path_actions: Any = None, ) -> dict[str, Any]: - report = self.validate(pipeline) - if not report.get("ok"): - raise StudioError("VALIDATION_FAILED", json.dumps(report, ensure_ascii=False)) path = self.managed_path(requested) if not save_as and path.name in self.generated_solutions: return self.update_solution(path, pipeline, expected_revision, profile_name, model_root, model_path_actions) + report = self.validate(pipeline) + if not report.get("ok"): + raise StudioError("VALIDATION_FAILED", json.dumps(report, ensure_ascii=False)) if path.exists() and not save_as: current = revision_for(path.read_bytes()) if not expected_revision or current != expected_revision: @@ -488,21 +488,24 @@ def check_unmanaged_deployment(self, path: Path, pipeline: Any) -> None: if not conf_path.is_file(): return try: - data = read_json(conf_path).get("data", {}) - reference = data.get("pipe_path") - overrides = data.get("model_paths") + conf = read_json(conf_path) + reference = conf.get("pipe_path") points_here = isinstance(reference, str) and any( (base / reference).resolve() == path.resolve() for base in (PROJECT_ROOT, conf_path.parent) ) + if not points_here: + return + original = read_json(path) + overrides = original.get("deployment", {}).get("model_paths") except (OSError, ValueError, AttributeError, TypeError): return - if points_here and isinstance(overrides, dict) and overrides: + if isinstance(overrides, dict) and overrides: def model_paths(document: Any) -> dict: models = document.get("models", []) if isinstance(document, dict) else [] return {model.get("model_id"): model.get("model_path", "") for model in models if isinstance(model, dict)} if isinstance(models, list) else {} - if model_paths(read_json(path)) == model_paths(pipeline): + if model_paths(original) == model_paths(pipeline): return raise StudioError( "DEPLOYMENT_CONFLICT", @@ -529,6 +532,9 @@ def check_revisions() -> tuple[bytes, bytes]: profile, conf = self.deployment_candidate( pipeline, profile_name, model_root, path.name, model_path_actions=model_path_actions) + report = self.validate(pipeline) + if not report.get("ok"): + raise StudioError("VALIDATION_FAILED", json.dumps(report, ensure_ascii=False)) encoded = (json.dumps(pipeline, ensure_ascii=False, indent=2) + "\n").encode() conf_encoded = (json.dumps(conf, ensure_ascii=False, indent=2) + "\n").encode() staging = Path(tempfile.mkdtemp(prefix=".studio-save-", dir=self.config_root)) @@ -538,8 +544,7 @@ def check_revisions() -> tuple[bytes, bytes]: staged_conf = staging / "pipeline.conf" backup_json = staging / "previous.json" staged_json.write_bytes(encoded) - staged_conf_data = copy.deepcopy(conf) - staged_conf_data["data"]["pipe_path"] = staged_json.name + staged_conf_data = {"pipe_path": staged_json.name} staged_conf.write_text(json.dumps(staged_conf_data, ensure_ascii=False, indent=2)) configuration = self.resolve_run_conf(staged_conf, profile) @@ -580,18 +585,11 @@ def profile_inputs(self, pipeline: Any, profile_name: str) -> tuple[dict, Any]: raise StudioError("UNKNOWN_PROFILE", profile_name) profile_conf = PROJECT_ROOT / profile["config"] conf = read_json(profile_conf) - if ( - not isinstance(conf, dict) - or set(conf) not in ({"data"}, {"schema_version", "data"}) - or not isinstance(conf.get("data"), dict) - or not isinstance(conf["data"].get("pipe_path"), str) - or not isinstance(conf["data"].get("outputs"), dict) - ): + if not isinstance(conf, dict) or "pipe_path" not in conf or not isinstance(conf["pipe_path"], str): raise StudioError( - "INVALID_PROFILE_CONFIG", "Profile .conf 必须包含 schema_version 与 data 对象" + "INVALID_PROFILE_CONFIG", "Profile .conf 必须包含 pipe_path" ) - data = conf["data"] - original_pipeline_path = Path(data["pipe_path"]) + original_pipeline_path = Path(conf["pipe_path"]) if not original_pipeline_path.is_absolute(): if (PROJECT_ROOT / original_pipeline_path).exists(): original_pipeline_path = PROJECT_ROOT / original_pipeline_path @@ -602,7 +600,8 @@ def profile_inputs(self, pipeline: Any, profile_name: str) -> tuple[dict, Any]: curr_biz = pipeline.get("biz_name") if orig_biz != curr_biz: raise StudioError("PROFILE_MISMATCH", "Profile 与业务契约不匹配") - return copy.deepcopy(profile), copy.deepcopy(data["outputs"]) + outputs = original.get("deployment", {}).get("io", {}).get("output_allocations", {}) + return copy.deepcopy(profile), copy.deepcopy(outputs) def run_conf(self, pipeline: Any, outputs: Any, pipe_path: Path, model_root: str) -> dict[str, Any]: if not isinstance(model_root, str) or not model_root or Path(model_root).is_absolute(): @@ -658,8 +657,7 @@ def deployment_candidate( if original.get("biz_name") != pipeline.get("biz_name"): raise StudioError("DEPLOYMENT_MISMATCH", "已关联方案不能改变业务契约，请另存方案") profile = self.profile_inputs(pipeline, profile_name)[0] if profile_name else copy.deepcopy(managed["profile"]) - conf = json.loads(conf_raw) - overrides = copy.deepcopy(conf["data"].get("model_paths", {})) + overrides = copy.deepcopy(original.get("deployment", {}).get("model_paths", {})) old_models = {m["model_id"]: m for m in original.get("models", [])} for mid in list(overrides): if mid not in models: @@ -667,15 +665,25 @@ def deployment_candidate( for mid, model in models.items(): choice = actions.get(mid) if choice and choice["action"] == "select_asset": - selected = self.run_conf({"models": [model]}, {}, path, model_root) - overrides[mid] = selected["data"]["model_paths"][mid] + temp_pipe = {"models": [model]} + self.run_conf(temp_pipe, {}, path, model_root) + overrides[mid] = temp_pipe.get("deployment", {}).get("model_paths", {}).get(mid, "") elif (not choice and mid in overrides and mid in old_models and model.get("model_path") != old_models[mid].get("model_path")): raise StudioError("DEPLOYMENT_PATH_INTENT_REQUIRED", f"模型 {mid} 的路径已改变，请明确选择保留部署覆盖或采用新资产路径", 409) - if "model_paths" in conf["data"] or overrides: - conf["data"]["model_paths"] = overrides - return profile, conf + if overrides: + pipeline.setdefault("deployment", {})["model_paths"] = overrides + elif "deployment" in pipeline and "model_paths" in pipeline["deployment"]: + del pipeline["deployment"]["model_paths"] + if "deployment" in original and "io" in original["deployment"]: + pipeline.setdefault("deployment", {})["io"] = copy.deepcopy(original["deployment"]["io"]) + elif "outputs" in managed and managed["outputs"]: + io_dict = pipeline.setdefault("deployment", {}).setdefault("io", {}) + io_dict["output_allocations"] = copy.deepcopy(managed["outputs"]) + if managed.get("io_binding"): + io_dict["io_binding"] = managed["io_binding"] + return profile, {"pipe_path": path.name} def resolve_run_conf(self, conf_path: Path, profile: dict[str, Any]) -> dict[str, Any]: depth = max(int(profile.get("batch_size", 1)), int(profile.get("depth", 1))) @@ -706,7 +714,7 @@ def save_solution(self, requested: str, pipeline: Any, profile_name: str, model_ if target.exists(): raise StudioError("FILE_EXISTS", f"另存目标已存在：{target.name}", 409) profile, conf = self.deployment_candidate(pipeline, profile_name, model_root, path.name) - outputs = conf["data"]["outputs"] + outputs = pipeline.get("deployment", {}).get("io", {}).get("output_allocations", {}) encoded = (json.dumps(pipeline, ensure_ascii=False, indent=2) + "\n").encode() conf_encoded = (json.dumps(conf, ensure_ascii=False, indent=2) + "\n").encode() created = [] @@ -787,12 +795,8 @@ def associate_deployment( except json.JSONDecodeError as error: raise StudioError("INVALID_JSON", str(error)) from error - if not isinstance(conf, dict) or "data" not in conf or not isinstance(conf["data"], dict): - raise StudioError("INVALID_DEPLOYMENT_CONFIG", "部署配置缺少 data 节点") - - pipe_ref = conf["data"].get("pipe_path") - if not isinstance(pipe_ref, str): - raise StudioError("INVALID_DEPLOYMENT_CONFIG", "部署配置缺少 data.pipe_path") + if not isinstance(conf, dict) or "pipe_path" not in conf or not isinstance(conf["pipe_path"], str): + raise StudioError("INVALID_DEPLOYMENT_CONFIG", "部署配置缺少 pipe_path") report = self.invoke_tool([ "resolve-conf", @@ -829,13 +833,16 @@ def associate_deployment( if pipe_path.read_bytes() != pipe_raw or conf_path.read_bytes() != conf_raw: raise StudioError("REVISION_CONFLICT", "关联期间文件已改变，请重新关联", 409) + outputs = pipeline.get("deployment", {}).get("io", {}).get("output_allocations", {}) + io_binding = pipeline.get("deployment", {}).get("io", {}).get("io_binding", "") with self.solution_lock: self.generated_solutions[pipe_path.name] = { "conf_path": conf_path, "conf_name": conf_path.name, "conf_revision": revision_for(conf_raw), "pipeline_revision": revision_for(pipe_raw), - "outputs": conf["data"].get("outputs", {}), + "outputs": outputs, + "io_binding": io_binding, "model_root": model_root, "profile": prof, "is_associated": True, @@ -898,8 +905,9 @@ def preflight( profile_obj, conf_data = self.deployment_candidate( pipeline, profile_name, model_root, filename, conf_name, model_path_actions) - conf_data["data"]["pipe_path"] = staged_pipe.name - staged_conf.write_text(json.dumps(conf_data, ensure_ascii=False, indent=2)) + staged_pipe.write_text(json.dumps(pipeline, ensure_ascii=False, indent=2), encoding="utf-8") + staged_conf_data = {"pipe_path": staged_pipe.name} + staged_conf.write_text(json.dumps(staged_conf_data, ensure_ascii=False, indent=2)) configuration = self.resolve_run_conf(staged_conf, profile_obj) assets_status = [] @@ -1004,8 +1012,7 @@ def _run_job( pipeline_path.write_text( json.dumps(pipeline, ensure_ascii=False, indent=2), encoding="utf-8" ) - temp_conf = copy.deepcopy(conf) - temp_conf["data"]["pipe_path"] = pipeline_path.name + temp_conf = {"pipe_path": pipeline_path.name} conf_path = temp_root / "pipeline.conf" conf_path.write_text(json.dumps(temp_conf, indent=2), encoding="utf-8") configuration = self.resolve_run_conf(conf_path, profile) diff --git a/tools/verify_selection.py b/tools/verify_selection.py index 7e8d867c..04b3b7c0 100644 --- a/tools/verify_selection.py +++ b/tools/verify_selection.py @@ -86,12 +86,20 @@ def build_run_conf(pipeline, outputs, pipe_path, model_root, bundle_root, io_bin pipeline_path = Path(pipe_path).name model_paths = {model["model_id"]: str(within(model_root, model["model_path"]).relative_to(bundle_root)) for model in pipeline.get("models", [])} - binding = io_binding or BIZ_TO_OPERATOR_BINDING.get(pipeline.get("biz_name")) - data = {"pipe_path": str(pipeline_path), - "model_paths": model_paths, "outputs": outputs} + binding = ( + io_binding + or pipeline.get("deployment", {}).get("io", {}).get("io_binding") + or BIZ_TO_OPERATOR_BINDING.get(pipeline.get("biz_name")) + ) + deployment = pipeline.setdefault("deployment", {}) + io_obj = deployment.setdefault("io", {}) if binding: - data["io_binding"] = binding - return {"schema_version": 1, "data": data} + io_obj["io_binding"] = binding + if outputs: + io_obj["output_allocations"] = outputs + if model_paths: + deployment["model_paths"] = model_paths + return {"pipe_path": str(pipeline_path)} def validate_manifest(manifest): @@ -215,13 +223,26 @@ def effect_inputs(spec_path, conf_path, demo): spec = read_json(spec_path) dataset = (spec_path.parent / spec["dataset"]).resolve() conf = read_json(conf_path) - # The evaluator deliberately regenerates model_paths from the selected - # Pipeline; only deployment output capacities are inherited. - outputs = conf["data"]["outputs"] + if not isinstance(conf, dict) or "pipe_path" not in conf or not isinstance(conf["pipe_path"], str): + raise ValueError(f"Conf must contain non-empty 'pipe_path' (RFC-0061): {conf_path}") + if any(k in conf for k in ("schema_version", "data", "io_binding", "model_paths", "outputs")): + raise ValueError( + f"Deprecated deployment configuration format in {conf_path} (RFC-0061): " + "conf must contain only 'pipe_path'" + ) + pipe_path = conf["pipe_path"] + pipeline_file = (Path(conf_path).parent / pipe_path).resolve() + pipe_doc = read_json(pipeline_file) + deployment = pipe_doc.get("deployment", {}) + io_doc = deployment.get("io", {}) + outputs = io_doc.get("output_allocations", {}) + model_paths = deployment.get("model_paths", {}) + binding = io_doc.get("io_binding", "") demo = Path(demo).resolve() sdk_candidates = list(demo.parent.glob("libcompany_alg_sdk.*")) sdk_files = sorted({path.resolve() for path in sdk_candidates if path.is_file()}) identity = {"spec": spec, "dataset_sha256": file_digest(dataset), "outputs": outputs, + "model_paths": model_paths, "io_binding": binding, "demo_sha256": file_digest(demo), "sdk": {p.name: file_digest(p) for p in sdk_files}, "chip": "cpu", "device_id": 0} return spec, dataset, outputs, identity @@ -243,8 +264,8 @@ def evaluate(pipeline, selection, tool, model_root, spec_path, conf_path, demo): with tempfile.TemporaryDirectory(prefix=".selection-", dir=bundle_root) as directory: temporary = Path(directory) relative = temporary.relative_to(bundle_root) - (temporary / "pipeline.json").write_text(json.dumps(pipeline)) generated_conf = build_run_conf(pipeline, outputs, relative / "pipeline.json", model_root, bundle_root) + (temporary / "pipeline.json").write_text(json.dumps(pipeline)) (temporary / "pipeline.conf").write_text(json.dumps(generated_conf)) command = [str(Path(demo).resolve()), "--biz", biz, "--config", str(relative / "pipeline.conf"), "--dataset", str(dataset), "--output-dir", str(temporary / "results"),