From a13961e68eb6d796d62abfa4dd4495d593d36dfa Mon Sep 17 00:00:00 2001 From: peachisai <2581009893@qq.com> Date: Sun, 12 Jul 2026 16:03:00 +0800 Subject: [PATCH 1/8] add the genai evaluation feature --- docs/en/swip/SWIP-16.md | 260 +++++++++++++++++ docs/en/swip/readme.md | 3 +- oap-server/ai-evaluation/pom.xml | 58 ++++ .../ai/evaluation/AIEvaluationConfig.java | 41 +++ .../evaluation/AIEvaluationConfigLoader.java | 151 ++++++++++ .../ai/evaluation/AIEvaluationModule.java | 37 +++ .../ai/evaluation/AIEvaluationProvider.java | 191 ++++++++++++ .../context/AIEvaluationContext.java | 44 +++ .../context/GenAIContextResolver.java | 51 ++++ .../context/GenAISemanticAttributes.java | 34 +++ .../evaluation/judge/JudgeModelProvider.java | 31 ++ .../evaluation/judge/JudgeModelRequest.java | 29 ++ .../evaluation/judge/JudgeModelResponse.java | 31 ++ .../provider/OpenAICompatibleProvider.java | 276 ++++++++++++++++++ .../level/EvaluationLevelConfig.java | 33 +++ .../level/EvaluationLevelResolver.java | 86 ++++++ .../ai/evaluation/level/ScoreLevelRule.java | 34 +++ .../listener/AIEvaluationSpanListener.java | 132 +++++++++ .../plan/EvaluationInputExtractor.java | 49 ++++ .../ai/evaluation/plan/EvaluationPlan.java | 34 +++ .../ai/evaluation/plan/EvaluationPlanner.java | 55 ++++ .../plan/EvaluationPromptBuilder.java | 65 +++++ .../ai/evaluation/plan/EvaluationResult.java | 34 +++ .../plan/EvaluationResultParser.java | 112 +++++++ .../service/AIEvaluationMetricReporter.java | 88 ++++++ .../service/AIEvaluationService.java | 117 ++++++++ .../service/IAIEvaluationService.java | 29 ++ .../sample/AIEvaluationSamplingPolicy.java | 24 ++ .../DefaultAIEvaluationSamplingPolicy.java | 39 +++ .../strategy/AIEvaluationStrategy.java | 32 ++ .../span/SpanAIEvaluationStrategy.java | 174 +++++++++++ .../ai/evaluation/task/EvaluationTask.java | 35 +++ .../task/EvaluationTaskRegistry.java | 54 ++++ .../evaluation/value/SpanEvaluationType.java | 23 ++ .../server/ai/evaluation/value/ValueType.java | 26 ++ ...walking.oap.server.core.trace.SpanListener | 18 ++ ...ing.oap.server.library.module.ModuleDefine | 19 ++ ...g.oap.server.library.module.ModuleProvider | 19 ++ .../OpenAICompatibleProviderTest.java | 95 ++++++ oap-server/analyzer/agent-analyzer/pom.xml | 7 +- .../provider/AnalyzerModuleProvider.java | 5 + .../GenAIEvaluationAnalysisListener.java | 116 ++++++++ oap-server/pom.xml | 1 + .../oap/server/core/CoreModule.java | 2 + .../oap/server/core/CoreModuleProvider.java | 4 + .../manual/genai/GenAIEvaluationRecord.java | 162 ++++++++++ .../core/analysis/meter/MeterEntity.java | 17 +- .../GenAIEvaluationRecordQueryService.java | 107 +++++++ .../GenAIEvaluationRecordQueryCondition.java | 40 +++ .../query/type/GenAIEvaluationRecords.java | 45 +++ .../core/source/DefaultScopeDefine.java | 1 + .../server/core/storage/StorageModule.java | 2 + .../query/IGenAIEvaluationRecordQueryDAO.java | 103 +++++++ .../oap/server/core/trace/OTLPSpanReader.java | 10 + .../query/graphql/GraphQLQueryProvider.java | 3 + .../resolver/GenAIEvaluationRecordQuery.java | 110 +++++++ .../otel/otlp/OTLPSpanReaderImpl.java | 21 ++ .../src/main/resources/ai-evaluation.yml | 108 +++++++ .../src/main/resources/application.yml | 5 + .../gen-ai-evaluation-rules/gen-ai-model.yaml | 21 ++ .../banyandb/BanyanDBStorageProvider.java | 4 + ...nDBGenAIGenAIEvaluationRecordQueryDAO.java | 135 +++++++++ .../StorageModuleElasticsearchProvider.java | 4 + .../GenAIEvaluationRecordQueryEsDAO.java | 162 ++++++++++ .../jdbc/common/JDBCStorageProvider.java | 5 + .../JDBCGenAIEvaluationRecordQueryDAO.java | 257 ++++++++++++++++ 66 files changed, 4112 insertions(+), 8 deletions(-) create mode 100644 docs/en/swip/SWIP-16.md create mode 100644 oap-server/ai-evaluation/pom.xml create mode 100644 oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/AIEvaluationConfig.java create mode 100644 oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/AIEvaluationConfigLoader.java create mode 100644 oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/AIEvaluationModule.java create mode 100644 oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/AIEvaluationProvider.java create mode 100644 oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/context/AIEvaluationContext.java create mode 100644 oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/context/GenAIContextResolver.java create mode 100644 oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/context/GenAISemanticAttributes.java create mode 100644 oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/judge/JudgeModelProvider.java create mode 100644 oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/judge/JudgeModelRequest.java create mode 100644 oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/judge/JudgeModelResponse.java create mode 100644 oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/judge/provider/OpenAICompatibleProvider.java create mode 100644 oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/level/EvaluationLevelConfig.java create mode 100644 oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/level/EvaluationLevelResolver.java create mode 100644 oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/level/ScoreLevelRule.java create mode 100644 oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/listener/AIEvaluationSpanListener.java create mode 100644 oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/plan/EvaluationInputExtractor.java create mode 100644 oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/plan/EvaluationPlan.java create mode 100644 oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/plan/EvaluationPlanner.java create mode 100644 oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/plan/EvaluationPromptBuilder.java create mode 100644 oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/plan/EvaluationResult.java create mode 100644 oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/plan/EvaluationResultParser.java create mode 100644 oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/AIEvaluationMetricReporter.java create mode 100644 oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/AIEvaluationService.java create mode 100644 oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/IAIEvaluationService.java create mode 100644 oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/sample/AIEvaluationSamplingPolicy.java create mode 100644 oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/sample/DefaultAIEvaluationSamplingPolicy.java create mode 100644 oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/strategy/AIEvaluationStrategy.java create mode 100644 oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/strategy/span/SpanAIEvaluationStrategy.java create mode 100644 oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/task/EvaluationTask.java create mode 100644 oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/task/EvaluationTaskRegistry.java create mode 100644 oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/value/SpanEvaluationType.java create mode 100644 oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/value/ValueType.java create mode 100644 oap-server/ai-evaluation/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.core.trace.SpanListener create mode 100644 oap-server/ai-evaluation/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine create mode 100644 oap-server/ai-evaluation/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider create mode 100644 oap-server/ai-evaluation/src/test/java/org/apache/skywalking/oap/server/ai/evaluation/judge/provider/OpenAICompatibleProviderTest.java create mode 100644 oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/GenAIEvaluationAnalysisListener.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/genai/GenAIEvaluationRecord.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/GenAIEvaluationRecordQueryService.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/GenAIEvaluationRecordQueryCondition.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/GenAIEvaluationRecords.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IGenAIEvaluationRecordQueryDAO.java create mode 100644 oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/GenAIEvaluationRecordQuery.java create mode 100644 oap-server/server-starter/src/main/resources/ai-evaluation.yml create mode 100644 oap-server/server-starter/src/main/resources/gen-ai-evaluation-rules/gen-ai-model.yaml create mode 100644 oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBGenAIGenAIEvaluationRecordQueryDAO.java create mode 100644 oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/GenAIEvaluationRecordQueryEsDAO.java create mode 100644 oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCGenAIEvaluationRecordQueryDAO.java diff --git a/docs/en/swip/SWIP-16.md b/docs/en/swip/SWIP-16.md new file mode 100644 index 000000000000..8450ffc069c4 --- /dev/null +++ b/docs/en/swip/SWIP-16.md @@ -0,0 +1,260 @@ +# SWIP-16 Support LLM-as-Judge on Top of GenAI Observability + +## Motivation + +SkyWalking already provides GenAI observability capabilities. Based on the existing GenAI semantic conventions and analysis pipeline, SkyWalking can recognize GenAI spans from multiple data sources, including SkyWalking native traces, OTLP, and Zipkin, extract GenAI-related attributes, build Virtual GenAI entities, and display runtime metrics such as traffic, latency, token usage, TTFT, TPOT, and estimated cost in the GenAI dashboard. + +These capabilities answer "what happened during the invocation" and "how are performance and cost," but they do not yet answer "what is the quality of the model output." For GenAI applications in production, users usually also need to continuously observe quality signals such as: + +- Whether the response is faithful to the given context +- Whether the response is relevant to the user query +- Whether the model completes the expected task +- Whether the response contains hallucinations + +Today, such evaluation usually depends on external evaluation platforms, custom business-side scripts, or manual sampling workflows. This causes several problems: + +- Evaluation results are disconnected from SkyWalking trace and span observability data +- Evaluation results cannot be stored as unified structured data in SkyWalking +- Evaluation tasks lack a unified OAP-side configuration model +- Evaluation results cannot be further aggregated into metrics and displayed in the existing GenAI dashboard + +This SWIP proposes introducing `LLM-as-Judge` into SkyWalking OAP. On top of the existing GenAI observability capabilities, the whole evaluation feature reuses the current trace ingestion and GenAI span analysis pipeline and supports SkyWalking native traces, OTLP, and Zipkin as input sources. OAP samples runtime GenAI spans, extracts evaluation inputs, invokes a configurable judge model, and writes evaluation results into SkyWalking structured records. For `SCORE`-type evaluation results, OAP further generates metrics and displays them in the GenAI dashboard. Meanwhile, the UI adds an evaluation result page for detailed result browsing and supports jumping from a single evaluation result to the related trace, so that quality observability is integrated into the existing GenAI observability system. + +## Architecture Graph + +```text +GenAI spans / traces SkyWalking OAP Storage / Query / UI +------------------- -------------- -------------------- +SkyWalking native / OTLP / Zipkin -> GenAI span analysis + |- recognize GenAI spans + |- extract context and tags + '- trigger AI evaluation + | + v + ai-evaluation module + |- PPM sampling strategy + |- evaluation planning + |- prompt building + |- Judge Provider + '- result parsing + | + v + External Judge Model + | + v + Evaluation result + |- write structured record + '- generate MAL labeled metrics for SCORE results + | + +---------------------+----------------------+ + | | + v v + records storage (`ai_evaluation_result`) GraphQL / query API / GenAI dashboard / evaluation result view +``` + +## Proposed Changes + +### 1. Introduce an independent AI evaluation module + +OAP adds a dedicated `ai-evaluation` module to host runtime AI evaluation capabilities. In the current implementation, evaluation data is decoupled through an asynchronous local in-memory queue so that judge invocation is not executed synchronously on the trace analysis critical path. This module is responsible for: + +- Loading evaluation-related configuration +- Validating judge model configuration +- Creating the judge provider +- Applying the sampling strategy +- Executing the evaluation strategy +- Persisting evaluation results +- Converting score-type evaluation results into MAL-based labeled metrics + +This keeps evaluation logic modular and avoids coupling judge-related logic directly into the existing analyzer core. + +### 2. Reuse the existing GenAI observability analysis entry + +This capability does not introduce a new parallel collection pipeline. Instead, it is built on top of the existing GenAI observability capability and directly reuses the current trace ingestion paths, including SkyWalking native traces, OTLP, and Zipkin. + +When OAP parses spans, the existing pipeline already recognizes GenAI spans. On top of that, the new analysis listener reuses runtime context and extracts the following information: + +- `traceId` +- `spanId` +- `serviceName` +- `serviceInstanceName` +- `operationName` +- `providerName` +- `modelName` +- `startTimeMillis` +- `endTimeMillis` +- `error` +- `GenAI-related tags` + +This information is packaged as the evaluation context and passed to the AI evaluation service, making evaluation results a natural extension of the existing GenAI observability model. + +### 3. Introduce task-based LLM-as-Judge evaluation + +The evaluation logic is task-driven rather than hardcoded around fixed dimensions. Each task defines the following fields: + +- `name` +- `valueType` +- `instruction` + +The initial default tasks include: + +- `Faithfulness` +- `Relevance` +- `TaskCompletion` +- `Hallucination` + +Based on the configured `system-prompt`, extracted GenAI context, and task list, OAP builds the prompt and sends it to the external judge model. The judge model returns structured JSON, and each task result contains at least: + +- `value` +- `reason` + +This design keeps evaluation dimensions configurable rather than hardcoded in the implementation. + +### 4. Support an OpenAI-compatible judge provider + +This SWIP introduces the `JudgeModelProvider` abstraction and provides the first implementation, `OpenAICompatibleProvider`. + +Runtime configuration includes: + +- `provider` +- `endpoint` +- `model` +- `api-key` + +This allows OAP to call judge endpoints compatible with the OpenAI API format while leaving room for future provider extensions. + +### 5. Introduce evaluation planning, prompt building, asynchronous queueing, and result parsing + +The implementation introduces a clear runtime evaluation pipeline. In the current implementation, after a GenAI span hits sampling, OAP does not synchronously invoke evaluation. Instead, it first places the evaluation task into a local asynchronous in-memory queue, and a background evaluation consumer executes the remaining steps: + +- `EvaluationInputExtractor` +- `EvaluationPlanner` +- `EvaluationPlan` +- `EvaluationPromptBuilder` +- `local async in-memory queue` +- `evaluation consumer` +- `EvaluationResultParser` +- `EvaluationResult` + +The end-to-end flow is: + +1. Extract evaluation input from the GenAI span context +2. Build the evaluation plan according to configured tasks +3. Put the evaluation task into the asynchronous local in-memory queue +4. The background evaluation consumer takes the task from the queue and builds the judge prompt +5. Invoke the judge model +6. Parse the returned JSON into structured evaluation results +7. Persist the evaluation results and generate MAL labeled metrics when applicable + +This decouples evaluation orchestration from transport and storage logic and avoids blocking external model invocation on the trace analysis critical path. + +### 6. Use PPM sampling for runtime evaluation + +Because LLM-as-Judge introduces additional model invocation cost and runtime overhead, this SWIP does not evaluate all GenAI spans. Instead, it introduces a sampling strategy. + +The sampling rate uses PPM, `parts per million`: + +- `1_000_000` means 100% evaluation +- `100_000` means 10% evaluation +- `10_000` means 1% evaluation +- `0` means runtime evaluation is disabled + +The module validates that the sampling rate is within `[0, 1_000_000]` and applies the configured sampling strategy before invoking the judge model. + +### 7. Write evaluation results into SkyWalking structured records + +Each evaluation result is written into SkyWalking storage through `AIEvaluationResultRecord` as a structured record. + +The record includes the following core fields: + +- `trace_id` +- `segment_id` +- `span_id` +- `span_type` +- `task_name` +- `value_type` +- `value` +- `evaluation_level` +- `reason` +- `judge_model` +- `evaluation_time` +- `time_bucket` + +This lets each evaluation result directly link back to existing trace and span data. OAP also derives a normalized `evaluation_level` from the returned result when the value type supports level resolution, so later query and UI layers can filter and group records by coarse quality level in addition to raw value. In merged record storage mode, the data is written into the logical record table `ai_evaluation_result`. + +### 8. Generate MAL labeled metrics from SCORE-type evaluation results + +In addition to persisting evaluation results as structured records, this SWIP also proposes converting `SCORE`-type evaluation results into MAL-based labeled metrics. + +For tasks where `valueType = SCORE`, the judge returns a numeric result in `[0.0, 1.0]`. OAP converts the result into a MAL `SampleFamily` and uses a MAL rule to generate the final metric. The task name is kept as a metric label instead of being encoded into the metric name, so newly configured evaluation tasks do not require additional OAL statements or new hardcoded metrics. + +The initial metric is: + +- `gen_ai_evaluation_score_ppm` + +The metric is attached to the Virtual GenAI service instance dimension, using `service_name` as the service key and `model_name` as the instance key. The `task_name` remains as a labeled value dimension, allowing the same metric to represent scores for `Faithfulness`, `Relevance`, `TaskCompletion`, `Hallucination`, or any user-defined task. + +Because SkyWalking MAL labeled values are stored as long values, the score is scaled before entering the MAL pipeline: + +```text +stored value = score * 1,000,000 +``` + +For example, a judge score of `0.86` is stored as `860000`. Query or UI code should divide the metric value by `1,000,000` when displaying the original score. + +This labeled metric supports: + +- Observing score trends by evaluation task +- Aggregating evaluation results by existing GenAI observability dimensions, especially service and model +- Displaying quality-related metrics grouped by `task_name` in the GenAI dashboard + +This means the new capability is not only a record persistence feature, but also extends the GenAI dashboard from performance and cost observability to quality observability. + +### 9. Add an evaluation result page and trace jump capability + +In addition to aggregated dashboard metrics, the UI adds an evaluation result page for displaying structured evaluation details. + +The page displays at least the following fields: + +- `traceId` +- `segmentId` +- `spanId` +- `serviceName` +- `operationName` +- `taskName` +- `valueType` +- `value` +- `evaluationLevel` +- `reason` +- `judgeModel` +- `evaluationTime` + +Users can filter the page by service, task name, evaluation level, time range, and other tag-based conditions, making it easier to investigate low scores, anomalies, or suspicious results. + +Most importantly, each evaluation result keeps its association with the original trace. Users can click `traceId` or a jump button in the evaluation result page to open the related trace detail page directly and continue investigating the full call chain, contextual spans, and related GenAI tags. + +This allows SkyWalking not only to show an evaluation result, but also to connect that result with runtime trace analysis and form a closed-loop troubleshooting experience from quality signal to execution context. + +## Compatibility + +This SWIP introduces a new OAP capability and a new record data model. The main compatibility impacts include: + +- A new structured record type `ai_evaluation_result`, including the normalized `evaluation_level` field +- `SCORE`-type evaluation results additionally generate a MAL labeled metric for dashboard display +- The `gen_ai_evaluation_score_ppm` metric stores scores scaled by `1,000,000`; query and UI layers need to divide by `1,000,000` to display the original `[0.0, 1.0]` score +- The capability depends on the existing GenAI observability pipeline being able to recognize GenAI spans from SkyWalking native traces, OTLP, and Zipkin +- The UI adds an evaluation result page and trace jump based on `traceId` +- The current implementation uses an asynchronous local in-memory queue to carry evaluation tasks, and queue data is not part of any persistent protocol +- Runtime behavior is controlled jointly by the module switch, judge configuration, task configuration, and PPM sampling rate + +## General usage docs + +1. Enable the `ai-evaluation` module in OAP. +2. Configure the judge provider, endpoint, model, API key, system prompt, task list, and PPM sampling rate. +3. Ensure the existing GenAI observability pipeline is already receiving supported GenAI spans from SkyWalking native traces, OTLP, and Zipkin. +4. OAP generates evaluation tasks for sampled GenAI spans according to the PPM sampling rate and puts them into the asynchronous local in-memory queue. +5. A background evaluation consumer takes tasks from the queue and sends requests to the configured judge model. +6. OAP writes each evaluation result into SkyWalking structured records. +7. For `SCORE`-type tasks, OAP generates the MAL labeled metric `gen_ai_evaluation_score_ppm` from evaluation results. +8. Users observe both existing GenAI runtime metrics and newly added quality metrics in the GenAI dashboard. The UI should divide `gen_ai_evaluation_score_ppm` values by `1,000,000` to display the original score. +9. Users can also open the evaluation result page, inspect evaluation details, and jump from a single result to the related trace. \ No newline at end of file diff --git a/docs/en/swip/readme.md b/docs/en/swip/readme.md index eac834e4a184..a9ca425de24d 100644 --- a/docs/en/swip/readme.md +++ b/docs/en/swip/readme.md @@ -68,7 +68,7 @@ All accepted and proposed SWIPs can be found in [here](https://github.com/apache ## Known SWIPs -Next SWIP Number: 16 +Next SWIP Number: 17 ### Proposed SWIPs @@ -78,6 +78,7 @@ Next SWIP Number: 16 ### Accepted SWIPs +- [SWIP-16 Support LLM-as-Judge on Top of GenAI Observability](SWIP-16.md) - [SWIP-12 Support WeChat & Alipay Mini Program Monitoring](SWIP-12.md) - [SWIP-11 Support iOS App Monitoring via OpenTelemetry](SWIP-11.md) - [SWIP-10 Support Envoy AI Gateway Observability](SWIP-10/SWIP.md) diff --git a/oap-server/ai-evaluation/pom.xml b/oap-server/ai-evaluation/pom.xml new file mode 100644 index 000000000000..43887e3a8261 --- /dev/null +++ b/oap-server/ai-evaluation/pom.xml @@ -0,0 +1,58 @@ + + + + + 4.0.0 + + + org.apache.skywalking + oap-server + ${revision} + + + org.apache.skywalking + ai-evaluation + + + AI evaluation module samples telemetry data from the OAP kernel, builds evaluation tasks for AI scenarios, + and calls a judge model to evaluate the observed AI behavior. + + + + + org.apache.skywalking + library-module + ${project.version} + + + org.apache.skywalking + server-core + ${project.version} + + + org.apache.skywalking + meter-analyzer + ${project.version} + + + com.google.code.gson + gson + + + diff --git a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/AIEvaluationConfig.java b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/AIEvaluationConfig.java new file mode 100644 index 000000000000..319fcd392dd0 --- /dev/null +++ b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/AIEvaluationConfig.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.ai.evaluation; + +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +import lombok.Getter; +import lombok.Setter; +import org.apache.skywalking.oap.server.ai.evaluation.level.EvaluationLevelConfig; +import org.apache.skywalking.oap.server.ai.evaluation.task.EvaluationTask; +import org.apache.skywalking.oap.server.library.module.ModuleConfig; + +@Getter +@Setter +public class AIEvaluationConfig extends ModuleConfig { + private int sampleRate; + private Properties judge = new Properties(); + private String systemPrompt; + private Double temperature; + private Integer maxTokens; + private List tasks = new ArrayList<>(); + private EvaluationLevelConfig level = new EvaluationLevelConfig(); +} diff --git a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/AIEvaluationConfigLoader.java b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/AIEvaluationConfigLoader.java new file mode 100644 index 000000000000..f31f814f9849 --- /dev/null +++ b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/AIEvaluationConfigLoader.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.ai.evaluation; + +import java.io.FileNotFoundException; +import java.io.Reader; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.function.Consumer; +import org.apache.skywalking.oap.server.ai.evaluation.level.EvaluationLevelConfig; +import org.apache.skywalking.oap.server.ai.evaluation.level.ScoreLevelRule; +import org.apache.skywalking.oap.server.ai.evaluation.task.EvaluationTask; +import org.apache.skywalking.oap.server.ai.evaluation.value.ValueType; +import org.apache.skywalking.oap.server.library.module.ModuleStartException; +import org.apache.skywalking.oap.server.library.util.ResourceUtils; +import org.apache.skywalking.oap.server.library.util.YamlConfigLoaderUtils; +import org.yaml.snakeyaml.Yaml; + +public class AIEvaluationConfigLoader { + private static final String CONFIG_FILE = "ai-evaluation.yml"; + + public AIEvaluationConfig load() throws ModuleStartException { + try { + final Reader reader = ResourceUtils.read(CONFIG_FILE); + final Map loaded = new Yaml().loadAs(reader, Map.class); + if (loaded == null || loaded.isEmpty()) { + return new AIEvaluationConfig(); + } + return buildConfig(loaded); + } catch (FileNotFoundException e) { + throw new ModuleStartException("Cannot find the AI evaluation configuration file [" + + CONFIG_FILE + "].", e); + } + } + + private AIEvaluationConfig buildConfig(final Map loaded) { + final AIEvaluationConfig config = new AIEvaluationConfig(); + + final Object judge = loaded.get("judge"); + if (judge instanceof Map) { + config.setJudge(buildProperties((Map) judge)); + } + + final Object systemPrompt = loaded.get("system-prompt"); + if (systemPrompt != null) { + config.setSystemPrompt(String.valueOf(systemPrompt)); + } + + final Object level = loaded.get("level"); + if (level instanceof Map) { + config.setLevel(buildLevelConfig((Map) level)); + } + + final Object tasks = loaded.get("tasks"); + if (tasks instanceof List) { + for (Map taskConfig : (List>) tasks) { + config.getTasks().add(buildTask(taskConfig)); + } + } + return config; + } + + private EvaluationLevelConfig buildLevelConfig(final Map levelConfig) { + final EvaluationLevelConfig level = new EvaluationLevelConfig(); + setString(levelConfig, "undefined", level::setUndefined); + + final Object score = levelConfig.get("score"); + if (score instanceof List) { + for (Map ruleConfig : (List>) score) { + level.getScore().add(buildScoreLevelRule(ruleConfig)); + } + } + + final Object bool = levelConfig.get("boolean"); + if (bool instanceof Map) { + final Map booleanConfig = (Map) bool; + setString(booleanConfig, "true", level::setBooleanTrue); + setString(booleanConfig, "false", level::setBooleanFalse); + } + return level; + } + + private ScoreLevelRule buildScoreLevelRule(final Map ruleConfig) { + final ScoreLevelRule rule = new ScoreLevelRule(); + final Object min = ruleConfig.get("min"); + if (min != null) { + rule.setMin(Double.parseDouble(String.valueOf(min))); + } + final Object max = ruleConfig.get("max"); + if (max != null) { + rule.setMax(Double.parseDouble(String.valueOf(max))); + } + setString(ruleConfig, "level", rule::setLevel); + return rule; + } + + private Properties buildProperties(final Map config) { + final Properties properties = new Properties(); + config.forEach((key, value) -> properties.put(key, value)); + final Yaml yaml = new Yaml(); + for (String key : new ArrayList<>(properties.stringPropertyNames())) { + YamlConfigLoaderUtils.replacePropertyAndLog( + key, properties.get(key), properties, "ai-evaluation", yaml); + } + return properties; + } + + private EvaluationTask buildTask(final Map taskConfig) { + final EvaluationTask task = new EvaluationTask(); + setString(taskConfig, "name", task::setName); + setString(taskConfig, "instruction", task::setInstruction); + + final Object valueType = taskConfig.get("valueType"); + if (valueType != null) { + task.setValueType(ValueType.valueOf(String.valueOf(valueType))); + } + + final Object allowedValues = taskConfig.get("allowedValues"); + if (allowedValues instanceof List) { + task.setAllowedValues((List) allowedValues); + } + return task; + } + + private void setString(final Map config, + final String key, + final Consumer setter) { + final Object value = config.get(key); + if (value != null) { + setter.accept(String.valueOf(value)); + } + } +} diff --git a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/AIEvaluationModule.java b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/AIEvaluationModule.java new file mode 100644 index 000000000000..62eb0107b146 --- /dev/null +++ b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/AIEvaluationModule.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.ai.evaluation; + +import org.apache.skywalking.oap.server.ai.evaluation.service.IAIEvaluationService; +import org.apache.skywalking.oap.server.library.module.ModuleDefine; + +public class AIEvaluationModule extends ModuleDefine { + public static final String NAME = "ai-evaluation"; + + public AIEvaluationModule() { + super(NAME); + } + + @Override + public Class[] services() { + return new Class[] { + IAIEvaluationService.class + }; + } +} diff --git a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/AIEvaluationProvider.java b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/AIEvaluationProvider.java new file mode 100644 index 000000000000..e23fbb93a4e2 --- /dev/null +++ b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/AIEvaluationProvider.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.ai.evaluation; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import java.util.stream.Collectors; +import org.apache.skywalking.oap.meter.analyzer.v2.MetricConvert; +import org.apache.skywalking.oap.meter.analyzer.v2.dsldebug.MalStaticBindingHook; +import org.apache.skywalking.oap.meter.analyzer.v2.prometheus.rule.Rule; +import org.apache.skywalking.oap.meter.analyzer.v2.prometheus.rule.Rules; +import org.apache.skywalking.oap.server.ai.evaluation.plan.EvaluationInputExtractor; +import org.apache.skywalking.oap.server.ai.evaluation.plan.EvaluationPlanner; +import org.apache.skywalking.oap.server.ai.evaluation.plan.EvaluationPromptBuilder; +import org.apache.skywalking.oap.server.ai.evaluation.plan.EvaluationResultParser; +import org.apache.skywalking.oap.server.ai.evaluation.task.EvaluationTaskRegistry; +import org.apache.skywalking.oap.server.ai.evaluation.judge.JudgeModelProvider; +import org.apache.skywalking.oap.server.ai.evaluation.judge.provider.OpenAICompatibleProvider; +import org.apache.skywalking.oap.server.ai.evaluation.level.EvaluationLevelResolver; +import org.apache.skywalking.oap.server.ai.evaluation.service.AIEvaluationMetricReporter; +import org.apache.skywalking.oap.server.ai.evaluation.service.AIEvaluationService; +import org.apache.skywalking.oap.server.ai.evaluation.service.sample.DefaultAIEvaluationSamplingPolicy; +import org.apache.skywalking.oap.server.ai.evaluation.service.IAIEvaluationService; +import org.apache.skywalking.oap.server.ai.evaluation.service.strategy.AIEvaluationStrategy; +import org.apache.skywalking.oap.server.ai.evaluation.service.strategy.span.SpanAIEvaluationStrategy; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.analysis.meter.MeterSystem; +import org.apache.skywalking.oap.server.core.config.NamingControl; +import org.apache.skywalking.oap.server.library.module.ModuleConfig; +import org.apache.skywalking.oap.server.library.module.ModuleDefine; +import org.apache.skywalking.oap.server.library.module.ModuleProvider; +import org.apache.skywalking.oap.server.library.module.ModuleStartException; +import org.apache.skywalking.oap.server.library.module.ServiceNotProvidedException; +import org.apache.skywalking.oap.server.library.util.StringUtil; + +public class AIEvaluationProvider extends ModuleProvider { + private static final int MAX_SAMPLE_RATE = 1_000_000; + private AIEvaluationConfig config = new AIEvaluationConfig(); + private AIEvaluationService aiEvaluationService; + private AIEvaluationMetricReporter metricReporter; + + @Override + public String name() { + return "default"; + } + + @Override + public Class module() { + return AIEvaluationModule.class; + } + + @Override + public ConfigCreator newConfigCreator() { + return new ConfigCreator() { + @Override + public Class type() { + return AIEvaluationConfig.class; + } + + @Override + public void onInitialized(final AIEvaluationConfig initialized) { + config = initialized; + } + }; + } + + @Override + public void prepare() throws ServiceNotProvidedException, ModuleStartException { + final int sampleRate = config.getSampleRate(); + config = new AIEvaluationConfigLoader().load(); + config.setSampleRate(sampleRate); + validateConfig(config); + if (config.getSampleRate() < 0 || config.getSampleRate() > MAX_SAMPLE_RATE) { + throw new IllegalArgumentException( + "sampleRate: " + config.getSampleRate() + ", should be between 0 and " + MAX_SAMPLE_RATE); + } + aiEvaluationService = new AIEvaluationService( + new DefaultAIEvaluationSamplingPolicy(config.getSampleRate()), + createJudgeProvider() + ); + registerServiceImplementation(IAIEvaluationService.class, aiEvaluationService); + } + + @Override + public void start() throws ServiceNotProvidedException, ModuleStartException { + metricReporter = createMetricReporter(); + aiEvaluationService.setStrategies(createStrategies()); + } + + @Override + public void notifyAfterCompleted() throws ServiceNotProvidedException, ModuleStartException { + } + + @Override + public String[] requiredModules() { + return new String[] { + CoreModule.NAME + }; + } + + private JudgeModelProvider createJudgeProvider() throws ModuleStartException { + final Properties judge = config.getJudge(); + final String provider = getString(judge, "provider"); + if ("openai".equalsIgnoreCase(provider)) { + return new OpenAICompatibleProvider(judge); + } + throw new ModuleStartException("Unsupported AI evaluation judge provider: " + provider); + } + + private List createStrategies() { + final EvaluationTaskRegistry taskRegistry = new EvaluationTaskRegistry(config.getTasks()); + final EvaluationInputExtractor inputExtractor = new EvaluationInputExtractor(); + final NamingControl namingControl = getManager().find(CoreModule.NAME).provider().getService(NamingControl.class); + return Collections.singletonList(new SpanAIEvaluationStrategy( + taskRegistry, + new EvaluationPlanner(inputExtractor), + new EvaluationPromptBuilder(config.getSystemPrompt()), + new EvaluationResultParser(), + metricReporter, + namingControl, + new EvaluationLevelResolver(config.getLevel()) + )); + } + + private AIEvaluationMetricReporter createMetricReporter() throws ModuleStartException { + final List rules; + try { + rules = Rules.loadRules( + AIEvaluationMetricReporter.RULE_CATALOG, + Collections.singletonList(AIEvaluationMetricReporter.RULE_NAME), + getManager() + ); + } catch (IOException e) { + throw new ModuleStartException("Failed to load AI evaluation MAL rules.", e); + } + final MeterSystem meterSystem = getManager().find(CoreModule.NAME).provider().getService(MeterSystem.class); + final NamingControl namingControl = getManager().find(CoreModule.NAME).provider().getService(NamingControl.class); + final List converts = rules.stream() + .map(rule -> { + final MetricConvert convert = new MetricConvert(rule, meterSystem); + MalStaticBindingHook.publish( + AIEvaluationMetricReporter.RULE_CATALOG, + rule.getName(), + convert + ); + return convert; + }) + .collect(Collectors.toList()); + return new AIEvaluationMetricReporter(converts, namingControl); + } + + private static void validateConfig(final AIEvaluationConfig config) throws ModuleStartException { + final Properties judge = config.getJudge(); + if (judge == null || judge.isEmpty()) { + throw new ModuleStartException("AI evaluation judge config is required."); + } + if (StringUtil.isBlank(getString(judge, "provider"))) { + throw new ModuleStartException("AI evaluation judge config [provider] is required."); + } + if (StringUtil.isBlank(config.getSystemPrompt())) { + throw new ModuleStartException("AI evaluation system-prompt is required."); + } + } + + private static String getString(final Properties properties, final String key) { + if (properties == null) { + return null; + } + final Object value = properties.get(key); + return value == null ? null : String.valueOf(value); + } + +} diff --git a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/context/AIEvaluationContext.java b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/context/AIEvaluationContext.java new file mode 100644 index 000000000000..28c7e4d2d365 --- /dev/null +++ b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/context/AIEvaluationContext.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.ai.evaluation.context; + +import lombok.Builder; +import lombok.Getter; + +import java.util.Collections; +import java.util.Map; + +@Getter +@Builder +public class AIEvaluationContext { + private final String traceId; + private final String spanId; + private final String segmentId; + private final String serviceName; + private final String serviceInstanceName; + private final String operationName; + private final String providerName; + private final String modelName; + private final long startTimeMillis; + private final long endTimeMillis; + private final boolean error; + @Builder.Default + private final Map tags = Collections.emptyMap(); + +} diff --git a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/context/GenAIContextResolver.java b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/context/GenAIContextResolver.java new file mode 100644 index 000000000000..d54aeb2dd9ab --- /dev/null +++ b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/context/GenAIContextResolver.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.ai.evaluation.context; + +import java.util.Map; +import lombok.Data; +import org.apache.skywalking.oap.server.library.util.StringUtil; +import org.apache.skywalking.oap.server.library.util.genai.GenAIModelMatcher; + +public final class GenAIContextResolver { + + private GenAIContextResolver() { + } + + public static Result resolve(final Map tags) { + String modelName = tags.get(GenAISemanticAttributes.RESPONSE_MODEL); + String providerName = tags.get(GenAISemanticAttributes.PROVIDER_NAME); + + if (StringUtil.isBlank(providerName)) { + providerName = tags.get(GenAISemanticAttributes.SYSTEM); + } + + if (StringUtil.isBlank(providerName) && StringUtil.isNotBlank(modelName)) { + providerName = GenAIModelMatcher.getInstance().match(modelName).getProvider(); + } + + return new Result(providerName, modelName); + } + + @Data + public static class Result { + private final String providerName; + private final String modelName; + } +} diff --git a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/context/GenAISemanticAttributes.java b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/context/GenAISemanticAttributes.java new file mode 100644 index 000000000000..3eb2f95670e9 --- /dev/null +++ b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/context/GenAISemanticAttributes.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.ai.evaluation.context; + +public final class GenAISemanticAttributes { + public static final String INPUT_MESSAGES = "gen_ai.input.messages"; + public static final String OPERATION_NAME = "gen_ai.operation.name"; + public static final String OUTPUT_MESSAGES = "gen_ai.output.messages"; + public static final String PROVIDER_NAME = "gen_ai.provider.name"; + public static final String RESPONSE_MODEL = "gen_ai.response.model"; + public static final String SERVER_TIME_TO_FIRST_TOKEN = "gen_ai.server.time_to_first_token"; + public static final String SYSTEM = "gen_ai.system"; + public static final String USAGE_INPUT_TOKENS = "gen_ai.usage.input_tokens"; + public static final String USAGE_OUTPUT_TOKENS = "gen_ai.usage.output_tokens"; + + private GenAISemanticAttributes() { + } +} diff --git a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/judge/JudgeModelProvider.java b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/judge/JudgeModelProvider.java new file mode 100644 index 000000000000..57b63851e482 --- /dev/null +++ b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/judge/JudgeModelProvider.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.ai.evaluation.judge; + +import java.io.IOException; +import java.util.Optional; + +public interface JudgeModelProvider { + + Optional judge(JudgeModelRequest request) throws IOException, InterruptedException; + + default String model() { + return ""; + } +} diff --git a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/judge/JudgeModelRequest.java b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/judge/JudgeModelRequest.java new file mode 100644 index 000000000000..0e7bf2fce36f --- /dev/null +++ b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/judge/JudgeModelRequest.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.ai.evaluation.judge; + +import lombok.Builder; +import lombok.Getter; + +@Getter +@Builder +public class JudgeModelRequest { + private final String systemPrompt; + private final String userPrompt; +} diff --git a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/judge/JudgeModelResponse.java b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/judge/JudgeModelResponse.java new file mode 100644 index 000000000000..bccaa99a1b32 --- /dev/null +++ b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/judge/JudgeModelResponse.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.ai.evaluation.judge; + +import lombok.Builder; +import lombok.Getter; + +@Getter +@Builder +public class JudgeModelResponse { + private final String content; + private final int promptTokens; + private final int completionTokens; + private final int totalTokens; +} diff --git a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/judge/provider/OpenAICompatibleProvider.java b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/judge/provider/OpenAICompatibleProvider.java new file mode 100644 index 000000000000..1e4e9b503c8e --- /dev/null +++ b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/judge/provider/OpenAICompatibleProvider.java @@ -0,0 +1,276 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.ai.evaluation.judge.provider; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.Optional; +import java.util.Properties; + +import org.apache.skywalking.oap.server.ai.evaluation.judge.JudgeModelProvider; +import org.apache.skywalking.oap.server.ai.evaluation.judge.JudgeModelRequest; +import org.apache.skywalking.oap.server.ai.evaluation.judge.JudgeModelResponse; +import org.apache.skywalking.oap.server.library.module.ModuleStartException; +import org.apache.skywalking.oap.server.library.util.StringUtil; + +public class OpenAICompatibleProvider implements JudgeModelProvider { + private static final Gson GSON = new Gson(); + private static final long DEFAULT_REQUEST_TIMEOUT_SECONDS = 30L; + private static final double MIN_TEMPERATURE = 0.0; + private static final double MAX_TEMPERATURE = 1.0; + + private final HttpClient httpClient; + private final String endpoint; + private final String apiKey; + private final String model; + private final Duration requestTimeout; + private final Double temperature; + private final Integer maxTokens; + + public OpenAICompatibleProvider(final Properties config) throws ModuleStartException { + this(HttpClient.newHttpClient(), config); + } + + OpenAICompatibleProvider(final HttpClient httpClient, final Properties config) throws ModuleStartException { + validate(config); + this.httpClient = httpClient; + this.endpoint = getString(config, "endpoint"); + this.apiKey = getString(config, "api-key"); + this.model = getString(config, "model"); + this.requestTimeout = Duration.ofSeconds(getRequestTimeoutSeconds(config)); + this.temperature = getDouble(config, "temperature"); + this.maxTokens = getInteger(config, "max_tokens"); + } + + @Override + public Optional judge(final JudgeModelRequest request) + throws IOException, InterruptedException { + final HttpRequest httpRequest = HttpRequest.newBuilder() + .uri(URI.create(endpoint)) + .timeout(requestTimeout) + .header("Authorization", "Bearer " + apiKey) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(buildRequestBody(request))) + .build(); + final HttpResponse response = httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofString()); + if (response.statusCode() < 200 || response.statusCode() >= 300) { + throw new IOException("OpenAI compatible judge API request failed, status: " + response.statusCode()); + } + final JudgeModelResponse judgeResponse = parseResponse(response.body()); + if (StringUtil.isBlank(judgeResponse.getContent())) { + throw new IOException("OpenAI compatible judge API response has no completion content."); + } + return Optional.of(judgeResponse); + } + + @Override + public String model() { + return model; + } + + private static void validate(final Properties config) throws ModuleStartException { + if (StringUtil.isBlank(getString(config, "endpoint"))) { + throw new ModuleStartException("AI evaluation judge config [endpoint] is required."); + } + if (StringUtil.isBlank(getString(config, "model"))) { + throw new ModuleStartException("AI evaluation judge config [model] is required."); + } + if (StringUtil.isBlank(getString(config, "api-key"))) { + throw new ModuleStartException("AI evaluation judge config [api-key] is required."); + } + validateRequestTimeoutSeconds(getString(config, "request-timeout-seconds")); + validateTemperature(getString(config, "temperature")); + validateMaxTokens(getString(config, "max_tokens")); + } + + private String buildRequestBody(final JudgeModelRequest request) { + final JsonObject body = new JsonObject(); + body.addProperty("model", model); + body.addProperty("stream", false); + if (temperature != null) { + body.addProperty("temperature", temperature); + } + if (maxTokens != null) { + body.addProperty("max_tokens", maxTokens); + } + + final JsonArray messages = new JsonArray(); + addMessage(messages, "system", request.getSystemPrompt()); + addMessage(messages, "user", request.getUserPrompt()); + body.add("messages", messages); + return GSON.toJson(body); + } + + private static void addMessage(final JsonArray messages, final String role, final String content) { + if (StringUtil.isBlank(content)) { + return; + } + final JsonObject message = new JsonObject(); + message.addProperty("role", role); + message.addProperty("content", content); + messages.add(message); + } + + private static JudgeModelResponse parseResponse(final String body) { + final JsonObject root = JsonParser.parseString(body).getAsJsonObject(); + final JsonArray choices = root.getAsJsonArray("choices"); + String content = ""; + if (choices != null && choices.size() > 0) { + final JsonObject choice = choices.get(0).getAsJsonObject(); + final JsonObject message = choice.getAsJsonObject("message"); + if (message != null) { + content = getAsString(message, "content"); + } + } + + final JsonObject usage = root.getAsJsonObject("usage"); + return JudgeModelResponse.builder() + .content(content) + .promptTokens(getAsInt(usage, "prompt_tokens")) + .completionTokens(getAsInt(usage, "completion_tokens")) + .totalTokens(getAsInt(usage, "total_tokens")) + .build(); + } + + private static String getAsString(final JsonObject object, final String memberName) { + if (object == null) { + return ""; + } + final JsonElement element = object.get(memberName); + return element == null || element.isJsonNull() ? "" : element.getAsString(); + } + + private static int getAsInt(final JsonObject object, final String memberName) { + if (object == null) { + return 0; + } + final JsonElement element = object.get(memberName); + return element == null || element.isJsonNull() ? 0 : element.getAsInt(); + } + + private static String getString(final Properties properties, final String key) { + if (properties == null) { + return null; + } + final Object value = properties.get(key); + return value == null ? null : String.valueOf(value); + } + + private static Double getDouble(final Properties properties, final String key) throws ModuleStartException { + final String value = getString(properties, key); + if (StringUtil.isBlank(value)) { + return null; + } + try { + return Double.valueOf(value); + } catch (NumberFormatException e) { + throw new ModuleStartException("AI evaluation judge config [" + key + "] must be a number.", e); + } + } + + private static Integer getInteger(final Properties properties, final String key) throws ModuleStartException { + final String value = getString(properties, key); + if (StringUtil.isBlank(value)) { + return null; + } + try { + return Integer.valueOf(value); + } catch (NumberFormatException e) { + throw new ModuleStartException("AI evaluation judge config [" + key + "] must be an integer.", e); + } + } + + private static long getRequestTimeoutSeconds(final Properties properties) throws ModuleStartException { + final String value = getString(properties, "request-timeout-seconds"); + if (StringUtil.isBlank(value)) { + return DEFAULT_REQUEST_TIMEOUT_SECONDS; + } + try { + return Long.parseLong(value); + } catch (NumberFormatException e) { + throw new ModuleStartException( + "AI evaluation judge config [request-timeout-seconds] must be an integer.", + e + ); + } + } + + private static void validateRequestTimeoutSeconds(final String value) throws ModuleStartException { + if (StringUtil.isBlank(value)) { + return; + } + final long parsed; + try { + parsed = Long.parseLong(value); + } catch (NumberFormatException e) { + throw new ModuleStartException( + "AI evaluation judge config [request-timeout-seconds] must be an integer.", + e + ); + } + if (parsed <= 0) { + throw new ModuleStartException( + "AI evaluation judge config [request-timeout-seconds] must be greater than 0." + ); + } + } + + private static void validateTemperature(final String value) throws ModuleStartException { + if (StringUtil.isBlank(value)) { + return; + } + final double parsed; + try { + parsed = Double.parseDouble(value); + } catch (NumberFormatException e) { + throw new ModuleStartException("AI evaluation judge config [temperature] must be a number.", e); + } + if (parsed < MIN_TEMPERATURE || parsed > MAX_TEMPERATURE) { + throw new ModuleStartException( + "AI evaluation judge config [temperature] must be greater than 0 and less than or equal to " + + MAX_TEMPERATURE + '.' + ); + } + } + + private static void validateMaxTokens(final String value) throws ModuleStartException { + if (StringUtil.isBlank(value)) { + return; + } + final int parsed; + try { + parsed = Integer.parseInt(value); + } catch (NumberFormatException e) { + throw new ModuleStartException("AI evaluation judge config [max_tokens] must be an integer.", e); + } + if (parsed <= 0) { + throw new ModuleStartException("AI evaluation judge config [max_tokens] must be greater than 0."); + } + } +} diff --git a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/level/EvaluationLevelConfig.java b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/level/EvaluationLevelConfig.java new file mode 100644 index 000000000000..06c83764c82f --- /dev/null +++ b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/level/EvaluationLevelConfig.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.ai.evaluation.level; + +import java.util.ArrayList; +import java.util.List; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class EvaluationLevelConfig { + private String undefined = EvaluationLevelResolver.UNDEFINED; + private List score = new ArrayList<>(); + private String booleanTrue = EvaluationLevelResolver.EXCELLENT; + private String booleanFalse = EvaluationLevelResolver.FAIL; +} diff --git a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/level/EvaluationLevelResolver.java b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/level/EvaluationLevelResolver.java new file mode 100644 index 000000000000..3e0656c42e16 --- /dev/null +++ b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/level/EvaluationLevelResolver.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.ai.evaluation.level; + +import java.util.List; +import org.apache.skywalking.oap.server.ai.evaluation.value.ValueType; +import org.apache.skywalking.oap.server.library.util.StringUtil; + +public class EvaluationLevelResolver { + public static final String FAIL = "fail"; + public static final String EXCELLENT = "excellent"; + public static final String UNDEFINED = "undefined"; + + private final EvaluationLevelConfig config; + + public EvaluationLevelResolver(final EvaluationLevelConfig config) { + this.config = config == null ? new EvaluationLevelConfig() : config; + } + + public String resolve(final ValueType valueType, final String value) { + if (valueType == null || value == null) { + return undefinedLevel(); + } + switch (valueType) { + case SCORE: + return resolveScore(value); + case BOOLEAN: + return resolveBoolean(value); + default: + return undefinedLevel(); + } + } + + private String resolveScore(final String value) { + final double score; + try { + score = Double.parseDouble(value); + } catch (NumberFormatException e) { + return undefinedLevel(); + } + final List rules = config.getScore(); + if (rules == null || rules.isEmpty()) { + return undefinedLevel(); + } + return rules.stream() + .filter(rule -> rule.matches(score)) + .map(ScoreLevelRule::getLevel) + .filter(StringUtil::isNotEmpty) + .findFirst() + .orElseGet(this::undefinedLevel); + } + + private String resolveBoolean(final String value) { + if ("true".equalsIgnoreCase(value)) { + return defaultIfEmpty(config.getBooleanTrue(), undefinedLevel()); + } + if ("false".equalsIgnoreCase(value)) { + return defaultIfEmpty(config.getBooleanFalse(), undefinedLevel()); + } + return undefinedLevel(); + } + + private String undefinedLevel() { + return defaultIfEmpty(config.getUndefined(), UNDEFINED); + } + + private static String defaultIfEmpty(final String value, final String defaultValue) { + return StringUtil.isNotEmpty(value) ? value : defaultValue; + } +} diff --git a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/level/ScoreLevelRule.java b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/level/ScoreLevelRule.java new file mode 100644 index 000000000000..87181dd68ac8 --- /dev/null +++ b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/level/ScoreLevelRule.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.ai.evaluation.level; + +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class ScoreLevelRule { + private double min; + private double max; + private String level; + + boolean matches(final double score) { + return (score >= min && score < max) || (score == max && max == 1.0D); + } +} diff --git a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/listener/AIEvaluationSpanListener.java b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/listener/AIEvaluationSpanListener.java new file mode 100644 index 000000000000..b5300243470d --- /dev/null +++ b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/listener/AIEvaluationSpanListener.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.ai.evaluation.listener; + +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import org.apache.skywalking.oap.server.ai.evaluation.AIEvaluationModule; +import org.apache.skywalking.oap.server.ai.evaluation.context.AIEvaluationContext; +import org.apache.skywalking.oap.server.ai.evaluation.context.GenAIContextResolver; +import org.apache.skywalking.oap.server.ai.evaluation.context.GenAISemanticAttributes; +import org.apache.skywalking.oap.server.ai.evaluation.service.IAIEvaluationService; +import org.apache.skywalking.oap.server.core.trace.SpanListener; +import org.apache.skywalking.oap.server.core.trace.SpanListenerResult; +import org.apache.skywalking.oap.server.core.zipkin.source.ZipkinSpan; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.util.StringUtil; + +import java.util.HashMap; +import java.util.Map; + +public class AIEvaluationSpanListener implements SpanListener { + private static final String ERROR_TAG = "error"; + private static final long MILLIS_PER_MICRO = 1_000L; + private static final String[] REQUIRED_TAG_KEYS = new String[]{ + GenAISemanticAttributes.RESPONSE_MODEL, + GenAISemanticAttributes.PROVIDER_NAME, + GenAISemanticAttributes.OPERATION_NAME, + GenAISemanticAttributes.INPUT_MESSAGES, + GenAISemanticAttributes.OUTPUT_MESSAGES, + ERROR_TAG + }; + + private IAIEvaluationService evaluationService; + + @Override + public String[] requiredModules() { + return new String[]{ + AIEvaluationModule.NAME + }; + } + + @Override + public void init(final ModuleManager moduleManager) { + evaluationService = moduleManager.find(AIEvaluationModule.NAME) + .provider() + .getService(IAIEvaluationService.class); + } + + @Override + public SpanListenerResult onZipkinSpan(final ZipkinSpan span) { + final Map tags = toRequiredTagMap(span.getTags()); + if (!isGenAISpan(tags)) { + return SpanListenerResult.CONTINUE; + } + if (!shouldSample(span.getTraceId())) { + return SpanListenerResult.CONTINUE; + } + sample(span.getTraceId(), + span.getSpanId(), + span.getLocalEndpointServiceName(), + span.getName(), + span.getTimestampMillis(), + span.getTimestampMillis() + span.getDuration() / MILLIS_PER_MICRO, + tags, + "true".equalsIgnoreCase(tags.get(ERROR_TAG))); + return SpanListenerResult.CONTINUE; + } + + private void sample(final String traceId, + final String spanId, + final String serviceName, + final String operationName, + final long startTimeMillis, + final long endTimeMillis, + final Map tags, + final boolean error) { + final GenAIContextResolver.Result genAIContext = GenAIContextResolver.resolve(tags); + evaluationService.sample(AIEvaluationContext.builder() + .traceId(traceId) + .spanId(spanId) + .serviceName(serviceName) + .operationName(operationName) + .providerName(genAIContext.getProviderName()) + .modelName(genAIContext.getModelName()) + .startTimeMillis(startTimeMillis) + .endTimeMillis(endTimeMillis) + .error(error) + .tags(tags) + .build()); + } + + private boolean shouldSample(final String traceId) { + return evaluationService.shouldSample(traceId); + } + + private static boolean isGenAISpan(final Map tags) { + return StringUtil.isNotBlank(tags.get(GenAISemanticAttributes.RESPONSE_MODEL)); + } + + private static Map toRequiredTagMap(final JsonObject tags) { + final Map result = new HashMap<>(); + if (tags == null) { + return result; + } + for (String requiredKey : REQUIRED_TAG_KEYS) { + final JsonElement value = tags.get(requiredKey); + if (value != null && !value.isJsonNull()) { + final String tagValue = value.getAsString(); + if (!tagValue.isEmpty()) { + result.put(requiredKey, tagValue); + } + } + } + return result; + } +} diff --git a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/plan/EvaluationInputExtractor.java b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/plan/EvaluationInputExtractor.java new file mode 100644 index 000000000000..b22b9177ca15 --- /dev/null +++ b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/plan/EvaluationInputExtractor.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.ai.evaluation.plan; + +import java.util.LinkedHashMap; +import java.util.Map; +import org.apache.skywalking.oap.server.ai.evaluation.context.AIEvaluationContext; +import org.apache.skywalking.oap.server.ai.evaluation.context.GenAISemanticAttributes; +import org.apache.skywalking.oap.server.ai.evaluation.value.SpanEvaluationType; + +public class EvaluationInputExtractor { + public Map extract(final AIEvaluationContext context, + final SpanEvaluationType spanType) { + final Map inputs = new LinkedHashMap<>(); + if (context == null || spanType == null) { + return inputs; + } + + switch (spanType) { + case LLM_CALL: + inputs.put("input_messages", tag(context, GenAISemanticAttributes.INPUT_MESSAGES)); + inputs.put("output_messages", tag(context, GenAISemanticAttributes.OUTPUT_MESSAGES)); + break; + default: + break; + } + return inputs; + } + + private static String tag(final AIEvaluationContext context, final String key) { + return context.getTags().getOrDefault(key, ""); + } +} diff --git a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/plan/EvaluationPlan.java b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/plan/EvaluationPlan.java new file mode 100644 index 000000000000..90fb84a72830 --- /dev/null +++ b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/plan/EvaluationPlan.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.ai.evaluation.plan; + +import java.util.List; +import java.util.Map; +import lombok.Builder; +import lombok.Getter; +import org.apache.skywalking.oap.server.ai.evaluation.value.SpanEvaluationType; +import org.apache.skywalking.oap.server.ai.evaluation.task.EvaluationTask; + +@Getter +@Builder +public class EvaluationPlan { + private final SpanEvaluationType spanType; + private final Map context; + private final List tasks; +} diff --git a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/plan/EvaluationPlanner.java b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/plan/EvaluationPlanner.java new file mode 100644 index 000000000000..b055091a9fb4 --- /dev/null +++ b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/plan/EvaluationPlanner.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.ai.evaluation.plan; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.skywalking.oap.server.ai.evaluation.context.AIEvaluationContext; +import org.apache.skywalking.oap.server.ai.evaluation.task.EvaluationTask; +import org.apache.skywalking.oap.server.ai.evaluation.value.SpanEvaluationType; + +public class EvaluationPlanner { + private final EvaluationInputExtractor inputExtractor; + + public EvaluationPlanner(final EvaluationInputExtractor inputExtractor) { + this.inputExtractor = inputExtractor; + } + + public List plan(final AIEvaluationContext context, + final List tasks) { + final List effectiveTasks = new ArrayList<>(); + for (EvaluationTask task : tasks) { + if (task == null) { + continue; + } + effectiveTasks.add(task); + } + + final List plans = new ArrayList<>(); + if (!effectiveTasks.isEmpty()) { + plans.add(EvaluationPlan.builder() + .spanType(SpanEvaluationType.LLM_CALL) + .tasks(effectiveTasks) + .context(inputExtractor.extract(context, SpanEvaluationType.LLM_CALL)) + .build()); + } + return plans; + } +} diff --git a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/plan/EvaluationPromptBuilder.java b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/plan/EvaluationPromptBuilder.java new file mode 100644 index 000000000000..426494ae220d --- /dev/null +++ b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/plan/EvaluationPromptBuilder.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.ai.evaluation.plan; + +import org.apache.skywalking.oap.server.ai.evaluation.judge.JudgeModelRequest; +import org.apache.skywalking.oap.server.ai.evaluation.task.EvaluationTask; + +public class EvaluationPromptBuilder { + private static final String INPUT_MESSAGES = "input_messages"; + private static final String OUTPUT_MESSAGES = "output_messages"; + + private final String systemPrompt; + + public EvaluationPromptBuilder(final String systemPrompt) { + this.systemPrompt = defaultString(systemPrompt); + } + + public JudgeModelRequest build(final EvaluationPlan plan) { + return JudgeModelRequest.builder() + .systemPrompt(systemPrompt) + .userPrompt(buildUserPrompt(plan)) + .build(); + } + + private String buildUserPrompt(final EvaluationPlan plan) { + final StringBuilder prompt = new StringBuilder(); + prompt.append("Input messages:\n"); + prompt.append(defaultString(plan.getContext().get(INPUT_MESSAGES))).append("\n\n"); + + prompt.append("Output messages:\n"); + prompt.append(defaultString(plan.getContext().get(OUTPUT_MESSAGES))).append("\n\n"); + + prompt.append("Evaluation tasks:\n"); + int index = 1; + for (EvaluationTask task : plan.getTasks()) { + prompt.append(index++).append(". ").append(task.getName()).append("\n"); + prompt.append("valueType: ").append(task.getValueType()).append("\n"); + if (task.getAllowedValues() != null && !task.getAllowedValues().isEmpty()) { + prompt.append("allowedValues: ").append(task.getAllowedValues()).append("\n"); + } + prompt.append(defaultString(task.getInstruction())).append("\n\n"); + } + return prompt.toString(); + } + + private static String defaultString(final String value) { + return value == null ? "" : value; + } +} diff --git a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/plan/EvaluationResult.java b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/plan/EvaluationResult.java new file mode 100644 index 000000000000..d781c61196fc --- /dev/null +++ b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/plan/EvaluationResult.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.ai.evaluation.plan; + +import lombok.Builder; +import lombok.Getter; +import lombok.ToString; +import org.apache.skywalking.oap.server.ai.evaluation.value.ValueType; + +@Getter +@Builder +@ToString +public class EvaluationResult { + private final String name; + private final ValueType valueType; + private final String value; + private final String reason; +} diff --git a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/plan/EvaluationResultParser.java b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/plan/EvaluationResultParser.java new file mode 100644 index 000000000000..7606e8dbbf0b --- /dev/null +++ b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/plan/EvaluationResultParser.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.ai.evaluation.plan; + +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import org.apache.skywalking.oap.server.ai.evaluation.task.EvaluationTask; + +import java.util.ArrayList; +import java.util.List; + +public class EvaluationResultParser { + public List parse(final EvaluationPlan plan, final String content) { + final JsonObject root = JsonParser.parseString(content).getAsJsonObject(); + final List results = new ArrayList<>(); + for (EvaluationTask task : plan.getTasks()) { + final JsonObject metric = root.getAsJsonObject(task.getName()); + if (metric == null) { + throw new IllegalArgumentException("Missing evaluation result: " + task.getName()); + } + + final String value = getAsString(metric, "value"); + final String reason = getAsString(metric, "reason"); + validate(task, value); + results.add(EvaluationResult.builder() + .name(task.getName()) + .valueType(task.getValueType()) + .value(value) + .reason(reason) + .build()); + } + return results; + } + + private static void validate(final EvaluationTask task, final String value) { + switch (task.getValueType()) { + case SCORE: + validateScore(task, value); + break; + case BOOLEAN: + validateBoolean(task, value); + break; + case STRING: + validateString(task, value); + break; + case JSON: + validateJson(task, value); + break; + default: + break; + } + } + + private static void validateScore(final EvaluationTask task, final String value) { + try { + final double score = Double.parseDouble(value); + if (score < 0 || score > 1) { + throw new IllegalArgumentException("Invalid score value of " + task.getName() + ": " + value); + } + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid score value of " + task.getName() + ": " + value, e); + } + } + + private static void validateBoolean(final EvaluationTask task, final String value) { + if (!"true".equalsIgnoreCase(value) && !"false".equalsIgnoreCase(value)) { + throw new IllegalArgumentException("Invalid boolean value of " + task.getName() + ": " + value); + } + } + + private static void validateString(final EvaluationTask task, final String value) { + if (task.getAllowedValues() == null || task.getAllowedValues().isEmpty()) { + return; + } + for (String allowedValue : task.getAllowedValues()) { + if (allowedValue.equalsIgnoreCase(value)) { + return; + } + } + throw new IllegalArgumentException("Invalid string value of " + task.getName() + ": " + value); + } + + private static void validateJson(final EvaluationTask task, final String value) { + try { + JsonParser.parseString(value); + } catch (Exception e) { + throw new IllegalArgumentException("Invalid JSON value of " + task.getName() + ": " + value, e); + } + } + + private static String getAsString(final JsonObject object, final String memberName) { + final JsonElement element = object.get(memberName); + return element == null || element.isJsonNull() ? "" : element.getAsString(); + } +} diff --git a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/AIEvaluationMetricReporter.java b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/AIEvaluationMetricReporter.java new file mode 100644 index 000000000000..0f14ec9e054b --- /dev/null +++ b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/AIEvaluationMetricReporter.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.ai.evaluation.service; + +import com.google.common.collect.ImmutableMap; + +import java.util.List; +import java.util.Map; + +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.meter.analyzer.v2.MetricConvert; +import org.apache.skywalking.oap.meter.analyzer.v2.dsl.Sample; +import org.apache.skywalking.oap.meter.analyzer.v2.dsl.SampleFamily; +import org.apache.skywalking.oap.meter.analyzer.v2.dsl.SampleFamilyBuilder; +import org.apache.skywalking.oap.server.ai.evaluation.context.AIEvaluationContext; +import org.apache.skywalking.oap.server.ai.evaluation.plan.EvaluationResult; +import org.apache.skywalking.oap.server.core.config.NamingControl; + +@Slf4j +public class AIEvaluationMetricReporter { + public static final String RULE_CATALOG = "gen-ai-evaluation-rules"; + public static final String RULE_NAME = "gen-ai-model"; + public static final String SAMPLE_SCORE_NAME = "gen_ai_model_evaluation_score_ppm"; + private static final double SCORE_SCALE = 1_000_000D; + + private final List metricConverts; + private final NamingControl namingControl; + + public AIEvaluationMetricReporter(final List metricConverts, + final NamingControl namingControl) { + this.metricConverts = metricConverts; + this.namingControl = namingControl; + } + + public void reportScore(final AIEvaluationContext context, + final EvaluationResult result, + final long sampleTime) { + final double score; + try { + score = Double.parseDouble(result.getValue()); + } catch (NumberFormatException e) { + log.warn("Skip AI evaluation score metric, invalid score: {}", result.getValue(), e); + return; + } + + final Sample sample = Sample.builder() + .name(SAMPLE_SCORE_NAME) + .timestamp(sampleTime) + .value(score * SCORE_SCALE) + .labels(ImmutableMap.copyOf(labels(context, result, namingControl))) + .build(); + final ImmutableMap sampleFamilies = ImmutableMap.of( + SAMPLE_SCORE_NAME, + SampleFamilyBuilder.newBuilder(sample).build() + ); + metricConverts.forEach(convert -> convert.toMeter(sampleFamilies)); + } + + private static Map labels(final AIEvaluationContext context, + final EvaluationResult result, + final NamingControl namingControl) { + return ImmutableMap.of( + "provider_name", defaultString(namingControl.formatServiceName(context.getProviderName())), + "model_name", defaultString(namingControl.formatInstanceName(context.getModelName())), + "task_name", defaultString(result.getName()) + ); + } + + private static String defaultString(final String value) { + return value == null ? "" : value; + } +} diff --git a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/AIEvaluationService.java b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/AIEvaluationService.java new file mode 100644 index 000000000000..a2af8ed5fc2d --- /dev/null +++ b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/AIEvaluationService.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.ai.evaluation.service; + +import java.util.List; +import java.util.Set; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.ai.evaluation.context.AIEvaluationContext; +import org.apache.skywalking.oap.server.ai.evaluation.judge.JudgeModelProvider; +import org.apache.skywalking.oap.server.ai.evaluation.service.sample.AIEvaluationSamplingPolicy; +import org.apache.skywalking.oap.server.ai.evaluation.service.strategy.AIEvaluationStrategy; +import org.apache.skywalking.oap.server.library.util.StringUtil; + +@Slf4j +public class AIEvaluationService implements IAIEvaluationService { + private final AIEvaluationSamplingPolicy samplingPolicy; + private final JudgeModelProvider judgeModelProvider; + private volatile List strategies; + private final Set pendingTaskIds = ConcurrentHashMap.newKeySet(); + private final ThreadPoolExecutor evaluationExecutor = + new ThreadPoolExecutor( + 4, + 4, + 0, + TimeUnit.SECONDS, + new ArrayBlockingQueue<>(100) + ); + + public AIEvaluationService(final AIEvaluationSamplingPolicy samplingPolicy, + final JudgeModelProvider judgeModelProvider) { + this.samplingPolicy = samplingPolicy; + this.judgeModelProvider = judgeModelProvider; + this.strategies = List.of(); + } + + public void setStrategies(final List strategies) { + this.strategies = strategies == null ? List.of() : List.copyOf(strategies); + } + + @Override + public boolean shouldSample(final String traceId) { + return StringUtil.isNotEmpty(traceId) && samplingPolicy.shouldSample(traceId); + } + + @Override + public void sample(final AIEvaluationContext context) { + if (context == null || StringUtil.isEmpty(context.getTraceId())) { + return; + } + + final AIEvaluationStrategy strategy = findStrategy(context); + if (strategy == null) { + return; + } + + final String taskId = strategy.taskId(context); + if (!pendingTaskIds.add(taskId)) { + return; + } + + try { + evaluationExecutor.execute(() -> { + try { + evaluate(context, strategy, taskId); + } finally { + pendingTaskIds.remove(taskId); + } + }); + } catch (RejectedExecutionException e) { + pendingTaskIds.remove(taskId); + log.warn("GenAI span evaluation task rejected, taskId: {}", taskId, e); + } + } + + private void evaluate(final AIEvaluationContext context, + final AIEvaluationStrategy strategy, + final String taskId) { + try { + strategy.evaluate(context, judgeModelProvider); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + log.error("GenAI evaluation interrupted, taskId: {}", taskId, e); + } catch (Exception e) { + log.error("GenAI evaluation failed, taskId: {}", taskId, e); + } + } + + private AIEvaluationStrategy findStrategy(final AIEvaluationContext context) { + for (AIEvaluationStrategy strategy : strategies) { + if (strategy.support(context)) { + return strategy; + } + } + return null; + } +} diff --git a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/IAIEvaluationService.java b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/IAIEvaluationService.java new file mode 100644 index 000000000000..3f9f883889aa --- /dev/null +++ b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/IAIEvaluationService.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.ai.evaluation.service; + +import org.apache.skywalking.oap.server.ai.evaluation.context.AIEvaluationContext; +import org.apache.skywalking.oap.server.library.module.Service; + +public interface IAIEvaluationService extends Service { + + boolean shouldSample(String traceId); + + void sample(AIEvaluationContext context); +} diff --git a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/sample/AIEvaluationSamplingPolicy.java b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/sample/AIEvaluationSamplingPolicy.java new file mode 100644 index 000000000000..b125cd5b020c --- /dev/null +++ b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/sample/AIEvaluationSamplingPolicy.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.ai.evaluation.service.sample; + +public interface AIEvaluationSamplingPolicy { + + boolean shouldSample(String traceId); +} diff --git a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/sample/DefaultAIEvaluationSamplingPolicy.java b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/sample/DefaultAIEvaluationSamplingPolicy.java new file mode 100644 index 000000000000..7a3452397521 --- /dev/null +++ b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/sample/DefaultAIEvaluationSamplingPolicy.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.ai.evaluation.service.sample; + +import com.google.common.hash.Hashing; +import java.nio.charset.StandardCharsets; + +public class DefaultAIEvaluationSamplingPolicy implements AIEvaluationSamplingPolicy { + private static final long HASH_SPACE = 1_000_000L; + private final int sampleRate; + + public DefaultAIEvaluationSamplingPolicy(final int sampleRate) { + this.sampleRate = sampleRate; + } + + @Override + public boolean shouldSample(final String traceId) { + final long hash = Hashing.murmur3_128() + .hashString(traceId, StandardCharsets.UTF_8) + .asLong(); + return Math.floorMod(hash, HASH_SPACE) < sampleRate; + } +} diff --git a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/strategy/AIEvaluationStrategy.java b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/strategy/AIEvaluationStrategy.java new file mode 100644 index 000000000000..660f8e8cb3d9 --- /dev/null +++ b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/strategy/AIEvaluationStrategy.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.ai.evaluation.service.strategy; + +import java.io.IOException; +import org.apache.skywalking.oap.server.ai.evaluation.judge.JudgeModelProvider; +import org.apache.skywalking.oap.server.ai.evaluation.context.AIEvaluationContext; + +public interface AIEvaluationStrategy { + boolean support(AIEvaluationContext context); + + String taskId(AIEvaluationContext context); + + void evaluate(AIEvaluationContext context, JudgeModelProvider judgeModelProvider) + throws IOException, InterruptedException; +} diff --git a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/strategy/span/SpanAIEvaluationStrategy.java b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/strategy/span/SpanAIEvaluationStrategy.java new file mode 100644 index 000000000000..41fb95aceb34 --- /dev/null +++ b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/strategy/span/SpanAIEvaluationStrategy.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.ai.evaluation.service.strategy.span; + +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.ai.evaluation.context.AIEvaluationContext; +import org.apache.skywalking.oap.server.ai.evaluation.context.GenAISemanticAttributes; +import org.apache.skywalking.oap.server.ai.evaluation.judge.JudgeModelProvider; +import org.apache.skywalking.oap.server.ai.evaluation.judge.JudgeModelResponse; +import org.apache.skywalking.oap.server.ai.evaluation.level.EvaluationLevelResolver; +import org.apache.skywalking.oap.server.ai.evaluation.plan.EvaluationPlan; +import org.apache.skywalking.oap.server.ai.evaluation.plan.EvaluationPlanner; +import org.apache.skywalking.oap.server.ai.evaluation.plan.EvaluationPromptBuilder; +import org.apache.skywalking.oap.server.ai.evaluation.plan.EvaluationResult; +import org.apache.skywalking.oap.server.ai.evaluation.plan.EvaluationResultParser; +import org.apache.skywalking.oap.server.ai.evaluation.service.AIEvaluationMetricReporter; +import org.apache.skywalking.oap.server.ai.evaluation.service.strategy.AIEvaluationStrategy; +import org.apache.skywalking.oap.server.ai.evaluation.task.EvaluationTaskRegistry; +import org.apache.skywalking.oap.server.ai.evaluation.value.ValueType; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.Layer; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.manual.genai.GenAIEvaluationRecord; +import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor; +import org.apache.skywalking.oap.server.core.config.NamingControl; +import org.apache.skywalking.oap.server.library.util.StringUtil; + +import java.io.IOException; +import java.util.List; +import java.util.Optional; + +@Slf4j +public class SpanAIEvaluationStrategy implements AIEvaluationStrategy { + private static final String CHAT_OPERATION = "chat"; + + private final EvaluationTaskRegistry taskRegistry; + private final EvaluationPlanner evaluationPlanner; + private final EvaluationPromptBuilder promptBuilder; + private final EvaluationResultParser resultParser; + private final AIEvaluationMetricReporter metricReporter; + private final NamingControl namingControl; + private final EvaluationLevelResolver levelResolver; + + public SpanAIEvaluationStrategy(final EvaluationTaskRegistry taskRegistry, + final EvaluationPlanner evaluationPlanner, + final EvaluationPromptBuilder promptBuilder, + final EvaluationResultParser resultParser, + final AIEvaluationMetricReporter metricReporter, + final NamingControl namingControl, + final EvaluationLevelResolver levelResolver) { + this.taskRegistry = taskRegistry; + this.evaluationPlanner = evaluationPlanner; + this.promptBuilder = promptBuilder; + this.resultParser = resultParser; + this.metricReporter = metricReporter; + this.namingControl = namingControl; + this.levelResolver = levelResolver; + } + + @Override + public boolean support(final AIEvaluationContext context) { + return context != null && StringUtil.isNotEmpty(context.getTraceId()) + && StringUtil.isNotEmpty(context.getSpanId()); + } + + @Override + public String taskId(final AIEvaluationContext context) { + return context.getTraceId() + "-" + context.getSpanId(); + } + + @Override + public void evaluate(final AIEvaluationContext context, + final JudgeModelProvider judgeModelProvider) throws IOException, InterruptedException { + if (taskRegistry.isEmpty()) { + log.debug("Skip GenAI span evaluation, no evaluation task configured, taskId: {}", taskId(context)); + return; + } + + if (validLLMCallSpan(context)) { + evaluateLLMCallSpan(context, judgeModelProvider); + } + } + + private void evaluateLLMCallSpan(final AIEvaluationContext context, + final JudgeModelProvider judgeModelProvider) + throws IOException, InterruptedException { + final String judgeModel = judgeModelProvider.model(); + final List plans = evaluationPlanner.plan(context, taskRegistry.tasks()); + for (EvaluationPlan plan : plans) { + final Optional response = judgeModelProvider.judge(promptBuilder.build(plan)); + if (response.isEmpty()) { + continue; + } + + final JudgeModelResponse judgeResponse = response.get(); + final List results = resultParser.parse(plan, judgeResponse.getContent()); + persistResults(context, plan, results, judgeModel); + } + } + + private void persistResults(final AIEvaluationContext context, + final EvaluationPlan plan, + final List results, + final String judgeModel) { + final long evaluationTime = System.currentTimeMillis(); + for (EvaluationResult result : results) { + final String serviceId = IDManager.ServiceID.buildId( + namingControl.formatServiceName(context.getProviderName()), + Layer.VIRTUAL_GENAI.isNormal() + ); + final GenAIEvaluationRecord record = new GenAIEvaluationRecord(); + record.setTraceId(context.getTraceId()); + record.setServiceId(serviceId); + record.setServiceInstanceId(IDManager.ServiceInstanceID.buildId( + serviceId, + namingControl.formatServiceName(context.getModelName()) + )); + record.setSegmentId(context.getSegmentId()); + record.setSpanId(context.getSpanId()); + record.setSpanType(plan.getSpanType() == null ? "" : plan.getSpanType().name()); + record.setTaskName(result.getName()); + record.setValueType(result.getValueType() == null ? "" : result.getValueType().name()); + record.setValue(result.getValue()); + record.setEvaluationLevel(levelResolver.resolve(result.getValueType(), result.getValue())); + record.setReason(result.getReason()); + record.setJudgeModel(judgeModel); + record.setEvaluationTime(evaluationTime); + record.setTimeBucket(TimeBucket.getRecordTimeBucket(evaluationTime)); + RecordStreamProcessor.getInstance().in(record); + + if (result.getValueType() == ValueType.SCORE) { + metricReporter.reportScore(context, result, evaluationTime); + } + } + } + + private static boolean validLLMCallSpan(final AIEvaluationContext context) { + if (!CHAT_OPERATION.equalsIgnoreCase(operationName(context))) { + return false; + } + + final String inputMessages = context.getTags().get(GenAISemanticAttributes.INPUT_MESSAGES); + final String outputMessages = context.getTags().get(GenAISemanticAttributes.OUTPUT_MESSAGES); + if (StringUtil.isEmpty(inputMessages) || StringUtil.isEmpty(outputMessages)) { + log.warn( + "Skip GenAI span evaluation, missing input or output messages,trace id :{}", + context.getTraceId() + ); + return false; + } + return true; + } + + private static String operationName(final AIEvaluationContext context) { + return context.getTags().get(GenAISemanticAttributes.OPERATION_NAME); + } + +} diff --git a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/task/EvaluationTask.java b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/task/EvaluationTask.java new file mode 100644 index 000000000000..6b5d7b80d834 --- /dev/null +++ b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/task/EvaluationTask.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.ai.evaluation.task; + +import java.util.ArrayList; +import java.util.List; + +import lombok.Getter; +import lombok.Setter; +import org.apache.skywalking.oap.server.ai.evaluation.value.ValueType; + +@Getter +@Setter +public class EvaluationTask { + private String name; + private ValueType valueType = ValueType.SCORE; + private List allowedValues = new ArrayList<>(); + private String instruction; +} diff --git a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/task/EvaluationTaskRegistry.java b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/task/EvaluationTaskRegistry.java new file mode 100644 index 000000000000..e06e662908cc --- /dev/null +++ b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/task/EvaluationTaskRegistry.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.ai.evaluation.task; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.apache.skywalking.oap.server.library.util.StringUtil; + +public class EvaluationTaskRegistry { + private final List tasks; + + public EvaluationTaskRegistry(final List tasks) { + if (tasks == null || tasks.isEmpty()) { + this.tasks = Collections.emptyList(); + return; + } + + final Map registeredTasks = new LinkedHashMap<>(); + for (EvaluationTask task : tasks) { + if (task == null || StringUtil.isEmpty(task.getName())) { + continue; + } + registeredTasks.put(task.getName(), task); + } + this.tasks = List.copyOf(registeredTasks.values()); + } + + public List tasks() { + return tasks; + } + + public boolean isEmpty() { + return tasks.isEmpty(); + } + +} diff --git a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/value/SpanEvaluationType.java b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/value/SpanEvaluationType.java new file mode 100644 index 000000000000..f0755ead894a --- /dev/null +++ b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/value/SpanEvaluationType.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.ai.evaluation.value; + +public enum SpanEvaluationType { + LLM_CALL +} diff --git a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/value/ValueType.java b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/value/ValueType.java new file mode 100644 index 000000000000..90f90d451da8 --- /dev/null +++ b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/value/ValueType.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.ai.evaluation.value; + +public enum ValueType { + SCORE, + BOOLEAN, + STRING, + JSON +} diff --git a/oap-server/ai-evaluation/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.core.trace.SpanListener b/oap-server/ai-evaluation/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.core.trace.SpanListener new file mode 100644 index 000000000000..09ef7981b055 --- /dev/null +++ b/oap-server/ai-evaluation/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.core.trace.SpanListener @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.apache.skywalking.oap.server.ai.evaluation.listener.AIEvaluationSpanListener diff --git a/oap-server/ai-evaluation/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine b/oap-server/ai-evaluation/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine new file mode 100644 index 000000000000..566c2d963303 --- /dev/null +++ b/oap-server/ai-evaluation/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +org.apache.skywalking.oap.server.ai.evaluation.AIEvaluationModule diff --git a/oap-server/ai-evaluation/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider b/oap-server/ai-evaluation/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider new file mode 100644 index 000000000000..fc324d7d0efe --- /dev/null +++ b/oap-server/ai-evaluation/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +org.apache.skywalking.oap.server.ai.evaluation.AIEvaluationProvider diff --git a/oap-server/ai-evaluation/src/test/java/org/apache/skywalking/oap/server/ai/evaluation/judge/provider/OpenAICompatibleProviderTest.java b/oap-server/ai-evaluation/src/test/java/org/apache/skywalking/oap/server/ai/evaluation/judge/provider/OpenAICompatibleProviderTest.java new file mode 100644 index 000000000000..8c4e32d17bad --- /dev/null +++ b/oap-server/ai-evaluation/src/test/java/org/apache/skywalking/oap/server/ai/evaluation/judge/provider/OpenAICompatibleProviderTest.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.ai.evaluation.judge.provider; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.lang.reflect.Method; +import java.util.Properties; +import org.apache.skywalking.oap.server.ai.evaluation.judge.JudgeModelRequest; +import org.apache.skywalking.oap.server.library.module.ModuleStartException; +import org.junit.jupiter.api.Test; + +class OpenAICompatibleProviderTest { + + @Test + void shouldRejectInvalidTemperatureOnStartup() { + final Properties config = validConfig(); + config.setProperty("temperature", "3.1"); + final ModuleStartException exception = assertThrows( + ModuleStartException.class, + () -> new OpenAICompatibleProvider(config) + ); + assertTrue(exception.getMessage().contains("temperature")); + } + + @Test + void shouldRejectInvalidMaxTokensOnStartup() { + final Properties config = validConfig(); + config.setProperty("max_tokens", "0"); + final ModuleStartException exception = assertThrows( + ModuleStartException.class, + () -> new OpenAICompatibleProvider(config) + ); + assertTrue(exception.getMessage().contains("max_tokens")); + } + + @Test + void shouldRejectInvalidRequestTimeoutOnStartup() { + final Properties config = validConfig(); + config.setProperty("request-timeout-seconds", "0"); + final ModuleStartException exception = assertThrows( + ModuleStartException.class, + () -> new OpenAICompatibleProvider(config) + ); + assertTrue(exception.getMessage().contains("request-timeout-seconds")); + } + + @Test + void shouldIncludeTemperatureAndMaxTokensInRequestBody() throws Exception { + final Properties config = validConfig(); + config.setProperty("request-timeout-seconds", "45"); + config.setProperty("temperature", "0.25"); + config.setProperty("max_tokens", "2048"); + final OpenAICompatibleProvider provider = new OpenAICompatibleProvider(config); + final Method method = OpenAICompatibleProvider.class.getDeclaredMethod( + "buildRequestBody", + JudgeModelRequest.class + ); + method.setAccessible(true); + final String body = (String) method.invoke( + provider, + JudgeModelRequest.builder() + .systemPrompt("sys") + .userPrompt("usr") + .build() + ); + assertTrue(body.contains("\"temperature\":0.25")); + assertTrue(body.contains("\"max_tokens\":2048")); + assertTrue(body.contains("\"model\":\"gpt-5.4-mini\"")); + } + + private static Properties validConfig() { + final Properties config = new Properties(); + config.setProperty("endpoint", "https://example.com/v1/chat/completions"); + config.setProperty("model", "gpt-5.4-mini"); + config.setProperty("api-key", "test-key"); + return config; + } +} diff --git a/oap-server/analyzer/agent-analyzer/pom.xml b/oap-server/analyzer/agent-analyzer/pom.xml index 1fe05db26720..25f176f9eaef 100644 --- a/oap-server/analyzer/agent-analyzer/pom.xml +++ b/oap-server/analyzer/agent-analyzer/pom.xml @@ -48,6 +48,11 @@ gen-ai-analyzer ${project.version} + + org.apache.skywalking + ai-evaluation + ${project.version} + org.apache.skywalking server-testing @@ -55,4 +60,4 @@ test - \ No newline at end of file + diff --git a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/AnalyzerModuleProvider.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/AnalyzerModuleProvider.java index 0c19854bb0fd..28881099dada 100644 --- a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/AnalyzerModuleProvider.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/AnalyzerModuleProvider.java @@ -34,10 +34,12 @@ import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.SegmentParserListenerManager; import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.SegmentParserServiceImpl; import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener.EndpointDepFromCrossThreadAnalysisListener; +import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener.GenAIEvaluationAnalysisListener; import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener.NetworkAddressAliasMappingListener; import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener.RPCAnalysisListener; import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener.SegmentAnalysisListener; import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener.VirtualServiceAnalysisListener; +import org.apache.skywalking.oap.server.ai.evaluation.AIEvaluationModule; import org.apache.skywalking.oap.server.configuration.api.ConfigurationModule; import org.apache.skywalking.oap.server.configuration.api.DynamicConfigurationService; import org.apache.skywalking.oap.server.core.CoreModule; @@ -161,6 +163,9 @@ private SegmentParserListenerManager listenerManager() { } listenerManager.add(new SegmentAnalysisListener.Factory(getManager(), moduleConfig)); listenerManager.add(new VirtualServiceAnalysisListener.Factory(getManager())); + if (getManager().has(AIEvaluationModule.NAME)) { + listenerManager.add(new GenAIEvaluationAnalysisListener.Factory(getManager())); + } return listenerManager; } diff --git a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/GenAIEvaluationAnalysisListener.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/GenAIEvaluationAnalysisListener.java new file mode 100644 index 000000000000..2a3b3668c34a --- /dev/null +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/GenAIEvaluationAnalysisListener.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener; + +import lombok.RequiredArgsConstructor; +import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair; +import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject; +import org.apache.skywalking.apm.network.language.agent.v3.SpanLayer; +import org.apache.skywalking.apm.network.language.agent.v3.SpanObject; +import org.apache.skywalking.oap.server.ai.evaluation.context.AIEvaluationContext; +import org.apache.skywalking.oap.server.ai.evaluation.AIEvaluationModule; +import org.apache.skywalking.oap.server.ai.evaluation.context.GenAIContextResolver; +import org.apache.skywalking.oap.server.ai.evaluation.context.GenAISemanticAttributes; +import org.apache.skywalking.oap.server.ai.evaluation.service.IAIEvaluationService; +import org.apache.skywalking.oap.server.analyzer.provider.AnalyzerModuleConfig; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.util.StringUtil; + +import java.util.Map; +import java.util.stream.Collectors; + +@RequiredArgsConstructor +public class GenAIEvaluationAnalysisListener implements EntryAnalysisListener, ExitAnalysisListener, LocalAnalysisListener { + private final IAIEvaluationService evaluationService; + + @Override + public void parseEntry(final SpanObject span, final SegmentObject segmentObject) { + sample(span, segmentObject); + } + + @Override + public void parseExit(final SpanObject span, final SegmentObject segmentObject) { + sample(span, segmentObject); + } + + @Override + public void parseLocal(final SpanObject span, final SegmentObject segmentObject) { + sample(span, segmentObject); + } + + @Override + public void build() { + } + + @Override + public boolean containsPoint(final Point point) { + return point == Point.Entry || point == Point.Exit || point == Point.Local; + } + + private void sample(final SpanObject span, final SegmentObject segmentObject) { + if (span.getSkipAnalysis() || span.getSpanLayer() != SpanLayer.GenAI) { + return; + } + if (!evaluationService.shouldSample(segmentObject.getTraceId())) { + return; + } + + final Map tags = span.getTagsList().stream() + .collect(Collectors.toMap( + KeyStringValuePair::getKey, + KeyStringValuePair::getValue, + (left, right) -> left + )); + + if (StringUtil.isBlank(tags.get(GenAISemanticAttributes.RESPONSE_MODEL))) { + return; + } + + final GenAIContextResolver.Result genAIContext = GenAIContextResolver.resolve(tags); + evaluationService.sample(AIEvaluationContext.builder() + .traceId(segmentObject.getTraceId()) + .segmentId(segmentObject.getTraceSegmentId()) + .spanId(String.valueOf(span.getSpanId())) + .serviceName(segmentObject.getService()) + .serviceInstanceName(segmentObject.getServiceInstance()) + .operationName(span.getOperationName()) + .providerName(genAIContext.getProviderName()) + .modelName(genAIContext.getModelName()) + .startTimeMillis(span.getStartTime()) + .endTimeMillis(span.getEndTime()) + .error(span.getIsError()) + .tags(tags) + .build()); + } + + public static class Factory implements AnalysisListenerFactory { + private final IAIEvaluationService evaluationService; + + public Factory(final ModuleManager moduleManager) { + this.evaluationService = moduleManager.find(AIEvaluationModule.NAME) + .provider() + .getService(IAIEvaluationService.class); + } + + @Override + public AnalysisListener create(final ModuleManager moduleManager, final AnalyzerModuleConfig config) { + return new GenAIEvaluationAnalysisListener(evaluationService); + } + } +} diff --git a/oap-server/pom.xml b/oap-server/pom.xml index bb88dde9f1ba..dc21a504fdca 100755 --- a/oap-server/pom.xml +++ b/oap-server/pom.xml @@ -48,6 +48,7 @@ server-fetcher-plugin server-health-checker ai-pipeline + ai-evaluation mqe-grammar mqe-rt diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModule.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModule.java index 6e3702b89364..3ed2cb168c55 100755 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModule.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModule.java @@ -49,6 +49,7 @@ import org.apache.skywalking.oap.server.core.query.AlarmQueryService; import org.apache.skywalking.oap.server.core.query.BrowserLogQueryService; import org.apache.skywalking.oap.server.core.query.EventQueryService; +import org.apache.skywalking.oap.server.core.query.GenAIEvaluationRecordQueryService; import org.apache.skywalking.oap.server.core.query.HierarchyQueryService; import org.apache.skywalking.oap.server.core.query.LogQueryService; import org.apache.skywalking.oap.server.core.query.MetadataQueryService; @@ -168,6 +169,7 @@ private void addQueryService(List classes) { classes.add(RecordQueryService.class); classes.add(HierarchyQueryService.class); classes.add(TTLStatusQuery.class); + classes.add(GenAIEvaluationRecordQueryService.class); } private void addServerInterface(List classes) { diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java index bd6115c04199..a4ecdfd5bc4b 100755 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java @@ -23,6 +23,7 @@ import org.apache.skywalking.oap.server.configuration.api.ConfigurationModule; import org.apache.skywalking.oap.server.configuration.api.DynamicConfigurationService; import org.apache.skywalking.oap.server.core.analysis.ApdexThresholdConfig; +import org.apache.skywalking.oap.server.core.query.GenAIEvaluationRecordQueryService; import org.apache.skywalking.oap.server.core.rule.ext.RuleSetMerger; import org.apache.skywalking.oap.server.core.analysis.DisableRegister; import org.apache.skywalking.oap.server.core.analysis.Layer; @@ -366,6 +367,9 @@ TTLStatusQuery.class, new TTLStatusQuery( this.registerServiceImplementation( ContinuousProfilingQueryService.class, new ContinuousProfilingQueryService(getManager())); + this.registerServiceImplementation( + GenAIEvaluationRecordQueryService.class, new GenAIEvaluationRecordQueryService(getManager())); + this.registerServiceImplementation(CommandService.class, new CommandService(getManager())); // add oal engine loader service implementations diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/genai/GenAIEvaluationRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/genai/GenAIEvaluationRecord.java new file mode 100644 index 000000000000..ce5d14163780 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/genai/GenAIEvaluationRecord.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.analysis.manual.genai; + +import lombok.Getter; +import lombok.Setter; +import org.apache.skywalking.oap.server.core.analysis.Stream; +import org.apache.skywalking.oap.server.core.analysis.record.Record; +import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor; +import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; +import org.apache.skywalking.oap.server.core.source.ScopeDeclaration; +import org.apache.skywalking.oap.server.core.storage.StorageID; +import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDB; +import org.apache.skywalking.oap.server.core.storage.annotation.Column; +import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearch; +import org.apache.skywalking.oap.server.core.storage.annotation.SuperDataset; +import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity; +import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage; +import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder; + +@Getter +@Setter +@SuperDataset +@ScopeDeclaration(id = DefaultScopeDefine.GEN_AI_EVALUATION_RECORD, name = "GenAIEvaluationRecord") +@Stream(name = GenAIEvaluationRecord.INDEX_NAME, scopeId = DefaultScopeDefine.GEN_AI_EVALUATION_RECORD, + builder = GenAIEvaluationRecord.Builder.class, processor = RecordStreamProcessor.class) +@BanyanDB.TimestampColumn(GenAIEvaluationRecord.EVALUATION_TIME) +@BanyanDB.Group(streamGroup = BanyanDB.StreamGroup.RECORDS) +public class GenAIEvaluationRecord extends Record { + + public static final String INDEX_NAME = "gen_ai_evaluation_record"; + public static final String TRACE_ID = "trace_id"; + public static final String SERVICE_ID = "service_id"; + public static final String SERVICE_INSTANCE_ID = "service_instance_id"; + public static final String SEGMENT_ID = "segment_id"; + public static final String SPAN_ID = "span_id"; + public static final String SPAN_TYPE = "span_type"; + public static final String TASK_NAME = "task_name"; + public static final String VALUE_TYPE = "value_type"; + public static final String VALUE = "value"; + public static final String EVALUATION_LEVEL = "evaluation_level"; + public static final String REASON = "reason"; + public static final String JUDGE_MODEL = "judge_model"; + public static final String EVALUATION_TIME = "evaluation_time"; + + @Column(name = TRACE_ID, length = 150) + @BanyanDB.IndexRule(indexType = BanyanDB.IndexRule.IndexType.SKIPPING) + private String traceId; + + @ElasticSearch.EnableDocValues + @Column(name = SERVICE_ID, length = 150, storageOnly = true) + @BanyanDB.SeriesID(index = 0) + private String serviceId; + + @ElasticSearch.EnableDocValues + @Column(name = SERVICE_INSTANCE_ID, length = 150, storageOnly = true) + @BanyanDB.SeriesID(index = 1) + private String serviceInstanceId; + + @Column(name = SEGMENT_ID, length = 150, storageOnly = true) + private String segmentId; + + @Column(name = SPAN_ID, length = 150, storageOnly = true) + private String spanId; + + @Column(name = SPAN_TYPE, length = 64) + private String spanType; + + @ElasticSearch.EnableDocValues + @Column(name = TASK_NAME, length = 512) + private String taskName; + + @Column(name = VALUE_TYPE, length = 64, storageOnly = true) + private String valueType; + + @Column(name = VALUE, length = 4096, storageOnly = true) + private String value; + + @ElasticSearch.EnableDocValues + @Column(name = EVALUATION_LEVEL, length = 64) + @BanyanDB.IndexRule(indexType = BanyanDB.IndexRule.IndexType.SKIPPING) + private String evaluationLevel; + + @Column(name = REASON, length = 4096, storageOnly = true) + private String reason; + + @Column(name = JUDGE_MODEL, length = 64, storageOnly = true) + private String judgeModel; + + @ElasticSearch.EnableDocValues + @Column(name = EVALUATION_TIME) + private long evaluationTime; + + @Override + public StorageID id() { + return new StorageID() + .append(TRACE_ID, traceId) + .append(SERVICE_ID, serviceId) + .append(SERVICE_INSTANCE_ID, serviceInstanceId) + .append(SPAN_ID, spanId) + .append(SPAN_TYPE, spanType) + .append(TASK_NAME, taskName) + .append(EVALUATION_LEVEL, evaluationLevel) + .append(EVALUATION_TIME, evaluationTime); + } + + public static class Builder implements StorageBuilder { + @Override + public GenAIEvaluationRecord storage2Entity(final Convert2Entity converter) { + final GenAIEvaluationRecord record = new GenAIEvaluationRecord(); + record.setTraceId((String) converter.get(TRACE_ID)); + record.setServiceId((String) converter.get(SERVICE_ID)); + record.setServiceInstanceId((String) converter.get(SERVICE_INSTANCE_ID)); + record.setSegmentId((String) converter.get(SEGMENT_ID)); + record.setSpanId((String) converter.get(SPAN_ID)); + record.setSpanType((String) converter.get(SPAN_TYPE)); + record.setTaskName((String) converter.get(TASK_NAME)); + record.setValueType((String) converter.get(VALUE_TYPE)); + record.setValue((String) converter.get(VALUE)); + record.setEvaluationLevel((String) converter.get(EVALUATION_LEVEL)); + record.setReason((String) converter.get(REASON)); + record.setJudgeModel((String) converter.get(JUDGE_MODEL)); + record.setEvaluationTime(((Number) converter.get(EVALUATION_TIME)).longValue()); + record.setTimeBucket(((Number) converter.get(TIME_BUCKET)).longValue()); + return record; + } + + @Override + public void entity2Storage(final GenAIEvaluationRecord storageData, final Convert2Storage converter) { + converter.accept(TRACE_ID, storageData.getTraceId()); + converter.accept(SERVICE_ID, storageData.getServiceId()); + converter.accept(SERVICE_INSTANCE_ID, storageData.getServiceInstanceId()); + converter.accept(SEGMENT_ID, storageData.getSegmentId()); + converter.accept(SPAN_ID, storageData.getSpanId()); + converter.accept(SPAN_TYPE, storageData.getSpanType()); + converter.accept(TASK_NAME, storageData.getTaskName()); + converter.accept(VALUE_TYPE, storageData.getValueType()); + converter.accept(VALUE, storageData.getValue()); + converter.accept(EVALUATION_LEVEL, storageData.getEvaluationLevel()); + converter.accept(REASON, storageData.getReason()); + converter.accept(JUDGE_MODEL, storageData.getJudgeModel()); + converter.accept(EVALUATION_TIME, storageData.getEvaluationTime()); + converter.accept(TIME_BUCKET, storageData.getTimeBucket()); + } + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/MeterEntity.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/MeterEntity.java index 3845bd56d25b..57759578481d 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/MeterEntity.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/MeterEntity.java @@ -73,15 +73,16 @@ private MeterEntity() { public String id() { switch (scopeType) { case SERVICE: - // In Meter system, only normal service, because we don't conjecture any node. - return IDManager.ServiceID.buildId(serviceName, true); + return IDManager.ServiceID.buildId(serviceName, normal()); case SERVICE_INSTANCE: return IDManager.ServiceInstanceID.buildId( - IDManager.ServiceID.buildId(serviceName, true), instanceName); + IDManager.ServiceID.buildId(serviceName, normal()), instanceName); case ENDPOINT: - return IDManager.EndpointID.buildId(IDManager.ServiceID.buildId(serviceName, true), endpointName); + return IDManager.EndpointID.buildId(IDManager.ServiceID.buildId(serviceName, normal()), endpointName); case PROCESS: - return IDManager.ProcessID.buildId(IDManager.ServiceInstanceID.buildId(IDManager.ServiceID.buildId(serviceName, true), instanceName), processName); + return IDManager.ProcessID.buildId( + IDManager.ServiceInstanceID.buildId(IDManager.ServiceID.buildId(serviceName, normal()), instanceName), + processName); case SERVICE_RELATION: return IDManager.ServiceID.buildRelationId(new IDManager.ServiceID.ServiceRelationDefine( sourceServiceId(), @@ -103,7 +104,7 @@ public String id() { } public String serviceId() { - return IDManager.ServiceID.buildId(serviceName, true); + return IDManager.ServiceID.buildId(serviceName, normal()); } public String serviceInstanceId() { @@ -118,6 +119,10 @@ public String destServiceId() { return IDManager.ServiceID.buildId(destServiceName, true); } + private boolean normal() { + return layer == null || layer.isNormal(); + } + public String sourceServiceInstanceId() { return IDManager.ServiceInstanceID.buildId(sourceServiceId(), sourceInstanceName); } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/GenAIEvaluationRecordQueryService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/GenAIEvaluationRecordQueryService.java new file mode 100644 index 000000000000..4fb443881c29 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/GenAIEvaluationRecordQueryService.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query; + +import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag; +import org.apache.skywalking.oap.server.core.query.enumeration.Order; +import org.apache.skywalking.oap.server.core.query.input.Duration; +import org.apache.skywalking.oap.server.core.query.input.TraceScopeCondition; +import org.apache.skywalking.oap.server.core.query.type.GenAIEvaluationRecords; +import org.apache.skywalking.oap.server.core.query.type.Pagination; +import org.apache.skywalking.oap.server.core.query.type.debugging.DebuggingSpan; +import org.apache.skywalking.oap.server.core.query.type.debugging.DebuggingTraceContext; +import org.apache.skywalking.oap.server.core.storage.StorageModule; +import org.apache.skywalking.oap.server.core.storage.query.IGenAIEvaluationRecordQueryDAO; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.module.Service; + +import java.io.IOException; +import java.util.List; +import static org.apache.skywalking.oap.server.core.query.type.debugging.DebuggingTraceContext.TRACE_CONTEXT; + +public class GenAIEvaluationRecordQueryService implements Service { + + private final ModuleManager moduleManager; + private IGenAIEvaluationRecordQueryDAO genAIEvaluationRecordQueryDAO; + + public GenAIEvaluationRecordQueryService(ModuleManager moduleManager) { + this.moduleManager = moduleManager; + } + + private IGenAIEvaluationRecordQueryDAO getGenAIEvaluationRecordQueryDAO() { + if (genAIEvaluationRecordQueryDAO == null) { + this.genAIEvaluationRecordQueryDAO = moduleManager.find(StorageModule.NAME).provider().getService(IGenAIEvaluationRecordQueryDAO.class); + } + return genAIEvaluationRecordQueryDAO; + } + + public boolean supportQueryGenAIEvaluationRecordByKeywords() { + return getGenAIEvaluationRecordQueryDAO().supportQueryGenAIEvaluationRecordByKeywords(); + } + + public GenAIEvaluationRecords queryGenAIEvaluationRecord(String serviceId, + String serviceInstanceId, + TraceScopeCondition relatedTrace, + Pagination paging, + Order queryOrder, + final Duration duration, + final List tags) throws IOException { + DebuggingTraceContext traceContext = TRACE_CONTEXT.get(); + DebuggingSpan span = null; + try { + if (traceContext != null) { + StringBuilder msg = new StringBuilder(); + span = traceContext.createSpan("Query Service: queryGenAIEvaluationRecord"); + msg.append("ServiceId: ").append(serviceId).append(", "); + msg.append("ServiceInstanceId: ").append(serviceInstanceId).append(", "); + msg.append("RelatedTrace: ").append(relatedTrace).append(", "); + msg.append("Pagination: ").append(paging).append(", "); + msg.append("QueryOrder: ").append(queryOrder).append(", "); + msg.append("Duration: ").append(duration).append(", "); + msg.append("Tags: ").append(tags); + span.setMsg(msg.toString()); + } + return queryGenAIEvaluationRecordInternal( + serviceId, serviceInstanceId, relatedTrace, paging, queryOrder, duration, tags + ); + } finally { + if (traceContext != null) { + traceContext.stopSpan(span); + } + } + } + + private GenAIEvaluationRecords queryGenAIEvaluationRecordInternal(String serviceId, + String serviceInstanceId, + TraceScopeCondition relatedTrace, + Pagination paging, + Order queryOrder, + final Duration duration, + final List tags) throws IOException { + PaginationUtils.Page page = PaginationUtils.INSTANCE.exchange(paging); + + return getGenAIEvaluationRecordQueryDAO().queryGenAIEvaluationRecordDebuggable(serviceId, + serviceInstanceId, + relatedTrace, + queryOrder, + page.getFrom(), page.getLimit(), + duration, tags + ); + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/GenAIEvaluationRecordQueryCondition.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/GenAIEvaluationRecordQueryCondition.java new file mode 100644 index 000000000000..e61f735dab94 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/GenAIEvaluationRecordQueryCondition.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.input; + +import java.util.List; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag; +import org.apache.skywalking.oap.server.core.query.enumeration.Order; +import org.apache.skywalking.oap.server.core.query.type.Pagination; + +@Getter +@Setter +@ToString +public class GenAIEvaluationRecordQueryCondition { + private String serviceId; + private String serviceInstanceId; + private TraceScopeCondition relatedTrace; + private Duration queryDuration; + private Pagination paging; + private List tags; + private Order queryOrder; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/GenAIEvaluationRecords.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/GenAIEvaluationRecords.java new file mode 100644 index 000000000000..1d8f20b61d50 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/GenAIEvaluationRecords.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.type; + +import lombok.Getter; +import lombok.Setter; +import lombok.experimental.Accessors; +import org.apache.skywalking.oap.server.core.analysis.manual.genai.GenAIEvaluationRecord; +import org.apache.skywalking.oap.server.core.query.type.debugging.DebuggingTrace; + +import java.util.ArrayList; +import java.util.List; + +@Setter +@Getter +@Accessors(chain = true) +public class GenAIEvaluationRecords { + private final List genAIEvaluationRecordList; + private String errorReason; + private DebuggingTrace debuggingTrace; + + public GenAIEvaluationRecords() { + this.genAIEvaluationRecordList = new ArrayList<>(); + } + + public GenAIEvaluationRecords(final List genAIEvaluationRecords) { + this.genAIEvaluationRecordList = genAIEvaluationRecords; + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java index f768aac32eb6..3e1ddbe24f80 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java @@ -159,6 +159,7 @@ public class DefaultScopeDefine { public static final int GEN_AI_PROVIDER_ACCESS = 96; public static final int GEN_AI_MODEL_ACCESS = 97; public static final int RUNTIME_RULE = 98; + public static final int GEN_AI_EVALUATION_RECORD = 99; /** * Catalog of scope, the metrics processor could use this to group all generated metrics by oal rt. diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/StorageModule.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/StorageModule.java index ec52d2a0b2a2..dd0a98d4dddf 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/StorageModule.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/StorageModule.java @@ -40,6 +40,7 @@ import org.apache.skywalking.oap.server.core.storage.profiling.ebpf.IEBPFProfilingScheduleDAO; import org.apache.skywalking.oap.server.core.storage.profiling.ebpf.IEBPFProfilingTaskDAO; import org.apache.skywalking.oap.server.core.storage.query.IEventQueryDAO; +import org.apache.skywalking.oap.server.core.storage.query.IGenAIEvaluationRecordQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IHierarchyQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.ILogQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IMetadataQueryDAO; @@ -81,6 +82,7 @@ public Class[] services() { IAlarmQueryDAO.class, IRecordsQueryDAO.class, ILogQueryDAO.class, + IGenAIEvaluationRecordQueryDAO.class, IProfileTaskQueryDAO.class, IProfileTaskLogQueryDAO.class, IProfileThreadSnapshotQueryDAO.class, diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IGenAIEvaluationRecordQueryDAO.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IGenAIEvaluationRecordQueryDAO.java new file mode 100644 index 000000000000..e2ce96d12ac3 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IGenAIEvaluationRecordQueryDAO.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.storage.query; + +import com.google.protobuf.InvalidProtocolBufferException; +import org.apache.skywalking.apm.network.logging.v3.LogTags; +import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag; +import org.apache.skywalking.oap.server.core.query.enumeration.Order; +import org.apache.skywalking.oap.server.core.query.input.Duration; +import org.apache.skywalking.oap.server.core.query.input.TraceScopeCondition; +import org.apache.skywalking.oap.server.core.query.type.GenAIEvaluationRecords; +import org.apache.skywalking.oap.server.core.query.type.KeyValue; +import org.apache.skywalking.oap.server.core.query.type.debugging.DebuggingSpan; +import org.apache.skywalking.oap.server.core.query.type.debugging.DebuggingTraceContext; +import org.apache.skywalking.oap.server.library.module.Service; + +import java.io.IOException; +import java.util.Base64; +import java.util.List; + +import static org.apache.skywalking.oap.server.core.query.type.debugging.DebuggingTraceContext.TRACE_CONTEXT; + +public interface IGenAIEvaluationRecordQueryDAO extends Service { + + default boolean supportQueryGenAIEvaluationRecordByKeywords() { + return false; + } + + default GenAIEvaluationRecords queryGenAIEvaluationRecordDebuggable(String serviceId, + String serviceInstanceId, + TraceScopeCondition relatedTrace, + Order queryOrder, + int from, + int limit, + final Duration duration, + final List tags) throws IOException { + DebuggingTraceContext traceContext = TRACE_CONTEXT.get(); + DebuggingSpan span = null; + try { + if (traceContext != null) { + span = traceContext.createSpan("Query Dao: queryGenAIEvaluationRecord"); + StringBuilder msg = new StringBuilder(); + msg.append("ServiceId: ").append(serviceId) + .append(", ServiceInstanceId: ").append(serviceInstanceId) + .append(", RelatedTrace: ").append(relatedTrace) + .append(", QueryOrder: ").append(queryOrder) + .append(", From: ").append(from) + .append(", Limit: ").append(limit) + .append(", Duration: ").append(duration) + .append(", Tags: ").append(tags); + span.setMsg(msg.toString()); + } + return queryGenAIEvaluationRecord( + serviceId, serviceInstanceId, relatedTrace, queryOrder, from, limit, duration, tags + ); + } finally { + if (traceContext != null && span != null) { + traceContext.stopSpan(span); + } + } + } + + GenAIEvaluationRecords queryGenAIEvaluationRecord(String serviceId, + String serviceInstanceId, + TraceScopeCondition relatedTrace, + Order queryOrder, + int from, + int limit, + final Duration duration, + final List tags) throws IOException; + + /** + * Parse the raw tags with base64 representation of data binary + */ + default void parserDataBinary(String dataBinaryBase64, List tags) { + parserDataBinary(Base64.getDecoder().decode(dataBinaryBase64), tags); + } + + default void parserDataBinary(byte[] dataBinary, List tags) { + try { + LogTags logTags = LogTags.parseFrom(dataBinary); + logTags.getDataList().forEach(pair -> tags.add(new KeyValue(pair.getKey(), pair.getValue()))); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/trace/OTLPSpanReader.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/trace/OTLPSpanReader.java index 80731cfbe3ff..b625afb35e53 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/trace/OTLPSpanReader.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/trace/OTLPSpanReader.java @@ -25,6 +25,16 @@ *

Implementations in the OTLP receiver module wrap the real proto object. */ public interface OTLPSpanReader { + /** + * @return the trace id encoded as lowercase hex + */ + String traceId(); + + /** + * @return the span id encoded as lowercase hex + */ + String spanId(); + /** * @return the span name (e.g., "HTTP GET", "MXMetricPayload") */ diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java index 390d7ab3a348..7e95f09c5baa 100644 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java @@ -34,6 +34,7 @@ import org.apache.skywalking.oap.query.graphql.resolver.EBPFProcessProfilingMutation; import org.apache.skywalking.oap.query.graphql.resolver.EBPFProcessProfilingQuery; import org.apache.skywalking.oap.query.graphql.resolver.EventQuery; +import org.apache.skywalking.oap.query.graphql.resolver.GenAIEvaluationRecordQuery; import org.apache.skywalking.oap.query.graphql.resolver.HealthQuery; import org.apache.skywalking.oap.query.graphql.resolver.HierarchyQuery; import org.apache.skywalking.oap.query.graphql.resolver.LogQuery; @@ -136,6 +137,8 @@ public void prepare() throws ServiceNotProvidedException { new LogQuery(getManager()), new LogTestQuery(getManager(), config) ) + .file("query-protocol/gen-ai-evaluation-record.graphqls") + .resolvers(new GenAIEvaluationRecordQuery(getManager())) .file("query-protocol/profile.graphqls") .resolvers(new ProfileQuery(getManager()), new ProfileMutation(getManager())) .file("query-protocol/browser-log.graphqls") diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/GenAIEvaluationRecordQuery.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/GenAIEvaluationRecordQuery.java new file mode 100644 index 000000000000..53acca440bb1 --- /dev/null +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/GenAIEvaluationRecordQuery.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.query.graphql.resolver; + +import graphql.kickstart.tools.GraphQLQueryResolver; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.UnexpectedException; +import org.apache.skywalking.oap.server.core.query.GenAIEvaluationRecordQueryService; +import org.apache.skywalking.oap.server.core.query.enumeration.Order; +import org.apache.skywalking.oap.server.core.query.input.GenAIEvaluationRecordQueryCondition; +import org.apache.skywalking.oap.server.core.query.type.GenAIEvaluationRecords; +import org.apache.skywalking.oap.server.core.query.type.debugging.DebuggingSpan; +import org.apache.skywalking.oap.server.core.query.type.debugging.DebuggingTraceContext; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.util.CollectionUtils; +import org.apache.skywalking.oap.server.library.util.StringUtil; + +import java.io.IOException; +import java.util.concurrent.CompletableFuture; + +import static java.util.Objects.isNull; +import static org.apache.skywalking.oap.query.graphql.AsyncQueryUtils.queryAsync; +import static org.apache.skywalking.oap.server.core.query.type.debugging.DebuggingTraceContext.TRACE_CONTEXT; + +public class GenAIEvaluationRecordQuery implements GraphQLQueryResolver { + private final ModuleManager moduleManager; + private GenAIEvaluationRecordQueryService genAIEvaluationRecordQueryService; + + public GenAIEvaluationRecordQuery(ModuleManager moduleManager) { + this.moduleManager = moduleManager; + } + + public boolean supportGenAIEvaluationRecordQueryByKeywords() { + return getQueryService().supportQueryGenAIEvaluationRecordByKeywords(); + } + + private GenAIEvaluationRecordQueryService getQueryService() { + if (genAIEvaluationRecordQueryService == null) { + this.genAIEvaluationRecordQueryService = moduleManager.find(CoreModule.NAME).provider().getService(GenAIEvaluationRecordQueryService.class); + } + return genAIEvaluationRecordQueryService; + } + + public CompletableFuture queryGenAIEvaluationRecord( + GenAIEvaluationRecordQueryCondition condition, boolean debug) { + return queryAsync(() -> { + DebuggingTraceContext traceContext = new DebuggingTraceContext( + "GenAIEvaluationRecordCondition: " + condition, debug, false); + DebuggingTraceContext.TRACE_CONTEXT.set(traceContext); + DebuggingSpan span = traceContext.createSpan("Query gen AI evaluation records"); + try { + GenAIEvaluationRecords evaluationRecords = queryGenAIEvaluationRecord(condition); + if (debug) { + evaluationRecords.setDebuggingTrace(traceContext.getExecTrace()); + } + return evaluationRecords; + } finally { + traceContext.stopSpan(span); + traceContext.stopTrace(); + TRACE_CONTEXT.remove(); + } + }); + } + + private GenAIEvaluationRecords queryGenAIEvaluationRecord( + GenAIEvaluationRecordQueryCondition condition) throws IOException { + if (isNull(condition.getQueryDuration()) && isNull(condition.getRelatedTrace())) { + throw new UnexpectedException("The condition must contains either queryDuration or relatedTrace."); + } + + Order queryOrder = isNull(condition.getQueryOrder()) ? Order.DES : condition.getQueryOrder(); + if (CollectionUtils.isNotEmpty(condition.getTags())) { + condition.getTags().forEach(tag -> { + if (tag != null) { + if (StringUtil.isNotEmpty(tag.getKey())) { + tag.setKey(tag.getKey().trim()); + } + if (StringUtil.isNotEmpty(tag.getValue())) { + tag.setValue(tag.getValue().trim()); + } + } + }); + } + return getQueryService().queryGenAIEvaluationRecord( + condition.getServiceId(), + condition.getServiceInstanceId(), + condition.getRelatedTrace(), + condition.getPaging(), + queryOrder, + condition.getQueryDuration(), + condition.getTags() + ); + } +} diff --git a/oap-server/server-receiver-plugin/otel-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/otel/otlp/OTLPSpanReaderImpl.java b/oap-server/server-receiver-plugin/otel-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/otel/otlp/OTLPSpanReaderImpl.java index 91f926e24ed0..d77c41600ac5 100644 --- a/oap-server/server-receiver-plugin/otel-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/otel/otlp/OTLPSpanReaderImpl.java +++ b/oap-server/server-receiver-plugin/otel-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/otel/otlp/OTLPSpanReaderImpl.java @@ -17,21 +17,35 @@ package org.apache.skywalking.oap.server.receiver.otel.otlp; +import com.google.protobuf.ByteString; import io.opentelemetry.proto.common.v1.AnyValue; import io.opentelemetry.proto.common.v1.KeyValue; import io.opentelemetry.proto.trace.v1.Span; import org.apache.skywalking.oap.server.core.trace.OTLPSpanReader; +import java.math.BigInteger; + /** * {@link OTLPSpanReader} implementation wrapping the real OTLP {@link Span} proto. */ public class OTLPSpanReaderImpl implements OTLPSpanReader { + private final Span span; public OTLPSpanReaderImpl(final Span span) { this.span = span; } + @Override + public String traceId() { + return idToHexString(span.getTraceId()); + } + + @Override + public String spanId() { + return idToHexString(span.getSpanId()); + } + @Override public String spanName() { return span.getName(); @@ -77,4 +91,11 @@ private static String convertValue(final AnyValue value) { } return ""; } + + private String idToHexString(ByteString id) { + if (id == null) { + return ""; + } + return new BigInteger(1, id.toByteArray()).toString(); + } } diff --git a/oap-server/server-starter/src/main/resources/ai-evaluation.yml b/oap-server/server-starter/src/main/resources/ai-evaluation.yml new file mode 100644 index 000000000000..69d2b85afa94 --- /dev/null +++ b/oap-server/server-starter/src/main/resources/ai-evaluation.yml @@ -0,0 +1,108 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# AI evaluation runtime configuration. `application.yml` only enables/disables +# the ai-evaluation module; judge model and tasks are maintained here. + +judge: + provider: openai + endpoint: ${AI_EVALUATION_ENDPOINT:} + model: ${AI_EVALUATION_MODEL:gpt-5.5} + api-key: ${AI_EVALUATION_API_KEY:} + request-timeout-seconds: 30 + temperature: 0.2 + max_tokens: 100000 + +level: + undefined: undefined + score: + - min: 0.0 + max: 0.3 + level: fail + - min: 0.3 + max: 0.6 + level: warning + - min: 0.6 + max: 0.85 + level: good + - min: 0.85 + max: 1.0 + level: excellent + boolean: + true: excellent + false: fail + +system-prompt: | + You are an AI observability evaluator. + + Your task is to evaluate GenAI spans according to the evaluation tasks provided by the user. + + General Rules: + + 1. Return ONLY valid JSON. + 2. Do NOT return markdown. + 3. Do NOT return code fences. + 4. Do NOT return explanations outside the JSON result. + 5. Do NOT add any fields that are not requested. + 6. Do NOT omit any evaluation task. + 7. The JSON keys MUST appear in exactly the same order as the evaluation tasks provided by the user. + 8. Each evaluation task MUST contain: + - value + - reason + 9. The reason field should be concise and explain the evaluation result in one or two sentences. + 10. If the available context is insufficient, provide the best evaluation possible and explain the limitation in the reason field. + + Value Type Rules: + + SCORE + - value must be a number between 0.0 and 1.0. + - 1.0 represents the best possible evaluation result. + - 0.0 represents the worst possible evaluation result. + + BOOLEAN + - value must be either true or false. + + STRING + - value must be a string. + + JSON + - value must be a valid JSON object. + + Evaluation Principles: + + - Evaluate only using the information provided in the span context. + - Do not assume facts that are not present in the input. + - Do not invent missing information. + - Be objective and consistent. + - Use the task instruction as the primary evaluation criterion. + + Return ONLY the JSON object. + +tasks: + - name: Faithfulness + valueType: SCORE + instruction: Evaluate factual grounding. + + - name: Relevance + valueType: SCORE + instruction: Evaluate response relevance. + + - name: TaskCompletion + valueType: SCORE + instruction: Evaluate task completion. + + - name: Hallucination + valueType: SCORE + instruction: Evaluate hallucination severity. diff --git a/oap-server/server-starter/src/main/resources/application.yml b/oap-server/server-starter/src/main/resources/application.yml index 9fa3eeb31525..3b603a808f31 100644 --- a/oap-server/server-starter/src/main/resources/application.yml +++ b/oap-server/server-starter/src/main/resources/application.yml @@ -253,6 +253,11 @@ gen-ai-analyzer: selector: ${SW_GENAI_ANALYZER:default} default: +ai-evaluation: + selector: ${SW_AI_EVALUATION:default} + default: + sampleRate: ${SW_AI_EVALUTION_SAMPLE_RATE:1000000} + receiver-sharing-server: selector: ${SW_RECEIVER_SHARING_SERVER:default} default: diff --git a/oap-server/server-starter/src/main/resources/gen-ai-evaluation-rules/gen-ai-model.yaml b/oap-server/server-starter/src/main/resources/gen-ai-evaluation-rules/gen-ai-model.yaml new file mode 100644 index 000000000000..0031bbcc7b83 --- /dev/null +++ b/oap-server/server-starter/src/main/resources/gen-ai-evaluation-rules/gen-ai-model.yaml @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +metricPrefix: gen_ai_model_evaluation +expSuffix: instance(['provider_name'], ['model_name'], Layer.VIRTUAL_GENAI) +metricsRules: + # Scores are scaled by 1,000,000 before entering MAL because labeled meter values are stored as long values. + - name: score_ppm + exp: gen_ai_model_evaluation_score_ppm.avg(['provider_name', 'model_name', 'task_name']) diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageProvider.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageProvider.java index dbb2b7f3dffc..5c3e07e54f88 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageProvider.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageProvider.java @@ -56,6 +56,7 @@ import org.apache.skywalking.oap.server.core.storage.query.IAlarmQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IBrowserLogQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IEventQueryDAO; +import org.apache.skywalking.oap.server.core.storage.query.IGenAIEvaluationRecordQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IHierarchyQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.ILogQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IMetadataQueryDAO; @@ -75,6 +76,7 @@ import org.apache.skywalking.oap.server.library.util.StringUtil; import org.apache.skywalking.oap.server.storage.plugin.banyandb.measure.BanyanDBEBPFProfilingScheduleQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBEventQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBGenAIGenAIEvaluationRecordQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.banyandb.measure.BanyanDBHierarchyQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.banyandb.measure.BanyanDBMetadataQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.banyandb.measure.BanyanDBMetricsQueryDAO; @@ -175,6 +177,8 @@ IBatchDAO.class, new BanyanDBBatchDAO(client, config.getGlobal().getMaxBulkSize( this.registerServiceImplementation(IMetadataQueryDAO.class, new BanyanDBMetadataQueryDAO(client, this.config)); this.registerServiceImplementation(IAlarmQueryDAO.class, new BanyanDBAlarmQueryDAO(client)); this.registerServiceImplementation(ILogQueryDAO.class, new BanyanDBLogQueryDAO(client)); + this.registerServiceImplementation( + IGenAIEvaluationRecordQueryDAO.class, new BanyanDBGenAIGenAIEvaluationRecordQueryDAO(client)); this.registerServiceImplementation( IProfileTaskQueryDAO.class, new BanyanDBProfileTaskQueryDAO(client, this.config.getGlobal().getProfileTaskQueryMaxSize() diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBGenAIGenAIEvaluationRecordQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBGenAIGenAIEvaluationRecordQueryDAO.java new file mode 100644 index 000000000000..d0c3cadfaa0a --- /dev/null +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBGenAIGenAIEvaluationRecordQueryDAO.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.banyandb.stream; + +import com.google.common.collect.ImmutableSet; +import org.apache.skywalking.library.banyandb.v1.client.AbstractQuery; +import org.apache.skywalking.library.banyandb.v1.client.RowEntity; +import org.apache.skywalking.library.banyandb.v1.client.StreamQuery; +import org.apache.skywalking.library.banyandb.v1.client.StreamQueryResponse; +import org.apache.skywalking.oap.server.core.analysis.manual.genai.GenAIEvaluationRecord; +import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag; +import org.apache.skywalking.oap.server.core.query.enumeration.Order; +import org.apache.skywalking.oap.server.core.query.input.Duration; +import org.apache.skywalking.oap.server.core.query.input.TraceScopeCondition; +import org.apache.skywalking.oap.server.core.query.type.GenAIEvaluationRecords; +import org.apache.skywalking.oap.server.core.storage.query.IGenAIEvaluationRecordQueryDAO; +import org.apache.skywalking.oap.server.library.util.CollectionUtils; +import org.apache.skywalking.oap.server.library.util.StringUtil; +import org.apache.skywalking.oap.server.storage.plugin.banyandb.BanyanDBStorageClient; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; +import java.util.Set; + +/** + * {@link org.apache.skywalking.oap.server.core.analysis.manual.log.LogRecord} is a gen-ai evaluation result + */ +public class BanyanDBGenAIGenAIEvaluationRecordQueryDAO extends AbstractBanyanDBDAO implements IGenAIEvaluationRecordQueryDAO { + private static final Set TAGS = ImmutableSet.of( + GenAIEvaluationRecord.TRACE_ID, + GenAIEvaluationRecord.SERVICE_ID, + GenAIEvaluationRecord.SERVICE_INSTANCE_ID, + GenAIEvaluationRecord.SEGMENT_ID, + GenAIEvaluationRecord.SPAN_ID, + GenAIEvaluationRecord.SPAN_TYPE, + GenAIEvaluationRecord.TASK_NAME, + GenAIEvaluationRecord.VALUE_TYPE, + GenAIEvaluationRecord.VALUE, + GenAIEvaluationRecord.EVALUATION_LEVEL, + GenAIEvaluationRecord.REASON, + GenAIEvaluationRecord.JUDGE_MODEL, + GenAIEvaluationRecord.EVALUATION_TIME + ); + + public BanyanDBGenAIGenAIEvaluationRecordQueryDAO(BanyanDBStorageClient client) { + super(client); + } + + @Override + public GenAIEvaluationRecords queryGenAIEvaluationRecord(String serviceId, String serviceInstanceId, + TraceScopeCondition relatedTrace, Order queryOrder, int from, int limit, + Duration duration, List tags) throws IOException { + final boolean isColdStage = duration != null && duration.isColdStage(); + final QueryBuilder query = new QueryBuilder<>() { + @Override + public void apply(StreamQuery query) { + if (StringUtil.isNotEmpty(serviceId)) { + query.and(eq(GenAIEvaluationRecord.SERVICE_ID, serviceId)); + } + if (StringUtil.isNotEmpty(serviceInstanceId)) { + query.and(eq(GenAIEvaluationRecord.SERVICE_INSTANCE_ID, serviceInstanceId)); + } + + if (Objects.nonNull(relatedTrace)) { + if (StringUtil.isNotEmpty(relatedTrace.getTraceId())) { + query.and(eq(GenAIEvaluationRecord.TRACE_ID, relatedTrace.getTraceId())); + } + if (StringUtil.isNotEmpty(relatedTrace.getSegmentId())) { + query.and(eq(GenAIEvaluationRecord.SEGMENT_ID, relatedTrace.getSegmentId())); + } + if (Objects.nonNull(relatedTrace.getSpanId())) { + query.and(eq(GenAIEvaluationRecord.SPAN_ID, (long) relatedTrace.getSpanId())); + } + } + + if (CollectionUtils.isNotEmpty(tags)) { + for (final Tag tag : tags) { + if (StringUtil.isNotEmpty(tag.getKey()) && StringUtil.isNotEmpty(tag.getValue())) { + query.and(eq(tag.getKey(), tag.getValue())); + } + } + } + if (queryOrder == Order.ASC) { + query.setOrderBy( + new AbstractQuery.OrderBy(AbstractQuery.Sort.ASC)); + } else { + query.setOrderBy( + new AbstractQuery.OrderBy(AbstractQuery.Sort.DESC)); + } + query.setLimit(limit); + query.setOffset(from); + } + }; + + StreamQueryResponse resp = queryDebuggable(isColdStage, GenAIEvaluationRecord.INDEX_NAME, TAGS, getTimestampRange(duration), query); + + GenAIEvaluationRecords genAIEvaluationRecords = new GenAIEvaluationRecords(); + + for (final RowEntity rowEntity : resp.getElements()) { + GenAIEvaluationRecord evaluationRecord = new GenAIEvaluationRecord(); + evaluationRecord.setTraceId(rowEntity.getTagValue(GenAIEvaluationRecord.TRACE_ID)); + evaluationRecord.setServiceId(rowEntity.getTagValue(GenAIEvaluationRecord.SERVICE_ID)); + evaluationRecord.setServiceInstanceId(rowEntity.getTagValue(GenAIEvaluationRecord.SERVICE_INSTANCE_ID)); + evaluationRecord.setSegmentId(rowEntity.getTagValue(GenAIEvaluationRecord.SEGMENT_ID)); + evaluationRecord.setSpanId(rowEntity.getTagValue(GenAIEvaluationRecord.SPAN_ID)); + evaluationRecord.setSpanType(rowEntity.getTagValue(GenAIEvaluationRecord.SPAN_TYPE)); + evaluationRecord.setTaskName(rowEntity.getTagValue(GenAIEvaluationRecord.TASK_NAME)); + evaluationRecord.setEvaluationTime(((Number) rowEntity.getTagValue(GenAIEvaluationRecord.EVALUATION_TIME)).longValue()); + evaluationRecord.setValueType(rowEntity.getTagValue(GenAIEvaluationRecord.VALUE_TYPE)); + evaluationRecord.setValue(rowEntity.getTagValue(GenAIEvaluationRecord.VALUE)); + evaluationRecord.setEvaluationLevel(rowEntity.getTagValue(GenAIEvaluationRecord.EVALUATION_LEVEL)); + evaluationRecord.setReason(rowEntity.getTagValue(GenAIEvaluationRecord.REASON)); + evaluationRecord.setJudgeModel(rowEntity.getTagValue(GenAIEvaluationRecord.JUDGE_MODEL)); + genAIEvaluationRecords.getGenAIEvaluationRecordList().add(evaluationRecord); + } + return genAIEvaluationRecords; + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java index 7f456b2e6f98..e6856e2d4c26 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java @@ -55,6 +55,7 @@ import org.apache.skywalking.oap.server.core.storage.query.IAlarmQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IBrowserLogQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IEventQueryDAO; +import org.apache.skywalking.oap.server.core.storage.query.IGenAIEvaluationRecordQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IHierarchyQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.ILogQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IMetadataQueryDAO; @@ -93,6 +94,7 @@ import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.EBPFProfilingTaskEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.ESEventQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.HierarchyQueryEsDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.GenAIEvaluationRecordQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.JFRDataQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.PprofDataQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.LogQueryEsDAO; @@ -241,6 +243,8 @@ public void prepare() throws ServiceNotProvidedException { this.registerServiceImplementation(IAlarmQueryDAO.class, new AlarmQueryEsDAO(elasticSearchClient)); this.registerServiceImplementation(IRecordsQueryDAO.class, new RecordsQueryEsDAO(elasticSearchClient)); this.registerServiceImplementation(ILogQueryDAO.class, new LogQueryEsDAO(elasticSearchClient)); + this.registerServiceImplementation( + IGenAIEvaluationRecordQueryDAO.class, new GenAIEvaluationRecordQueryEsDAO(elasticSearchClient)); this.registerServiceImplementation( IProfileTaskQueryDAO.class, new ProfileTaskQueryEsDAO(elasticSearchClient, config .getProfileTaskQueryMaxSize())); diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/GenAIEvaluationRecordQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/GenAIEvaluationRecordQueryEsDAO.java new file mode 100644 index 000000000000..6ec0f3e7641c --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/GenAIEvaluationRecordQueryEsDAO.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import java.io.IOException; +import java.util.List; +import java.util.Set; +import org.apache.skywalking.library.elasticsearch.requests.search.BoolQueryBuilder; +import org.apache.skywalking.library.elasticsearch.requests.search.Query; +import org.apache.skywalking.library.elasticsearch.requests.search.Search; +import org.apache.skywalking.library.elasticsearch.requests.search.SearchBuilder; +import org.apache.skywalking.library.elasticsearch.requests.search.Sort; +import org.apache.skywalking.library.elasticsearch.response.search.SearchHit; +import org.apache.skywalking.library.elasticsearch.response.search.SearchResponse; +import org.apache.skywalking.oap.server.core.analysis.manual.genai.GenAIEvaluationRecord; +import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag; +import org.apache.skywalking.oap.server.core.analysis.record.Record; +import org.apache.skywalking.oap.server.core.query.enumeration.Order; +import org.apache.skywalking.oap.server.core.query.input.Duration; +import org.apache.skywalking.oap.server.core.query.input.TraceScopeCondition; +import org.apache.skywalking.oap.server.core.query.type.GenAIEvaluationRecords; +import org.apache.skywalking.oap.server.core.storage.query.IGenAIEvaluationRecordQueryDAO; +import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient; +import org.apache.skywalking.oap.server.library.util.CollectionUtils; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.EsDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.IndexController; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.TimeRangeIndexNameGenerator; + +import static java.util.Objects.nonNull; +import static org.apache.skywalking.oap.server.library.util.StringUtil.isNotEmpty; + +public class GenAIEvaluationRecordQueryEsDAO extends EsDAO implements IGenAIEvaluationRecordQueryDAO { + private static final Set QUERYABLE_TAG_KEYS = Set.of( + GenAIEvaluationRecord.TRACE_ID, + GenAIEvaluationRecord.SERVICE_ID, + GenAIEvaluationRecord.SERVICE_INSTANCE_ID, + GenAIEvaluationRecord.SEGMENT_ID, + GenAIEvaluationRecord.SPAN_ID, + GenAIEvaluationRecord.SPAN_TYPE, + GenAIEvaluationRecord.TASK_NAME, + GenAIEvaluationRecord.VALUE_TYPE, + GenAIEvaluationRecord.VALUE, + GenAIEvaluationRecord.EVALUATION_LEVEL, + GenAIEvaluationRecord.REASON, + GenAIEvaluationRecord.JUDGE_MODEL + ); + + public GenAIEvaluationRecordQueryEsDAO(final ElasticSearchClient client) { + super(client); + } + + @Override + public GenAIEvaluationRecords queryGenAIEvaluationRecord(final String serviceId, + final String serviceInstanceId, + final TraceScopeCondition relatedTrace, + final Order queryOrder, + final int from, + final int limit, + final Duration duration, + final List tags) throws IOException { + long startSecondTB = 0; + long endSecondTB = 0; + if (nonNull(duration)) { + startSecondTB = duration.getStartTimeBucketInSec(); + endSecondTB = duration.getEndTimeBucketInSec(); + } + + final BoolQueryBuilder query = Query.bool(); + if (IndexController.LogicIndicesRegister.isMergedTable(GenAIEvaluationRecord.INDEX_NAME)) { + query.must(Query.term( + IndexController.LogicIndicesRegister.RECORD_TABLE_NAME, + GenAIEvaluationRecord.INDEX_NAME + )); + } + if (startSecondTB != 0 && endSecondTB != 0) { + query.must(Query.range(Record.TIME_BUCKET).gte(startSecondTB).lte(endSecondTB)); + } + if (isNotEmpty(serviceId)) { + query.must(Query.term(GenAIEvaluationRecord.SERVICE_ID, serviceId)); + } + if (isNotEmpty(serviceInstanceId)) { + query.must(Query.term(GenAIEvaluationRecord.SERVICE_INSTANCE_ID, serviceInstanceId)); + } + if (nonNull(relatedTrace)) { + if (isNotEmpty(relatedTrace.getTraceId())) { + query.must(Query.term(GenAIEvaluationRecord.TRACE_ID, relatedTrace.getTraceId())); + } + if (isNotEmpty(relatedTrace.getSegmentId())) { + query.must(Query.term(GenAIEvaluationRecord.SEGMENT_ID, relatedTrace.getSegmentId())); + } + if (nonNull(relatedTrace.getSpanId())) { + query.must(Query.term(GenAIEvaluationRecord.SPAN_ID, String.valueOf(relatedTrace.getSpanId()))); + } + } + if (CollectionUtils.isNotEmpty(tags)) { + for (final Tag tag : tags) { + if (isNotEmpty(tag.getKey()) && isNotEmpty(tag.getValue())) { + if (!QUERYABLE_TAG_KEYS.contains(tag.getKey())) { + return new GenAIEvaluationRecords(); + } + query.must(Query.term(tag.getKey(), tag.getValue())); + } + } + } + + final SearchBuilder search = Search.builder() + .query(query) + .sort( + GenAIEvaluationRecord.EVALUATION_TIME, + Order.DES.equals(queryOrder) ? Sort.Order.DESC : Sort.Order.ASC + ) + .size(limit) + .from(from); + + final SearchResponse response = searchDebuggable(new TimeRangeIndexNameGenerator( + IndexController.LogicIndicesRegister.getPhysicalTableName(GenAIEvaluationRecord.INDEX_NAME), + startSecondTB, + endSecondTB + ), search.build()); + + final GenAIEvaluationRecords records = new GenAIEvaluationRecords(); + for (SearchHit searchHit : response.getHits().getHits()) { + records.getGenAIEvaluationRecordList().add(parseRecord(searchHit)); + } + return records; + } + + private GenAIEvaluationRecord parseRecord(final SearchHit searchHit) { + final var source = searchHit.getSource(); + final GenAIEvaluationRecord record = new GenAIEvaluationRecord(); + record.setTraceId((String) source.get(GenAIEvaluationRecord.TRACE_ID)); + record.setServiceId((String) source.get(GenAIEvaluationRecord.SERVICE_ID)); + record.setServiceInstanceId((String) source.get(GenAIEvaluationRecord.SERVICE_INSTANCE_ID)); + record.setSegmentId((String) source.get(GenAIEvaluationRecord.SEGMENT_ID)); + record.setSpanId((String) source.get(GenAIEvaluationRecord.SPAN_ID)); + record.setSpanType((String) source.get(GenAIEvaluationRecord.SPAN_TYPE)); + record.setTaskName((String) source.get(GenAIEvaluationRecord.TASK_NAME)); + record.setValueType((String) source.get(GenAIEvaluationRecord.VALUE_TYPE)); + record.setValue((String) source.get(GenAIEvaluationRecord.VALUE)); + record.setEvaluationLevel((String) source.get(GenAIEvaluationRecord.EVALUATION_LEVEL)); + record.setReason((String) source.get(GenAIEvaluationRecord.REASON)); + record.setJudgeModel((String) source.get(GenAIEvaluationRecord.JUDGE_MODEL)); + record.setEvaluationTime(((Number) source.get(GenAIEvaluationRecord.EVALUATION_TIME)).longValue()); + return record; + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/JDBCStorageProvider.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/JDBCStorageProvider.java index d947fd3eb24b..43a80b1d558e 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/JDBCStorageProvider.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/JDBCStorageProvider.java @@ -48,6 +48,7 @@ import org.apache.skywalking.oap.server.core.storage.query.IAlarmQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IBrowserLogQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IEventQueryDAO; +import org.apache.skywalking.oap.server.core.storage.query.IGenAIEvaluationRecordQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IHierarchyQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.ILogQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IMetadataQueryDAO; @@ -79,6 +80,7 @@ import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCEBPFProfilingScheduleDAO; import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCEBPFProfilingTaskDAO; import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCEventQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCGenAIEvaluationRecordQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCHierarchyQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCHistoryDeleteDAO; import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCJFRDataQueryDAO; @@ -198,6 +200,9 @@ public void prepare() throws ServiceNotProvidedException, ModuleStartException { this.registerServiceImplementation( ILogQueryDAO.class, new JDBCLogQueryDAO(jdbcClient, getManager(), tableHelper)); + this.registerServiceImplementation( + IGenAIEvaluationRecordQueryDAO.class, + new JDBCGenAIEvaluationRecordQueryDAO(jdbcClient, tableHelper)); this.registerServiceImplementation( IProfileTaskQueryDAO.class, diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCGenAIEvaluationRecordQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCGenAIEvaluationRecordQueryDAO.java new file mode 100644 index 000000000000..0b85466bb297 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCGenAIEvaluationRecordQueryDAO.java @@ -0,0 +1,257 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import org.apache.skywalking.oap.server.core.analysis.manual.genai.GenAIEvaluationRecord; +import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag; +import org.apache.skywalking.oap.server.core.query.enumeration.Order; +import org.apache.skywalking.oap.server.core.query.input.Duration; +import org.apache.skywalking.oap.server.core.query.input.TraceScopeCondition; +import org.apache.skywalking.oap.server.core.query.type.GenAIEvaluationRecords; +import org.apache.skywalking.oap.server.core.storage.model.ColumnName; +import org.apache.skywalking.oap.server.core.storage.model.ModelColumn; +import org.apache.skywalking.oap.server.core.storage.query.IGenAIEvaluationRecordQueryDAO; +import org.apache.skywalking.oap.server.library.client.jdbc.hikaricp.JDBCClient; +import org.apache.skywalking.oap.server.library.util.CollectionUtils; +import org.apache.skywalking.oap.server.library.util.StringUtil; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.JDBCTableInstaller; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.SQLAndParameters; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.TableHelper; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.TableMetaInfo; + +import static java.util.Comparator.comparing; +import static java.util.Objects.nonNull; +import static java.util.stream.Collectors.toList; + +@RequiredArgsConstructor +public class JDBCGenAIEvaluationRecordQueryDAO implements IGenAIEvaluationRecordQueryDAO { + private static final Set QUERYABLE_TAG_KEYS = Set.of( + GenAIEvaluationRecord.TRACE_ID, + GenAIEvaluationRecord.SERVICE_ID, + GenAIEvaluationRecord.SERVICE_INSTANCE_ID, + GenAIEvaluationRecord.SEGMENT_ID, + GenAIEvaluationRecord.SPAN_ID, + GenAIEvaluationRecord.SPAN_TYPE, + GenAIEvaluationRecord.TASK_NAME, + GenAIEvaluationRecord.VALUE_TYPE, + GenAIEvaluationRecord.VALUE, + GenAIEvaluationRecord.EVALUATION_LEVEL, + GenAIEvaluationRecord.REASON, + GenAIEvaluationRecord.JUDGE_MODEL + ); + + private static final List SELECTED_COLUMNS = List.of( + GenAIEvaluationRecord.TRACE_ID, + GenAIEvaluationRecord.SERVICE_ID, + GenAIEvaluationRecord.SERVICE_INSTANCE_ID, + GenAIEvaluationRecord.SEGMENT_ID, + GenAIEvaluationRecord.SPAN_ID, + GenAIEvaluationRecord.SPAN_TYPE, + GenAIEvaluationRecord.TASK_NAME, + GenAIEvaluationRecord.VALUE_TYPE, + GenAIEvaluationRecord.VALUE, + GenAIEvaluationRecord.EVALUATION_LEVEL, + GenAIEvaluationRecord.REASON, + GenAIEvaluationRecord.JUDGE_MODEL, + GenAIEvaluationRecord.EVALUATION_TIME + ); + + private final JDBCClient jdbcClient; + private final TableHelper tableHelper; + + @Override + @SneakyThrows + public GenAIEvaluationRecords queryGenAIEvaluationRecord(final String serviceId, + final String serviceInstanceId, + final TraceScopeCondition relatedTrace, + final Order queryOrder, + final int from, + final int limit, + final Duration duration, + final List tags) { + if (CollectionUtils.isNotEmpty(tags)) { + for (final Tag tag : tags) { + if (StringUtil.isNotEmpty(tag.getKey()) + && StringUtil.isNotEmpty(tag.getValue()) + && !QUERYABLE_TAG_KEYS.contains(tag.getKey())) { + return new GenAIEvaluationRecords(); + } + } + } + + final List tables; + if (nonNull(duration)) { + tables = tableHelper.getTablesForRead( + GenAIEvaluationRecord.INDEX_NAME, + duration.getStartTimeBucket(), + duration.getEndTimeBucket() + ); + } else { + tables = tableHelper.getTablesWithinTTL(GenAIEvaluationRecord.INDEX_NAME); + } + + final var records = new ArrayList(); + for (final var table : tables) { + final var sqlAndParameters = buildSQL( + serviceId, serviceInstanceId, relatedTrace, queryOrder, from, limit, duration, tags, table); + records.addAll( + jdbcClient.executeQuery( + sqlAndParameters.sql(), + this::parseResults, + sqlAndParameters.parameters() + ) + ); + } + + final var comparator = Order.ASC.equals(queryOrder) ? + comparing(GenAIEvaluationRecord::getEvaluationTime) : + comparing(GenAIEvaluationRecord::getEvaluationTime).reversed(); + return new GenAIEvaluationRecords( + records.stream().sorted(comparator).skip(from).limit(limit).collect(toList()) + ); + } + + protected ArrayList parseResults(final ResultSet resultSet) throws SQLException { + final var records = new ArrayList(); + while (resultSet.next()) { + final GenAIEvaluationRecord record = new GenAIEvaluationRecord(); + record.setTraceId(resultSet.getString(GenAIEvaluationRecord.TRACE_ID)); + record.setServiceId(resultSet.getString(GenAIEvaluationRecord.SERVICE_ID)); + record.setServiceInstanceId(resultSet.getString(GenAIEvaluationRecord.SERVICE_INSTANCE_ID)); + record.setSegmentId(resultSet.getString(GenAIEvaluationRecord.SEGMENT_ID)); + record.setSpanId(resultSet.getString(GenAIEvaluationRecord.SPAN_ID)); + record.setSpanType(resultSet.getString(GenAIEvaluationRecord.SPAN_TYPE)); + record.setTaskName(resultSet.getString(GenAIEvaluationRecord.TASK_NAME)); + record.setValueType(resultSet.getString(GenAIEvaluationRecord.VALUE_TYPE)); + record.setValue(resultSet.getString(GenAIEvaluationRecord.VALUE)); + record.setEvaluationLevel(resultSet.getString(GenAIEvaluationRecord.EVALUATION_LEVEL)); + record.setReason(resultSet.getString(GenAIEvaluationRecord.REASON)); + record.setJudgeModel(resultSet.getString(GenAIEvaluationRecord.JUDGE_MODEL)); + record.setEvaluationTime(resultSet.getLong(GenAIEvaluationRecord.EVALUATION_TIME)); + records.add(record); + } + return records; + } + + protected SQLAndParameters buildSQL(final String serviceId, + final String serviceInstanceId, + final TraceScopeCondition relatedTrace, + final Order queryOrder, + final int from, + final int limit, + final Duration duration, + final List tags, + final String table) { + long startSecondTB = 0; + long endSecondTB = 0; + if (nonNull(duration)) { + startSecondTB = duration.getStartTimeBucketInSec(); + endSecondTB = duration.getEndTimeBucketInSec(); + } + + final StringBuilder sql = new StringBuilder("select "); + final List parameters = new ArrayList<>(10); + sql.append(selectColumns()) + .append(" from ") + .append(table) + .append(" where ") + .append(JDBCTableInstaller.TABLE_COLUMN) + .append(" = ?"); + parameters.add(GenAIEvaluationRecord.INDEX_NAME); + + if (startSecondTB != 0 && endSecondTB != 0) { + sql.append(" and ").append(storageColumn(GenAIEvaluationRecord.TIME_BUCKET)).append(" >= ?"); + parameters.add(startSecondTB); + sql.append(" and ").append(storageColumn(GenAIEvaluationRecord.TIME_BUCKET)).append(" <= ?"); + parameters.add(endSecondTB); + } + if (StringUtil.isNotEmpty(serviceId)) { + sql.append(" and ").append(storageColumn(GenAIEvaluationRecord.SERVICE_ID)).append(" = ?"); + parameters.add(serviceId); + } + if (StringUtil.isNotEmpty(serviceInstanceId)) { + sql.append(" and ").append(storageColumn(GenAIEvaluationRecord.SERVICE_INSTANCE_ID)).append(" = ?"); + parameters.add(serviceInstanceId); + } + if (nonNull(relatedTrace)) { + if (StringUtil.isNotEmpty(relatedTrace.getTraceId())) { + sql.append(" and ").append(storageColumn(GenAIEvaluationRecord.TRACE_ID)).append(" = ?"); + parameters.add(relatedTrace.getTraceId()); + } + if (StringUtil.isNotEmpty(relatedTrace.getSegmentId())) { + sql.append(" and ").append(storageColumn(GenAIEvaluationRecord.SEGMENT_ID)).append(" = ?"); + parameters.add(relatedTrace.getSegmentId()); + } + if (nonNull(relatedTrace.getSpanId())) { + sql.append(" and ").append(storageColumn(GenAIEvaluationRecord.SPAN_ID)).append(" = ?"); + parameters.add(String.valueOf(relatedTrace.getSpanId())); + } + } + if (CollectionUtils.isNotEmpty(tags)) { + for (final Tag tag : tags) { + if (StringUtil.isNotEmpty(tag.getKey()) + && StringUtil.isNotEmpty(tag.getValue()) + && QUERYABLE_TAG_KEYS.contains(tag.getKey())) { + sql.append(" and ").append(storageColumn(tag.getKey())).append(" = ?"); + parameters.add(tag.getValue()); + } + } + } + + sql.append(" order by ") + .append(storageColumn(GenAIEvaluationRecord.EVALUATION_TIME)) + .append(" ") + .append(Order.DES.equals(queryOrder) ? "desc" : "asc"); + sql.append(" limit ").append(from + limit); + + return new SQLAndParameters(sql.toString(), parameters); + } + + private String selectColumns() { + return SELECTED_COLUMNS.stream() + .map(this::selectColumn) + .collect(java.util.stream.Collectors.joining(", ")); + } + + private String selectColumn(final String logicalColumn) { + final String storageColumn = storageColumn(logicalColumn); + if (storageColumn.equals(logicalColumn)) { + return storageColumn; + } + return storageColumn + " as " + logicalColumn; + } + + private String storageColumn(final String logicalColumn) { + return TableMetaInfo.get(GenAIEvaluationRecord.INDEX_NAME) + .getColumns() + .stream() + .map(ModelColumn::getColumnName) + .filter(it -> logicalColumn.equals(it.getName())) + .findFirst() + .map(ColumnName::getStorageName) + .orElse(logicalColumn); + } +} From 84233e5f52e8008d33231a56cb1342478185d7c1 Mon Sep 17 00:00:00 2001 From: peachisai <2581009893@qq.com> Date: Sun, 12 Jul 2026 17:32:09 +0800 Subject: [PATCH 2/8] enabling openai config from env --- .../evaluation/judge/provider/OpenAICompatibleProvider.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/judge/provider/OpenAICompatibleProvider.java b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/judge/provider/OpenAICompatibleProvider.java index 1e4e9b503c8e..67a89dea4b64 100644 --- a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/judge/provider/OpenAICompatibleProvider.java +++ b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/judge/provider/OpenAICompatibleProvider.java @@ -37,6 +37,7 @@ import org.apache.skywalking.oap.server.ai.evaluation.judge.JudgeModelRequest; import org.apache.skywalking.oap.server.ai.evaluation.judge.JudgeModelResponse; import org.apache.skywalking.oap.server.library.module.ModuleStartException; +import org.apache.skywalking.oap.server.library.util.PropertyPlaceholderHelper; import org.apache.skywalking.oap.server.library.util.StringUtil; public class OpenAICompatibleProvider implements JudgeModelProvider { @@ -179,7 +180,10 @@ private static String getString(final Properties properties, final String key) { return null; } final Object value = properties.get(key); - return value == null ? null : String.valueOf(value); + if (value == null) { + return null; + } + return PropertyPlaceholderHelper.INSTANCE.replacePlaceholders(String.valueOf(value), properties); } private static Double getDouble(final Properties properties, final String key) throws ModuleStartException { From 95d4437b45c9009c5e6aa92643966e63f22c61ee Mon Sep 17 00:00:00 2001 From: peachisai <2581009893@qq.com> Date: Sun, 12 Jul 2026 22:30:44 +0800 Subject: [PATCH 3/8] docs: add AI evaluation release notes --- docs/en/changes/changes.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index 15839723ecef..f7c937a41103 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -333,8 +333,10 @@ * Fix Envoy ALS rendering for the LAL live-debugger and the persisted log `content`: an Istio metadata-exchange peer in `common_properties.filter_state_objects` (legacy Wasm `wasm.*_peer` = `Any{BytesValue}` wrapping a FlatBuffer, or modern `*_peer` = `Any{Struct}`) is now decoded into the readable peer metadata (pod / namespace / labels) instead of an opaque `jsonformat-failed` envelope or base64. The serialization is hardened so a single un-printable field can no longer blank the whole entry — the `LalPayloadDebugDump` printer carries a well-known-type `TypeRegistry` and sanitizes every value `JsonFormat` would reject (an unresolvable, no-slash, or corrupt-bytes `Any` degrades to an `@unresolved` placeholder; a non-finite `Value` double `NaN`/`Infinity` is rendered as a string), keeping the rest of the entry readable. Because the LAL output builder's `bindInput` runs eagerly before the debug capture, this also stops an unregistered `filter_state_objects` type from throwing and aborting the whole rule (dropping the mesh log). Decoding is wired through a new `LalInputDebugRenderer` SPI (`EnvoyAlsHttpDebugRenderer` / `EnvoyAlsTcpDebugRenderer`) so `log-analyzer` reaches the receiver-side decoders without depending on the Envoy receiver, and covers both HTTP and TCP access logs. * Surface the effective BanyanDB configuration (`bydb.yml` / `bydb-topn.yml`) in the `/debugging/config/dump` admin API. Because the BanyanDB config moved to a separate file in 10.2.0, a BanyanDB deployment previously showed an empty `storage.banyandb` block in the dump; its post-environment-resolution values are now merged into the same response under `storage.banyandb.*` (TopN rules under `storage.banyandb.topN.*`), masked by the same secret-keyword list, via a generic `ConfigDumpExtension` SPI on `ServerStatusService` that any module loading config from a secondary file can implement. * Fix: an MQE `top_n(metric, N, order, attrX='value')` query whose attribute is not a column of the target metric now returns a descriptive MQE error instead of a raw storage `IOException` surfaced as `Internal IO exception, query metrics error.`. Attribute columns (`attr0..attrN`) exist only on decorated metrics (`service_*` / `endpoint_*` / `kubernetes_service_*`, set to the layer name via OAL `.decorator(...)`) and the MAL meter base; metrics such as relations or database / cache / mq access carry none, so passing an attribute condition previously reached the storage engine with a tag it does not define and failed there. `MQEVisitor` now validates each attribute key against the metric's registered queryable columns before the storage call and raises `IllegalExpressionException` (naming the attribute and the metric) when it is absent. +* Add customizable LLM-as-judge support for AI evaluation, with OpenAI-compatible endpoint / model / API key configuration, and persist the evaluation result as queryable `GenAIEvaluationRecord` rows for later inspection. #### UI +* Add a Virtual GenAI evaluation-record page and evaluation-score chart in Horizon UI, so operators can inspect evaluation result, level, reason, judge model, timestamp, trace linkage, and the `gen_ai_model_evaluation_score_ppm` trend for evaluated records. * Add Airflow layer dashboards and menu i18n under Workflow Scheduler in Horizon UI (SWIP-7). * Add mobile menu icon and i18n labels for the iOS layer. * Fix metric label rendering in multi-expression dashboard widgets. From 7f1c72b648feeed8f528cb1e4bc0d330f8d2e079 Mon Sep 17 00:00:00 2001 From: peachisai <2581009893@qq.com> Date: Mon, 13 Jul 2026 22:29:23 +0800 Subject: [PATCH 4/8] Use unique id for GenAI evaluation records --- .../strategy/span/SpanAIEvaluationStrategy.java | 2 ++ .../manual/genai/GenAIEvaluationRecord.java | 16 +++++++--------- .../query/GenAIEvaluationRecordQueryEsDAO.java | 1 + .../dao/JDBCGenAIEvaluationRecordQueryDAO.java | 2 ++ 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/strategy/span/SpanAIEvaluationStrategy.java b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/strategy/span/SpanAIEvaluationStrategy.java index 41fb95aceb34..b0b6bd6cae28 100644 --- a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/strategy/span/SpanAIEvaluationStrategy.java +++ b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/strategy/span/SpanAIEvaluationStrategy.java @@ -44,6 +44,7 @@ import java.io.IOException; import java.util.List; import java.util.Optional; +import java.util.UUID; @Slf4j public class SpanAIEvaluationStrategy implements AIEvaluationStrategy { @@ -125,6 +126,7 @@ private void persistResults(final AIEvaluationContext context, Layer.VIRTUAL_GENAI.isNormal() ); final GenAIEvaluationRecord record = new GenAIEvaluationRecord(); + record.setUniqueId(UUID.randomUUID().toString().replace("-", "")); record.setTraceId(context.getTraceId()); record.setServiceId(serviceId); record.setServiceInstanceId(IDManager.ServiceInstanceID.buildId( diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/genai/GenAIEvaluationRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/genai/GenAIEvaluationRecord.java index ce5d14163780..7310f8f39538 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/genai/GenAIEvaluationRecord.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/genai/GenAIEvaluationRecord.java @@ -45,6 +45,7 @@ public class GenAIEvaluationRecord extends Record { public static final String INDEX_NAME = "gen_ai_evaluation_record"; + public static final String UNIQUE_ID = "unique_id"; public static final String TRACE_ID = "trace_id"; public static final String SERVICE_ID = "service_id"; public static final String SERVICE_INSTANCE_ID = "service_instance_id"; @@ -59,6 +60,9 @@ public class GenAIEvaluationRecord extends Record { public static final String JUDGE_MODEL = "judge_model"; public static final String EVALUATION_TIME = "evaluation_time"; + @Column(name = UNIQUE_ID) + private String uniqueId; + @Column(name = TRACE_ID, length = 150) @BanyanDB.IndexRule(indexType = BanyanDB.IndexRule.IndexType.SKIPPING) private String traceId; @@ -109,21 +113,14 @@ public class GenAIEvaluationRecord extends Record { @Override public StorageID id() { - return new StorageID() - .append(TRACE_ID, traceId) - .append(SERVICE_ID, serviceId) - .append(SERVICE_INSTANCE_ID, serviceInstanceId) - .append(SPAN_ID, spanId) - .append(SPAN_TYPE, spanType) - .append(TASK_NAME, taskName) - .append(EVALUATION_LEVEL, evaluationLevel) - .append(EVALUATION_TIME, evaluationTime); + return new StorageID().append(UNIQUE_ID, uniqueId); } public static class Builder implements StorageBuilder { @Override public GenAIEvaluationRecord storage2Entity(final Convert2Entity converter) { final GenAIEvaluationRecord record = new GenAIEvaluationRecord(); + record.setUniqueId((String) converter.get(UNIQUE_ID)); record.setTraceId((String) converter.get(TRACE_ID)); record.setServiceId((String) converter.get(SERVICE_ID)); record.setServiceInstanceId((String) converter.get(SERVICE_INSTANCE_ID)); @@ -143,6 +140,7 @@ public GenAIEvaluationRecord storage2Entity(final Convert2Entity converter) { @Override public void entity2Storage(final GenAIEvaluationRecord storageData, final Convert2Storage converter) { + converter.accept(UNIQUE_ID, storageData.getUniqueId()); converter.accept(TRACE_ID, storageData.getTraceId()); converter.accept(SERVICE_ID, storageData.getServiceId()); converter.accept(SERVICE_INSTANCE_ID, storageData.getServiceInstanceId()); diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/GenAIEvaluationRecordQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/GenAIEvaluationRecordQueryEsDAO.java index 6ec0f3e7641c..2fcfcb1eed09 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/GenAIEvaluationRecordQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/GenAIEvaluationRecordQueryEsDAO.java @@ -144,6 +144,7 @@ public GenAIEvaluationRecords queryGenAIEvaluationRecord(final String serviceId, private GenAIEvaluationRecord parseRecord(final SearchHit searchHit) { final var source = searchHit.getSource(); final GenAIEvaluationRecord record = new GenAIEvaluationRecord(); + record.setUniqueId((String) source.get(GenAIEvaluationRecord.UNIQUE_ID)); record.setTraceId((String) source.get(GenAIEvaluationRecord.TRACE_ID)); record.setServiceId((String) source.get(GenAIEvaluationRecord.SERVICE_ID)); record.setServiceInstanceId((String) source.get(GenAIEvaluationRecord.SERVICE_INSTANCE_ID)); diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCGenAIEvaluationRecordQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCGenAIEvaluationRecordQueryDAO.java index 0b85466bb297..2101afe75adb 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCGenAIEvaluationRecordQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCGenAIEvaluationRecordQueryDAO.java @@ -64,6 +64,7 @@ public class JDBCGenAIEvaluationRecordQueryDAO implements IGenAIEvaluationRecord ); private static final List SELECTED_COLUMNS = List.of( + GenAIEvaluationRecord.UNIQUE_ID, GenAIEvaluationRecord.TRACE_ID, GenAIEvaluationRecord.SERVICE_ID, GenAIEvaluationRecord.SERVICE_INSTANCE_ID, @@ -138,6 +139,7 @@ protected ArrayList parseResults(final ResultSet resultSe final var records = new ArrayList(); while (resultSet.next()) { final GenAIEvaluationRecord record = new GenAIEvaluationRecord(); + record.setUniqueId(resultSet.getString(GenAIEvaluationRecord.UNIQUE_ID)); record.setTraceId(resultSet.getString(GenAIEvaluationRecord.TRACE_ID)); record.setServiceId(resultSet.getString(GenAIEvaluationRecord.SERVICE_ID)); record.setServiceInstanceId(resultSet.getString(GenAIEvaluationRecord.SERVICE_INSTANCE_ID)); From 536485f6043f3f07d2f8effba406ffdb061e7617 Mon Sep 17 00:00:00 2001 From: peachisai <2581009893@qq.com> Date: Tue, 4 Aug 2026 00:28:58 +0800 Subject: [PATCH 5/8] refactor the genai evaluation record structure --- docs/en/swip/SWIP-16.md | 8 +- .../span/SpanAIEvaluationStrategy.java | 24 +++--- .../manual/genai/GenAIEvaluationRecord.java | 73 ++++++++++++++++--- .../GenAIEvaluationRecordQueryService.java | 32 ++++---- .../GenAIEvaluationRecordQueryCondition.java | 7 +- .../query/IGenAIEvaluationRecordQueryDAO.java | 24 +++--- .../resolver/GenAIEvaluationRecordQuery.java | 11 ++- .../src/main/resources/ai-evaluation.yml | 38 +++++----- .../banyandb/BanyanDBStorageProvider.java | 4 +- .../GenAIEvaluationRecordQueryEsDAO.java | 36 ++++++--- .../JDBCGenAIEvaluationRecordQueryDAO.java | 59 ++++++++++----- 11 files changed, 204 insertions(+), 112 deletions(-) diff --git a/docs/en/swip/SWIP-16.md b/docs/en/swip/SWIP-16.md index 8450ffc069c4..5ab310a26c7f 100644 --- a/docs/en/swip/SWIP-16.md +++ b/docs/en/swip/SWIP-16.md @@ -190,7 +190,7 @@ For tasks where `valueType = SCORE`, the judge returns a numeric result in `[0.0 The initial metric is: -- `gen_ai_evaluation_score_ppm` +- `gen_ai_model_evaluation_score_ppm` The metric is attached to the Virtual GenAI service instance dimension, using `service_name` as the service key and `model_name` as the instance key. The `task_name` remains as a labeled value dimension, allowing the same metric to represent scores for `Faithfulness`, `Relevance`, `TaskCompletion`, `Hallucination`, or any user-defined task. @@ -241,7 +241,7 @@ This SWIP introduces a new OAP capability and a new record data model. The main - A new structured record type `ai_evaluation_result`, including the normalized `evaluation_level` field - `SCORE`-type evaluation results additionally generate a MAL labeled metric for dashboard display -- The `gen_ai_evaluation_score_ppm` metric stores scores scaled by `1,000,000`; query and UI layers need to divide by `1,000,000` to display the original `[0.0, 1.0]` score +- The `gen_ai_model_evaluation_score_ppm` metric stores scores scaled by `1,000,000`; query and UI layers need to divide by `1,000,000` to display the original `[0.0, 1.0]` score - The capability depends on the existing GenAI observability pipeline being able to recognize GenAI spans from SkyWalking native traces, OTLP, and Zipkin - The UI adds an evaluation result page and trace jump based on `traceId` - The current implementation uses an asynchronous local in-memory queue to carry evaluation tasks, and queue data is not part of any persistent protocol @@ -255,6 +255,6 @@ This SWIP introduces a new OAP capability and a new record data model. The main 4. OAP generates evaluation tasks for sampled GenAI spans according to the PPM sampling rate and puts them into the asynchronous local in-memory queue. 5. A background evaluation consumer takes tasks from the queue and sends requests to the configured judge model. 6. OAP writes each evaluation result into SkyWalking structured records. -7. For `SCORE`-type tasks, OAP generates the MAL labeled metric `gen_ai_evaluation_score_ppm` from evaluation results. -8. Users observe both existing GenAI runtime metrics and newly added quality metrics in the GenAI dashboard. The UI should divide `gen_ai_evaluation_score_ppm` values by `1,000,000` to display the original score. +7. For `SCORE`-type tasks, OAP generates the MAL labeled metric `gen_ai_model_evaluation_score_ppm` from evaluation results. +8. Users observe both existing GenAI runtime metrics and newly added quality metrics in the GenAI dashboard. The UI should divide `gen_ai_model_evaluation_score_ppm` values by `1,000,000` to display the original score. 9. Users can also open the evaluation result page, inspect evaluation details, and jump from a single result to the related trace. \ No newline at end of file diff --git a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/strategy/span/SpanAIEvaluationStrategy.java b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/strategy/span/SpanAIEvaluationStrategy.java index b0b6bd6cae28..987c01402bf3 100644 --- a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/strategy/span/SpanAIEvaluationStrategy.java +++ b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/strategy/span/SpanAIEvaluationStrategy.java @@ -33,8 +33,6 @@ import org.apache.skywalking.oap.server.ai.evaluation.service.strategy.AIEvaluationStrategy; import org.apache.skywalking.oap.server.ai.evaluation.task.EvaluationTaskRegistry; import org.apache.skywalking.oap.server.ai.evaluation.value.ValueType; -import org.apache.skywalking.oap.server.core.analysis.IDManager; -import org.apache.skywalking.oap.server.core.analysis.Layer; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.analysis.manual.genai.GenAIEvaluationRecord; import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor; @@ -121,24 +119,20 @@ private void persistResults(final AIEvaluationContext context, final String judgeModel) { final long evaluationTime = System.currentTimeMillis(); for (EvaluationResult result : results) { - final String serviceId = IDManager.ServiceID.buildId( - namingControl.formatServiceName(context.getProviderName()), - Layer.VIRTUAL_GENAI.isNormal() - ); final GenAIEvaluationRecord record = new GenAIEvaluationRecord(); record.setUniqueId(UUID.randomUUID().toString().replace("-", "")); record.setTraceId(context.getTraceId()); - record.setServiceId(serviceId); - record.setServiceInstanceId(IDManager.ServiceInstanceID.buildId( - serviceId, - namingControl.formatServiceName(context.getModelName()) - )); + record.setServiceName(context.getServiceName()); + record.setProviderName(namingControl.formatServiceName(context.getProviderName())); + record.setModelName(namingControl.formatInstanceName(context.getModelName())); + record.setOperationName(operationName(context)); record.setSegmentId(context.getSegmentId()); record.setSpanId(context.getSpanId()); record.setSpanType(plan.getSpanType() == null ? "" : plan.getSpanType().name()); record.setTaskName(result.getName()); record.setValueType(result.getValueType() == null ? "" : result.getValueType().name()); record.setValue(result.getValue()); + record.setScoreValuePpm(GenAIEvaluationRecord.toScoreValuePpm(numericValue(result.getValue()))); record.setEvaluationLevel(levelResolver.resolve(result.getValueType(), result.getValue())); record.setReason(result.getReason()); record.setJudgeModel(judgeModel); @@ -173,4 +167,12 @@ private static String operationName(final AIEvaluationContext context) { return context.getTags().get(GenAISemanticAttributes.OPERATION_NAME); } + private static Double numericValue(final String value) { + try { + return Double.valueOf(value); + } catch (NumberFormatException e) { + return null; + } + } + } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/genai/GenAIEvaluationRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/genai/GenAIEvaluationRecord.java index 7310f8f39538..1c7f0e71205c 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/genai/GenAIEvaluationRecord.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/genai/GenAIEvaluationRecord.java @@ -18,6 +18,8 @@ package org.apache.skywalking.oap.server.core.analysis.manual.genai; +import java.math.BigDecimal; +import java.math.RoundingMode; import lombok.Getter; import lombok.Setter; import org.apache.skywalking.oap.server.core.analysis.Stream; @@ -47,8 +49,12 @@ public class GenAIEvaluationRecord extends Record { public static final String INDEX_NAME = "gen_ai_evaluation_record"; public static final String UNIQUE_ID = "unique_id"; public static final String TRACE_ID = "trace_id"; - public static final String SERVICE_ID = "service_id"; - public static final String SERVICE_INSTANCE_ID = "service_instance_id"; + public static final String PROVIDER_NAME = "provider_name"; + public static final String MODEL_NAME = "model_name"; + public static final String SERVICE_NAME = "service_name"; + public static final String OPERATION_NAME = "operation_name"; + public static final String SCORE_VALUE = "score_value"; + public static final long SCORE_SCALE = 1_000_000L; public static final String SEGMENT_ID = "segment_id"; public static final String SPAN_ID = "span_id"; public static final String SPAN_TYPE = "span_type"; @@ -68,14 +74,52 @@ public class GenAIEvaluationRecord extends Record { private String traceId; @ElasticSearch.EnableDocValues - @Column(name = SERVICE_ID, length = 150, storageOnly = true) + @Column(name = SERVICE_NAME, length = 150) + private String serviceName; + + @ElasticSearch.EnableDocValues + @Column(name = PROVIDER_NAME, length = 150) @BanyanDB.SeriesID(index = 0) - private String serviceId; + private String providerName; @ElasticSearch.EnableDocValues - @Column(name = SERVICE_INSTANCE_ID, length = 150, storageOnly = true) + @Column(name = MODEL_NAME, length = 150) @BanyanDB.SeriesID(index = 1) - private String serviceInstanceId; + private String modelName; + + @ElasticSearch.EnableDocValues + @Column(name = OPERATION_NAME, length = 150) + private String operationName; + + @ElasticSearch.EnableDocValues + @BanyanDB.EnableSort + @Column(name = SCORE_VALUE) + private Long scoreValuePpm; + + public Double getScoreValue() { + return scoreValuePpm == null ? null : scoreValuePpm / (double) SCORE_SCALE; + } + + public static Long toScoreValuePpm(final Double score) { + return score == null ? null : BigDecimal.valueOf(score) + .movePointRight(6) + .setScale(0, RoundingMode.HALF_UP) + .longValueExact(); + } + + public static long minScoreValuePpm(final double score) { + return BigDecimal.valueOf(score) + .movePointRight(6) + .setScale(0, RoundingMode.CEILING) + .longValueExact(); + } + + public static long maxScoreValuePpm(final double score) { + return BigDecimal.valueOf(score) + .movePointRight(6) + .setScale(0, RoundingMode.FLOOR) + .longValueExact(); + } @Column(name = SEGMENT_ID, length = 150, storageOnly = true) private String segmentId; @@ -83,7 +127,7 @@ public class GenAIEvaluationRecord extends Record { @Column(name = SPAN_ID, length = 150, storageOnly = true) private String spanId; - @Column(name = SPAN_TYPE, length = 64) + @Column(name = SPAN_TYPE, length = 64, storageOnly = true) private String spanType; @ElasticSearch.EnableDocValues @@ -122,8 +166,12 @@ public GenAIEvaluationRecord storage2Entity(final Convert2Entity converter) { final GenAIEvaluationRecord record = new GenAIEvaluationRecord(); record.setUniqueId((String) converter.get(UNIQUE_ID)); record.setTraceId((String) converter.get(TRACE_ID)); - record.setServiceId((String) converter.get(SERVICE_ID)); - record.setServiceInstanceId((String) converter.get(SERVICE_INSTANCE_ID)); + record.setServiceName((String) converter.get(SERVICE_NAME)); + record.setProviderName((String) converter.get(PROVIDER_NAME)); + record.setModelName((String) converter.get(MODEL_NAME)); + record.setOperationName((String) converter.get(OPERATION_NAME)); + final Number scoreValue = (Number) converter.get(SCORE_VALUE); + record.setScoreValuePpm(scoreValue == null ? null : scoreValue.longValue()); record.setSegmentId((String) converter.get(SEGMENT_ID)); record.setSpanId((String) converter.get(SPAN_ID)); record.setSpanType((String) converter.get(SPAN_TYPE)); @@ -142,8 +190,11 @@ public GenAIEvaluationRecord storage2Entity(final Convert2Entity converter) { public void entity2Storage(final GenAIEvaluationRecord storageData, final Convert2Storage converter) { converter.accept(UNIQUE_ID, storageData.getUniqueId()); converter.accept(TRACE_ID, storageData.getTraceId()); - converter.accept(SERVICE_ID, storageData.getServiceId()); - converter.accept(SERVICE_INSTANCE_ID, storageData.getServiceInstanceId()); + converter.accept(SERVICE_NAME, storageData.getServiceName()); + converter.accept(PROVIDER_NAME, storageData.getProviderName()); + converter.accept(MODEL_NAME, storageData.getModelName()); + converter.accept(OPERATION_NAME, storageData.getOperationName()); + converter.accept(SCORE_VALUE, storageData.getScoreValuePpm()); converter.accept(SEGMENT_ID, storageData.getSegmentId()); converter.accept(SPAN_ID, storageData.getSpanId()); converter.accept(SPAN_TYPE, storageData.getSpanType()); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/GenAIEvaluationRecordQueryService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/GenAIEvaluationRecordQueryService.java index 4fb443881c29..9aadacd9ae37 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/GenAIEvaluationRecordQueryService.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/GenAIEvaluationRecordQueryService.java @@ -51,12 +51,11 @@ private IGenAIEvaluationRecordQueryDAO getGenAIEvaluationRecordQueryDAO() { return genAIEvaluationRecordQueryDAO; } - public boolean supportQueryGenAIEvaluationRecordByKeywords() { - return getGenAIEvaluationRecordQueryDAO().supportQueryGenAIEvaluationRecordByKeywords(); - } - - public GenAIEvaluationRecords queryGenAIEvaluationRecord(String serviceId, - String serviceInstanceId, + public GenAIEvaluationRecords queryGenAIEvaluationRecord(String providerName, + String modelName, + Double minScore, + Double maxScore, + String sortField, TraceScopeCondition relatedTrace, Pagination paging, Order queryOrder, @@ -68,8 +67,8 @@ public GenAIEvaluationRecords queryGenAIEvaluationRecord(String serviceId, if (traceContext != null) { StringBuilder msg = new StringBuilder(); span = traceContext.createSpan("Query Service: queryGenAIEvaluationRecord"); - msg.append("ServiceId: ").append(serviceId).append(", "); - msg.append("ServiceInstanceId: ").append(serviceInstanceId).append(", "); + msg.append("ProviderName: ").append(providerName).append(", "); + msg.append("ModelName: ").append(modelName).append(", "); msg.append("RelatedTrace: ").append(relatedTrace).append(", "); msg.append("Pagination: ").append(paging).append(", "); msg.append("QueryOrder: ").append(queryOrder).append(", "); @@ -78,7 +77,7 @@ public GenAIEvaluationRecords queryGenAIEvaluationRecord(String serviceId, span.setMsg(msg.toString()); } return queryGenAIEvaluationRecordInternal( - serviceId, serviceInstanceId, relatedTrace, paging, queryOrder, duration, tags + providerName, modelName, minScore, maxScore, sortField, relatedTrace, paging, queryOrder, duration, tags ); } finally { if (traceContext != null) { @@ -87,8 +86,11 @@ public GenAIEvaluationRecords queryGenAIEvaluationRecord(String serviceId, } } - private GenAIEvaluationRecords queryGenAIEvaluationRecordInternal(String serviceId, - String serviceInstanceId, + private GenAIEvaluationRecords queryGenAIEvaluationRecordInternal(String providerName, + String modelName, + Double minScore, + Double maxScore, + String sortField, TraceScopeCondition relatedTrace, Pagination paging, Order queryOrder, @@ -96,12 +98,8 @@ private GenAIEvaluationRecords queryGenAIEvaluationRecordInternal(String service final List tags) throws IOException { PaginationUtils.Page page = PaginationUtils.INSTANCE.exchange(paging); - return getGenAIEvaluationRecordQueryDAO().queryGenAIEvaluationRecordDebuggable(serviceId, - serviceInstanceId, - relatedTrace, - queryOrder, - page.getFrom(), page.getLimit(), - duration, tags + return getGenAIEvaluationRecordQueryDAO().queryGenAIEvaluationRecordDebuggable( + providerName, modelName, minScore, maxScore, sortField, relatedTrace, queryOrder, page.getFrom(), page.getLimit(), duration, tags ); } } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/GenAIEvaluationRecordQueryCondition.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/GenAIEvaluationRecordQueryCondition.java index e61f735dab94..7ba1856deaf7 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/GenAIEvaluationRecordQueryCondition.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/GenAIEvaluationRecordQueryCondition.java @@ -30,8 +30,11 @@ @Setter @ToString public class GenAIEvaluationRecordQueryCondition { - private String serviceId; - private String serviceInstanceId; + private String providerName; + private String modelName; + private Double minScore; + private Double maxScore; + private String sortField; private TraceScopeCondition relatedTrace; private Duration queryDuration; private Pagination paging; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IGenAIEvaluationRecordQueryDAO.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IGenAIEvaluationRecordQueryDAO.java index e2ce96d12ac3..9f2eb6ecad78 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IGenAIEvaluationRecordQueryDAO.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IGenAIEvaluationRecordQueryDAO.java @@ -38,12 +38,11 @@ public interface IGenAIEvaluationRecordQueryDAO extends Service { - default boolean supportQueryGenAIEvaluationRecordByKeywords() { - return false; - } - - default GenAIEvaluationRecords queryGenAIEvaluationRecordDebuggable(String serviceId, - String serviceInstanceId, + default GenAIEvaluationRecords queryGenAIEvaluationRecordDebuggable(String providerName, + String modelName, + Double minScore, + Double maxScore, + String sortField, TraceScopeCondition relatedTrace, Order queryOrder, int from, @@ -56,8 +55,8 @@ default GenAIEvaluationRecords queryGenAIEvaluationRecordDebuggable(String servi if (traceContext != null) { span = traceContext.createSpan("Query Dao: queryGenAIEvaluationRecord"); StringBuilder msg = new StringBuilder(); - msg.append("ServiceId: ").append(serviceId) - .append(", ServiceInstanceId: ").append(serviceInstanceId) + msg.append("ProviderName: ").append(providerName) + .append(", ModelName: ").append(modelName) .append(", RelatedTrace: ").append(relatedTrace) .append(", QueryOrder: ").append(queryOrder) .append(", From: ").append(from) @@ -67,7 +66,7 @@ default GenAIEvaluationRecords queryGenAIEvaluationRecordDebuggable(String servi span.setMsg(msg.toString()); } return queryGenAIEvaluationRecord( - serviceId, serviceInstanceId, relatedTrace, queryOrder, from, limit, duration, tags + providerName, modelName, minScore, maxScore, sortField, relatedTrace, queryOrder, from, limit, duration, tags ); } finally { if (traceContext != null && span != null) { @@ -76,8 +75,11 @@ default GenAIEvaluationRecords queryGenAIEvaluationRecordDebuggable(String servi } } - GenAIEvaluationRecords queryGenAIEvaluationRecord(String serviceId, - String serviceInstanceId, + GenAIEvaluationRecords queryGenAIEvaluationRecord(String providerName, + String modelName, + Double minScore, + Double maxScore, + String sortField, TraceScopeCondition relatedTrace, Order queryOrder, int from, diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/GenAIEvaluationRecordQuery.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/GenAIEvaluationRecordQuery.java index 53acca440bb1..1588f302450b 100644 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/GenAIEvaluationRecordQuery.java +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/GenAIEvaluationRecordQuery.java @@ -46,10 +46,6 @@ public GenAIEvaluationRecordQuery(ModuleManager moduleManager) { this.moduleManager = moduleManager; } - public boolean supportGenAIEvaluationRecordQueryByKeywords() { - return getQueryService().supportQueryGenAIEvaluationRecordByKeywords(); - } - private GenAIEvaluationRecordQueryService getQueryService() { if (genAIEvaluationRecordQueryService == null) { this.genAIEvaluationRecordQueryService = moduleManager.find(CoreModule.NAME).provider().getService(GenAIEvaluationRecordQueryService.class); @@ -98,8 +94,11 @@ private GenAIEvaluationRecords queryGenAIEvaluationRecord( }); } return getQueryService().queryGenAIEvaluationRecord( - condition.getServiceId(), - condition.getServiceInstanceId(), + condition.getProviderName(), + condition.getModelName(), + condition.getMinScore(), + condition.getMaxScore(), + condition.getSortField(), condition.getRelatedTrace(), condition.getPaging(), queryOrder, diff --git a/oap-server/server-starter/src/main/resources/ai-evaluation.yml b/oap-server/server-starter/src/main/resources/ai-evaluation.yml index 69d2b85afa94..7ce14e231593 100644 --- a/oap-server/server-starter/src/main/resources/ai-evaluation.yml +++ b/oap-server/server-starter/src/main/resources/ai-evaluation.yml @@ -25,25 +25,6 @@ judge: temperature: 0.2 max_tokens: 100000 -level: - undefined: undefined - score: - - min: 0.0 - max: 0.3 - level: fail - - min: 0.3 - max: 0.6 - level: warning - - min: 0.6 - max: 0.85 - level: good - - min: 0.85 - max: 1.0 - level: excellent - boolean: - true: excellent - false: fail - system-prompt: | You are an AI observability evaluator. @@ -90,6 +71,25 @@ system-prompt: | Return ONLY the JSON object. +level: + undefined: undefined + score: + - min: 0.0 + max: 0.3 + level: fail + - min: 0.3 + max: 0.6 + level: warning + - min: 0.6 + max: 0.85 + level: good + - min: 0.85 + max: 1.0 + level: excellent + boolean: + true: excellent + false: fail + tasks: - name: Faithfulness valueType: SCORE diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageProvider.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageProvider.java index 5c3e07e54f88..1941316e14a0 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageProvider.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageProvider.java @@ -76,7 +76,7 @@ import org.apache.skywalking.oap.server.library.util.StringUtil; import org.apache.skywalking.oap.server.storage.plugin.banyandb.measure.BanyanDBEBPFProfilingScheduleQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBEventQueryDAO; -import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBGenAIGenAIEvaluationRecordQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBGenAIEvaluationRecordQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.banyandb.measure.BanyanDBHierarchyQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.banyandb.measure.BanyanDBMetadataQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.banyandb.measure.BanyanDBMetricsQueryDAO; @@ -178,7 +178,7 @@ IBatchDAO.class, new BanyanDBBatchDAO(client, config.getGlobal().getMaxBulkSize( this.registerServiceImplementation(IAlarmQueryDAO.class, new BanyanDBAlarmQueryDAO(client)); this.registerServiceImplementation(ILogQueryDAO.class, new BanyanDBLogQueryDAO(client)); this.registerServiceImplementation( - IGenAIEvaluationRecordQueryDAO.class, new BanyanDBGenAIGenAIEvaluationRecordQueryDAO(client)); + IGenAIEvaluationRecordQueryDAO.class, new BanyanDBGenAIEvaluationRecordQueryDAO(client)); this.registerServiceImplementation( IProfileTaskQueryDAO.class, new BanyanDBProfileTaskQueryDAO(client, this.config.getGlobal().getProfileTaskQueryMaxSize() diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/GenAIEvaluationRecordQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/GenAIEvaluationRecordQueryEsDAO.java index 2fcfcb1eed09..2fcc139f4271 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/GenAIEvaluationRecordQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/GenAIEvaluationRecordQueryEsDAO.java @@ -48,8 +48,11 @@ public class GenAIEvaluationRecordQueryEsDAO extends EsDAO implements IGenAIEvaluationRecordQueryDAO { private static final Set QUERYABLE_TAG_KEYS = Set.of( GenAIEvaluationRecord.TRACE_ID, - GenAIEvaluationRecord.SERVICE_ID, - GenAIEvaluationRecord.SERVICE_INSTANCE_ID, + GenAIEvaluationRecord.SERVICE_NAME, + GenAIEvaluationRecord.PROVIDER_NAME, + GenAIEvaluationRecord.MODEL_NAME, + GenAIEvaluationRecord.OPERATION_NAME, + GenAIEvaluationRecord.SCORE_VALUE, GenAIEvaluationRecord.SEGMENT_ID, GenAIEvaluationRecord.SPAN_ID, GenAIEvaluationRecord.SPAN_TYPE, @@ -66,8 +69,9 @@ public GenAIEvaluationRecordQueryEsDAO(final ElasticSearchClient client) { } @Override - public GenAIEvaluationRecords queryGenAIEvaluationRecord(final String serviceId, - final String serviceInstanceId, + public GenAIEvaluationRecords queryGenAIEvaluationRecord(final String providerName, + final String modelName, + final Double minScore, final Double maxScore, final String sortField, final TraceScopeCondition relatedTrace, final Order queryOrder, final int from, @@ -91,11 +95,17 @@ public GenAIEvaluationRecords queryGenAIEvaluationRecord(final String serviceId, if (startSecondTB != 0 && endSecondTB != 0) { query.must(Query.range(Record.TIME_BUCKET).gte(startSecondTB).lte(endSecondTB)); } - if (isNotEmpty(serviceId)) { - query.must(Query.term(GenAIEvaluationRecord.SERVICE_ID, serviceId)); + if (isNotEmpty(providerName)) { + query.must(Query.term(GenAIEvaluationRecord.PROVIDER_NAME, providerName)); } - if (isNotEmpty(serviceInstanceId)) { - query.must(Query.term(GenAIEvaluationRecord.SERVICE_INSTANCE_ID, serviceInstanceId)); + if (isNotEmpty(modelName)) { + query.must(Query.term(GenAIEvaluationRecord.MODEL_NAME, modelName)); + } + if (minScore != null || maxScore != null) { + final var scoreRange = Query.range(GenAIEvaluationRecord.SCORE_VALUE); + if (minScore != null) scoreRange.gte(GenAIEvaluationRecord.minScoreValuePpm(minScore)); + if (maxScore != null) scoreRange.lte(GenAIEvaluationRecord.maxScoreValuePpm(maxScore)); + query.must(scoreRange); } if (nonNull(relatedTrace)) { if (isNotEmpty(relatedTrace.getTraceId())) { @@ -122,7 +132,7 @@ public GenAIEvaluationRecords queryGenAIEvaluationRecord(final String serviceId, final SearchBuilder search = Search.builder() .query(query) .sort( - GenAIEvaluationRecord.EVALUATION_TIME, + GenAIEvaluationRecord.SCORE_VALUE.equals(sortField) ? GenAIEvaluationRecord.SCORE_VALUE : GenAIEvaluationRecord.EVALUATION_TIME, Order.DES.equals(queryOrder) ? Sort.Order.DESC : Sort.Order.ASC ) .size(limit) @@ -146,8 +156,12 @@ private GenAIEvaluationRecord parseRecord(final SearchHit searchHit) { final GenAIEvaluationRecord record = new GenAIEvaluationRecord(); record.setUniqueId((String) source.get(GenAIEvaluationRecord.UNIQUE_ID)); record.setTraceId((String) source.get(GenAIEvaluationRecord.TRACE_ID)); - record.setServiceId((String) source.get(GenAIEvaluationRecord.SERVICE_ID)); - record.setServiceInstanceId((String) source.get(GenAIEvaluationRecord.SERVICE_INSTANCE_ID)); + record.setServiceName((String) source.get(GenAIEvaluationRecord.SERVICE_NAME)); + record.setProviderName((String) source.get(GenAIEvaluationRecord.PROVIDER_NAME)); + record.setModelName((String) source.get(GenAIEvaluationRecord.MODEL_NAME)); + record.setOperationName((String) source.get(GenAIEvaluationRecord.OPERATION_NAME)); + final Number scoreValue = (Number) source.get(GenAIEvaluationRecord.SCORE_VALUE); + record.setScoreValuePpm(scoreValue == null ? null : scoreValue.longValue()); record.setSegmentId((String) source.get(GenAIEvaluationRecord.SEGMENT_ID)); record.setSpanId((String) source.get(GenAIEvaluationRecord.SPAN_ID)); record.setSpanType((String) source.get(GenAIEvaluationRecord.SPAN_TYPE)); diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCGenAIEvaluationRecordQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCGenAIEvaluationRecordQueryDAO.java index 2101afe75adb..7fc98f49c02a 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCGenAIEvaluationRecordQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCGenAIEvaluationRecordQueryDAO.java @@ -50,8 +50,11 @@ public class JDBCGenAIEvaluationRecordQueryDAO implements IGenAIEvaluationRecordQueryDAO { private static final Set QUERYABLE_TAG_KEYS = Set.of( GenAIEvaluationRecord.TRACE_ID, - GenAIEvaluationRecord.SERVICE_ID, - GenAIEvaluationRecord.SERVICE_INSTANCE_ID, + GenAIEvaluationRecord.SERVICE_NAME, + GenAIEvaluationRecord.PROVIDER_NAME, + GenAIEvaluationRecord.MODEL_NAME, + GenAIEvaluationRecord.OPERATION_NAME, + GenAIEvaluationRecord.SCORE_VALUE, GenAIEvaluationRecord.SEGMENT_ID, GenAIEvaluationRecord.SPAN_ID, GenAIEvaluationRecord.SPAN_TYPE, @@ -66,8 +69,11 @@ public class JDBCGenAIEvaluationRecordQueryDAO implements IGenAIEvaluationRecord private static final List SELECTED_COLUMNS = List.of( GenAIEvaluationRecord.UNIQUE_ID, GenAIEvaluationRecord.TRACE_ID, - GenAIEvaluationRecord.SERVICE_ID, - GenAIEvaluationRecord.SERVICE_INSTANCE_ID, + GenAIEvaluationRecord.SERVICE_NAME, + GenAIEvaluationRecord.PROVIDER_NAME, + GenAIEvaluationRecord.MODEL_NAME, + GenAIEvaluationRecord.OPERATION_NAME, + GenAIEvaluationRecord.SCORE_VALUE, GenAIEvaluationRecord.SEGMENT_ID, GenAIEvaluationRecord.SPAN_ID, GenAIEvaluationRecord.SPAN_TYPE, @@ -85,8 +91,9 @@ public class JDBCGenAIEvaluationRecordQueryDAO implements IGenAIEvaluationRecord @Override @SneakyThrows - public GenAIEvaluationRecords queryGenAIEvaluationRecord(final String serviceId, - final String serviceInstanceId, + public GenAIEvaluationRecords queryGenAIEvaluationRecord(final String providerName, + final String modelName, + final Double minScore, final Double maxScore, final String sortField, final TraceScopeCondition relatedTrace, final Order queryOrder, final int from, @@ -117,7 +124,7 @@ public GenAIEvaluationRecords queryGenAIEvaluationRecord(final String serviceId, final var records = new ArrayList(); for (final var table : tables) { final var sqlAndParameters = buildSQL( - serviceId, serviceInstanceId, relatedTrace, queryOrder, from, limit, duration, tags, table); + providerName, modelName, minScore, maxScore, sortField, relatedTrace, queryOrder, from, limit, duration, tags, table); records.addAll( jdbcClient.executeQuery( sqlAndParameters.sql(), @@ -141,8 +148,12 @@ protected ArrayList parseResults(final ResultSet resultSe final GenAIEvaluationRecord record = new GenAIEvaluationRecord(); record.setUniqueId(resultSet.getString(GenAIEvaluationRecord.UNIQUE_ID)); record.setTraceId(resultSet.getString(GenAIEvaluationRecord.TRACE_ID)); - record.setServiceId(resultSet.getString(GenAIEvaluationRecord.SERVICE_ID)); - record.setServiceInstanceId(resultSet.getString(GenAIEvaluationRecord.SERVICE_INSTANCE_ID)); + record.setServiceName(resultSet.getString(GenAIEvaluationRecord.SERVICE_NAME)); + record.setProviderName(resultSet.getString(GenAIEvaluationRecord.PROVIDER_NAME)); + record.setModelName(resultSet.getString(GenAIEvaluationRecord.MODEL_NAME)); + record.setOperationName(resultSet.getString(GenAIEvaluationRecord.OPERATION_NAME)); + final long scoreValue = resultSet.getLong(GenAIEvaluationRecord.SCORE_VALUE); + record.setScoreValuePpm(resultSet.wasNull() ? null : scoreValue); record.setSegmentId(resultSet.getString(GenAIEvaluationRecord.SEGMENT_ID)); record.setSpanId(resultSet.getString(GenAIEvaluationRecord.SPAN_ID)); record.setSpanType(resultSet.getString(GenAIEvaluationRecord.SPAN_TYPE)); @@ -158,8 +169,11 @@ protected ArrayList parseResults(final ResultSet resultSe return records; } - protected SQLAndParameters buildSQL(final String serviceId, - final String serviceInstanceId, + protected SQLAndParameters buildSQL(final String providerName, + final String modelName, + final Double minScore, + final Double maxScore, + final String sortField, final TraceScopeCondition relatedTrace, final Order queryOrder, final int from, @@ -190,13 +204,21 @@ protected SQLAndParameters buildSQL(final String serviceId, sql.append(" and ").append(storageColumn(GenAIEvaluationRecord.TIME_BUCKET)).append(" <= ?"); parameters.add(endSecondTB); } - if (StringUtil.isNotEmpty(serviceId)) { - sql.append(" and ").append(storageColumn(GenAIEvaluationRecord.SERVICE_ID)).append(" = ?"); - parameters.add(serviceId); + if (StringUtil.isNotEmpty(providerName)) { + sql.append(" and ").append(storageColumn(GenAIEvaluationRecord.PROVIDER_NAME)).append(" = ?"); + parameters.add(providerName); } - if (StringUtil.isNotEmpty(serviceInstanceId)) { - sql.append(" and ").append(storageColumn(GenAIEvaluationRecord.SERVICE_INSTANCE_ID)).append(" = ?"); - parameters.add(serviceInstanceId); + if (StringUtil.isNotEmpty(modelName)) { + sql.append(" and ").append(storageColumn(GenAIEvaluationRecord.MODEL_NAME)).append(" = ?"); + parameters.add(modelName); + } + if (minScore != null) { + sql.append(" and ").append(storageColumn(GenAIEvaluationRecord.SCORE_VALUE)).append(" >= ?"); + parameters.add(GenAIEvaluationRecord.minScoreValuePpm(minScore)); + } + if (maxScore != null) { + sql.append(" and ").append(storageColumn(GenAIEvaluationRecord.SCORE_VALUE)).append(" <= ?"); + parameters.add(GenAIEvaluationRecord.maxScoreValuePpm(maxScore)); } if (nonNull(relatedTrace)) { if (StringUtil.isNotEmpty(relatedTrace.getTraceId())) { @@ -224,7 +246,8 @@ protected SQLAndParameters buildSQL(final String serviceId, } sql.append(" order by ") - .append(storageColumn(GenAIEvaluationRecord.EVALUATION_TIME)) + .append(storageColumn(GenAIEvaluationRecord.SCORE_VALUE.equals(sortField) + ? GenAIEvaluationRecord.SCORE_VALUE : GenAIEvaluationRecord.EVALUATION_TIME)) .append(" ") .append(Order.DES.equals(queryOrder) ? "desc" : "asc"); sql.append(" limit ").append(from + limit); From 6e21b8cc6aa39dc19c46b6a1526ffb4c4cf7ceba Mon Sep 17 00:00:00 2001 From: peachisai <2581009893@qq.com> Date: Tue, 4 Aug 2026 00:29:05 +0800 Subject: [PATCH 6/8] fix --- ...anyanDBGenAIEvaluationRecordQueryDAO.java} | 101 ++++++++++-------- 1 file changed, 55 insertions(+), 46 deletions(-) rename oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/{BanyanDBGenAIGenAIEvaluationRecordQueryDAO.java => BanyanDBGenAIEvaluationRecordQueryDAO.java} (60%) diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBGenAIGenAIEvaluationRecordQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBGenAIEvaluationRecordQueryDAO.java similarity index 60% rename from oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBGenAIGenAIEvaluationRecordQueryDAO.java rename to oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBGenAIEvaluationRecordQueryDAO.java index d0c3cadfaa0a..39afc138bfd9 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBGenAIGenAIEvaluationRecordQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBGenAIEvaluationRecordQueryDAO.java @@ -19,9 +19,7 @@ package org.apache.skywalking.oap.server.storage.plugin.banyandb.stream; import com.google.common.collect.ImmutableSet; -import org.apache.skywalking.library.banyandb.v1.client.AbstractQuery; import org.apache.skywalking.library.banyandb.v1.client.RowEntity; -import org.apache.skywalking.library.banyandb.v1.client.StreamQuery; import org.apache.skywalking.library.banyandb.v1.client.StreamQueryResponse; import org.apache.skywalking.oap.server.core.analysis.manual.genai.GenAIEvaluationRecord; import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag; @@ -42,11 +40,14 @@ /** * {@link org.apache.skywalking.oap.server.core.analysis.manual.log.LogRecord} is a gen-ai evaluation result */ -public class BanyanDBGenAIGenAIEvaluationRecordQueryDAO extends AbstractBanyanDBDAO implements IGenAIEvaluationRecordQueryDAO { +public class BanyanDBGenAIEvaluationRecordQueryDAO extends AbstractBanyanDBDAO implements IGenAIEvaluationRecordQueryDAO { private static final Set TAGS = ImmutableSet.of( GenAIEvaluationRecord.TRACE_ID, - GenAIEvaluationRecord.SERVICE_ID, - GenAIEvaluationRecord.SERVICE_INSTANCE_ID, + GenAIEvaluationRecord.SERVICE_NAME, + GenAIEvaluationRecord.PROVIDER_NAME, + GenAIEvaluationRecord.MODEL_NAME, + GenAIEvaluationRecord.OPERATION_NAME, + GenAIEvaluationRecord.SCORE_VALUE, GenAIEvaluationRecord.SEGMENT_ID, GenAIEvaluationRecord.SPAN_ID, GenAIEvaluationRecord.SPAN_TYPE, @@ -59,65 +60,68 @@ public class BanyanDBGenAIGenAIEvaluationRecordQueryDAO extends AbstractBanyanDB GenAIEvaluationRecord.EVALUATION_TIME ); - public BanyanDBGenAIGenAIEvaluationRecordQueryDAO(BanyanDBStorageClient client) { + public BanyanDBGenAIEvaluationRecordQueryDAO(BanyanDBStorageClient client) { super(client); } @Override - public GenAIEvaluationRecords queryGenAIEvaluationRecord(String serviceId, String serviceInstanceId, + public GenAIEvaluationRecords queryGenAIEvaluationRecord(String providerName, String modelName, Double minScore, Double maxScore, String sortField, TraceScopeCondition relatedTrace, Order queryOrder, int from, int limit, Duration duration, List tags) throws IOException { final boolean isColdStage = duration != null && duration.isColdStage(); - final QueryBuilder query = new QueryBuilder<>() { - @Override - public void apply(StreamQuery query) { - if (StringUtil.isNotEmpty(serviceId)) { - query.and(eq(GenAIEvaluationRecord.SERVICE_ID, serviceId)); - } - if (StringUtil.isNotEmpty(serviceInstanceId)) { - query.and(eq(GenAIEvaluationRecord.SERVICE_INSTANCE_ID, serviceInstanceId)); - } + final Conditions where = Conditions.create(); + if (StringUtil.isNotEmpty(providerName)) { + where.eq(GenAIEvaluationRecord.PROVIDER_NAME, providerName); + } + if (StringUtil.isNotEmpty(modelName)) { + where.eq(GenAIEvaluationRecord.MODEL_NAME, modelName); + } + if (minScore != null) { + where.gte(GenAIEvaluationRecord.SCORE_VALUE, GenAIEvaluationRecord.minScoreValuePpm(minScore)); + } + if (maxScore != null) { + where.lte(GenAIEvaluationRecord.SCORE_VALUE, GenAIEvaluationRecord.maxScoreValuePpm(maxScore)); + } - if (Objects.nonNull(relatedTrace)) { - if (StringUtil.isNotEmpty(relatedTrace.getTraceId())) { - query.and(eq(GenAIEvaluationRecord.TRACE_ID, relatedTrace.getTraceId())); - } - if (StringUtil.isNotEmpty(relatedTrace.getSegmentId())) { - query.and(eq(GenAIEvaluationRecord.SEGMENT_ID, relatedTrace.getSegmentId())); - } - if (Objects.nonNull(relatedTrace.getSpanId())) { - query.and(eq(GenAIEvaluationRecord.SPAN_ID, (long) relatedTrace.getSpanId())); - } - } + if (Objects.nonNull(relatedTrace)) { + if (StringUtil.isNotEmpty(relatedTrace.getTraceId())) { + where.eq(GenAIEvaluationRecord.TRACE_ID, relatedTrace.getTraceId()); + } + if (StringUtil.isNotEmpty(relatedTrace.getSegmentId())) { + where.eq(GenAIEvaluationRecord.SEGMENT_ID, relatedTrace.getSegmentId()); + } + if (Objects.nonNull(relatedTrace.getSpanId())) { + where.eq(GenAIEvaluationRecord.SPAN_ID, (long) relatedTrace.getSpanId()); + } + } - if (CollectionUtils.isNotEmpty(tags)) { - for (final Tag tag : tags) { - if (StringUtil.isNotEmpty(tag.getKey()) && StringUtil.isNotEmpty(tag.getValue())) { - query.and(eq(tag.getKey(), tag.getValue())); - } - } - } - if (queryOrder == Order.ASC) { - query.setOrderBy( - new AbstractQuery.OrderBy(AbstractQuery.Sort.ASC)); - } else { - query.setOrderBy( - new AbstractQuery.OrderBy(AbstractQuery.Sort.DESC)); + if (CollectionUtils.isNotEmpty(tags)) { + for (final Tag tag : tags) { + if (StringUtil.isNotEmpty(tag.getKey()) && StringUtil.isNotEmpty(tag.getValue())) { + where.eq(tag.getKey(), tag.getValue()); } - query.setLimit(limit); - query.setOffset(from); } - }; + } + if (queryOrder == Order.ASC) { + where.orderBy(sortColumn(sortField), "ASC"); + } else { + where.orderByDesc(sortColumn(sortField)); + } + where.limit(limit).offset(from); - StreamQueryResponse resp = queryDebuggable(isColdStage, GenAIEvaluationRecord.INDEX_NAME, TAGS, getTimestampRange(duration), query); + StreamQueryResponse resp = queryDebuggable(isColdStage, GenAIEvaluationRecord.INDEX_NAME, TAGS, getTimestampRange(duration), where); GenAIEvaluationRecords genAIEvaluationRecords = new GenAIEvaluationRecords(); for (final RowEntity rowEntity : resp.getElements()) { GenAIEvaluationRecord evaluationRecord = new GenAIEvaluationRecord(); evaluationRecord.setTraceId(rowEntity.getTagValue(GenAIEvaluationRecord.TRACE_ID)); - evaluationRecord.setServiceId(rowEntity.getTagValue(GenAIEvaluationRecord.SERVICE_ID)); - evaluationRecord.setServiceInstanceId(rowEntity.getTagValue(GenAIEvaluationRecord.SERVICE_INSTANCE_ID)); + evaluationRecord.setServiceName(rowEntity.getTagValue(GenAIEvaluationRecord.SERVICE_NAME)); + evaluationRecord.setProviderName(rowEntity.getTagValue(GenAIEvaluationRecord.PROVIDER_NAME)); + evaluationRecord.setModelName(rowEntity.getTagValue(GenAIEvaluationRecord.MODEL_NAME)); + evaluationRecord.setOperationName(rowEntity.getTagValue(GenAIEvaluationRecord.OPERATION_NAME)); + final Number scoreValue = (Number) rowEntity.getTagValue(GenAIEvaluationRecord.SCORE_VALUE); + evaluationRecord.setScoreValuePpm(scoreValue == null ? null : scoreValue.longValue()); evaluationRecord.setSegmentId(rowEntity.getTagValue(GenAIEvaluationRecord.SEGMENT_ID)); evaluationRecord.setSpanId(rowEntity.getTagValue(GenAIEvaluationRecord.SPAN_ID)); evaluationRecord.setSpanType(rowEntity.getTagValue(GenAIEvaluationRecord.SPAN_TYPE)); @@ -132,4 +136,9 @@ public void apply(StreamQuery query) { } return genAIEvaluationRecords; } + + private static String sortColumn(final String sortField) { + return GenAIEvaluationRecord.SCORE_VALUE.equals(sortField) + ? GenAIEvaluationRecord.SCORE_VALUE : GenAIEvaluationRecord.EVALUATION_TIME; + } } From 65f7381980a6690ca5947f5a996415590bb2143f Mon Sep 17 00:00:00 2001 From: peachisai <2581009893@qq.com> Date: Thu, 6 Aug 2026 00:33:08 +0800 Subject: [PATCH 7/8] fix --- .../span/SpanAIEvaluationStrategy.java | 2 +- .../manual/genai/GenAIEvaluationRecord.java | 128 ++++++++++++------ .../GenAIEvaluationRecordQueryService.java | 19 ++- .../GenAIEvaluationRecordQueryCondition.java | 4 + .../query/type/GenAIEvaluationRecords.java | 1 - .../query/IGenAIEvaluationRecordQueryDAO.java | 18 ++- .../resolver/GenAIEvaluationRecordQuery.java | 4 + ...BanyanDBGenAIEvaluationRecordQueryDAO.java | 48 ++++--- .../GenAIEvaluationRecordQueryEsDAO.java | 38 ++++-- .../JDBCGenAIEvaluationRecordQueryDAO.java | 71 +++++++--- 10 files changed, 226 insertions(+), 107 deletions(-) diff --git a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/strategy/span/SpanAIEvaluationStrategy.java b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/strategy/span/SpanAIEvaluationStrategy.java index 987c01402bf3..1e62f1eff7a9 100644 --- a/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/strategy/span/SpanAIEvaluationStrategy.java +++ b/oap-server/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/service/strategy/span/SpanAIEvaluationStrategy.java @@ -127,7 +127,7 @@ private void persistResults(final AIEvaluationContext context, record.setModelName(namingControl.formatInstanceName(context.getModelName())); record.setOperationName(operationName(context)); record.setSegmentId(context.getSegmentId()); - record.setSpanId(context.getSpanId()); + record.setSpanId(Integer.parseInt(context.getSpanId())); record.setSpanType(plan.getSpanType() == null ? "" : plan.getSpanType().name()); record.setTaskName(result.getName()); record.setValueType(result.getValueType() == null ? "" : result.getValueType().name()); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/genai/GenAIEvaluationRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/genai/GenAIEvaluationRecord.java index 1c7f0e71205c..88b6c91f587b 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/genai/GenAIEvaluationRecord.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/genai/GenAIEvaluationRecord.java @@ -20,8 +20,11 @@ import java.math.BigDecimal; import java.math.RoundingMode; + import lombok.Getter; import lombok.Setter; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.Layer; import org.apache.skywalking.oap.server.core.analysis.Stream; import org.apache.skywalking.oap.server.core.analysis.record.Record; import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor; @@ -49,9 +52,9 @@ public class GenAIEvaluationRecord extends Record { public static final String INDEX_NAME = "gen_ai_evaluation_record"; public static final String UNIQUE_ID = "unique_id"; public static final String TRACE_ID = "trace_id"; - public static final String PROVIDER_NAME = "provider_name"; - public static final String MODEL_NAME = "model_name"; - public static final String SERVICE_NAME = "service_name"; + public static final String PROVIDER_ID = "provider_id"; + public static final String MODEL_ID = "model_id"; + public static final String SERVICE_ID = "service_id"; public static final String OPERATION_NAME = "operation_name"; public static final String SCORE_VALUE = "score_value"; public static final long SCORE_SCALE = 1_000_000L; @@ -74,18 +77,18 @@ public class GenAIEvaluationRecord extends Record { private String traceId; @ElasticSearch.EnableDocValues - @Column(name = SERVICE_NAME, length = 150) - private String serviceName; + @Column(name = SERVICE_ID, length = 150) + private String serviceId; @ElasticSearch.EnableDocValues - @Column(name = PROVIDER_NAME, length = 150) + @Column(name = PROVIDER_ID, length = 150) @BanyanDB.SeriesID(index = 0) - private String providerName; + private String providerId; @ElasticSearch.EnableDocValues - @Column(name = MODEL_NAME, length = 150) + @Column(name = MODEL_ID, length = 150) @BanyanDB.SeriesID(index = 1) - private String modelName; + private String modelId; @ElasticSearch.EnableDocValues @Column(name = OPERATION_NAME, length = 150) @@ -96,36 +99,12 @@ public class GenAIEvaluationRecord extends Record { @Column(name = SCORE_VALUE) private Long scoreValuePpm; - public Double getScoreValue() { - return scoreValuePpm == null ? null : scoreValuePpm / (double) SCORE_SCALE; - } - - public static Long toScoreValuePpm(final Double score) { - return score == null ? null : BigDecimal.valueOf(score) - .movePointRight(6) - .setScale(0, RoundingMode.HALF_UP) - .longValueExact(); - } - - public static long minScoreValuePpm(final double score) { - return BigDecimal.valueOf(score) - .movePointRight(6) - .setScale(0, RoundingMode.CEILING) - .longValueExact(); - } - - public static long maxScoreValuePpm(final double score) { - return BigDecimal.valueOf(score) - .movePointRight(6) - .setScale(0, RoundingMode.FLOOR) - .longValueExact(); - } - @Column(name = SEGMENT_ID, length = 150, storageOnly = true) private String segmentId; - @Column(name = SPAN_ID, length = 150, storageOnly = true) - private String spanId; + @Column(name = SPAN_ID) + @BanyanDB.NoIndexing + private int spanId; @Column(name = SPAN_TYPE, length = 64, storageOnly = true) private String spanType; @@ -148,13 +127,74 @@ public static long maxScoreValuePpm(final double score) { @Column(name = REASON, length = 4096, storageOnly = true) private String reason; - @Column(name = JUDGE_MODEL, length = 64, storageOnly = true) + @Column(name = JUDGE_MODEL, length = 64) private String judgeModel; @ElasticSearch.EnableDocValues @Column(name = EVALUATION_TIME) private long evaluationTime; + public Double getScoreValue() { + return scoreValuePpm == null ? null : scoreValuePpm / (double) SCORE_SCALE; + } + + public static Long toScoreValuePpm(final Double score) { + return score == null ? null : BigDecimal.valueOf(score) + .movePointRight(6) + .setScale(0, RoundingMode.HALF_UP) + .longValueExact(); + } + + public static long minScoreValuePpm(final double score) { + return BigDecimal.valueOf(score) + .movePointRight(6) + .setScale(0, RoundingMode.CEILING) + .longValueExact(); + } + + public static long maxScoreValuePpm(final double score) { + return BigDecimal.valueOf(score) + .movePointRight(6) + .setScale(0, RoundingMode.FLOOR) + .longValueExact(); + } + + public static String toEntityId(final String name) { + return IDManager.ServiceID.buildId(name, Layer.VIRTUAL_GENAI.isNormal()); + } + + public static String toServiceId(final String name) { + return IDManager.ServiceID.buildId(name, Layer.GENAI.isNormal()); + } + + public String getServiceName() { + return decodeName(serviceId); + } + + public void setServiceName(final String serviceName) { + this.serviceId = toServiceId(serviceName); + } + + public String getProviderName() { + return decodeName(providerId); + } + + public void setProviderName(final String providerName) { + this.providerId = toEntityId(providerName); + } + + public String getModelName() { + return decodeName(modelId); + } + + public void setModelName(final String modelName) { + this.modelId = toEntityId(modelName); + } + + private static String decodeName(final String id) { + return id == null ? null : IDManager.ServiceID.analysisId(id).getName(); + } + @Override public StorageID id() { return new StorageID().append(UNIQUE_ID, uniqueId); @@ -166,14 +206,14 @@ public GenAIEvaluationRecord storage2Entity(final Convert2Entity converter) { final GenAIEvaluationRecord record = new GenAIEvaluationRecord(); record.setUniqueId((String) converter.get(UNIQUE_ID)); record.setTraceId((String) converter.get(TRACE_ID)); - record.setServiceName((String) converter.get(SERVICE_NAME)); - record.setProviderName((String) converter.get(PROVIDER_NAME)); - record.setModelName((String) converter.get(MODEL_NAME)); + record.setServiceId((String) converter.get(SERVICE_ID)); + record.setProviderId((String) converter.get(PROVIDER_ID)); + record.setModelId((String) converter.get(MODEL_ID)); record.setOperationName((String) converter.get(OPERATION_NAME)); final Number scoreValue = (Number) converter.get(SCORE_VALUE); record.setScoreValuePpm(scoreValue == null ? null : scoreValue.longValue()); record.setSegmentId((String) converter.get(SEGMENT_ID)); - record.setSpanId((String) converter.get(SPAN_ID)); + record.setSpanId(((Number) converter.get(SPAN_ID)).intValue()); record.setSpanType((String) converter.get(SPAN_TYPE)); record.setTaskName((String) converter.get(TASK_NAME)); record.setValueType((String) converter.get(VALUE_TYPE)); @@ -190,9 +230,9 @@ public GenAIEvaluationRecord storage2Entity(final Convert2Entity converter) { public void entity2Storage(final GenAIEvaluationRecord storageData, final Convert2Storage converter) { converter.accept(UNIQUE_ID, storageData.getUniqueId()); converter.accept(TRACE_ID, storageData.getTraceId()); - converter.accept(SERVICE_NAME, storageData.getServiceName()); - converter.accept(PROVIDER_NAME, storageData.getProviderName()); - converter.accept(MODEL_NAME, storageData.getModelName()); + converter.accept(SERVICE_ID, storageData.getServiceId()); + converter.accept(PROVIDER_ID, storageData.getProviderId()); + converter.accept(MODEL_ID, storageData.getModelId()); converter.accept(OPERATION_NAME, storageData.getOperationName()); converter.accept(SCORE_VALUE, storageData.getScoreValuePpm()); converter.accept(SEGMENT_ID, storageData.getSegmentId()); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/GenAIEvaluationRecordQueryService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/GenAIEvaluationRecordQueryService.java index 9aadacd9ae37..7a3c5f2167c3 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/GenAIEvaluationRecordQueryService.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/GenAIEvaluationRecordQueryService.java @@ -51,11 +51,15 @@ private IGenAIEvaluationRecordQueryDAO getGenAIEvaluationRecordQueryDAO() { return genAIEvaluationRecordQueryDAO; } - public GenAIEvaluationRecords queryGenAIEvaluationRecord(String providerName, + public GenAIEvaluationRecords queryGenAIEvaluationRecord(String serviceName, + String providerName, String modelName, Double minScore, Double maxScore, String sortField, + String taskName, + String evaluationLevel, + String judgeModel, TraceScopeCondition relatedTrace, Pagination paging, Order queryOrder, @@ -67,6 +71,7 @@ public GenAIEvaluationRecords queryGenAIEvaluationRecord(String providerName, if (traceContext != null) { StringBuilder msg = new StringBuilder(); span = traceContext.createSpan("Query Service: queryGenAIEvaluationRecord"); + msg.append("ServiceName: ").append(serviceName).append(", "); msg.append("ProviderName: ").append(providerName).append(", "); msg.append("ModelName: ").append(modelName).append(", "); msg.append("RelatedTrace: ").append(relatedTrace).append(", "); @@ -77,7 +82,8 @@ public GenAIEvaluationRecords queryGenAIEvaluationRecord(String providerName, span.setMsg(msg.toString()); } return queryGenAIEvaluationRecordInternal( - providerName, modelName, minScore, maxScore, sortField, relatedTrace, paging, queryOrder, duration, tags + serviceName, providerName, modelName, minScore, maxScore, sortField, taskName, evaluationLevel, judgeModel, + relatedTrace, paging, queryOrder, duration, tags ); } finally { if (traceContext != null) { @@ -86,11 +92,15 @@ public GenAIEvaluationRecords queryGenAIEvaluationRecord(String providerName, } } - private GenAIEvaluationRecords queryGenAIEvaluationRecordInternal(String providerName, + private GenAIEvaluationRecords queryGenAIEvaluationRecordInternal(String serviceName, + String providerName, String modelName, Double minScore, Double maxScore, String sortField, + String taskName, + String evaluationLevel, + String judgeModel, TraceScopeCondition relatedTrace, Pagination paging, Order queryOrder, @@ -99,7 +109,8 @@ private GenAIEvaluationRecords queryGenAIEvaluationRecordInternal(String provide PaginationUtils.Page page = PaginationUtils.INSTANCE.exchange(paging); return getGenAIEvaluationRecordQueryDAO().queryGenAIEvaluationRecordDebuggable( - providerName, modelName, minScore, maxScore, sortField, relatedTrace, queryOrder, page.getFrom(), page.getLimit(), duration, tags + serviceName, providerName, modelName, minScore, maxScore, sortField, taskName, evaluationLevel, judgeModel, + relatedTrace, queryOrder, page.getFrom(), page.getLimit(), duration, tags ); } } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/GenAIEvaluationRecordQueryCondition.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/GenAIEvaluationRecordQueryCondition.java index 7ba1856deaf7..64529eb32417 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/GenAIEvaluationRecordQueryCondition.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/GenAIEvaluationRecordQueryCondition.java @@ -30,11 +30,15 @@ @Setter @ToString public class GenAIEvaluationRecordQueryCondition { + private String serviceName; private String providerName; private String modelName; private Double minScore; private Double maxScore; private String sortField; + private String taskName; + private String evaluationLevel; + private String judgeModel; private TraceScopeCondition relatedTrace; private Duration queryDuration; private Pagination paging; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/GenAIEvaluationRecords.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/GenAIEvaluationRecords.java index 1d8f20b61d50..29d1b504b05f 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/GenAIEvaluationRecords.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/GenAIEvaluationRecords.java @@ -32,7 +32,6 @@ @Accessors(chain = true) public class GenAIEvaluationRecords { private final List genAIEvaluationRecordList; - private String errorReason; private DebuggingTrace debuggingTrace; public GenAIEvaluationRecords() { diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IGenAIEvaluationRecordQueryDAO.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IGenAIEvaluationRecordQueryDAO.java index 9f2eb6ecad78..239c937c9de4 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IGenAIEvaluationRecordQueryDAO.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IGenAIEvaluationRecordQueryDAO.java @@ -38,11 +38,15 @@ public interface IGenAIEvaluationRecordQueryDAO extends Service { - default GenAIEvaluationRecords queryGenAIEvaluationRecordDebuggable(String providerName, + default GenAIEvaluationRecords queryGenAIEvaluationRecordDebuggable(String serviceName, + String providerName, String modelName, Double minScore, Double maxScore, String sortField, + String taskName, + String evaluationLevel, + String judgeModel, TraceScopeCondition relatedTrace, Order queryOrder, int from, @@ -55,7 +59,8 @@ default GenAIEvaluationRecords queryGenAIEvaluationRecordDebuggable(String provi if (traceContext != null) { span = traceContext.createSpan("Query Dao: queryGenAIEvaluationRecord"); StringBuilder msg = new StringBuilder(); - msg.append("ProviderName: ").append(providerName) + msg.append("ServiceName: ").append(serviceName) + .append(", ProviderName: ").append(providerName) .append(", ModelName: ").append(modelName) .append(", RelatedTrace: ").append(relatedTrace) .append(", QueryOrder: ").append(queryOrder) @@ -66,7 +71,8 @@ default GenAIEvaluationRecords queryGenAIEvaluationRecordDebuggable(String provi span.setMsg(msg.toString()); } return queryGenAIEvaluationRecord( - providerName, modelName, minScore, maxScore, sortField, relatedTrace, queryOrder, from, limit, duration, tags + serviceName, providerName, modelName, minScore, maxScore, sortField, taskName, evaluationLevel, judgeModel, + relatedTrace, queryOrder, from, limit, duration, tags ); } finally { if (traceContext != null && span != null) { @@ -75,11 +81,15 @@ default GenAIEvaluationRecords queryGenAIEvaluationRecordDebuggable(String provi } } - GenAIEvaluationRecords queryGenAIEvaluationRecord(String providerName, + GenAIEvaluationRecords queryGenAIEvaluationRecord(String serviceName, + String providerName, String modelName, Double minScore, Double maxScore, String sortField, + String taskName, + String evaluationLevel, + String judgeModel, TraceScopeCondition relatedTrace, Order queryOrder, int from, diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/GenAIEvaluationRecordQuery.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/GenAIEvaluationRecordQuery.java index 1588f302450b..5b3f00f479c3 100644 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/GenAIEvaluationRecordQuery.java +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/GenAIEvaluationRecordQuery.java @@ -94,11 +94,15 @@ private GenAIEvaluationRecords queryGenAIEvaluationRecord( }); } return getQueryService().queryGenAIEvaluationRecord( + condition.getServiceName(), condition.getProviderName(), condition.getModelName(), condition.getMinScore(), condition.getMaxScore(), condition.getSortField(), + condition.getTaskName(), + condition.getEvaluationLevel(), + condition.getJudgeModel(), condition.getRelatedTrace(), condition.getPaging(), queryOrder, diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBGenAIEvaluationRecordQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBGenAIEvaluationRecordQueryDAO.java index 39afc138bfd9..d98d08349f5b 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBGenAIEvaluationRecordQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBGenAIEvaluationRecordQueryDAO.java @@ -43,9 +43,9 @@ public class BanyanDBGenAIEvaluationRecordQueryDAO extends AbstractBanyanDBDAO implements IGenAIEvaluationRecordQueryDAO { private static final Set TAGS = ImmutableSet.of( GenAIEvaluationRecord.TRACE_ID, - GenAIEvaluationRecord.SERVICE_NAME, - GenAIEvaluationRecord.PROVIDER_NAME, - GenAIEvaluationRecord.MODEL_NAME, + GenAIEvaluationRecord.SERVICE_ID, + GenAIEvaluationRecord.PROVIDER_ID, + GenAIEvaluationRecord.MODEL_ID, GenAIEvaluationRecord.OPERATION_NAME, GenAIEvaluationRecord.SCORE_VALUE, GenAIEvaluationRecord.SEGMENT_ID, @@ -65,16 +65,20 @@ public BanyanDBGenAIEvaluationRecordQueryDAO(BanyanDBStorageClient client) { } @Override - public GenAIEvaluationRecords queryGenAIEvaluationRecord(String providerName, String modelName, Double minScore, Double maxScore, String sortField, + public GenAIEvaluationRecords queryGenAIEvaluationRecord(String serviceName, String providerName, String modelName, Double minScore, Double maxScore, String sortField, + String taskName, String evaluationLevel, String judgeModel, TraceScopeCondition relatedTrace, Order queryOrder, int from, int limit, Duration duration, List tags) throws IOException { final boolean isColdStage = duration != null && duration.isColdStage(); final Conditions where = Conditions.create(); + if (StringUtil.isNotEmpty(serviceName)) { + where.eq(GenAIEvaluationRecord.SERVICE_ID, GenAIEvaluationRecord.toServiceId(serviceName)); + } if (StringUtil.isNotEmpty(providerName)) { - where.eq(GenAIEvaluationRecord.PROVIDER_NAME, providerName); + where.eq(GenAIEvaluationRecord.PROVIDER_ID, GenAIEvaluationRecord.toEntityId(providerName)); } if (StringUtil.isNotEmpty(modelName)) { - where.eq(GenAIEvaluationRecord.MODEL_NAME, modelName); + where.eq(GenAIEvaluationRecord.MODEL_ID, GenAIEvaluationRecord.toEntityId(modelName)); } if (minScore != null) { where.gte(GenAIEvaluationRecord.SCORE_VALUE, GenAIEvaluationRecord.minScoreValuePpm(minScore)); @@ -82,6 +86,15 @@ public GenAIEvaluationRecords queryGenAIEvaluationRecord(String providerName, St if (maxScore != null) { where.lte(GenAIEvaluationRecord.SCORE_VALUE, GenAIEvaluationRecord.maxScoreValuePpm(maxScore)); } + if (StringUtil.isNotEmpty(taskName)) { + where.eq(GenAIEvaluationRecord.TASK_NAME, taskName); + } + if (StringUtil.isNotEmpty(evaluationLevel)) { + where.eq(GenAIEvaluationRecord.EVALUATION_LEVEL, evaluationLevel); + } + if (StringUtil.isNotEmpty(judgeModel)) { + where.eq(GenAIEvaluationRecord.JUDGE_MODEL, judgeModel); + } if (Objects.nonNull(relatedTrace)) { if (StringUtil.isNotEmpty(relatedTrace.getTraceId())) { @@ -102,10 +115,12 @@ public GenAIEvaluationRecords queryGenAIEvaluationRecord(String providerName, St } } } - if (queryOrder == Order.ASC) { - where.orderBy(sortColumn(sortField), "ASC"); + if (GenAIEvaluationRecord.SCORE_VALUE.equalsIgnoreCase(sortField)) { + where.orderBy(GenAIEvaluationRecord.SCORE_VALUE, queryOrder == Order.ASC ? "ASC" : "DESC"); + } else if (queryOrder == Order.ASC) { + where.orderByAsc(); } else { - where.orderByDesc(sortColumn(sortField)); + where.orderByDesc(); } where.limit(limit).offset(from); @@ -116,14 +131,14 @@ public GenAIEvaluationRecords queryGenAIEvaluationRecord(String providerName, St for (final RowEntity rowEntity : resp.getElements()) { GenAIEvaluationRecord evaluationRecord = new GenAIEvaluationRecord(); evaluationRecord.setTraceId(rowEntity.getTagValue(GenAIEvaluationRecord.TRACE_ID)); - evaluationRecord.setServiceName(rowEntity.getTagValue(GenAIEvaluationRecord.SERVICE_NAME)); - evaluationRecord.setProviderName(rowEntity.getTagValue(GenAIEvaluationRecord.PROVIDER_NAME)); - evaluationRecord.setModelName(rowEntity.getTagValue(GenAIEvaluationRecord.MODEL_NAME)); + evaluationRecord.setServiceId(rowEntity.getTagValue(GenAIEvaluationRecord.SERVICE_ID)); + evaluationRecord.setProviderId(rowEntity.getTagValue(GenAIEvaluationRecord.PROVIDER_ID)); + evaluationRecord.setModelId(rowEntity.getTagValue(GenAIEvaluationRecord.MODEL_ID)); evaluationRecord.setOperationName(rowEntity.getTagValue(GenAIEvaluationRecord.OPERATION_NAME)); - final Number scoreValue = (Number) rowEntity.getTagValue(GenAIEvaluationRecord.SCORE_VALUE); + final Number scoreValue = rowEntity.getTagValue(GenAIEvaluationRecord.SCORE_VALUE); evaluationRecord.setScoreValuePpm(scoreValue == null ? null : scoreValue.longValue()); evaluationRecord.setSegmentId(rowEntity.getTagValue(GenAIEvaluationRecord.SEGMENT_ID)); - evaluationRecord.setSpanId(rowEntity.getTagValue(GenAIEvaluationRecord.SPAN_ID)); + evaluationRecord.setSpanId(((Number) rowEntity.getTagValue(GenAIEvaluationRecord.SPAN_ID)).intValue()); evaluationRecord.setSpanType(rowEntity.getTagValue(GenAIEvaluationRecord.SPAN_TYPE)); evaluationRecord.setTaskName(rowEntity.getTagValue(GenAIEvaluationRecord.TASK_NAME)); evaluationRecord.setEvaluationTime(((Number) rowEntity.getTagValue(GenAIEvaluationRecord.EVALUATION_TIME)).longValue()); @@ -136,9 +151,4 @@ public GenAIEvaluationRecords queryGenAIEvaluationRecord(String providerName, St } return genAIEvaluationRecords; } - - private static String sortColumn(final String sortField) { - return GenAIEvaluationRecord.SCORE_VALUE.equals(sortField) - ? GenAIEvaluationRecord.SCORE_VALUE : GenAIEvaluationRecord.EVALUATION_TIME; - } } diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/GenAIEvaluationRecordQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/GenAIEvaluationRecordQueryEsDAO.java index 2fcc139f4271..4b8ca3d18ab3 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/GenAIEvaluationRecordQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/GenAIEvaluationRecordQueryEsDAO.java @@ -48,9 +48,9 @@ public class GenAIEvaluationRecordQueryEsDAO extends EsDAO implements IGenAIEvaluationRecordQueryDAO { private static final Set QUERYABLE_TAG_KEYS = Set.of( GenAIEvaluationRecord.TRACE_ID, - GenAIEvaluationRecord.SERVICE_NAME, - GenAIEvaluationRecord.PROVIDER_NAME, - GenAIEvaluationRecord.MODEL_NAME, + GenAIEvaluationRecord.SERVICE_ID, + GenAIEvaluationRecord.PROVIDER_ID, + GenAIEvaluationRecord.MODEL_ID, GenAIEvaluationRecord.OPERATION_NAME, GenAIEvaluationRecord.SCORE_VALUE, GenAIEvaluationRecord.SEGMENT_ID, @@ -69,9 +69,11 @@ public GenAIEvaluationRecordQueryEsDAO(final ElasticSearchClient client) { } @Override - public GenAIEvaluationRecords queryGenAIEvaluationRecord(final String providerName, + public GenAIEvaluationRecords queryGenAIEvaluationRecord(final String serviceName, + final String providerName, final String modelName, final Double minScore, final Double maxScore, final String sortField, + final String taskName, final String evaluationLevel, final String judgeModel, final TraceScopeCondition relatedTrace, final Order queryOrder, final int from, @@ -95,11 +97,14 @@ public GenAIEvaluationRecords queryGenAIEvaluationRecord(final String providerNa if (startSecondTB != 0 && endSecondTB != 0) { query.must(Query.range(Record.TIME_BUCKET).gte(startSecondTB).lte(endSecondTB)); } + if (isNotEmpty(serviceName)) { + query.must(Query.term(GenAIEvaluationRecord.SERVICE_ID, GenAIEvaluationRecord.toServiceId(serviceName))); + } if (isNotEmpty(providerName)) { - query.must(Query.term(GenAIEvaluationRecord.PROVIDER_NAME, providerName)); + query.must(Query.term(GenAIEvaluationRecord.PROVIDER_ID, GenAIEvaluationRecord.toEntityId(providerName))); } if (isNotEmpty(modelName)) { - query.must(Query.term(GenAIEvaluationRecord.MODEL_NAME, modelName)); + query.must(Query.term(GenAIEvaluationRecord.MODEL_ID, GenAIEvaluationRecord.toEntityId(modelName))); } if (minScore != null || maxScore != null) { final var scoreRange = Query.range(GenAIEvaluationRecord.SCORE_VALUE); @@ -107,6 +112,15 @@ public GenAIEvaluationRecords queryGenAIEvaluationRecord(final String providerNa if (maxScore != null) scoreRange.lte(GenAIEvaluationRecord.maxScoreValuePpm(maxScore)); query.must(scoreRange); } + if (isNotEmpty(taskName)) { + query.must(Query.term(GenAIEvaluationRecord.TASK_NAME, taskName)); + } + if (isNotEmpty(evaluationLevel)) { + query.must(Query.term(GenAIEvaluationRecord.EVALUATION_LEVEL, evaluationLevel)); + } + if (isNotEmpty(judgeModel)) { + query.must(Query.term(GenAIEvaluationRecord.JUDGE_MODEL, judgeModel)); + } if (nonNull(relatedTrace)) { if (isNotEmpty(relatedTrace.getTraceId())) { query.must(Query.term(GenAIEvaluationRecord.TRACE_ID, relatedTrace.getTraceId())); @@ -115,7 +129,7 @@ public GenAIEvaluationRecords queryGenAIEvaluationRecord(final String providerNa query.must(Query.term(GenAIEvaluationRecord.SEGMENT_ID, relatedTrace.getSegmentId())); } if (nonNull(relatedTrace.getSpanId())) { - query.must(Query.term(GenAIEvaluationRecord.SPAN_ID, String.valueOf(relatedTrace.getSpanId()))); + query.must(Query.term(GenAIEvaluationRecord.SPAN_ID, relatedTrace.getSpanId())); } } if (CollectionUtils.isNotEmpty(tags)) { @@ -132,7 +146,7 @@ public GenAIEvaluationRecords queryGenAIEvaluationRecord(final String providerNa final SearchBuilder search = Search.builder() .query(query) .sort( - GenAIEvaluationRecord.SCORE_VALUE.equals(sortField) ? GenAIEvaluationRecord.SCORE_VALUE : GenAIEvaluationRecord.EVALUATION_TIME, + GenAIEvaluationRecord.SCORE_VALUE.equalsIgnoreCase(sortField) ? GenAIEvaluationRecord.SCORE_VALUE : GenAIEvaluationRecord.EVALUATION_TIME, Order.DES.equals(queryOrder) ? Sort.Order.DESC : Sort.Order.ASC ) .size(limit) @@ -156,14 +170,14 @@ private GenAIEvaluationRecord parseRecord(final SearchHit searchHit) { final GenAIEvaluationRecord record = new GenAIEvaluationRecord(); record.setUniqueId((String) source.get(GenAIEvaluationRecord.UNIQUE_ID)); record.setTraceId((String) source.get(GenAIEvaluationRecord.TRACE_ID)); - record.setServiceName((String) source.get(GenAIEvaluationRecord.SERVICE_NAME)); - record.setProviderName((String) source.get(GenAIEvaluationRecord.PROVIDER_NAME)); - record.setModelName((String) source.get(GenAIEvaluationRecord.MODEL_NAME)); + record.setServiceId((String) source.get(GenAIEvaluationRecord.SERVICE_ID)); + record.setProviderId((String) source.get(GenAIEvaluationRecord.PROVIDER_ID)); + record.setModelId((String) source.get(GenAIEvaluationRecord.MODEL_ID)); record.setOperationName((String) source.get(GenAIEvaluationRecord.OPERATION_NAME)); final Number scoreValue = (Number) source.get(GenAIEvaluationRecord.SCORE_VALUE); record.setScoreValuePpm(scoreValue == null ? null : scoreValue.longValue()); record.setSegmentId((String) source.get(GenAIEvaluationRecord.SEGMENT_ID)); - record.setSpanId((String) source.get(GenAIEvaluationRecord.SPAN_ID)); + record.setSpanId(((Number) source.get(GenAIEvaluationRecord.SPAN_ID)).intValue()); record.setSpanType((String) source.get(GenAIEvaluationRecord.SPAN_TYPE)); record.setTaskName((String) source.get(GenAIEvaluationRecord.TASK_NAME)); record.setValueType((String) source.get(GenAIEvaluationRecord.VALUE_TYPE)); diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCGenAIEvaluationRecordQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCGenAIEvaluationRecordQueryDAO.java index 7fc98f49c02a..6328bcd1ffdf 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCGenAIEvaluationRecordQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCGenAIEvaluationRecordQueryDAO.java @@ -21,6 +21,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import java.util.Set; import lombok.RequiredArgsConstructor; @@ -50,9 +51,9 @@ public class JDBCGenAIEvaluationRecordQueryDAO implements IGenAIEvaluationRecordQueryDAO { private static final Set QUERYABLE_TAG_KEYS = Set.of( GenAIEvaluationRecord.TRACE_ID, - GenAIEvaluationRecord.SERVICE_NAME, - GenAIEvaluationRecord.PROVIDER_NAME, - GenAIEvaluationRecord.MODEL_NAME, + GenAIEvaluationRecord.SERVICE_ID, + GenAIEvaluationRecord.PROVIDER_ID, + GenAIEvaluationRecord.MODEL_ID, GenAIEvaluationRecord.OPERATION_NAME, GenAIEvaluationRecord.SCORE_VALUE, GenAIEvaluationRecord.SEGMENT_ID, @@ -69,9 +70,9 @@ public class JDBCGenAIEvaluationRecordQueryDAO implements IGenAIEvaluationRecord private static final List SELECTED_COLUMNS = List.of( GenAIEvaluationRecord.UNIQUE_ID, GenAIEvaluationRecord.TRACE_ID, - GenAIEvaluationRecord.SERVICE_NAME, - GenAIEvaluationRecord.PROVIDER_NAME, - GenAIEvaluationRecord.MODEL_NAME, + GenAIEvaluationRecord.SERVICE_ID, + GenAIEvaluationRecord.PROVIDER_ID, + GenAIEvaluationRecord.MODEL_ID, GenAIEvaluationRecord.OPERATION_NAME, GenAIEvaluationRecord.SCORE_VALUE, GenAIEvaluationRecord.SEGMENT_ID, @@ -91,9 +92,11 @@ public class JDBCGenAIEvaluationRecordQueryDAO implements IGenAIEvaluationRecord @Override @SneakyThrows - public GenAIEvaluationRecords queryGenAIEvaluationRecord(final String providerName, + public GenAIEvaluationRecords queryGenAIEvaluationRecord(final String serviceName, + final String providerName, final String modelName, final Double minScore, final Double maxScore, final String sortField, + final String taskName, final String evaluationLevel, final String judgeModel, final TraceScopeCondition relatedTrace, final Order queryOrder, final int from, @@ -124,7 +127,8 @@ public GenAIEvaluationRecords queryGenAIEvaluationRecord(final String providerNa final var records = new ArrayList(); for (final var table : tables) { final var sqlAndParameters = buildSQL( - providerName, modelName, minScore, maxScore, sortField, relatedTrace, queryOrder, from, limit, duration, tags, table); + serviceName, providerName, modelName, minScore, maxScore, sortField, taskName, evaluationLevel, judgeModel, + relatedTrace, queryOrder, from, limit, duration, tags, table); records.addAll( jdbcClient.executeQuery( sqlAndParameters.sql(), @@ -134,9 +138,12 @@ public GenAIEvaluationRecords queryGenAIEvaluationRecord(final String providerNa ); } - final var comparator = Order.ASC.equals(queryOrder) ? - comparing(GenAIEvaluationRecord::getEvaluationTime) : - comparing(GenAIEvaluationRecord::getEvaluationTime).reversed(); + Comparator comparator = GenAIEvaluationRecord.SCORE_VALUE.equalsIgnoreCase(sortField) + ? comparing((GenAIEvaluationRecord record) -> record.getScoreValue() == null ? Double.NEGATIVE_INFINITY : record.getScoreValue()) + : comparing(GenAIEvaluationRecord::getEvaluationTime); + if (Order.DES.equals(queryOrder)) { + comparator = comparator.reversed(); + } return new GenAIEvaluationRecords( records.stream().sorted(comparator).skip(from).limit(limit).collect(toList()) ); @@ -148,14 +155,14 @@ protected ArrayList parseResults(final ResultSet resultSe final GenAIEvaluationRecord record = new GenAIEvaluationRecord(); record.setUniqueId(resultSet.getString(GenAIEvaluationRecord.UNIQUE_ID)); record.setTraceId(resultSet.getString(GenAIEvaluationRecord.TRACE_ID)); - record.setServiceName(resultSet.getString(GenAIEvaluationRecord.SERVICE_NAME)); - record.setProviderName(resultSet.getString(GenAIEvaluationRecord.PROVIDER_NAME)); - record.setModelName(resultSet.getString(GenAIEvaluationRecord.MODEL_NAME)); + record.setServiceId(resultSet.getString(GenAIEvaluationRecord.SERVICE_ID)); + record.setProviderId(resultSet.getString(GenAIEvaluationRecord.PROVIDER_ID)); + record.setModelId(resultSet.getString(GenAIEvaluationRecord.MODEL_ID)); record.setOperationName(resultSet.getString(GenAIEvaluationRecord.OPERATION_NAME)); final long scoreValue = resultSet.getLong(GenAIEvaluationRecord.SCORE_VALUE); record.setScoreValuePpm(resultSet.wasNull() ? null : scoreValue); record.setSegmentId(resultSet.getString(GenAIEvaluationRecord.SEGMENT_ID)); - record.setSpanId(resultSet.getString(GenAIEvaluationRecord.SPAN_ID)); + record.setSpanId(resultSet.getInt(GenAIEvaluationRecord.SPAN_ID)); record.setSpanType(resultSet.getString(GenAIEvaluationRecord.SPAN_TYPE)); record.setTaskName(resultSet.getString(GenAIEvaluationRecord.TASK_NAME)); record.setValueType(resultSet.getString(GenAIEvaluationRecord.VALUE_TYPE)); @@ -169,11 +176,15 @@ protected ArrayList parseResults(final ResultSet resultSe return records; } - protected SQLAndParameters buildSQL(final String providerName, + protected SQLAndParameters buildSQL(final String serviceName, + final String providerName, final String modelName, final Double minScore, final Double maxScore, final String sortField, + final String taskName, + final String evaluationLevel, + final String judgeModel, final TraceScopeCondition relatedTrace, final Order queryOrder, final int from, @@ -204,13 +215,17 @@ protected SQLAndParameters buildSQL(final String providerName, sql.append(" and ").append(storageColumn(GenAIEvaluationRecord.TIME_BUCKET)).append(" <= ?"); parameters.add(endSecondTB); } + if (StringUtil.isNotEmpty(serviceName)) { + sql.append(" and ").append(storageColumn(GenAIEvaluationRecord.SERVICE_ID)).append(" = ?"); + parameters.add(GenAIEvaluationRecord.toServiceId(serviceName)); + } if (StringUtil.isNotEmpty(providerName)) { - sql.append(" and ").append(storageColumn(GenAIEvaluationRecord.PROVIDER_NAME)).append(" = ?"); - parameters.add(providerName); + sql.append(" and ").append(storageColumn(GenAIEvaluationRecord.PROVIDER_ID)).append(" = ?"); + parameters.add(GenAIEvaluationRecord.toEntityId(providerName)); } if (StringUtil.isNotEmpty(modelName)) { - sql.append(" and ").append(storageColumn(GenAIEvaluationRecord.MODEL_NAME)).append(" = ?"); - parameters.add(modelName); + sql.append(" and ").append(storageColumn(GenAIEvaluationRecord.MODEL_ID)).append(" = ?"); + parameters.add(GenAIEvaluationRecord.toEntityId(modelName)); } if (minScore != null) { sql.append(" and ").append(storageColumn(GenAIEvaluationRecord.SCORE_VALUE)).append(" >= ?"); @@ -220,6 +235,18 @@ protected SQLAndParameters buildSQL(final String providerName, sql.append(" and ").append(storageColumn(GenAIEvaluationRecord.SCORE_VALUE)).append(" <= ?"); parameters.add(GenAIEvaluationRecord.maxScoreValuePpm(maxScore)); } + if (StringUtil.isNotEmpty(taskName)) { + sql.append(" and ").append(storageColumn(GenAIEvaluationRecord.TASK_NAME)).append(" = ?"); + parameters.add(taskName); + } + if (StringUtil.isNotEmpty(evaluationLevel)) { + sql.append(" and ").append(storageColumn(GenAIEvaluationRecord.EVALUATION_LEVEL)).append(" = ?"); + parameters.add(evaluationLevel); + } + if (StringUtil.isNotEmpty(judgeModel)) { + sql.append(" and ").append(storageColumn(GenAIEvaluationRecord.JUDGE_MODEL)).append(" = ?"); + parameters.add(judgeModel); + } if (nonNull(relatedTrace)) { if (StringUtil.isNotEmpty(relatedTrace.getTraceId())) { sql.append(" and ").append(storageColumn(GenAIEvaluationRecord.TRACE_ID)).append(" = ?"); @@ -231,7 +258,7 @@ protected SQLAndParameters buildSQL(final String providerName, } if (nonNull(relatedTrace.getSpanId())) { sql.append(" and ").append(storageColumn(GenAIEvaluationRecord.SPAN_ID)).append(" = ?"); - parameters.add(String.valueOf(relatedTrace.getSpanId())); + parameters.add(relatedTrace.getSpanId()); } } if (CollectionUtils.isNotEmpty(tags)) { @@ -246,7 +273,7 @@ protected SQLAndParameters buildSQL(final String providerName, } sql.append(" order by ") - .append(storageColumn(GenAIEvaluationRecord.SCORE_VALUE.equals(sortField) + .append(storageColumn(GenAIEvaluationRecord.SCORE_VALUE.equalsIgnoreCase(sortField) ? GenAIEvaluationRecord.SCORE_VALUE : GenAIEvaluationRecord.EVALUATION_TIME)) .append(" ") .append(Order.DES.equals(queryOrder) ? "desc" : "asc"); From d2528bc738c52805f050998b2f1c4a595bb30b40 Mon Sep 17 00:00:00 2001 From: peachisai <2581009893@qq.com> Date: Thu, 6 Aug 2026 22:39:24 +0800 Subject: [PATCH 8/8] fix --- .../GenAIEvaluationRecordQueryService.java | 36 +++++++++++++------ .../GenAIEvaluationRecordQueryCondition.java | 6 ++-- .../query/IGenAIEvaluationRecordQueryDAO.java | 20 +++++------ .../resolver/GenAIEvaluationRecordQuery.java | 6 ++-- ...BanyanDBGenAIEvaluationRecordQueryDAO.java | 14 ++++---- .../GenAIEvaluationRecordQueryEsDAO.java | 18 +++++----- .../JDBCGenAIEvaluationRecordQueryDAO.java | 26 +++++++------- 7 files changed, 70 insertions(+), 56 deletions(-) diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/GenAIEvaluationRecordQueryService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/GenAIEvaluationRecordQueryService.java index 7a3c5f2167c3..23520311dd4b 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/GenAIEvaluationRecordQueryService.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/GenAIEvaluationRecordQueryService.java @@ -18,6 +18,8 @@ package org.apache.skywalking.oap.server.core.query; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.manual.genai.GenAIEvaluationRecord; import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag; import org.apache.skywalking.oap.server.core.query.enumeration.Order; import org.apache.skywalking.oap.server.core.query.input.Duration; @@ -51,9 +53,9 @@ private IGenAIEvaluationRecordQueryDAO getGenAIEvaluationRecordQueryDAO() { return genAIEvaluationRecordQueryDAO; } - public GenAIEvaluationRecords queryGenAIEvaluationRecord(String serviceName, - String providerName, - String modelName, + public GenAIEvaluationRecords queryGenAIEvaluationRecord(String serviceId, + String providerId, + String modelId, Double minScore, Double maxScore, String sortField, @@ -71,9 +73,9 @@ public GenAIEvaluationRecords queryGenAIEvaluationRecord(String serviceName, if (traceContext != null) { StringBuilder msg = new StringBuilder(); span = traceContext.createSpan("Query Service: queryGenAIEvaluationRecord"); - msg.append("ServiceName: ").append(serviceName).append(", "); - msg.append("ProviderName: ").append(providerName).append(", "); - msg.append("ModelName: ").append(modelName).append(", "); + msg.append("ServiceId: ").append(serviceId).append(", "); + msg.append("ProviderId: ").append(providerId).append(", "); + msg.append("ModelId: ").append(modelId).append(", "); msg.append("RelatedTrace: ").append(relatedTrace).append(", "); msg.append("Pagination: ").append(paging).append(", "); msg.append("QueryOrder: ").append(queryOrder).append(", "); @@ -82,7 +84,7 @@ public GenAIEvaluationRecords queryGenAIEvaluationRecord(String serviceName, span.setMsg(msg.toString()); } return queryGenAIEvaluationRecordInternal( - serviceName, providerName, modelName, minScore, maxScore, sortField, taskName, evaluationLevel, judgeModel, + serviceId, providerId, toStoredModelId(modelId), minScore, maxScore, sortField, taskName, evaluationLevel, judgeModel, relatedTrace, paging, queryOrder, duration, tags ); } finally { @@ -92,9 +94,9 @@ public GenAIEvaluationRecords queryGenAIEvaluationRecord(String serviceName, } } - private GenAIEvaluationRecords queryGenAIEvaluationRecordInternal(String serviceName, - String providerName, - String modelName, + private GenAIEvaluationRecords queryGenAIEvaluationRecordInternal(String serviceId, + String providerId, + String modelId, Double minScore, Double maxScore, String sortField, @@ -109,8 +111,20 @@ private GenAIEvaluationRecords queryGenAIEvaluationRecordInternal(String service PaginationUtils.Page page = PaginationUtils.INSTANCE.exchange(paging); return getGenAIEvaluationRecordQueryDAO().queryGenAIEvaluationRecordDebuggable( - serviceName, providerName, modelName, minScore, maxScore, sortField, taskName, evaluationLevel, judgeModel, + serviceId, providerId, modelId, minScore, maxScore, sortField, taskName, evaluationLevel, judgeModel, relatedTrace, queryOrder, page.getFrom(), page.getLimit(), duration, tags ); } + + private String toStoredModelId(final String modelInstanceId) { + if (modelInstanceId == null || modelInstanceId.isEmpty()) { + return modelInstanceId; + } + // The UI model picker supplies a ServiceInstanceID. Accept an already + // stored model ServiceID as well for non-UI API callers. + if (!modelInstanceId.contains("_")) { + return modelInstanceId; + } + return GenAIEvaluationRecord.toEntityId(IDManager.ServiceInstanceID.analysisId(modelInstanceId).getName()); + } } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/GenAIEvaluationRecordQueryCondition.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/GenAIEvaluationRecordQueryCondition.java index 64529eb32417..4ac76d5a73e2 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/GenAIEvaluationRecordQueryCondition.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/GenAIEvaluationRecordQueryCondition.java @@ -30,9 +30,9 @@ @Setter @ToString public class GenAIEvaluationRecordQueryCondition { - private String serviceName; - private String providerName; - private String modelName; + private String serviceId; + private String providerId; + private String modelId; private Double minScore; private Double maxScore; private String sortField; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IGenAIEvaluationRecordQueryDAO.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IGenAIEvaluationRecordQueryDAO.java index 239c937c9de4..09cc9d11f303 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IGenAIEvaluationRecordQueryDAO.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IGenAIEvaluationRecordQueryDAO.java @@ -38,9 +38,9 @@ public interface IGenAIEvaluationRecordQueryDAO extends Service { - default GenAIEvaluationRecords queryGenAIEvaluationRecordDebuggable(String serviceName, - String providerName, - String modelName, + default GenAIEvaluationRecords queryGenAIEvaluationRecordDebuggable(String serviceId, + String providerId, + String modelId, Double minScore, Double maxScore, String sortField, @@ -59,9 +59,9 @@ default GenAIEvaluationRecords queryGenAIEvaluationRecordDebuggable(String servi if (traceContext != null) { span = traceContext.createSpan("Query Dao: queryGenAIEvaluationRecord"); StringBuilder msg = new StringBuilder(); - msg.append("ServiceName: ").append(serviceName) - .append(", ProviderName: ").append(providerName) - .append(", ModelName: ").append(modelName) + msg.append("ServiceId: ").append(serviceId) + .append(", ProviderId: ").append(providerId) + .append(", ModelId: ").append(modelId) .append(", RelatedTrace: ").append(relatedTrace) .append(", QueryOrder: ").append(queryOrder) .append(", From: ").append(from) @@ -71,7 +71,7 @@ default GenAIEvaluationRecords queryGenAIEvaluationRecordDebuggable(String servi span.setMsg(msg.toString()); } return queryGenAIEvaluationRecord( - serviceName, providerName, modelName, minScore, maxScore, sortField, taskName, evaluationLevel, judgeModel, + serviceId, providerId, modelId, minScore, maxScore, sortField, taskName, evaluationLevel, judgeModel, relatedTrace, queryOrder, from, limit, duration, tags ); } finally { @@ -81,9 +81,9 @@ default GenAIEvaluationRecords queryGenAIEvaluationRecordDebuggable(String servi } } - GenAIEvaluationRecords queryGenAIEvaluationRecord(String serviceName, - String providerName, - String modelName, + GenAIEvaluationRecords queryGenAIEvaluationRecord(String serviceId, + String providerId, + String modelId, Double minScore, Double maxScore, String sortField, diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/GenAIEvaluationRecordQuery.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/GenAIEvaluationRecordQuery.java index 5b3f00f479c3..267e0f257545 100644 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/GenAIEvaluationRecordQuery.java +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/GenAIEvaluationRecordQuery.java @@ -94,9 +94,9 @@ private GenAIEvaluationRecords queryGenAIEvaluationRecord( }); } return getQueryService().queryGenAIEvaluationRecord( - condition.getServiceName(), - condition.getProviderName(), - condition.getModelName(), + condition.getServiceId(), + condition.getProviderId(), + condition.getModelId(), condition.getMinScore(), condition.getMaxScore(), condition.getSortField(), diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBGenAIEvaluationRecordQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBGenAIEvaluationRecordQueryDAO.java index d98d08349f5b..168331dfbe84 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBGenAIEvaluationRecordQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBGenAIEvaluationRecordQueryDAO.java @@ -65,20 +65,20 @@ public BanyanDBGenAIEvaluationRecordQueryDAO(BanyanDBStorageClient client) { } @Override - public GenAIEvaluationRecords queryGenAIEvaluationRecord(String serviceName, String providerName, String modelName, Double minScore, Double maxScore, String sortField, + public GenAIEvaluationRecords queryGenAIEvaluationRecord(String serviceId, String providerId, String modelId, Double minScore, Double maxScore, String sortField, String taskName, String evaluationLevel, String judgeModel, TraceScopeCondition relatedTrace, Order queryOrder, int from, int limit, Duration duration, List tags) throws IOException { final boolean isColdStage = duration != null && duration.isColdStage(); final Conditions where = Conditions.create(); - if (StringUtil.isNotEmpty(serviceName)) { - where.eq(GenAIEvaluationRecord.SERVICE_ID, GenAIEvaluationRecord.toServiceId(serviceName)); + if (StringUtil.isNotEmpty(serviceId)) { + where.eq(GenAIEvaluationRecord.SERVICE_ID, serviceId); } - if (StringUtil.isNotEmpty(providerName)) { - where.eq(GenAIEvaluationRecord.PROVIDER_ID, GenAIEvaluationRecord.toEntityId(providerName)); + if (StringUtil.isNotEmpty(providerId)) { + where.eq(GenAIEvaluationRecord.PROVIDER_ID, providerId); } - if (StringUtil.isNotEmpty(modelName)) { - where.eq(GenAIEvaluationRecord.MODEL_ID, GenAIEvaluationRecord.toEntityId(modelName)); + if (StringUtil.isNotEmpty(modelId)) { + where.eq(GenAIEvaluationRecord.MODEL_ID, modelId); } if (minScore != null) { where.gte(GenAIEvaluationRecord.SCORE_VALUE, GenAIEvaluationRecord.minScoreValuePpm(minScore)); diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/GenAIEvaluationRecordQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/GenAIEvaluationRecordQueryEsDAO.java index 4b8ca3d18ab3..fe33cd614fff 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/GenAIEvaluationRecordQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/GenAIEvaluationRecordQueryEsDAO.java @@ -69,9 +69,9 @@ public GenAIEvaluationRecordQueryEsDAO(final ElasticSearchClient client) { } @Override - public GenAIEvaluationRecords queryGenAIEvaluationRecord(final String serviceName, - final String providerName, - final String modelName, + public GenAIEvaluationRecords queryGenAIEvaluationRecord(final String serviceId, + final String providerId, + final String modelId, final Double minScore, final Double maxScore, final String sortField, final String taskName, final String evaluationLevel, final String judgeModel, final TraceScopeCondition relatedTrace, @@ -97,14 +97,14 @@ public GenAIEvaluationRecords queryGenAIEvaluationRecord(final String serviceNam if (startSecondTB != 0 && endSecondTB != 0) { query.must(Query.range(Record.TIME_BUCKET).gte(startSecondTB).lte(endSecondTB)); } - if (isNotEmpty(serviceName)) { - query.must(Query.term(GenAIEvaluationRecord.SERVICE_ID, GenAIEvaluationRecord.toServiceId(serviceName))); + if (isNotEmpty(serviceId)) { + query.must(Query.term(GenAIEvaluationRecord.SERVICE_ID, serviceId)); } - if (isNotEmpty(providerName)) { - query.must(Query.term(GenAIEvaluationRecord.PROVIDER_ID, GenAIEvaluationRecord.toEntityId(providerName))); + if (isNotEmpty(providerId)) { + query.must(Query.term(GenAIEvaluationRecord.PROVIDER_ID, providerId)); } - if (isNotEmpty(modelName)) { - query.must(Query.term(GenAIEvaluationRecord.MODEL_ID, GenAIEvaluationRecord.toEntityId(modelName))); + if (isNotEmpty(modelId)) { + query.must(Query.term(GenAIEvaluationRecord.MODEL_ID, modelId)); } if (minScore != null || maxScore != null) { final var scoreRange = Query.range(GenAIEvaluationRecord.SCORE_VALUE); diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCGenAIEvaluationRecordQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCGenAIEvaluationRecordQueryDAO.java index 6328bcd1ffdf..17c7257dbf5c 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCGenAIEvaluationRecordQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCGenAIEvaluationRecordQueryDAO.java @@ -92,9 +92,9 @@ public class JDBCGenAIEvaluationRecordQueryDAO implements IGenAIEvaluationRecord @Override @SneakyThrows - public GenAIEvaluationRecords queryGenAIEvaluationRecord(final String serviceName, - final String providerName, - final String modelName, + public GenAIEvaluationRecords queryGenAIEvaluationRecord(final String serviceId, + final String providerId, + final String modelId, final Double minScore, final Double maxScore, final String sortField, final String taskName, final String evaluationLevel, final String judgeModel, final TraceScopeCondition relatedTrace, @@ -127,7 +127,7 @@ public GenAIEvaluationRecords queryGenAIEvaluationRecord(final String serviceNam final var records = new ArrayList(); for (final var table : tables) { final var sqlAndParameters = buildSQL( - serviceName, providerName, modelName, minScore, maxScore, sortField, taskName, evaluationLevel, judgeModel, + serviceId, providerId, modelId, minScore, maxScore, sortField, taskName, evaluationLevel, judgeModel, relatedTrace, queryOrder, from, limit, duration, tags, table); records.addAll( jdbcClient.executeQuery( @@ -176,9 +176,9 @@ protected ArrayList parseResults(final ResultSet resultSe return records; } - protected SQLAndParameters buildSQL(final String serviceName, - final String providerName, - final String modelName, + protected SQLAndParameters buildSQL(final String serviceId, + final String providerId, + final String modelId, final Double minScore, final Double maxScore, final String sortField, @@ -215,17 +215,17 @@ protected SQLAndParameters buildSQL(final String serviceName, sql.append(" and ").append(storageColumn(GenAIEvaluationRecord.TIME_BUCKET)).append(" <= ?"); parameters.add(endSecondTB); } - if (StringUtil.isNotEmpty(serviceName)) { + if (StringUtil.isNotEmpty(serviceId)) { sql.append(" and ").append(storageColumn(GenAIEvaluationRecord.SERVICE_ID)).append(" = ?"); - parameters.add(GenAIEvaluationRecord.toServiceId(serviceName)); + parameters.add(serviceId); } - if (StringUtil.isNotEmpty(providerName)) { + if (StringUtil.isNotEmpty(providerId)) { sql.append(" and ").append(storageColumn(GenAIEvaluationRecord.PROVIDER_ID)).append(" = ?"); - parameters.add(GenAIEvaluationRecord.toEntityId(providerName)); + parameters.add(providerId); } - if (StringUtil.isNotEmpty(modelName)) { + if (StringUtil.isNotEmpty(modelId)) { sql.append(" and ").append(storageColumn(GenAIEvaluationRecord.MODEL_ID)).append(" = ?"); - parameters.add(GenAIEvaluationRecord.toEntityId(modelName)); + parameters.add(modelId); } if (minScore != null) { sql.append(" and ").append(storageColumn(GenAIEvaluationRecord.SCORE_VALUE)).append(" >= ?");