diff --git a/web/packages/studio/src/components/evaluation/SubmitEvaluationModal.tsx b/web/packages/studio/src/components/evaluation/SubmitEvaluationModal.tsx index ccd00d6eea..49887d8727 100644 --- a/web/packages/studio/src/components/evaluation/SubmitEvaluationModal.tsx +++ b/web/packages/studio/src/components/evaluation/SubmitEvaluationModal.tsx @@ -23,6 +23,7 @@ import { filesDeleteFileset, filesDownloadFile, filesUploadFile, + useListEvaluations, useListExperiments, } from '@nemo/sdk/generated/platform/api'; import { SegmentedControl, Stack, Text } from '@nvidia/foundations-react-core'; @@ -32,9 +33,8 @@ import { isConflictError, type EvalSeedFile } from '@studio/api/evaluation/eval- import { createRunEvaluation, EVAL_CONFIG_FILENAME, - EVAL_CONFIG_FILESET_KEY, - experimentConfigError, - experimentFilesetName, + evaluationConfigError, + evaluationFilesetName, } from '@studio/components/evaluation/experimentEvalConfig'; import { JudgeModelSelect } from '@studio/components/evaluation/JudgeModelSelect'; import { @@ -63,17 +63,17 @@ import { z } from 'zod'; const EVAL_CONFIG_MODE_ITEMS = [ { value: MODE_DEFAULT, children: 'Use Example' }, - { value: MODE_EXPERIMENT, children: 'Choose Experiment' }, + { value: MODE_EXPERIMENT, children: 'Use existing evaluation' }, ]; const DATASET_FILENAME = 'dataset.jsonl'; /** Backend caps page_size at 100; the picker shows the most recent page. */ -const EXPERIMENT_PAGE_SIZE = 100; +const LIST_PAGE_SIZE = 100; const README_FILENAME = 'README.md'; -const NO_EXPERIMENTS_MESSAGE = - 'No experiments yet. Run one from "Use Example" first — it creates the experiment and its eval config, which you can then re-run here.'; +const NO_EVALUATIONS_MESSAGE = + 'No evaluations with a reusable eval config yet. Create one to run and re-use it.'; const NO_DEPLOYMENT_MESSAGE = 'This agent has no active deployment.'; const DEPLOYMENT_CHECK_FAILED_MESSAGE = @@ -88,8 +88,8 @@ const submitEvaluationBaseSchema = z.object({ newName: z.string(), /** Fileset created alongside it, holding eval-config.json and any data artifacts. */ filesetName: z.string(), - /** Name of the experiment to re-run in "Choose Experiment" mode. */ - experimentName: z.string(), + /** Name of the existing evaluation whose eval config is reused in "Use existing evaluation" mode. */ + evaluationName: z.string(), }); type SubmitEvaluationFormData = z.infer; @@ -121,11 +121,11 @@ const makeSubmitEvaluationSchema = (requiresJudgeModel: () => boolean) => }); } } - if (data.mode === MODE_EXPERIMENT && !data.experimentName) { + if (data.mode === MODE_EXPERIMENT && !data.evaluationName) { ctx.addIssue({ code: z.ZodIssueCode.custom, - message: 'Pick an experiment to run', - path: ['experimentName'], + message: 'Pick an evaluation to reuse', + path: ['evaluationName'], }); } }); @@ -147,7 +147,7 @@ const makeDefaultValues = (agent?: string): SubmitEvaluationFormData => { exampleKey: DATASET_EVAL_CONFIG_KEY, newName, filesetName: filesetNameForExperiment(newName), - experimentName: '', + evaluationName: '', }; }; @@ -202,12 +202,12 @@ const discardSeeded = async ( /** Resolves the persisted yardstick spec for this submission. In "Use Example" mode * it builds the spec from the sample template (fanning the metric onto every task with - * the picked judge baked in) and seeds it into a new fileset; in "Choose Fileset" mode - * it reads the saved spec back verbatim (no re-fan, no judge re-pick). */ + * the picked judge baked in) and seeds it into a new fileset; in "Use existing evaluation" + * mode it reads the saved spec back verbatim (no re-fan, no judge re-pick). */ const loadPersistedSpec = async ( workspace: string, formData: SubmitEvaluationFormData, - experimentFileset: string | null + configFileset: string | null ): Promise => { if (formData.mode === MODE_DEFAULT) { const signal = new AbortController().signal; @@ -277,14 +277,14 @@ const loadPersistedSpec = async ( } return spec; } - if (!experimentFileset) throw new Error('The selected experiment has no eval config fileset'); + if (!configFileset) throw new Error('The selected evaluation has no eval config fileset'); const blob = await filesDownloadFile( workspace, - experimentFileset, + configFileset, EVAL_CONFIG_FILENAME, new AbortController().signal ); - if (!blob) throw new Error("Failed to read the selected experiment's eval config"); + if (!blob) throw new Error("Failed to read the selected evaluation's eval config"); return parseEvalConfig(await blob.text()); }; @@ -353,36 +353,53 @@ export const SubmitEvaluationModal: FC = ({ const agentFieldError = errors.agent?.message ?? deploymentError; const exampleKey = useWatch({ control, name: 'exampleKey' }); - const experimentName = useWatch({ control, name: 'experimentName' }); + const evaluationName = useWatch({ control, name: 'evaluationName' }); - const { data: experimentsResponse, isLoading: isExperimentsLoading } = useListExperiments( + const { data: evaluationsResponse, isLoading: isEvaluationsLoading } = useListEvaluations( workspace, - { page_size: EXPERIMENT_PAGE_SIZE, sort: '-created_at' }, + { page_size: LIST_PAGE_SIZE, sort: '-created_at' }, { query: { enabled: open && mode === MODE_EXPERIMENT } } ); - const experiments = experimentsResponse?.data ?? []; - const selectedExperiment = experiments.find((item) => item.name === experimentName); - const hasNoExperiments = mode === MODE_EXPERIMENT && !isExperimentsLoading && !experiments.length; - const latestExperimentName = experiments[0]?.name; + const evaluations = evaluationsResponse?.data ?? []; + /* The eval-config.json is identified on each Evaluation by convention in Studio. + * It's not persisted by the CLI or API at all. Only Studio created jobs will + * have this field written to the Evaluation's metadata (dict[str,str]). + * Unfortunately there's no existing way for Evaluations to be matched to the + * artifacts that generated them by contract. */ + const compatibleEvaluations = evaluations.filter((item) => evaluationFilesetName(item) != null); + const selectedEvaluation = evaluations.find((item) => item.name === evaluationName); + const hasNoEvaluations = + mode === MODE_EXPERIMENT && !isEvaluationsLoading && !compatibleEvaluations.length; + const latestEvaluationName = compatibleEvaluations[0]?.name; + + // Parent ExperimentGroups, loaded in reuse mode only to resolve a selected evaluation's group + // name — so a reused run is named after its experiment (flat) instead of nesting the prior + // run's random suffix. Used for the name stem only, not for the dropdown or the filter. + const { data: experimentGroupsResponse } = useListExperiments( + workspace, + { page_size: LIST_PAGE_SIZE, sort: '-created_at' }, + { query: { enabled: open && mode === MODE_EXPERIMENT } } + ); + const experimentGroups = experimentGroupsResponse?.data ?? []; useEffect(() => { - if (mode !== MODE_EXPERIMENT || experimentName || !latestExperimentName) return; - setValue('experimentName', latestExperimentName, { shouldValidate: true }); - }, [mode, experimentName, latestExperimentName, setValue]); + if (mode !== MODE_EXPERIMENT || evaluationName || !latestEvaluationName) return; + setValue('evaluationName', latestEvaluationName, { shouldValidate: true }); + }, [mode, evaluationName, latestEvaluationName, setValue]); - const { data: experimentConfigIssue, isFetching: isValidatingExperiment } = useQuery({ - queryKey: ['experiment-eval-config', workspace, experimentName], + const { data: evaluationConfigIssue, isFetching: isValidatingEvaluation } = useQuery({ + queryKey: ['evaluation-eval-config', workspace, evaluationName], queryFn: ({ signal }) => - selectedExperiment ? experimentConfigError(workspace, selectedExperiment, signal) : null, - enabled: open && mode === MODE_EXPERIMENT && !!selectedExperiment, + selectedEvaluation ? evaluationConfigError(workspace, selectedEvaluation, signal) : null, + enabled: open && mode === MODE_EXPERIMENT && !!selectedEvaluation, }); - const experimentFileset = selectedExperiment ? experimentFilesetName(selectedExperiment) : null; - const experimentFieldError = errors.experimentName?.message ?? experimentConfigIssue ?? undefined; + const evaluationFileset = selectedEvaluation ? evaluationFilesetName(selectedEvaluation) : null; + const evaluationFieldError = errors.evaluationName?.message ?? evaluationConfigIssue ?? undefined; - const canRunSelectedExperiment = + const canRunSelectedEvaluation = mode !== MODE_EXPERIMENT || - (!isValidatingExperiment && !!selectedExperiment && !experimentConfigIssue); + (!isValidatingEvaluation && !!selectedEvaluation && !evaluationConfigIssue); // Fetch and parse the selected example config early to detect metric type and default model. const { data: exampleConfig } = useQuery({ @@ -439,27 +456,42 @@ export const SubmitEvaluationModal: FC = ({ reset: resetMutation, } = useMutation({ mutationFn: async (formData: SubmitEvaluationFormData) => { - const spec = await loadPersistedSpec(workspace, formData, experimentFileset); + const spec = await loadPersistedSpec(workspace, formData, evaluationFileset); const isNew = formData.mode === MODE_DEFAULT; - const filesetName = isNew ? formData.filesetName.trim() : (experimentFileset ?? ''); + const filesetName = isNew ? formData.filesetName.trim() : (evaluationFileset ?? ''); const seeded: SeededEntities = isNew ? { filesetName } : {}; try { - const experiment = isNew - ? await createExperiment(workspace, { - name: formData.newName.trim(), - metadata: { [EVAL_CONFIG_FILESET_KEY]: filesetName }, - }) - : selectedExperiment; - if (!experiment) throw new Error('No experiment to run this evaluation under'); - if (isNew) seeded.experimentName = experiment.name; + // "Use Example" creates a fresh ExperimentGroup to hold this run; "Use existing + // evaluation" reuses the picked evaluation's group(s) and records the lineage. + let experimentIds: string[]; + let nameStem: string; + let parentEvaluationId: string | undefined; + if (isNew) { + const experiment = await createExperiment(workspace, { name: formData.newName.trim() }); + seeded.experimentName = experiment.name; + experimentIds = [experiment.id]; + nameStem = experiment.name; + } else { + if (!selectedEvaluation) throw new Error('No evaluation to reuse'); + experimentIds = selectedEvaluation.experiment_ids; + // Name the run after its parent experiment (group), not the prior run — else the run's + // random suffix would nest and grow on every reuse. Fall back to the eval name with a + // trailing 8-char suffix stripped if the group isn't in the loaded page. + const parentGroup = experimentGroups.find( + (group) => group.id === selectedEvaluation.experiment_ids[0] + ); + nameStem = parentGroup?.name ?? selectedEvaluation.name.replace(/-[a-z0-9]{8}$/, ''); + parentEvaluationId = selectedEvaluation.id; + } const evaluationId = await createRunEvaluation(workspace, { - experimentId: experiment.id, - experimentName: experiment.name, + experimentIds, + nameStem, filesetName, + parentEvaluationId, }); seeded.evaluationName = evaluationId; @@ -467,7 +499,7 @@ export const SubmitEvaluationModal: FC = ({ workspace, agent: formData.agent, filesetName, - experimentName: experiment.name, + experimentName: nameStem, evaluationId, }; const created = isDatasetEvalSpec(spec) @@ -533,7 +565,7 @@ export const SubmitEvaluationModal: FC = ({ submitButtonText="Submit" onSubmit={handleSubmit(onSubmit)} disabled={isPending} - submitDisabled={!deploymentVerified || !canRunSelectedExperiment} + submitDisabled={!deploymentVerified || !canRunSelectedEvaluation} loading={isPending} errorText={errorMessage} className="w-[690px]! max-w-[95vw]!" @@ -576,7 +608,7 @@ export const SubmitEvaluationModal: FC = ({ setValue('mode', v as typeof MODE_DEFAULT | typeof MODE_EXPERIMENT, { shouldValidate: false, }); - clearErrors('experimentName'); + clearErrors('evaluationName'); }} items={EVAL_CONFIG_MODE_ITEMS} /> @@ -612,22 +644,22 @@ export const SubmitEvaluationModal: FC = ({ ) : ( <> - {hasNoExperiments ? ( + {hasNoEvaluations ? ( - {NO_EXPERIMENTS_MESSAGE} + {NO_EVALUATIONS_MESSAGE} ) : ( + useControllerProps={{ control, name: 'evaluationName' }} + loading={isEvaluationsLoading} + items={compatibleEvaluations.flatMap((item) => item.name ? [{ value: item.name, children: item.name }] : [] )} formFieldProps={{ - slotLabel: 'Experiment', - slotHelp: `Runs the ${EVAL_CONFIG_FILENAME} in the experiment's fileset.`, - slotError: experimentFieldError, - status: experimentFieldError ? 'error' : undefined, + slotLabel: 'Evaluation', + slotHelp: `Reuses the selected evaluation's ${EVAL_CONFIG_FILENAME}.`, + slotError: evaluationFieldError, + status: evaluationFieldError ? 'error' : undefined, }} /> )} diff --git a/web/packages/studio/src/components/evaluation/experimentEvalConfig.ts b/web/packages/studio/src/components/evaluation/experimentEvalConfig.ts index fc485d3450..21f5ef4c39 100644 --- a/web/packages/studio/src/components/evaluation/experimentEvalConfig.ts +++ b/web/packages/studio/src/components/evaluation/experimentEvalConfig.ts @@ -2,65 +2,82 @@ // SPDX-License-Identifier: Apache-2.0 import { createEvaluation, filesListFilesetFiles } from '@nemo/sdk/generated/platform/api'; -import type { ExperimentResponse } from '@nemo/sdk/generated/platform/schema'; +import type { EvaluationResponse } from '@nemo/sdk/generated/platform/schema'; import { buildEvalJobName } from '@studio/components/evaluation/submitEvaluationJob'; -/** Experiment metadata key holding the name of the fileset that stores its eval config. - * Metadata values are plain strings, which is all a fileset name needs to be. */ +/** Evaluation metadata key holding the name of the fileset that stores its eval config. + * Lives on the Evaluation, not its ExperimentGroup: the config is a property of a single + * run, and the Evaluation entity's ``metadata`` is documented for exactly this "config + * snapshot". Metadata values are plain strings, which is all a fileset name needs to be. */ export const EVAL_CONFIG_FILESET_KEY = 'eval_config_fileset'; -/** Flat filename the reusable config is stored as inside its fileset. Every Experiment's - * fileset must carry one at the root — there is no per-run file picker. */ +/** Flat filename the reusable config is stored as inside its fileset. Every reusable + * evaluation's fileset carries one at the root — there is no per-run file picker. */ export const EVAL_CONFIG_FILENAME = 'eval-config.json'; -/** The fileset an Experiment stores its eval config in, or null when it names none. */ -export const experimentFilesetName = (experiment: ExperimentResponse): string | null => - experiment.metadata?.[EVAL_CONFIG_FILESET_KEY] ?? null; +/** The fileset an Evaluation stores its eval config in, or null when it names none. A blank + * or whitespace-only metadata value counts as "none": it would otherwise pass the picker's + * filter and even become the default, only to be rejected by evaluationConfigError. + * This is stored by Studio UI convention only, not enforced by API or CLI */ +export const evaluationFilesetName = (evaluation: EvaluationResponse): string | null => { + const name = evaluation.metadata?.[EVAL_CONFIG_FILESET_KEY]?.trim(); + return name ? name : null; +}; -/** Why an Experiment cannot be run against, or null when it can. Two ways to be invalid: +/** Why an Evaluation cannot be reused, or null when it can. Two ways to be invalid: * it names no fileset, or the fileset it names has no eval-config.json at the root. */ -export const experimentConfigError = async ( +export const evaluationConfigError = async ( workspace: string, - experiment: ExperimentResponse, + evaluation: EvaluationResponse, signal?: AbortSignal ): Promise => { - const filesetName = experimentFilesetName(experiment); + const filesetName = evaluationFilesetName(evaluation); if (!filesetName) { - return `Experiment "${experiment.name}" has no eval config fileset. Pick another experiment, or create one from a template.`; + return `Evaluation "${evaluation.name}" has no eval config fileset. Pick another evaluation, or create one from a template.`; } const files = await filesListFilesetFiles(workspace, filesetName, undefined, signal).catch( () => null ); if (!files) { - return `Could not read fileset "${filesetName}" for experiment "${experiment.name}". Pick another experiment.`; + return `Could not read fileset "${filesetName}" for evaluation "${evaluation.name}". Pick another evaluation.`; } const hasConfig = (files.data ?? []).some((file) => file.path === EVAL_CONFIG_FILENAME); return hasConfig ? null - : `Fileset "${filesetName}" has no ${EVAL_CONFIG_FILENAME} at its root, so experiment "${experiment.name}" cannot be run. Pick another experiment.`; + : `Fileset "${filesetName}" has no ${EVAL_CONFIG_FILENAME} at its root, so evaluation "${evaluation.name}" cannot be reused. Pick another evaluation.`; }; /** Create the Intake Evaluation this run publishes under, returning its **name** — * which is what ``publication.intake.evaluation_id`` takes (the entity id is not it). - * ``experimentId`` conversely is the Experiment's **id**, not its name. */ + * The eval-config fileset pointer is written to the Evaluation's own ``metadata`` (the + * documented "config snapshot" home) so a later run can reuse it; ``parentEvaluationId`` + * records the evaluation this one was derived from, when reusing an existing one. */ export const createRunEvaluation = async ( workspace: string, { - experimentId, - experimentName, + experimentIds, + nameStem, filesetName, + parentEvaluationId, signal, }: { - experimentId: string; - experimentName: string; + experimentIds: string[]; + nameStem: string; filesetName: string; + parentEvaluationId?: string; signal?: AbortSignal; } ): Promise => { - const name = buildEvalJobName(experimentName); + const name = buildEvalJobName(nameStem); await createEvaluation( workspace, - { name, experiment_ids: [experimentId], dataset_name: filesetName }, + { + name, + experiment_ids: experimentIds, + dataset_name: filesetName, + metadata: { [EVAL_CONFIG_FILESET_KEY]: filesetName }, + ...(parentEvaluationId ? { parent_evaluation_id: parentEvaluationId } : {}), + }, signal ); return name;