From 02637ca1ebc806b7abdc6df0b134665f4f0e1ec1 Mon Sep 17 00:00:00 2001 From: KFilippopolitis Date: Mon, 9 Nov 2026 14:06:07 +0200 Subject: [PATCH 1/3] feat: UI support for federated preprocessing steps Implements the platform-ui tasks from the federated preprocessing spec (step names are the proposal: variable_transformer / variable_scaler; replace them if exaflow ships different ones): - most_frequent imputation for nominal variables (editor, validation, summary label, valid-action check); mean/median stay numerical-only - New per-variable transform station (log1p, sqrt, asinh, yeo_johnson, quantile_uniform, quantile_normal) with domain-limit notes for log1p/sqrt read off the loaded describe - New per-variable scaling station (standard, robust, minmax with feature_range, maxabs) - Rail sub-nodes, ghost add rows, batch menus, doc panels, hydration of saved experiments, and Remove for both new stations - Request building: variable filtering for the new steps, rail order (impute, clip, transform, scale), summary entries feeding experiment detail and PDF/CSV export - Specs: request shape, filtering, step order, summary text, restore --- .../statistic-analysis-panel.component.html | 327 ++++++++++++++ .../statistic-analysis-panel.component.ts | 415 +++++++++++++++++- .../statistic-analysis-panel.pipeline.spec.ts | 57 ++- .../experiment-studio.service.spec.ts | 58 +++ src/app/services/experiment-studio.service.ts | 90 +++- 5 files changed, 917 insertions(+), 30 deletions(-) diff --git a/src/app/pages/experiment-studio/statistic-analysis-panel/statistic-analysis-panel.component.html b/src/app/pages/experiment-studio/statistic-analysis-panel/statistic-analysis-panel.component.html index 514c9563c..cdd114efd 100644 --- a/src/app/pages/experiment-studio/statistic-analysis-panel/statistic-analysis-panel.component.html +++ b/src/app/pages/experiment-studio/statistic-analysis-panel/statistic-analysis-panel.component.html @@ -695,6 +695,32 @@

+ + } + @if (!scalerSubNodeApplied) { +
+ + +
+ } @if (isLongitudinalModel && !expStudioService.appliedPreprocessingConfig()?.['longitudinal_transformer']) {
@@ -1281,6 +1307,307 @@
Values that would be capped · per data } + @if (!transformSubNodeApplied) { + + } @else { + + + @if (preprocessingStepDocumentation('variable_transformer')) { + + } + @if (preprocessingStepDocumentation('variable_transformer') && preprocessingDocOpen().transform) { +
+
+
+ } + + @if (outlierPreprocessingVariables.length > 0) { +
+ + +
+ @if (selectedPrepVariable('transform'); as variable) { +
+
+ Numerical variable +

{{ variableLabel(variable) }}

+
+ + {{ prepVariableStateLabel('transform', variable) }} + +
+
+
+ + Transform +
+ + @if (transformStrategyFor(variable); as strategy) { +

{{ transformDomainWarning(variable, strategy) }}

+ } +
+ } @else { +
+ +

No variables match the current search.

+ Clear the search field to choose a transform variable. +
+ } +
+
+ } @else { +
+ +

No numerical variables selected.

+ Select numerical variables before configuring transforms. +
+ } + +
+ } + + @if (!scalerSubNodeApplied) { + + } @else { + + + @if (preprocessingStepDocumentation('variable_scaler')) { + + } + @if (preprocessingStepDocumentation('variable_scaler') && preprocessingDocOpen().scaler) { +
+
+
+ } + + @if (outlierPreprocessingVariables.length > 0) { +
+ + +
+ @if (selectedPrepVariable('scaler'); as variable) { +
+
+ Numerical variable +

{{ variableLabel(variable) }}

+
+ + {{ prepVariableStateLabel('scaler', variable) }} + +
+
+
+ + Scaling +
+ + @if (scalerStrategyFor(variable) === 'minmax') { +
+ + +
+ } +
+ } @else { +
+ +

No variables match the current search.

+ Clear the search field to choose a scaling variable. +
+ } +
+
+ } @else { +
+ +

No numerical variables selected.

+ Select numerical variables before configuring scaling. +
+ } + +
+ } + @if (isLongitudinalModel) { = [ + { value: 'log1p', label: 'log(1 + x)' }, + { value: 'sqrt', label: 'Square root' }, + { value: 'asinh', label: 'Inverse hyperbolic sine' }, + { value: 'yeo_johnson', label: 'Yeo-Johnson' }, + { value: 'quantile_uniform', label: 'Quantile → uniform' }, + { value: 'quantile_normal', label: 'Quantile → normal' }, +]; + +const SCALER_STRATEGIES: Array<{ value: ScalerStrategy; label: string }> = [ + { value: 'standard', label: 'Standard (mean 0, std 1)' }, + { value: 'robust', label: 'Robust (median/IQR)' }, + { value: 'minmax', label: 'Min-max' }, + { value: 'maxabs', label: 'Max absolute' }, +]; + +/** Domain the two monotone transforms require; exaflow enforces with a 400 BadInputError. */ +const TRANSFORM_DOMAINS: Partial> = { + log1p: 'requires values greater than -1', + sqrt: 'requires values greater than or equal to 0', +}; + const CREATOR_CHOOSER_COPY: Record = { categorical_column_creator: { icon: 'fas fa-tags', @@ -332,6 +357,14 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { appliedLongitudinalVisit2 = ''; pendingLongitudinalStrategies: Record = {}; appliedLongitudinalStrategies: Record = {}; + /** Full desired state per variable (null = no transform); hydration copies `applied` into it. */ + pendingTransformStrategies: Record = {}; + appliedTransformStrategies: Record = {}; + pendingScalerStrategies: Record = {}; + appliedScalerStrategies: Record = {}; + /** `feature_range` for minmax, as the two text inputs the editor binds; null = default. */ + pendingScalerFeatureRange: [string, string] = ['', '']; + appliedScalerFeatureRange: [number, number] | null = null; pendingOutlierRules: Record = {}; appliedOutlierRules: Record = {}; preprocessingStatus: PreprocessingStatus = 'none'; @@ -348,11 +381,15 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { missing: '', outlier: '', longitudinal: '', + transform: '', + scaler: '', }; selectedPrepCode: Record = { missing: null, outlier: null, longitudinal: null, + transform: null, + scaler: null, }; // Record: the shared summary template indexes it with its own kind. statisticsSearch: Record = { @@ -374,10 +411,12 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { filters: false, setup: false, }); - preprocessingStepOpen: Record<'missing' | 'outlier' | 'longitudinal' | 'transformation', boolean> = { + preprocessingStepOpen: Record<'missing' | 'outlier' | 'longitudinal' | 'transform' | 'scaler' | 'transformation', boolean> = { missing: true, outlier: true, longitudinal: true, + transform: true, + scaler: true, transformation: true, }; @@ -403,7 +442,9 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { return !!( applied['missing_values_handler'] || applied['outlier_winsorizer'] || - applied['longitudinal_transformer'] + applied['longitudinal_transformer'] || + applied['variable_transformer'] || + applied['variable_scaler'] ); } @@ -427,6 +468,12 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { this.pendingOutlierRules = {}; this.appliedOutlierRules = {}; this.userEnabledOutliers.set(false); + this.pendingTransformStrategies = {}; + this.appliedTransformStrategies = {}; + this.pendingScalerStrategies = {}; + this.appliedScalerStrategies = {}; + this.pendingScalerFeatureRange = ['', '']; + this.appliedScalerFeatureRange = null; this.persistAppliedDescriptivePreprocessing(null); this.preprocessingStatus = 'none'; this.userPreprocessingApplied = false; @@ -562,10 +609,28 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { } } for (const code of Object.keys(this.appliedOutlierRules)) { - if (this.appliedOutlierRules[code]) { - this.appliedOutlierRules[code].enabled = false; - } + this.appliedOutlierRules[code] = { ...this.appliedOutlierRules[code], enabled: false }; + } + this.cdr.markForCheck(); + } + + /** Drop one transform/scaler sub-step entirely and persist the request config without it. */ + removePreprocessingSubNode(step: 'transform' | 'scaler'): void { + if (step === 'transform') { + this.pendingTransformStrategies = {}; + this.appliedTransformStrategies = {}; + } else { + this.pendingScalerStrategies = {}; + this.appliedScalerStrategies = {}; + this.pendingScalerFeatureRange = ['', '']; + this.appliedScalerFeatureRange = null; } + this.batchMenuOpen.set(null); + this.persistAppliedDescriptivePreprocessing( + this.buildPreprocessingConfig(this.appliedPreprocessingRules) + ); + this.updatePreprocessingStatus(); + this.emitProgressState(); this.cdr.markForCheck(); } @@ -642,6 +707,11 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { }); } + const transformNode = this.transformSubNode(config); + if (transformNode) nodes.push(transformNode); + const scalerNode = this.scalerSubNode(config); + if (scalerNode) nodes.push(scalerNode); + const longitudinal = config['longitudinal_transformer'] as | { visit1?: unknown; visit2?: unknown; strategies?: Record } | undefined; @@ -733,7 +803,7 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { this.preprocessingStepOpen.transformation = true; return; } - for (const step of ['missing', 'outlier', 'longitudinal'] as const) { + for (const step of ['missing', 'outlier', 'transform', 'scaler', 'longitudinal'] as const) { this.preprocessingStepOpen[step] = step === subNode; } if (subNode === 'outlier') this.userEnabledOutliers.set(true); @@ -741,7 +811,7 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { } /** A dashed rail row adds and opens its sub-step in one click, writing nothing. */ - addPreprocessingSubNode(subNode: 'outlier' | 'longitudinal'): void { + addPreprocessingSubNode(subNode: 'outlier' | 'longitudinal' | 'transform' | 'scaler'): void { if (!this.isStepAdded('setup')) this.addStep('setup'); this.jumpToSubNode('setup', subNode); } @@ -896,6 +966,87 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { return copy.title; } + /** Collapsed rail node for the per-variable transform step, when it holds a strategy. */ + private transformSubNode(config: Record): PipelineSubNode | null { + const holder = config['variable_transformer'] as { strategies?: Record } | undefined; + const strategies = Object.entries(holder?.strategies ?? {}); + if (!strategies.length) return null; + const summary = strategies.slice(0, 3).map(([code, strategy]) => { + const variable = this.preprocessingVariables.find((v) => v.code === code); + return `${variable ? this.variableLabel(variable) : code}: ${this.transformStrategyLabel(String(strategy))}`; + }); + return { + id: 'transform', + icon: 'fas fa-arrow-up-right-dots', + title: 'Variable Transforms', + subtitle: `Transforms applied to ${strategies.length} ${strategies.length === 1 ? 'variable' : 'variables'} (${summary.join(', ')}${strategies.length > 3 ? ', …' : ''})`, + statusLabel: 'Applied', + statusTone: this.hasPendingTransformChanges ? 'pending' : 'applied', + }; + } + + /** Collapsed rail node for the per-variable scaling step, when it holds a strategy. */ + private scalerSubNode(config: Record): PipelineSubNode | null { + const holder = config['variable_scaler'] as { strategies?: Record } | undefined; + const strategies = Object.entries(holder?.strategies ?? {}); + if (!strategies.length) return null; + const summary = strategies.slice(0, 3).map(([code, strategy]) => { + const variable = this.preprocessingVariables.find((v) => v.code === code); + return `${variable ? this.variableLabel(variable) : code}: ${this.scalerStrategyLabel(String(strategy))}`; + }); + return { + id: 'scaler', + icon: 'fas fa-ruler-horizontal', + title: 'Variable Scaling', + subtitle: `Scaling applied to ${strategies.length} ${strategies.length === 1 ? 'variable' : 'variables'} (${summary.join(', ')}${strategies.length > 3 ? ', …' : ''})`, + statusLabel: 'Applied', + statusTone: this.hasPendingScalerChanges ? 'pending' : 'applied', + }; + } + + transformStrategyLabel(strategy: string): string { + return this.transformStrategies.find((item) => item.value === strategy)?.label ?? strategy.replace(/_/g, ' '); + } + + scalerStrategyLabel(strategy: string): string { + return this.scalerStrategies.find((item) => item.value === strategy)?.label ?? strategy.replace(/_/g, ' '); + } + + /** Domain note for the selected transform: the observed minimum when the describe has it. */ + transformDomainWarning(variable: VariableRow, strategy: TransformStrategy | null): string { + const domain = strategy ? TRANSFORM_DOMAINS[strategy] : undefined; + if (!domain) return ''; + const minText = this.distributionFor(variable).find((point) => point.label === 'Min')?.value; + const min = minText !== undefined ? parseNumber(minText) : null; + const violates = strategy === 'log1p' ? (min !== null && min <= -1) : strategy === 'sqrt' ? (min !== null && min < 0) : false; + if (min === null) return `This transform ${domain}.`; + return violates + ? `This transform ${domain}. Lowest observed value is ${minText} — exaflow will reject the run.` + : `This transform ${domain}. Lowest observed value is ${minText}.`; + } + + get transformSubNodeApplied(): boolean { + return this.appliedPreprocessingSubNodes.some((node) => node.id === 'transform'); + } + + get scalerSubNodeApplied(): boolean { + return this.appliedPreprocessingSubNodes.some((node) => node.id === 'scaler'); + } + + get transformPreprocessingStatusLabel(): string { + if (!this.outlierPreprocessingVariables.length) return 'Not available'; + if (this.hasPendingTransformChanges) return `${pluralize(this.pendingTransformChangeCount, 'change')} · not applied`; + if (Object.keys(this.appliedTransformStrategies).length) return 'Applied'; + return 'Optional'; + } + + get scalerPreprocessingStatusLabel(): string { + if (!this.outlierPreprocessingVariables.length) return 'Not available'; + if (this.hasPendingScalerChanges) return `${pluralize(this.pendingScalerChangeCount, 'change')} · not applied`; + if (Object.keys(this.appliedScalerStrategies).length) return 'Applied'; + return 'Optional'; + } + private missingValuesSubNode(config: Record): PipelineSubNode { const handler = config['missing_values_handler'] as | { strategies?: Record } @@ -1038,9 +1189,14 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { { value: 'drop', label: 'Remove rows' }, { value: 'mean', label: 'Mean imputation' }, { value: 'median', label: 'Median imputation' }, + { value: 'most_frequent', label: 'Most frequent imputation' }, { value: 'constant', label: 'Constant value' }, ]; + readonly transformStrategies = TRANSFORM_STRATEGIES; + readonly scalerStrategies = SCALER_STRATEGIES; + readonly transformDomains = TRANSFORM_DOMAINS; + /** The card whose category counts replace its editor ('kmeans' = the cluster card). */ previewDraftId: string | null = null; /** The categorical card whose Done loaded the current counts; it shows their loading/error state. */ @@ -1142,6 +1298,8 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { missing: layout.preprocessingStep === 'missing', outlier: layout.preprocessingStep === 'outlier', longitudinal: layout.preprocessingStep === 'longitudinal', + transform: false, + scaler: false, transformation: false, }; } @@ -1238,7 +1396,8 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { } get pendingChangeCount(): number { - return this.pendingMissingChangeCount + this.pendingOutlierChangeCount + this.pendingLongitudinalChangeCount; + return this.pendingMissingChangeCount + this.pendingOutlierChangeCount + this.pendingLongitudinalChangeCount + + this.pendingTransformChangeCount + this.pendingScalerChangeCount; } readonly filterRuleCount = computed(() => @@ -1387,7 +1546,9 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { filteredPrepVariables(kind: PrepKind): VariableRow[] { const query = this.prepSearch[kind].trim().toLowerCase(); - const pool = kind === 'outlier' ? this.outlierPreprocessingVariables : this.preprocessingVariables; + const pool = kind === 'outlier' || kind === 'transform' || kind === 'scaler' + ? this.outlierPreprocessingVariables + : this.preprocessingVariables; if (!query) return pool; return pool.filter((variable) => { const label = this.variableLabel(variable).toLowerCase(); @@ -1496,6 +1657,10 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { const appliedStrategy = this.appliedLongitudinalStrategies[variable.code] ?? null; return pendingStrategy !== appliedStrategy || this.longitudinalVisitPairHasPendingChange; } + case 'transform': + return (this.pendingTransformStrategies[variable.code] ?? null) !== (this.appliedTransformStrategies[variable.code] ?? null); + case 'scaler': + return (this.pendingScalerStrategies[variable.code] ?? null) !== (this.appliedScalerStrategies[variable.code] ?? null); default: { const _exhaustive: never = kind; return _exhaustive; @@ -1511,6 +1676,10 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { return this.hasAppliedOutlierPreprocessing(variable.code); case 'longitudinal': return this.hasAppliedLongitudinalPreprocessing(variable.code); + case 'transform': + return !!this.appliedTransformStrategies[variable.code]; + case 'scaler': + return !!this.appliedScalerStrategies[variable.code]; default: { const _exhaustive: never = kind; return _exhaustive; @@ -1906,18 +2075,20 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { this.cdr.markForCheck(); } - togglePreprocessingStep(step: 'missing' | 'outlier' | 'longitudinal' | 'transformation'): void { + togglePreprocessingStep(step: 'missing' | 'outlier' | 'longitudinal' | 'transform' | 'scaler' | 'transformation'): void { this.preprocessingStepOpen[step] = !this.preprocessingStepOpen[step]; } /** "How this works" disclosures, moved from the body summary into the card header. */ - readonly preprocessingDocOpen = signal>({ + readonly preprocessingDocOpen = signal>({ missing: false, outlier: false, longitudinal: false, + transform: false, + scaler: false, }); - togglePreprocessingDoc(step: 'missing' | 'outlier' | 'longitudinal'): void { + togglePreprocessingDoc(step: 'missing' | 'outlier' | 'longitudinal' | 'transform' | 'scaler'): void { this.preprocessingDocOpen.update((open) => ({ ...open, [step]: !open[step] })); this.cdr.markForCheck(); } @@ -2470,7 +2641,7 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { this.appliedLongitudinalEnabled = this.isLongitudinalModel; this.appliedLongitudinalVisit1 = this.longitudinalVisit1; this.appliedLongitudinalVisit2 = this.longitudinalVisit2; - this.appliedLongitudinalStrategies = { ...this.pendingLongitudinalStrategies }; + this.appliedLongitudinalStrategies = { ...this.pendingLongitudinalStrategies } as Record; }, reset: () => { this.longitudinalVisit1 = this.appliedLongitudinalVisit1; @@ -2479,10 +2650,41 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { this.ensureLongitudinalDefaults(); }, }, + transform: { + pending: () => this.pendingTransformChangeCount, + validate: () => ({}), + commit: () => { + this.appliedTransformStrategies = this.cleanStrategyMap(this.pendingTransformStrategies); + }, + reset: () => { + this.pendingTransformStrategies = { ...this.appliedTransformStrategies }; + }, + }, + scaler: { + pending: () => this.pendingScalerChangeCount, + validate: () => ({}), + commit: () => { + this.appliedScalerStrategies = this.cleanStrategyMap(this.pendingScalerStrategies); + this.appliedScalerFeatureRange = this.parseFeatureRange(this.pendingScalerFeatureRange); + }, + reset: () => { + this.pendingScalerStrategies = { ...this.appliedScalerStrategies }; + this.pendingScalerFeatureRange = this.appliedScalerFeatureRange + ? [String(this.appliedScalerFeatureRange[0]), String(this.appliedScalerFeatureRange[1])] + : ['', '']; + }, + }, }; + /** Drop the null "no strategy" entries so the stored map holds configured codes only. */ + private cleanStrategyMap(map: Record): Record { + return Object.fromEntries( + Object.entries(map).filter(([, strategy]) => strategy !== null && strategy !== undefined) + ) as Record; + } + private stepsIn(scope: PrepStep | 'all'): PrepStep[] { - return scope === 'all' ? ['missing', 'outlier', 'longitudinal'] : [scope]; + return scope === 'all' ? ['missing', 'outlier', 'transform', 'scaler', 'longitudinal'] : [scope]; } /** Pending edits of one preprocessing card; the stage count sums these. */ @@ -2521,6 +2723,8 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { } case 'constant': return `Fills ${n} values with a value you set`; + case 'most_frequent': + return `Fills ${n} values with the most frequent category`; default: return ''; } @@ -2607,6 +2811,78 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { this.emitProgressState(); } + transformStrategyFor(variable: VariableRow): TransformStrategy | null { + return this.pendingTransformStrategies[variable.code] ?? null; + } + + onTransformStrategyChange(variable: VariableRow, strategy: TransformStrategy | null): void { + this.pendingTransformStrategies = { + ...this.pendingTransformStrategies, + [variable.code]: strategy, + }; + this.updatePreprocessingStatus(); + this.emitProgressState(); + } + + scalerStrategyFor(variable: VariableRow): ScalerStrategy | null { + return this.pendingScalerStrategies[variable.code] ?? null; + } + + onScalerStrategyChange(variable: VariableRow, strategy: ScalerStrategy | null): void { + this.pendingScalerStrategies = { + ...this.pendingScalerStrategies, + [variable.code]: strategy, + }; + this.updatePreprocessingStatus(); + this.emitProgressState(); + } + + onScalerFeatureRangeChange(which: 'low' | 'high', value: string): void { + this.pendingScalerFeatureRange = which === 'low' + ? [value, this.pendingScalerFeatureRange[1]] + : [this.pendingScalerFeatureRange[0], value]; + this.updatePreprocessingStatus(); + this.emitProgressState(); + } + + get hasPendingTransformChanges(): boolean { + return this.serializeTransformScalerState('transform', false) !== this.serializeTransformScalerState('transform', true); + } + + get pendingTransformChangeCount(): number { + return this.hasPendingTransformChanges ? 1 : 0; + } + + get hasPendingScalerChanges(): boolean { + return this.serializeTransformScalerState('scaler', false) !== this.serializeTransformScalerState('scaler', true); + } + + get pendingScalerChangeCount(): number { + return this.hasPendingScalerChanges ? 1 : 0; + } + + /** Serialized transform/scaler state over the current numerical variables (pending or applied). */ + private serializeTransformScalerState(step: TransformOrScalerStep, applied: boolean): string { + const transformStrategies = applied ? this.appliedTransformStrategies : this.pendingTransformStrategies; + const scalerStrategies = applied ? this.appliedScalerStrategies : this.pendingScalerStrategies; + const range = applied ? this.appliedScalerFeatureRange : this.parseFeatureRange(this.pendingScalerFeatureRange); + const source = step === 'transform' ? transformStrategies : scalerStrategies; + const codes = this.outlierPreprocessingVariables.map((v) => v.code); + return JSON.stringify({ + strategies: Object.fromEntries(codes.sort().map((code) => [code, source[code] ?? null])), + ...(step === 'scaler' ? { featureRange: range } : {}), + }); + } + + private parseFeatureRange(range: [string, string]): [number, number] | null { + const low = Number(String(range[0] ?? '').trim()); + const high = Number(String(range[1] ?? '').trim()); + return range[0].trim() !== '' && range[1].trim() !== '' + && Number.isFinite(low) && Number.isFinite(high) && low < high + ? [low, high] + : null; + } + outlierStrategyDisplay(strategy: OutlierStrategy): string { return this.outlierStrategies.find((item) => item.value === strategy)?.label ?? strategy; } @@ -2805,10 +3081,10 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { this.emitProgressState(); } - batchMenuOpen = signal<'missing' | 'outlier' | null>(null); + batchMenuOpen = signal<'missing' | 'outlier' | 'transform' | 'scaler' | null>(null); summaryExportMenuKind = signal(null); - toggleBatchMenu(kind: 'missing' | 'outlier'): void { + toggleBatchMenu(kind: 'missing' | 'outlier' | 'transform' | 'scaler'): void { this.batchMenuOpen.update((open) => (open === kind ? null : kind)); this.cdr.markForCheck(); } @@ -2846,6 +3122,19 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { this.cdr.markForCheck(); } + /** Batch-set one transform/scaler strategy for every numerical variable. */ + applyBatchTransformStrategy(strategy: TransformStrategy): void { + this.outlierPreprocessingVariables.forEach((variable) => this.onTransformStrategyChange(variable, strategy)); + this.batchMenuOpen.set(null); + this.cdr.markForCheck(); + } + + applyBatchScalerStrategy(strategy: ScalerStrategy): void { + this.outlierPreprocessingVariables.forEach((variable) => this.onScalerStrategyChange(variable, strategy)); + this.batchMenuOpen.set(null); + this.cdr.markForCheck(); + } + toggleAllOutliers(enabled: boolean): void { this.outlierPreprocessingVariables.forEach((variable) => this.onOutlierEnabledChange(variable, enabled)); this.batchMenuOpen.set(null); @@ -2865,7 +3154,10 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { } isMissingActionDisabled(variable: VariableRow, action: MissingAction): boolean { - return (action === 'mean' || action === 'median') && !this.isNumericVariable(variable); + if (action === 'mean' || action === 'median') return !this.isNumericVariable(variable); + // The vocabulary comes from the CDE on the worker side, so most_frequent needs one. + if (action === 'most_frequent') return !this.isCategoricalVariable(variable); + return false; } isLongitudinalStrategyDisabled(variable: VariableRow, strategy: LongitudinalStrategy): boolean { @@ -3866,6 +4158,10 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { ? this.pendingOutlierRules : this.mergeOutlierRulesForCurrentSelection({}, this.appliedOutlierRules)); if (outlier) config['outlier_winsorizer'] = outlier; + const transform = this.buildTransformPreprocessingConfig(step === 'transform' ? false : true); + if (transform) config['variable_transformer'] = transform; + const scaler = this.buildScalerPreprocessingConfig(step === 'scaler' ? false : true); + if (scaler) config['variable_scaler'] = scaler; const longitudinal = this.buildLongitudinalPreprocessingConfig(step !== 'longitudinal'); if (longitudinal) config['longitudinal_transformer'] = longitudinal; return Object.keys(config).length ? config : null; @@ -3879,13 +4175,20 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { const missingValuesHandler = this.buildMissingPreprocessingConfig(rules, allowedCodes); if (missingValuesHandler) config['missing_values_handler'] = missingValuesHandler; + const applied = rules === this.appliedPreprocessingRules; const outlier = this.buildOutlierPreprocessingConfig( - rules === this.appliedPreprocessingRules ? this.appliedOutlierRules : this.pendingOutlierRules, + applied ? this.appliedOutlierRules : this.pendingOutlierRules, allowedCodes ); if (outlier) config['outlier_winsorizer'] = outlier; - const longitudinal = this.buildLongitudinalPreprocessingConfig(rules === this.appliedPreprocessingRules, allowedCodes); + // Request order mirrors the rail: impute, clip, transform, scale. + const transform = this.buildTransformPreprocessingConfig(applied, allowedCodes); + if (transform) config['variable_transformer'] = transform; + const scaler = this.buildScalerPreprocessingConfig(applied, allowedCodes); + if (scaler) config['variable_scaler'] = scaler; + + const longitudinal = this.buildLongitudinalPreprocessingConfig(applied, allowedCodes); if (longitudinal) config['longitudinal_transformer'] = longitudinal; return Object.keys(config).length ? config : null; @@ -4238,6 +4541,33 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { return serialized ? { ...serialized } : null; } + /** Request parameters for the transform/scaler steps; null leaves the step out of the request. */ + private buildTransformPreprocessingConfig( + applied: boolean, + allowedCodes = this.currentPreprocessingCodeSet() + ): Record | null { + const source = applied ? this.appliedTransformStrategies : this.cleanStrategyMap(this.pendingTransformStrategies); + const strategies = Object.fromEntries( + Object.entries(source).filter(([code]) => allowedCodes.has(code)) + ); + return Object.keys(strategies).length ? { strategies } : null; + } + + private buildScalerPreprocessingConfig( + applied: boolean, + allowedCodes = this.currentPreprocessingCodeSet() + ): Record | null { + const source = applied ? this.appliedScalerStrategies : this.cleanStrategyMap(this.pendingScalerStrategies); + const strategies = Object.fromEntries( + Object.entries(source).filter(([code]) => allowedCodes.has(code)) + ); + if (!Object.keys(strategies).length) return null; + // `feature_range` is a minmax parameter; keep the request shape flat and permissive. + const range = applied ? this.appliedScalerFeatureRange : this.parseFeatureRange(this.pendingScalerFeatureRange); + const usesMinMax = Object.values(strategies).includes('minmax'); + return range && usesMinMax ? { strategies, feature_range: range } : { strategies }; + } + private validatePendingRulesFor(scope: PrepStep | 'all'): Record { return Object.assign({}, ...this.stepsIn(scope).map((step) => this.prepStepOps[step].validate())); } @@ -4291,6 +4621,9 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { if ((rule.action === 'mean' || rule.action === 'median') && !this.isNumericVariable(variable)) { errors[rule.variableCode] = 'Mean and median imputation are only available for numeric variables.'; } + if (rule.action === 'most_frequent' && !this.isCategoricalVariable(variable)) { + errors[rule.variableCode] = 'Most frequent imputation is only available for nominal variables.'; + } if (rule.action === 'constant') { const value = String(rule.value ?? '').trim(); if (!value) { @@ -4463,6 +4796,34 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { this.pendingLongitudinalStrategies = { ...appliedStrategies }; } + // Transform/scaler restore: the stored step names are the proposal from the federated + // preprocessing spec; unknown shapes hydrate to nothing so the stage stays usable. + const hydrateStrategies = ( + step: 'variable_transformer' | 'variable_scaler', + isStrategy: (value: string) => value is S + ): Record => { + const holder = preprocessing[step] as { strategies?: Record } | undefined; + const strategies: Record = {}; + Object.entries(holder?.strategies ?? {}).forEach(([code, strategy]) => { + if (isStrategy(String(strategy))) strategies[code] = String(strategy) as S; + }); + return strategies; + }; + this.appliedTransformStrategies = hydrateStrategies('variable_transformer', this.isTransformStrategy); + this.pendingTransformStrategies = { ...this.appliedTransformStrategies }; + this.appliedScalerStrategies = hydrateStrategies('variable_scaler', this.isScalerStrategy); + this.pendingScalerStrategies = { ...this.appliedScalerStrategies }; + const scalerHolder = preprocessing['variable_scaler'] as { feature_range?: unknown } | undefined; + const rawRange = Array.isArray(scalerHolder?.feature_range) ? scalerHolder!.feature_range as unknown[] : []; + const low = Number(rawRange[0]); + const high = Number(rawRange[1]); + this.appliedScalerFeatureRange = rawRange.length === 2 && Number.isFinite(low) && Number.isFinite(high) + ? [low, high] + : null; + this.pendingScalerFeatureRange = this.appliedScalerFeatureRange + ? [String(this.appliedScalerFeatureRange[0]), String(this.appliedScalerFeatureRange[1])] + : ['', '']; + this.hydrateAppliedTransformation(); } @@ -4504,7 +4865,15 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { } private isMissingAction(value: string): value is MissingAction { - return ['drop', 'mean', 'median', 'constant'].includes(value); + return ['drop', 'mean', 'median', 'constant', 'most_frequent'].includes(value); + } + + private isTransformStrategy(value: string): value is TransformStrategy { + return ['log1p', 'sqrt', 'asinh', 'yeo_johnson', 'quantile_uniform', 'quantile_normal'].includes(value); + } + + private isScalerStrategy(value: string): value is ScalerStrategy { + return ['standard', 'robust', 'minmax', 'maxabs'].includes(value); } private isLongitudinalStrategy(value: string): value is LongitudinalStrategy { diff --git a/src/app/pages/experiment-studio/statistic-analysis-panel/statistic-analysis-panel.pipeline.spec.ts b/src/app/pages/experiment-studio/statistic-analysis-panel/statistic-analysis-panel.pipeline.spec.ts index d277390cb..7d4a35b87 100644 --- a/src/app/pages/experiment-studio/statistic-analysis-panel/statistic-analysis-panel.pipeline.spec.ts +++ b/src/app/pages/experiment-studio/statistic-analysis-panel/statistic-analysis-panel.pipeline.spec.ts @@ -338,10 +338,11 @@ describe('StatisticAnalysisPanelComponent pipeline presence', () => { expect(pipelineCard('analysis-preprocessing').querySelector('.pipeline-node-body')).toBeNull(); const rows = railRows(pipelineCard('analysis-preprocessing')); - expect(rows.ghost.map((row) => row.textContent?.trim())).toEqual([expect.stringMatching('Add outlier clipping')]); + const outlierGhost = rows.ghost.find((row) => row.textContent?.includes('Add outlier clipping')); + expect(outlierGhost).toBeTruthy(); mockExpService.setAppliedDescriptivePreprocessing.mockClear(); - (rows.ghost[0].querySelector('button') as HTMLButtonElement).click(); + (outlierGhost!.querySelector('button') as HTMLButtonElement).click(); fixture.detectChanges(); expect(component.sectionOpen().setup).toBe(true); @@ -350,7 +351,7 @@ describe('StatisticAnalysisPanelComponent pipeline presence', () => { expect(component.preprocessingStepOpen.missing).toBe(false); expect(pipelineCard('analysis-preprocessing').querySelector('.pipeline-node-body')).toBeTruthy(); expect(mockExpService.setAppliedDescriptivePreprocessing).not.toHaveBeenCalled(); - expect(railRows(pipelineCard('analysis-preprocessing')).ghost).toEqual([]); + expect(railRows(pipelineCard('analysis-preprocessing')).ghost.some((row) => row.textContent?.includes('Add outlier clipping'))).toBe(false); }); /** @@ -363,7 +364,8 @@ describe('StatisticAnalysisPanelComponent pipeline presence', () => { fixture.detectChanges(); const rail = railRows(pipelineCard('analysis-preprocessing')); - (rail.ghost[0].querySelector('button') as HTMLButtonElement).click(); + const outlierGhost = rail.ghost.find((row) => row.textContent?.includes('Add outlier clipping'))!; + (outlierGhost.querySelector('button') as HTMLButtonElement).click(); fixture.detectChanges(); expect(component.pendingChangeCount).toBe(0); @@ -381,9 +383,10 @@ describe('StatisticAnalysisPanelComponent pipeline presence', () => { fixture.detectChanges(); const rows = railRows(pipelineCard('analysis-preprocessing')); expect(rows.live.length).toBe(1); - expect(rows.ghost.map((row) => row.textContent?.trim())).toEqual([expect.stringMatching('Add outlier clipping')]); + expect(rows.ghost.some((row) => row.textContent?.includes('Add outlier clipping'))).toBe(true); - (rows.ghost[0].querySelector('button') as HTMLButtonElement).click(); + const outlierRow = rows.ghost.find((row) => row.textContent?.includes('Add outlier clipping'))!; + (outlierRow.querySelector('button') as HTMLButtonElement).click(); fixture.detectChanges(); expect(component.sectionOpen().setup).toBe(true); expect(component.preprocessingStepOpen.outlier).toBe(true); @@ -442,6 +445,48 @@ describe('StatisticAnalysisPanelComponent pipeline presence', () => { expect(component.preprocessingStatusLabel).toBe('2 steps applied ✓'); }); + it('lists the transform and scaler sub-nodes between outlier and longitudinal', () => { + (mockExpService.selectedVariables as any).set([age]); + seedAppliedConfig({ + missing_values_handler: { strategies: { age: 'drop' } }, + variable_transformer: { strategies: { age: 'log1p' } }, + variable_scaler: { strategies: { age: 'standard' } }, + }); + fixture.detectChanges(); + + const nodes = component.appliedPreprocessingSubNodes; + expect(nodes.map((n) => n.id)).toEqual(['missing', 'transform', 'scaler']); + expect(nodes[1].subtitle).toContain('log(1 + x)'); + expect(nodes[2].subtitle).toContain('Standard'); + }); + + it('offers transform and scaler ghost rows while they hold no strategies', () => { + (mockExpService.selectedVariables as any).set([age]); + fixture.detectChanges(); + + const card = pipelineCard('analysis-preprocessing'); + const { ghost } = railRows(card); + const texts = ghost.map((row) => row.textContent ?? ''); + expect(texts.some((t) => t.includes('Add variable transforms'))).toBe(true); + expect(texts.some((t) => t.includes('Add variable scaling'))).toBe(true); + }); + + it('restores the transform and scaler editors from a persisted config', () => { + (mockExpService.selectedVariables as any).set([age]); + seedAppliedConfig({ + variable_transformer: { strategies: { age: 'yeo_johnson' } }, + variable_scaler: { strategies: { age: 'minmax' }, feature_range: [-1, 1] }, + }); + fixture.detectChanges(); + + expect(component.isStepAdded('setup')).toBe(true); + expect(component.appliedTransformStrategies).toEqual({ age: 'yeo_johnson' }); + expect(component.pendingTransformStrategies).toEqual({ age: 'yeo_johnson' }); + expect(component.appliedScalerStrategies).toEqual({ age: 'minmax' }); + expect(component.appliedScalerFeatureRange).toEqual([-1, 1]); + expect(component.pendingScalerFeatureRange).toEqual(['-1', '1']); + }); + it('surfaces the applied derived column as a transformation sub-node', () => { seedAppliedConfig({ categorical_column_creator: [{ diff --git a/src/app/services/experiment-studio.service.spec.ts b/src/app/services/experiment-studio.service.spec.ts index eccee6d1f..455977eda 100644 --- a/src/app/services/experiment-studio.service.spec.ts +++ b/src/app/services/experiment-studio.service.spec.ts @@ -921,6 +921,64 @@ describe('ExperimentStudioService', () => { ); }); + it('summarizes most frequent imputation in plain language', () => { + const summary = service.formatPreprocessingConfig({ + missing_values_handler: { + strategies: { gender: 'most_frequent', age: 'mean' }, + }, + }, { gender: 'Gender', age: 'Age' }); + + expect(summary).toBe('Missing values: Gender: most frequent imputation, Age: mean imputation'); + }); + + it('summarizes transform and scaler steps with variable labels', () => { + const summary = service.formatPreprocessingConfig({ + missing_values_handler: { strategies: { age: 'drop' } }, + variable_transformer: { strategies: { age: 'log1p', bmi: 'quantile_uniform' } }, + variable_scaler: { strategies: { age: 'standard' }, feature_range: [0, 1] }, + }, { age: 'Age', bmi: 'BMI' }); + + expect(summary).toBe( + 'Missing values: Age: remove rows\n' + + 'Variable transforms: Age: log(1 + x), BMI: quantile to uniform\n' + + 'Variable scaling [0, 1]: Age: standard scaling' + ); + }); + + it('emits applied transform and scaler singleton steps in rail order', () => { + service.selectedDataModel.set(mockDataModel); + service.setSelectedDatasets(['ds1']); + const applied = { + missing_values_handler: { strategies: { age: 'drop' } }, + outlier_winsorizer: { strategies: { age: 'iqr' }, tails: { age: 'both' }, folds: { age: 1.5 } }, + variable_transformer: { strategies: { age: 'yeo_johnson' } }, + variable_scaler: { strategies: { age: 'standard' } }, + }; + + service.setAppliedDescriptivePreprocessing(applied); + const body = service.buildRequestBody('mock_algo', ['age']); + + expect(body.analysis.preprocessing).toEqual(preprocessingSteps(applied)); + }); + + it('filters transform and scaler strategies to the histogram y variables', () => { + service.selectedDataModel.set(mockDataModel); + service.setSelectedDatasets(['ds1']); + const applied = { + variable_transformer: { strategies: { age: 'log1p', bmi: 'sqrt' } }, + variable_scaler: { strategies: { age: 'standard', bmi: 'maxabs' }, feature_range: [0, 1] }, + }; + + const body = service.buildRequestBody('histogram', ['age'], null, null, null, null, applied); + + // An override without a missing-values step carries no drop default: the filtered + // config is forwarded as stored, matching the outlier-only override behavior. + expect(body.analysis.preprocessing).toEqual(preprocessingSteps({ + variable_transformer: { strategies: { age: 'log1p' } }, + variable_scaler: { strategies: { age: 'standard' }, feature_range: [0, 1] }, + })); + }); + it('loads outlier report previews with outlier parameters and upstream missing preprocessing', () => new Promise((done) => { service.selectedDataModel.set(mockDataModel); service.setVariables([ diff --git a/src/app/services/experiment-studio.service.ts b/src/app/services/experiment-studio.service.ts index 82dded505..49c33a391 100644 --- a/src/app/services/experiment-studio.service.ts +++ b/src/app/services/experiment-studio.service.ts @@ -1417,6 +1417,26 @@ export class ExperimentStudioService { if (next) filtered[OUTLIER_WINSORIZER] = next; } + const transform = preprocessing['variable_transformer']; + if (transform && typeof transform === 'object' && !Array.isArray(transform)) { + const next = this.filterPreprocessingStepRecordMaps( + transform as Record, + allowed, + ['strategies'] + ); + if (next) filtered['variable_transformer'] = next; + } + + const scaler = preprocessing['variable_scaler']; + if (scaler && typeof scaler === 'object' && !Array.isArray(scaler)) { + const next = this.filterPreprocessingStepRecordMaps( + scaler as Record, + allowed, + ['strategies'] + ); + if (next) filtered['variable_scaler'] = next; + } + const longitudinal = preprocessing['longitudinal_transformer']; if (longitudinal && typeof longitudinal === 'object' && !Array.isArray(longitudinal)) { const longitudinalRecord = longitudinal as Record; @@ -1548,6 +1568,32 @@ export class ExperimentStudioService { entries.push({ label: 'Outlier winsorizer', value: labels.join('; '), lines: labels }); } + const transform = preprocessing['variable_transformer'] as { + strategies?: Record; + } | undefined; + const transformEntries = Object.entries(transform?.strategies ?? {}); + if (transformEntries.length) { + const labels = transformEntries.map(([code, strategy]) => + `${this.preprocessingVariableLabel(code, labelMap)}: ${this.transformStrategyLabel(String(strategy))}` + ); + entries.push({ label: 'Variable transforms', value: labels.join(', '), lines: labels }); + } + + const scaler = preprocessing['variable_scaler'] as { + strategies?: Record; + feature_range?: unknown; + } | undefined; + const scalerEntries = Object.entries(scaler?.strategies ?? {}); + if (scalerEntries.length) { + const range = Array.isArray(scaler?.feature_range) && scaler!.feature_range.length === 2 + ? ` [${scaler!.feature_range.join(', ')}]` + : ''; + const labels = scalerEntries.map(([code, strategy]) => + `${this.preprocessingVariableLabel(code, labelMap)}: ${this.scalerStrategyLabel(String(strategy))}` + ); + entries.push({ label: `Variable scaling${range}`, value: labels.join(', '), lines: labels }); + } + const longitudinal = preprocessing['longitudinal_transformer'] as Record | undefined; if (longitudinal) { const visit1 = longitudinal['visit1']; @@ -1565,7 +1611,13 @@ export class ExperimentStudioService { entries.push({ label: 'Longitudinal transformation', value: `${strategySummary}${visitLabel}` }); } - const knownKeys = new Set([MISSING_VALUES_HANDLER, OUTLIER_WINSORIZER, 'longitudinal_transformer']); + const knownKeys = new Set([ + MISSING_VALUES_HANDLER, + OUTLIER_WINSORIZER, + 'longitudinal_transformer', + 'variable_transformer', + 'variable_scaler', + ]); Object.keys(preprocessing) .filter((key) => !knownKeys.has(key)) .forEach((key) => { @@ -1634,6 +1686,40 @@ export class ExperimentStudioService { return labelMap[code] ?? code; } + transformStrategyLabel(strategy: string): string { + switch (strategy) { + case 'log1p': + return 'log(1 + x)'; + case 'sqrt': + return 'square root'; + case 'asinh': + return 'inverse hyperbolic sine'; + case 'yeo_johnson': + return 'Yeo-Johnson'; + case 'quantile_uniform': + return 'quantile to uniform'; + case 'quantile_normal': + return 'quantile to normal'; + default: + return strategy.replace(/_/g, ' '); + } + } + + scalerStrategyLabel(strategy: string): string { + switch (strategy) { + case 'standard': + return 'standard scaling'; + case 'robust': + return 'robust scaling'; + case 'minmax': + return 'min-max scaling'; + case 'maxabs': + return 'max absolute scaling'; + default: + return strategy.replace(/_/g, ' '); + } + } + private humanizePreprocessingValue(value: string): string { switch (value) { case 'drop': @@ -1642,6 +1728,8 @@ export class ExperimentStudioService { return 'mean imputation'; case 'median': return 'median imputation'; + case 'most_frequent': + return 'most frequent imputation'; case 'constant': return 'constant value'; default: From 00df08a29b10fa359b4f64819ae2e0c5e7025123 Mon Sep 17 00:00:00 2001 From: KFilippopolitis Date: Mon, 9 Nov 2026 15:46:59 +0200 Subject: [PATCH 2/3] fix: gate transform and scaler stations on the preprocessing catalog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-ups on #73: - The transform and scaling stations, their rail nodes and ghost add rows are shown only while the catalog lists the step (optimistically before the catalog loads, like the creators), and the steps are dropped from every request once the catalog loads without them — exaflow rejects unknown steps with 'Transformer does not exist.' - The scaler step is dropped from a filtered request when no strategy survives, instead of a lone step-level feature_range keeping it - The minmax feature range now rides on the min-max summary lines only - Component spec coverage: most_frequent nominal-only gate, station availability against the catalog, pending/applied strategy lifecycle --- .../statistic-analysis-panel.component.html | 12 +-- ...statistic-analysis-panel.component.spec.ts | 48 +++++++++++ .../statistic-analysis-panel.component.ts | 25 +++++- .../statistic-analysis-panel.pipeline.spec.ts | 23 +++++- .../experiment-studio.service.spec.ts | 80 ++++++++++++++++--- src/app/services/experiment-studio.service.ts | 38 +++++++-- 6 files changed, 197 insertions(+), 29 deletions(-) diff --git a/src/app/pages/experiment-studio/statistic-analysis-panel/statistic-analysis-panel.component.html b/src/app/pages/experiment-studio/statistic-analysis-panel/statistic-analysis-panel.component.html index cdd114efd..1a4780632 100644 --- a/src/app/pages/experiment-studio/statistic-analysis-panel/statistic-analysis-panel.component.html +++ b/src/app/pages/experiment-studio/statistic-analysis-panel/statistic-analysis-panel.component.html @@ -695,7 +695,7 @@

- } @else { + } @else if (transformSubNodeApplied) { {{ variableLabel(variable) }}

} - @if (!scalerSubNodeApplied) { + @if (!scalerSubNodeApplied && scalerStepAvailable) { - } @else { + } @else if (scalerSubNodeApplied) { { }); }); + describe('federated preprocessing stations', () => { + it('offers most_frequent only to nominal variables', () => { + const age = { code: 'age', label: 'Age', type: 'real' }; + const sex = { code: 'sex', label: 'Sex', type: 'nominal', enumerations: [{ code: 'm', label: 'M' }] }; + + expect(component.isMissingActionDisabled(sex, 'most_frequent')).toBe(false); + expect(component.isMissingActionDisabled(age, 'most_frequent')).toBe(true); + // mean/median keep their numerical-only gate. + expect(component.isMissingActionDisabled(sex, 'mean')).toBe(true); + expect(component.isMissingActionDisabled(age, 'mean')).toBe(false); + }); + + it('hides the transform and scaler stations when the catalog lacks them', () => { + // beforeEach mocks catalogLoaded() true and hasPreprocessingSpec without the new names. + expect(component.transformStepAvailable).toBe(false); + expect(component.scalerStepAvailable).toBe(false); + expect(component.transformSubNodeApplied).toBe(false); + }); + + it('shows the transform and scaler stations once the catalog lists them', () => { + mockExpService.hasPreprocessingSpec.mockImplementation((name: string) => + ['categorical_column_creator', 'kmeans_cluster_creator', 'variable_transformer', 'variable_scaler'].includes(name)); + + expect(component.transformStepAvailable).toBe(true); + expect(component.scalerStepAvailable).toBe(true); + }); + + it('keeps the transform strategy on the pending state until applied', () => { + const age = { code: 'age', label: 'Age', type: 'real' }; + (mockExpService.selectedVariables as any).set([age]); + component.pendingTransformStrategies = {}; + component.appliedTransformStrategies = {}; + + component.onTransformStrategyChange(age, 'log1p'); + expect(component.hasPendingTransformChanges).toBe(true); + expect(component.transformStrategyFor(age)).toBe('log1p'); + + // Commit is what the card Apply runs. + (component as any).prepStepOps['transform'].commit(); + expect(component.appliedTransformStrategies).toEqual({ age: 'log1p' }); + expect(component.hasPendingTransformChanges).toBe(false); + + // Reverting to no transform is a pending change until applied. + component.onTransformStrategyChange(age, null); + expect(component.hasPendingTransformChanges).toBe(true); + }); + }); + describe('cohort retention strip', () => { it('shows the retention strip only when both raw and processed summaries are loaded', () => new Promise((done) => { const { age } = configureRawSummary(); diff --git a/src/app/pages/experiment-studio/statistic-analysis-panel/statistic-analysis-panel.component.ts b/src/app/pages/experiment-studio/statistic-analysis-panel/statistic-analysis-panel.component.ts index 244299401..6f955cb58 100644 --- a/src/app/pages/experiment-studio/statistic-analysis-panel/statistic-analysis-panel.component.ts +++ b/src/app/pages/experiment-studio/statistic-analysis-panel/statistic-analysis-panel.component.ts @@ -968,6 +968,7 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { /** Collapsed rail node for the per-variable transform step, when it holds a strategy. */ private transformSubNode(config: Record): PipelineSubNode | null { + if (!this.transformStepAvailable) return null; const holder = config['variable_transformer'] as { strategies?: Record } | undefined; const strategies = Object.entries(holder?.strategies ?? {}); if (!strategies.length) return null; @@ -987,6 +988,7 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { /** Collapsed rail node for the per-variable scaling step, when it holds a strategy. */ private scalerSubNode(config: Record): PipelineSubNode | null { + if (!this.scalerStepAvailable) return null; const holder = config['variable_scaler'] as { strategies?: Record } | undefined; const strategies = Object.entries(holder?.strategies ?? {}); if (!strategies.length) return null; @@ -1025,23 +1027,38 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { : `This transform ${domain}. Lowest observed value is ${minText}.`; } + /** + * Spec gate for the transform/scaler stations, mirroring the creators: before the + * catalog loads the availability is unknown (optimistically shown); once it loads + * without the step, the station is unavailable and its request step is dropped. + */ + get transformStepAvailable(): boolean { + return !this.expStudioService.catalogLoaded() || this.expStudioService.hasPreprocessingSpec('variable_transformer'); + } + + get scalerStepAvailable(): boolean { + return !this.expStudioService.catalogLoaded() || this.expStudioService.hasPreprocessingSpec('variable_scaler'); + } + get transformSubNodeApplied(): boolean { - return this.appliedPreprocessingSubNodes.some((node) => node.id === 'transform'); + return this.transformStepAvailable + && this.appliedPreprocessingSubNodes.some((node) => node.id === 'transform'); } get scalerSubNodeApplied(): boolean { - return this.appliedPreprocessingSubNodes.some((node) => node.id === 'scaler'); + return this.scalerStepAvailable + && this.appliedPreprocessingSubNodes.some((node) => node.id === 'scaler'); } get transformPreprocessingStatusLabel(): string { - if (!this.outlierPreprocessingVariables.length) return 'Not available'; + if (!this.transformStepAvailable || !this.outlierPreprocessingVariables.length) return 'Not available'; if (this.hasPendingTransformChanges) return `${pluralize(this.pendingTransformChangeCount, 'change')} · not applied`; if (Object.keys(this.appliedTransformStrategies).length) return 'Applied'; return 'Optional'; } get scalerPreprocessingStatusLabel(): string { - if (!this.outlierPreprocessingVariables.length) return 'Not available'; + if (!this.scalerStepAvailable || !this.outlierPreprocessingVariables.length) return 'Not available'; if (this.hasPendingScalerChanges) return `${pluralize(this.pendingScalerChangeCount, 'change')} · not applied`; if (Object.keys(this.appliedScalerStrategies).length) return 'Applied'; return 'Optional'; diff --git a/src/app/pages/experiment-studio/statistic-analysis-panel/statistic-analysis-panel.pipeline.spec.ts b/src/app/pages/experiment-studio/statistic-analysis-panel/statistic-analysis-panel.pipeline.spec.ts index 7d4a35b87..e6b5366ab 100644 --- a/src/app/pages/experiment-studio/statistic-analysis-panel/statistic-analysis-panel.pipeline.spec.ts +++ b/src/app/pages/experiment-studio/statistic-analysis-panel/statistic-analysis-panel.pipeline.spec.ts @@ -114,7 +114,7 @@ describe('StatisticAnalysisPanelComponent pipeline presence', () => { 'kmeans_cluster_creator', ]); mockExpService.hasPreprocessingSpec.mockImplementation((name: string) => - ['categorical_column_creator', 'kmeans_cluster_creator'].includes(name), + ['categorical_column_creator', 'kmeans_cluster_creator', 'variable_transformer', 'variable_scaler'].includes(name), ); mockExpService.catalogLoaded.mockReturnValue(true); mockExpService.transformationDependents.mockReturnValue([]); @@ -487,6 +487,27 @@ describe('StatisticAnalysisPanelComponent pipeline presence', () => { expect(component.pendingScalerFeatureRange).toEqual(['-1', '1']); }); + it('hides the transform and scaler stations when the catalog lacks them', () => { + (mockExpService.selectedVariables as any).set([age]); + seedAppliedConfig({ + variable_transformer: { strategies: { age: 'log1p' } }, + variable_scaler: { strategies: { age: 'standard' } }, + }); + mockExpService.hasPreprocessingSpec.mockImplementation((name: string) => + ['categorical_column_creator', 'kmeans_cluster_creator'].includes(name)); + fixture.detectChanges(); + + // No rail node and no ghost row: the deployment cannot run the step, so the + // UI must not offer it even though a persisted config still carries it. + expect(component.appliedPreprocessingSubNodes.map((n) => n.id)).toEqual(['missing']); + const card = pipelineCard('analysis-preprocessing'); + const { ghost } = railRows(card); + expect(ghost.some((row) => row.textContent?.includes('Add variable transforms'))).toBe(false); + expect(ghost.some((row) => row.textContent?.includes('Add variable scaling'))).toBe(false); + expect(card.textContent).not.toContain('Variable Transforms'); + expect(card.textContent).not.toContain('Variable Scaling'); + }); + it('surfaces the applied derived column as a transformation sub-node', () => { seedAppliedConfig({ categorical_column_creator: [{ diff --git a/src/app/services/experiment-studio.service.spec.ts b/src/app/services/experiment-studio.service.spec.ts index 455977eda..5ad3d08e7 100644 --- a/src/app/services/experiment-studio.service.spec.ts +++ b/src/app/services/experiment-studio.service.spec.ts @@ -199,6 +199,16 @@ describe('ExperimentStudioService', () => { released: true, }; + const mockAlgorithmSpecs = [ + toAlgorithmSpec(mockRawAlgo), + toAlgorithmSpec(mockHistogramAlgo), + toAlgorithmSpec(mockDescribeAlgo), + toAlgorithmSpec(mockOutlierReportAlgo), + toAlgorithmSpec(mockLinearSvmAlgo), + toAlgorithmSpec(mockKmeansAlgo), + toAlgorithmSpec(mockPcaAlgo), + ]; + beforeEach(() => { sessionStorage.clear(); TestBed.configureTestingModule({ @@ -210,15 +220,7 @@ describe('ExperimentStudioService', () => { httpMock = TestBed.inject(HttpTestingController); flushSpecifications( - [ - toAlgorithmSpec(mockRawAlgo), - toAlgorithmSpec(mockHistogramAlgo), - toAlgorithmSpec(mockDescribeAlgo), - toAlgorithmSpec(mockOutlierReportAlgo), - toAlgorithmSpec(mockLinearSvmAlgo), - toAlgorithmSpec(mockKmeansAlgo), - toAlgorithmSpec(mockPcaAlgo), - ], + mockAlgorithmSpecs, [ { name: 'categorical_column_creator', @@ -281,6 +283,22 @@ describe('ExperimentStudioService', () => { ], }, }, + // The proposal names from the federated preprocessing spec; the gating test + // covers the case where the catalog ships without them. + { + name: 'variable_transformer', + label: 'Variable Transformer', + desc: '', + parameters: {}, + outputs: null, + }, + { + name: 'variable_scaler', + label: 'Variable Scaler', + desc: '', + parameters: {}, + outputs: null, + }, ], ); }); @@ -935,16 +953,56 @@ describe('ExperimentStudioService', () => { const summary = service.formatPreprocessingConfig({ missing_values_handler: { strategies: { age: 'drop' } }, variable_transformer: { strategies: { age: 'log1p', bmi: 'quantile_uniform' } }, - variable_scaler: { strategies: { age: 'standard' }, feature_range: [0, 1] }, + variable_scaler: { strategies: { age: 'standard', bmi: 'minmax' }, feature_range: [0, 1] }, }, { age: 'Age', bmi: 'BMI' }); expect(summary).toBe( 'Missing values: Age: remove rows\n' + 'Variable transforms: Age: log(1 + x), BMI: quantile to uniform\n' - + 'Variable scaling [0, 1]: Age: standard scaling' + + 'Variable scaling: Age: standard scaling, BMI: min-max scaling [0, 1]' ); }); + it('drops transform and scaler steps the loaded catalog does not list', () => { + service.selectedDataModel.set(mockDataModel); + service.setSelectedDatasets(['ds1']); + const applied = { + missing_values_handler: { strategies: { age: 'drop' } }, + variable_transformer: { strategies: { age: 'log1p' } }, + variable_scaler: { strategies: { age: 'standard' } }, + }; + + // Simulate a deployment whose catalog ships without the new steps: reload the + // specifications with a reduced preprocessing catalog. + service.loadBackendAlgorithms().subscribe(); + httpMock.expectOne('/services/specifications/inputdata').flush(mockInputdataSpec); + httpMock.expectOne('/services/specifications/preprocessing').flush([]); + httpMock.expectOne('/services/specifications/algorithms').flush(mockAlgorithmSpecs); + + service.setAppliedDescriptivePreprocessing(applied); + const body = service.buildRequestBody('mock_algo', ['age']); + + expect(body.analysis.preprocessing).toEqual(preprocessingSteps({ + missing_values_handler: { strategies: { age: 'drop' } }, + })); + }); + + it('drops the scaler step when filtering leaves no strategy, not just feature_range', () => { + service.selectedDataModel.set(mockDataModel); + service.setSelectedDatasets(['ds1']); + const applied = { + variable_scaler: { strategies: { bmi: 'standard' }, feature_range: [0, 1] }, + }; + + const body = service.buildRequestBody('histogram', ['age'], null, null, null, null, applied); + + // The lone step-level `feature_range` must not keep the step in the request; the + // histogram fallback then supplies its usual drop default. + expect(body.analysis.preprocessing).toEqual(preprocessingSteps({ + missing_values_handler: { strategies: { age: 'drop' } }, + })); + }); + it('emits applied transform and scaler singleton steps in rail order', () => { service.selectedDataModel.set(mockDataModel); service.setSelectedDatasets(['ds1']); diff --git a/src/app/services/experiment-studio.service.ts b/src/app/services/experiment-studio.service.ts index 49c33a391..c47375b1a 100644 --- a/src/app/services/experiment-studio.service.ts +++ b/src/app/services/experiment-studio.service.ts @@ -63,6 +63,13 @@ const APPLIED_DESCRIPTIVE_PREPROCESSING = '__applied_descriptive_preprocessing__ /** Session-storage key for the ordered column-creating transformation steps. */ const APPLIED_TRANSFORMATIONS = '__applied_transformations__'; +/** + * Singleton steps gated on the preprocessing catalog: once the catalog has loaded without + * them, the deployment cannot run them, so they are hidden in the UI and dropped from every + * request (exaflow rejects unknown steps with `Transformer '' does not exist.`). + */ +const GATED_SINGLETON_STEPS = new Set(['variable_transformer', 'variable_scaler']); + /** Quick-preview / diagnostic algorithms hidden from the experiment algorithm picker. */ const ALGORITHM_PANEL_EXCLUDED = new Set([ AlgorithmNames.HISTOGRAM, @@ -996,6 +1003,14 @@ export class ExperimentStudioService { ); } + /** + * A gated singleton step is runnable only once the catalog loaded and still lists it — + * before that, empty specs mean "unknown", matching how creators are gated. + */ + private singletonStepRunnable(name: string): boolean { + return !this.preprocessingCatalogLoaded() || !!this.preprocessingSpecs()[name]; + } + private preprocessingConfigToSteps( config: PreprocessingConfig | null | undefined, ): AnalysisPreprocessingStep[] | null { @@ -1004,6 +1019,9 @@ export class ExperimentStudioService { // one ordered step per entry; the engine accepts repeated steps with the same name. const steps: AnalysisPreprocessingStep[] = []; for (const [name, parameters] of Object.entries(config)) { + // A gated step the deployment cannot run is dropped from every request, + // the same gate the column-creating creators go through. + if (GATED_SINGLETON_STEPS.has(name) && !this.singletonStepRunnable(name)) continue; if (Array.isArray(parameters)) { for (const item of parameters) { if (item && typeof item === 'object') { @@ -1434,7 +1452,11 @@ export class ExperimentStudioService { allowed, ['strategies'] ); - if (next) filtered['variable_scaler'] = next; + // A step whose strategies were all filtered out is dropped entirely: a lone + // step-level parameter such as `feature_range` must not keep it in the request. + if (next && Object.keys((next['strategies'] as Record) ?? {}).length > 0) { + filtered['variable_scaler'] = next; + } } const longitudinal = preprocessing['longitudinal_transformer']; @@ -1585,13 +1607,15 @@ export class ExperimentStudioService { } | undefined; const scalerEntries = Object.entries(scaler?.strategies ?? {}); if (scalerEntries.length) { - const range = Array.isArray(scaler?.feature_range) && scaler!.feature_range.length === 2 - ? ` [${scaler!.feature_range.join(', ')}]` + // `feature_range` belongs to minmax only, so it rides on that strategy's lines. + const rawRange = Array.isArray(scaler?.feature_range) && scaler!.feature_range.length === 2 + ? `[${scaler!.feature_range.join(', ')}]` : ''; - const labels = scalerEntries.map(([code, strategy]) => - `${this.preprocessingVariableLabel(code, labelMap)}: ${this.scalerStrategyLabel(String(strategy))}` - ); - entries.push({ label: `Variable scaling${range}`, value: labels.join(', '), lines: labels }); + const labels = scalerEntries.map(([code, strategy]) => { + const rangeText = String(strategy) === 'minmax' && rawRange ? ` ${rawRange}` : ''; + return `${this.preprocessingVariableLabel(code, labelMap)}: ${this.scalerStrategyLabel(String(strategy))}${rangeText}`; + }); + entries.push({ label: 'Variable scaling', value: labels.join(', '), lines: labels }); } const longitudinal = preprocessing['longitudinal_transformer'] as Record | undefined; From 34c5859e7ac84b69e902d4758b407a98f5ed0fda Mon Sep 17 00:00:00 2001 From: KFilippopolitis Date: Mon, 9 Nov 2026 16:10:45 +0200 Subject: [PATCH 3/3] docs: correct the most_frequent help comment Current exaflow fills each dataset with its own local mode; no CDE vocabulary is used yet. Say so in the gate comment instead of 'the vocabulary comes from the CDE on the worker side'. --- .../statistic-analysis-panel.component.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/app/pages/experiment-studio/statistic-analysis-panel/statistic-analysis-panel.component.ts b/src/app/pages/experiment-studio/statistic-analysis-panel/statistic-analysis-panel.component.ts index 6f955cb58..d5b2e94ae 100644 --- a/src/app/pages/experiment-studio/statistic-analysis-panel/statistic-analysis-panel.component.ts +++ b/src/app/pages/experiment-studio/statistic-analysis-panel/statistic-analysis-panel.component.ts @@ -3172,7 +3172,8 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { isMissingActionDisabled(variable: VariableRow, action: MissingAction): boolean { if (action === 'mean' || action === 'median') return !this.isNumericVariable(variable); - // The vocabulary comes from the CDE on the worker side, so most_frequent needs one. + // Exaflow's most_frequent fills with each worker's local mode (no category list is + // sent), so it only makes sense on categorical variables. if (action === 'most_frequent') return !this.isCategoricalVariable(variable); return false; }