diff --git a/src/app/pages/experiment-studio/statistic-analysis-panel/statistic-analysis-panel.component.html b/src/app/pages/experiment-studio/statistic-analysis-panel/statistic-analysis-panel.component.html index 514c9563c..1a4780632 100644 --- a/src/app/pages/experiment-studio/statistic-analysis-panel/statistic-analysis-panel.component.html +++ b/src/app/pages/experiment-studio/statistic-analysis-panel/statistic-analysis-panel.component.html @@ -695,6 +695,32 @@

+ + } + @if (!scalerSubNodeApplied && scalerStepAvailable) { +
+ + +
+ } @if (isLongitudinalModel && !expStudioService.appliedPreprocessingConfig()?.['longitudinal_transformer']) {
@@ -1281,6 +1307,307 @@
Values that would be capped · per data } + @if (!transformSubNodeApplied && transformStepAvailable) { + + } @else if (transformSubNodeApplied) { + + + @if (preprocessingStepDocumentation('variable_transformer')) { + + } + @if (preprocessingStepDocumentation('variable_transformer') && preprocessingDocOpen().transform) { +
+
+
+ } + + @if (outlierPreprocessingVariables.length > 0) { +
+ + +
+ @if (selectedPrepVariable('transform'); as variable) { +
+
+ Numerical variable +

{{ variableLabel(variable) }}

+
+ + {{ prepVariableStateLabel('transform', variable) }} + +
+
+
+ + Transform +
+ + @if (transformStrategyFor(variable); as strategy) { +

{{ transformDomainWarning(variable, strategy) }}

+ } +
+ } @else { +
+ +

No variables match the current search.

+ Clear the search field to choose a transform variable. +
+ } +
+
+ } @else { +
+ +

No numerical variables selected.

+ Select numerical variables before configuring transforms. +
+ } + +
+ } + + @if (!scalerSubNodeApplied && scalerStepAvailable) { + + } @else if (scalerSubNodeApplied) { + + + @if (preprocessingStepDocumentation('variable_scaler')) { + + } + @if (preprocessingStepDocumentation('variable_scaler') && preprocessingDocOpen().scaler) { +
+
+
+ } + + @if (outlierPreprocessingVariables.length > 0) { +
+ + +
+ @if (selectedPrepVariable('scaler'); as variable) { +
+
+ Numerical variable +

{{ variableLabel(variable) }}

+
+ + {{ prepVariableStateLabel('scaler', variable) }} + +
+
+
+ + Scaling +
+ + @if (scalerStrategyFor(variable) === 'minmax') { +
+ + +
+ } +
+ } @else { +
+ +

No variables match the current search.

+ Clear the search field to choose a scaling variable. +
+ } +
+
+ } @else { +
+ +

No numerical variables selected.

+ Select numerical variables before configuring scaling. +
+ } + +
+ } + @if (isLongitudinalModel) { { }); }); + describe('federated preprocessing stations', () => { + it('offers most_frequent only to nominal variables', () => { + const age = { code: 'age', label: 'Age', type: 'real' }; + const sex = { code: 'sex', label: 'Sex', type: 'nominal', enumerations: [{ code: 'm', label: 'M' }] }; + + expect(component.isMissingActionDisabled(sex, 'most_frequent')).toBe(false); + expect(component.isMissingActionDisabled(age, 'most_frequent')).toBe(true); + // mean/median keep their numerical-only gate. + expect(component.isMissingActionDisabled(sex, 'mean')).toBe(true); + expect(component.isMissingActionDisabled(age, 'mean')).toBe(false); + }); + + it('hides the transform and scaler stations when the catalog lacks them', () => { + // beforeEach mocks catalogLoaded() true and hasPreprocessingSpec without the new names. + expect(component.transformStepAvailable).toBe(false); + expect(component.scalerStepAvailable).toBe(false); + expect(component.transformSubNodeApplied).toBe(false); + }); + + it('shows the transform and scaler stations once the catalog lists them', () => { + mockExpService.hasPreprocessingSpec.mockImplementation((name: string) => + ['categorical_column_creator', 'kmeans_cluster_creator', 'variable_transformer', 'variable_scaler'].includes(name)); + + expect(component.transformStepAvailable).toBe(true); + expect(component.scalerStepAvailable).toBe(true); + }); + + it('keeps the transform strategy on the pending state until applied', () => { + const age = { code: 'age', label: 'Age', type: 'real' }; + (mockExpService.selectedVariables as any).set([age]); + component.pendingTransformStrategies = {}; + component.appliedTransformStrategies = {}; + + component.onTransformStrategyChange(age, 'log1p'); + expect(component.hasPendingTransformChanges).toBe(true); + expect(component.transformStrategyFor(age)).toBe('log1p'); + + // Commit is what the card Apply runs. + (component as any).prepStepOps['transform'].commit(); + expect(component.appliedTransformStrategies).toEqual({ age: 'log1p' }); + expect(component.hasPendingTransformChanges).toBe(false); + + // Reverting to no transform is a pending change until applied. + component.onTransformStrategyChange(age, null); + expect(component.hasPendingTransformChanges).toBe(true); + }); + }); + describe('cohort retention strip', () => { it('shows the retention strip only when both raw and processed summaries are loaded', () => new Promise((done) => { const { age } = configureRawSummary(); diff --git a/src/app/pages/experiment-studio/statistic-analysis-panel/statistic-analysis-panel.component.ts b/src/app/pages/experiment-studio/statistic-analysis-panel/statistic-analysis-panel.component.ts index 6e9afd316..d5b2e94ae 100644 --- a/src/app/pages/experiment-studio/statistic-analysis-panel/statistic-analysis-panel.component.ts +++ b/src/app/pages/experiment-studio/statistic-analysis-panel/statistic-analysis-panel.component.ts @@ -88,11 +88,11 @@ type TabKey = 'Statistics' | 'Charts' | 'Histogram'; type SummaryKind = 'source' | 'raw' | 'processed'; type SectionKey = 'source' | 'raw' | 'setup' | 'filters' | 'processed' | 'transformation'; type PipelineStep = 'source' | 'filters' | 'setup' | 'transformation'; -type PrepStep = 'missing' | 'outlier' | 'longitudinal'; +type PrepStep = 'missing' | 'outlier' | 'longitudinal' | 'transform' | 'scaler'; type DistributionSubTab = 'Numeric' | 'Nominal'; type StatisticVariableType = 'numeric' | 'nominal'; type PreprocessingStatus = 'none' | 'pending' | 'applied'; -type PrepKind = 'missing' | 'outlier' | 'longitudinal'; +type PrepKind = 'missing' | 'outlier' | 'longitudinal' | 'transform' | 'scaler'; type SummaryAnchorKey = 'charts' | 'details' | 'export'; @@ -112,8 +112,11 @@ const SUMMARY_GUIDE_ANCHORS: Record = [ + { value: 'log1p', label: 'log(1 + x)' }, + { value: 'sqrt', label: 'Square root' }, + { value: 'asinh', label: 'Inverse hyperbolic sine' }, + { value: 'yeo_johnson', label: 'Yeo-Johnson' }, + { value: 'quantile_uniform', label: 'Quantile → uniform' }, + { value: 'quantile_normal', label: 'Quantile → normal' }, +]; + +const SCALER_STRATEGIES: Array<{ value: ScalerStrategy; label: string }> = [ + { value: 'standard', label: 'Standard (mean 0, std 1)' }, + { value: 'robust', label: 'Robust (median/IQR)' }, + { value: 'minmax', label: 'Min-max' }, + { value: 'maxabs', label: 'Max absolute' }, +]; + +/** Domain the two monotone transforms require; exaflow enforces with a 400 BadInputError. */ +const TRANSFORM_DOMAINS: Partial> = { + log1p: 'requires values greater than -1', + sqrt: 'requires values greater than or equal to 0', +}; + const CREATOR_CHOOSER_COPY: Record = { categorical_column_creator: { icon: 'fas fa-tags', @@ -332,6 +357,14 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { appliedLongitudinalVisit2 = ''; pendingLongitudinalStrategies: Record = {}; appliedLongitudinalStrategies: Record = {}; + /** Full desired state per variable (null = no transform); hydration copies `applied` into it. */ + pendingTransformStrategies: Record = {}; + appliedTransformStrategies: Record = {}; + pendingScalerStrategies: Record = {}; + appliedScalerStrategies: Record = {}; + /** `feature_range` for minmax, as the two text inputs the editor binds; null = default. */ + pendingScalerFeatureRange: [string, string] = ['', '']; + appliedScalerFeatureRange: [number, number] | null = null; pendingOutlierRules: Record = {}; appliedOutlierRules: Record = {}; preprocessingStatus: PreprocessingStatus = 'none'; @@ -348,11 +381,15 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { missing: '', outlier: '', longitudinal: '', + transform: '', + scaler: '', }; selectedPrepCode: Record = { missing: null, outlier: null, longitudinal: null, + transform: null, + scaler: null, }; // Record: the shared summary template indexes it with its own kind. statisticsSearch: Record = { @@ -374,10 +411,12 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { filters: false, setup: false, }); - preprocessingStepOpen: Record<'missing' | 'outlier' | 'longitudinal' | 'transformation', boolean> = { + preprocessingStepOpen: Record<'missing' | 'outlier' | 'longitudinal' | 'transform' | 'scaler' | 'transformation', boolean> = { missing: true, outlier: true, longitudinal: true, + transform: true, + scaler: true, transformation: true, }; @@ -403,7 +442,9 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { return !!( applied['missing_values_handler'] || applied['outlier_winsorizer'] || - applied['longitudinal_transformer'] + applied['longitudinal_transformer'] || + applied['variable_transformer'] || + applied['variable_scaler'] ); } @@ -427,6 +468,12 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { this.pendingOutlierRules = {}; this.appliedOutlierRules = {}; this.userEnabledOutliers.set(false); + this.pendingTransformStrategies = {}; + this.appliedTransformStrategies = {}; + this.pendingScalerStrategies = {}; + this.appliedScalerStrategies = {}; + this.pendingScalerFeatureRange = ['', '']; + this.appliedScalerFeatureRange = null; this.persistAppliedDescriptivePreprocessing(null); this.preprocessingStatus = 'none'; this.userPreprocessingApplied = false; @@ -562,13 +609,31 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { } } for (const code of Object.keys(this.appliedOutlierRules)) { - if (this.appliedOutlierRules[code]) { - this.appliedOutlierRules[code].enabled = false; - } + this.appliedOutlierRules[code] = { ...this.appliedOutlierRules[code], enabled: false }; } this.cdr.markForCheck(); } + /** Drop one transform/scaler sub-step entirely and persist the request config without it. */ + removePreprocessingSubNode(step: 'transform' | 'scaler'): void { + if (step === 'transform') { + this.pendingTransformStrategies = {}; + this.appliedTransformStrategies = {}; + } else { + this.pendingScalerStrategies = {}; + this.appliedScalerStrategies = {}; + this.pendingScalerFeatureRange = ['', '']; + this.appliedScalerFeatureRange = null; + } + this.batchMenuOpen.set(null); + this.persistAppliedDescriptivePreprocessing( + this.buildPreprocessingConfig(this.appliedPreprocessingRules) + ); + this.updatePreprocessingStatus(); + this.emitProgressState(); + this.cdr.markForCheck(); + } + finishDataHandling(): void { if (this.pendingChangeCount > 0 || this.transformationHasPendingChange) { this.showUnappliedChangesWarning.set(true); @@ -642,6 +707,11 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { }); } + const transformNode = this.transformSubNode(config); + if (transformNode) nodes.push(transformNode); + const scalerNode = this.scalerSubNode(config); + if (scalerNode) nodes.push(scalerNode); + const longitudinal = config['longitudinal_transformer'] as | { visit1?: unknown; visit2?: unknown; strategies?: Record } | undefined; @@ -733,7 +803,7 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { this.preprocessingStepOpen.transformation = true; return; } - for (const step of ['missing', 'outlier', 'longitudinal'] as const) { + for (const step of ['missing', 'outlier', 'transform', 'scaler', 'longitudinal'] as const) { this.preprocessingStepOpen[step] = step === subNode; } if (subNode === 'outlier') this.userEnabledOutliers.set(true); @@ -741,7 +811,7 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { } /** A dashed rail row adds and opens its sub-step in one click, writing nothing. */ - addPreprocessingSubNode(subNode: 'outlier' | 'longitudinal'): void { + addPreprocessingSubNode(subNode: 'outlier' | 'longitudinal' | 'transform' | 'scaler'): void { if (!this.isStepAdded('setup')) this.addStep('setup'); this.jumpToSubNode('setup', subNode); } @@ -896,6 +966,104 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { return copy.title; } + /** Collapsed rail node for the per-variable transform step, when it holds a strategy. */ + private transformSubNode(config: Record): PipelineSubNode | null { + if (!this.transformStepAvailable) return null; + const holder = config['variable_transformer'] as { strategies?: Record } | undefined; + const strategies = Object.entries(holder?.strategies ?? {}); + if (!strategies.length) return null; + const summary = strategies.slice(0, 3).map(([code, strategy]) => { + const variable = this.preprocessingVariables.find((v) => v.code === code); + return `${variable ? this.variableLabel(variable) : code}: ${this.transformStrategyLabel(String(strategy))}`; + }); + return { + id: 'transform', + icon: 'fas fa-arrow-up-right-dots', + title: 'Variable Transforms', + subtitle: `Transforms applied to ${strategies.length} ${strategies.length === 1 ? 'variable' : 'variables'} (${summary.join(', ')}${strategies.length > 3 ? ', …' : ''})`, + statusLabel: 'Applied', + statusTone: this.hasPendingTransformChanges ? 'pending' : 'applied', + }; + } + + /** Collapsed rail node for the per-variable scaling step, when it holds a strategy. */ + private scalerSubNode(config: Record): PipelineSubNode | null { + if (!this.scalerStepAvailable) return null; + const holder = config['variable_scaler'] as { strategies?: Record } | undefined; + const strategies = Object.entries(holder?.strategies ?? {}); + if (!strategies.length) return null; + const summary = strategies.slice(0, 3).map(([code, strategy]) => { + const variable = this.preprocessingVariables.find((v) => v.code === code); + return `${variable ? this.variableLabel(variable) : code}: ${this.scalerStrategyLabel(String(strategy))}`; + }); + return { + id: 'scaler', + icon: 'fas fa-ruler-horizontal', + title: 'Variable Scaling', + subtitle: `Scaling applied to ${strategies.length} ${strategies.length === 1 ? 'variable' : 'variables'} (${summary.join(', ')}${strategies.length > 3 ? ', …' : ''})`, + statusLabel: 'Applied', + statusTone: this.hasPendingScalerChanges ? 'pending' : 'applied', + }; + } + + transformStrategyLabel(strategy: string): string { + return this.transformStrategies.find((item) => item.value === strategy)?.label ?? strategy.replace(/_/g, ' '); + } + + scalerStrategyLabel(strategy: string): string { + return this.scalerStrategies.find((item) => item.value === strategy)?.label ?? strategy.replace(/_/g, ' '); + } + + /** Domain note for the selected transform: the observed minimum when the describe has it. */ + transformDomainWarning(variable: VariableRow, strategy: TransformStrategy | null): string { + const domain = strategy ? TRANSFORM_DOMAINS[strategy] : undefined; + if (!domain) return ''; + const minText = this.distributionFor(variable).find((point) => point.label === 'Min')?.value; + const min = minText !== undefined ? parseNumber(minText) : null; + const violates = strategy === 'log1p' ? (min !== null && min <= -1) : strategy === 'sqrt' ? (min !== null && min < 0) : false; + if (min === null) return `This transform ${domain}.`; + return violates + ? `This transform ${domain}. Lowest observed value is ${minText} — exaflow will reject the run.` + : `This transform ${domain}. Lowest observed value is ${minText}.`; + } + + /** + * Spec gate for the transform/scaler stations, mirroring the creators: before the + * catalog loads the availability is unknown (optimistically shown); once it loads + * without the step, the station is unavailable and its request step is dropped. + */ + get transformStepAvailable(): boolean { + return !this.expStudioService.catalogLoaded() || this.expStudioService.hasPreprocessingSpec('variable_transformer'); + } + + get scalerStepAvailable(): boolean { + return !this.expStudioService.catalogLoaded() || this.expStudioService.hasPreprocessingSpec('variable_scaler'); + } + + get transformSubNodeApplied(): boolean { + return this.transformStepAvailable + && this.appliedPreprocessingSubNodes.some((node) => node.id === 'transform'); + } + + get scalerSubNodeApplied(): boolean { + return this.scalerStepAvailable + && this.appliedPreprocessingSubNodes.some((node) => node.id === 'scaler'); + } + + get transformPreprocessingStatusLabel(): string { + if (!this.transformStepAvailable || !this.outlierPreprocessingVariables.length) return 'Not available'; + if (this.hasPendingTransformChanges) return `${pluralize(this.pendingTransformChangeCount, 'change')} · not applied`; + if (Object.keys(this.appliedTransformStrategies).length) return 'Applied'; + return 'Optional'; + } + + get scalerPreprocessingStatusLabel(): string { + if (!this.scalerStepAvailable || !this.outlierPreprocessingVariables.length) return 'Not available'; + if (this.hasPendingScalerChanges) return `${pluralize(this.pendingScalerChangeCount, 'change')} · not applied`; + if (Object.keys(this.appliedScalerStrategies).length) return 'Applied'; + return 'Optional'; + } + private missingValuesSubNode(config: Record): PipelineSubNode { const handler = config['missing_values_handler'] as | { strategies?: Record } @@ -1038,9 +1206,14 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { { value: 'drop', label: 'Remove rows' }, { value: 'mean', label: 'Mean imputation' }, { value: 'median', label: 'Median imputation' }, + { value: 'most_frequent', label: 'Most frequent imputation' }, { value: 'constant', label: 'Constant value' }, ]; + readonly transformStrategies = TRANSFORM_STRATEGIES; + readonly scalerStrategies = SCALER_STRATEGIES; + readonly transformDomains = TRANSFORM_DOMAINS; + /** The card whose category counts replace its editor ('kmeans' = the cluster card). */ previewDraftId: string | null = null; /** The categorical card whose Done loaded the current counts; it shows their loading/error state. */ @@ -1142,6 +1315,8 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { missing: layout.preprocessingStep === 'missing', outlier: layout.preprocessingStep === 'outlier', longitudinal: layout.preprocessingStep === 'longitudinal', + transform: false, + scaler: false, transformation: false, }; } @@ -1238,7 +1413,8 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { } get pendingChangeCount(): number { - return this.pendingMissingChangeCount + this.pendingOutlierChangeCount + this.pendingLongitudinalChangeCount; + return this.pendingMissingChangeCount + this.pendingOutlierChangeCount + this.pendingLongitudinalChangeCount + + this.pendingTransformChangeCount + this.pendingScalerChangeCount; } readonly filterRuleCount = computed(() => @@ -1387,7 +1563,9 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { filteredPrepVariables(kind: PrepKind): VariableRow[] { const query = this.prepSearch[kind].trim().toLowerCase(); - const pool = kind === 'outlier' ? this.outlierPreprocessingVariables : this.preprocessingVariables; + const pool = kind === 'outlier' || kind === 'transform' || kind === 'scaler' + ? this.outlierPreprocessingVariables + : this.preprocessingVariables; if (!query) return pool; return pool.filter((variable) => { const label = this.variableLabel(variable).toLowerCase(); @@ -1496,6 +1674,10 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { const appliedStrategy = this.appliedLongitudinalStrategies[variable.code] ?? null; return pendingStrategy !== appliedStrategy || this.longitudinalVisitPairHasPendingChange; } + case 'transform': + return (this.pendingTransformStrategies[variable.code] ?? null) !== (this.appliedTransformStrategies[variable.code] ?? null); + case 'scaler': + return (this.pendingScalerStrategies[variable.code] ?? null) !== (this.appliedScalerStrategies[variable.code] ?? null); default: { const _exhaustive: never = kind; return _exhaustive; @@ -1511,6 +1693,10 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { return this.hasAppliedOutlierPreprocessing(variable.code); case 'longitudinal': return this.hasAppliedLongitudinalPreprocessing(variable.code); + case 'transform': + return !!this.appliedTransformStrategies[variable.code]; + case 'scaler': + return !!this.appliedScalerStrategies[variable.code]; default: { const _exhaustive: never = kind; return _exhaustive; @@ -1906,18 +2092,20 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { this.cdr.markForCheck(); } - togglePreprocessingStep(step: 'missing' | 'outlier' | 'longitudinal' | 'transformation'): void { + togglePreprocessingStep(step: 'missing' | 'outlier' | 'longitudinal' | 'transform' | 'scaler' | 'transformation'): void { this.preprocessingStepOpen[step] = !this.preprocessingStepOpen[step]; } /** "How this works" disclosures, moved from the body summary into the card header. */ - readonly preprocessingDocOpen = signal>({ + readonly preprocessingDocOpen = signal>({ missing: false, outlier: false, longitudinal: false, + transform: false, + scaler: false, }); - togglePreprocessingDoc(step: 'missing' | 'outlier' | 'longitudinal'): void { + togglePreprocessingDoc(step: 'missing' | 'outlier' | 'longitudinal' | 'transform' | 'scaler'): void { this.preprocessingDocOpen.update((open) => ({ ...open, [step]: !open[step] })); this.cdr.markForCheck(); } @@ -2470,7 +2658,7 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { this.appliedLongitudinalEnabled = this.isLongitudinalModel; this.appliedLongitudinalVisit1 = this.longitudinalVisit1; this.appliedLongitudinalVisit2 = this.longitudinalVisit2; - this.appliedLongitudinalStrategies = { ...this.pendingLongitudinalStrategies }; + this.appliedLongitudinalStrategies = { ...this.pendingLongitudinalStrategies } as Record; }, reset: () => { this.longitudinalVisit1 = this.appliedLongitudinalVisit1; @@ -2479,10 +2667,41 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { this.ensureLongitudinalDefaults(); }, }, + transform: { + pending: () => this.pendingTransformChangeCount, + validate: () => ({}), + commit: () => { + this.appliedTransformStrategies = this.cleanStrategyMap(this.pendingTransformStrategies); + }, + reset: () => { + this.pendingTransformStrategies = { ...this.appliedTransformStrategies }; + }, + }, + scaler: { + pending: () => this.pendingScalerChangeCount, + validate: () => ({}), + commit: () => { + this.appliedScalerStrategies = this.cleanStrategyMap(this.pendingScalerStrategies); + this.appliedScalerFeatureRange = this.parseFeatureRange(this.pendingScalerFeatureRange); + }, + reset: () => { + this.pendingScalerStrategies = { ...this.appliedScalerStrategies }; + this.pendingScalerFeatureRange = this.appliedScalerFeatureRange + ? [String(this.appliedScalerFeatureRange[0]), String(this.appliedScalerFeatureRange[1])] + : ['', '']; + }, + }, }; + /** Drop the null "no strategy" entries so the stored map holds configured codes only. */ + private cleanStrategyMap(map: Record): Record { + return Object.fromEntries( + Object.entries(map).filter(([, strategy]) => strategy !== null && strategy !== undefined) + ) as Record; + } + private stepsIn(scope: PrepStep | 'all'): PrepStep[] { - return scope === 'all' ? ['missing', 'outlier', 'longitudinal'] : [scope]; + return scope === 'all' ? ['missing', 'outlier', 'transform', 'scaler', 'longitudinal'] : [scope]; } /** Pending edits of one preprocessing card; the stage count sums these. */ @@ -2521,6 +2740,8 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { } case 'constant': return `Fills ${n} values with a value you set`; + case 'most_frequent': + return `Fills ${n} values with the most frequent category`; default: return ''; } @@ -2607,6 +2828,78 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { this.emitProgressState(); } + transformStrategyFor(variable: VariableRow): TransformStrategy | null { + return this.pendingTransformStrategies[variable.code] ?? null; + } + + onTransformStrategyChange(variable: VariableRow, strategy: TransformStrategy | null): void { + this.pendingTransformStrategies = { + ...this.pendingTransformStrategies, + [variable.code]: strategy, + }; + this.updatePreprocessingStatus(); + this.emitProgressState(); + } + + scalerStrategyFor(variable: VariableRow): ScalerStrategy | null { + return this.pendingScalerStrategies[variable.code] ?? null; + } + + onScalerStrategyChange(variable: VariableRow, strategy: ScalerStrategy | null): void { + this.pendingScalerStrategies = { + ...this.pendingScalerStrategies, + [variable.code]: strategy, + }; + this.updatePreprocessingStatus(); + this.emitProgressState(); + } + + onScalerFeatureRangeChange(which: 'low' | 'high', value: string): void { + this.pendingScalerFeatureRange = which === 'low' + ? [value, this.pendingScalerFeatureRange[1]] + : [this.pendingScalerFeatureRange[0], value]; + this.updatePreprocessingStatus(); + this.emitProgressState(); + } + + get hasPendingTransformChanges(): boolean { + return this.serializeTransformScalerState('transform', false) !== this.serializeTransformScalerState('transform', true); + } + + get pendingTransformChangeCount(): number { + return this.hasPendingTransformChanges ? 1 : 0; + } + + get hasPendingScalerChanges(): boolean { + return this.serializeTransformScalerState('scaler', false) !== this.serializeTransformScalerState('scaler', true); + } + + get pendingScalerChangeCount(): number { + return this.hasPendingScalerChanges ? 1 : 0; + } + + /** Serialized transform/scaler state over the current numerical variables (pending or applied). */ + private serializeTransformScalerState(step: TransformOrScalerStep, applied: boolean): string { + const transformStrategies = applied ? this.appliedTransformStrategies : this.pendingTransformStrategies; + const scalerStrategies = applied ? this.appliedScalerStrategies : this.pendingScalerStrategies; + const range = applied ? this.appliedScalerFeatureRange : this.parseFeatureRange(this.pendingScalerFeatureRange); + const source = step === 'transform' ? transformStrategies : scalerStrategies; + const codes = this.outlierPreprocessingVariables.map((v) => v.code); + return JSON.stringify({ + strategies: Object.fromEntries(codes.sort().map((code) => [code, source[code] ?? null])), + ...(step === 'scaler' ? { featureRange: range } : {}), + }); + } + + private parseFeatureRange(range: [string, string]): [number, number] | null { + const low = Number(String(range[0] ?? '').trim()); + const high = Number(String(range[1] ?? '').trim()); + return range[0].trim() !== '' && range[1].trim() !== '' + && Number.isFinite(low) && Number.isFinite(high) && low < high + ? [low, high] + : null; + } + outlierStrategyDisplay(strategy: OutlierStrategy): string { return this.outlierStrategies.find((item) => item.value === strategy)?.label ?? strategy; } @@ -2805,10 +3098,10 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { this.emitProgressState(); } - batchMenuOpen = signal<'missing' | 'outlier' | null>(null); + batchMenuOpen = signal<'missing' | 'outlier' | 'transform' | 'scaler' | null>(null); summaryExportMenuKind = signal(null); - toggleBatchMenu(kind: 'missing' | 'outlier'): void { + toggleBatchMenu(kind: 'missing' | 'outlier' | 'transform' | 'scaler'): void { this.batchMenuOpen.update((open) => (open === kind ? null : kind)); this.cdr.markForCheck(); } @@ -2846,6 +3139,19 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { this.cdr.markForCheck(); } + /** Batch-set one transform/scaler strategy for every numerical variable. */ + applyBatchTransformStrategy(strategy: TransformStrategy): void { + this.outlierPreprocessingVariables.forEach((variable) => this.onTransformStrategyChange(variable, strategy)); + this.batchMenuOpen.set(null); + this.cdr.markForCheck(); + } + + applyBatchScalerStrategy(strategy: ScalerStrategy): void { + this.outlierPreprocessingVariables.forEach((variable) => this.onScalerStrategyChange(variable, strategy)); + this.batchMenuOpen.set(null); + this.cdr.markForCheck(); + } + toggleAllOutliers(enabled: boolean): void { this.outlierPreprocessingVariables.forEach((variable) => this.onOutlierEnabledChange(variable, enabled)); this.batchMenuOpen.set(null); @@ -2865,7 +3171,11 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { } isMissingActionDisabled(variable: VariableRow, action: MissingAction): boolean { - return (action === 'mean' || action === 'median') && !this.isNumericVariable(variable); + if (action === 'mean' || action === 'median') return !this.isNumericVariable(variable); + // Exaflow's most_frequent fills with each worker's local mode (no category list is + // sent), so it only makes sense on categorical variables. + if (action === 'most_frequent') return !this.isCategoricalVariable(variable); + return false; } isLongitudinalStrategyDisabled(variable: VariableRow, strategy: LongitudinalStrategy): boolean { @@ -3866,6 +4176,10 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { ? this.pendingOutlierRules : this.mergeOutlierRulesForCurrentSelection({}, this.appliedOutlierRules)); if (outlier) config['outlier_winsorizer'] = outlier; + const transform = this.buildTransformPreprocessingConfig(step === 'transform' ? false : true); + if (transform) config['variable_transformer'] = transform; + const scaler = this.buildScalerPreprocessingConfig(step === 'scaler' ? false : true); + if (scaler) config['variable_scaler'] = scaler; const longitudinal = this.buildLongitudinalPreprocessingConfig(step !== 'longitudinal'); if (longitudinal) config['longitudinal_transformer'] = longitudinal; return Object.keys(config).length ? config : null; @@ -3879,13 +4193,20 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { const missingValuesHandler = this.buildMissingPreprocessingConfig(rules, allowedCodes); if (missingValuesHandler) config['missing_values_handler'] = missingValuesHandler; + const applied = rules === this.appliedPreprocessingRules; const outlier = this.buildOutlierPreprocessingConfig( - rules === this.appliedPreprocessingRules ? this.appliedOutlierRules : this.pendingOutlierRules, + applied ? this.appliedOutlierRules : this.pendingOutlierRules, allowedCodes ); if (outlier) config['outlier_winsorizer'] = outlier; - const longitudinal = this.buildLongitudinalPreprocessingConfig(rules === this.appliedPreprocessingRules, allowedCodes); + // Request order mirrors the rail: impute, clip, transform, scale. + const transform = this.buildTransformPreprocessingConfig(applied, allowedCodes); + if (transform) config['variable_transformer'] = transform; + const scaler = this.buildScalerPreprocessingConfig(applied, allowedCodes); + if (scaler) config['variable_scaler'] = scaler; + + const longitudinal = this.buildLongitudinalPreprocessingConfig(applied, allowedCodes); if (longitudinal) config['longitudinal_transformer'] = longitudinal; return Object.keys(config).length ? config : null; @@ -4238,6 +4559,33 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { return serialized ? { ...serialized } : null; } + /** Request parameters for the transform/scaler steps; null leaves the step out of the request. */ + private buildTransformPreprocessingConfig( + applied: boolean, + allowedCodes = this.currentPreprocessingCodeSet() + ): Record | null { + const source = applied ? this.appliedTransformStrategies : this.cleanStrategyMap(this.pendingTransformStrategies); + const strategies = Object.fromEntries( + Object.entries(source).filter(([code]) => allowedCodes.has(code)) + ); + return Object.keys(strategies).length ? { strategies } : null; + } + + private buildScalerPreprocessingConfig( + applied: boolean, + allowedCodes = this.currentPreprocessingCodeSet() + ): Record | null { + const source = applied ? this.appliedScalerStrategies : this.cleanStrategyMap(this.pendingScalerStrategies); + const strategies = Object.fromEntries( + Object.entries(source).filter(([code]) => allowedCodes.has(code)) + ); + if (!Object.keys(strategies).length) return null; + // `feature_range` is a minmax parameter; keep the request shape flat and permissive. + const range = applied ? this.appliedScalerFeatureRange : this.parseFeatureRange(this.pendingScalerFeatureRange); + const usesMinMax = Object.values(strategies).includes('minmax'); + return range && usesMinMax ? { strategies, feature_range: range } : { strategies }; + } + private validatePendingRulesFor(scope: PrepStep | 'all'): Record { return Object.assign({}, ...this.stepsIn(scope).map((step) => this.prepStepOps[step].validate())); } @@ -4291,6 +4639,9 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { if ((rule.action === 'mean' || rule.action === 'median') && !this.isNumericVariable(variable)) { errors[rule.variableCode] = 'Mean and median imputation are only available for numeric variables.'; } + if (rule.action === 'most_frequent' && !this.isCategoricalVariable(variable)) { + errors[rule.variableCode] = 'Most frequent imputation is only available for nominal variables.'; + } if (rule.action === 'constant') { const value = String(rule.value ?? '').trim(); if (!value) { @@ -4463,6 +4814,34 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { this.pendingLongitudinalStrategies = { ...appliedStrategies }; } + // Transform/scaler restore: the stored step names are the proposal from the federated + // preprocessing spec; unknown shapes hydrate to nothing so the stage stays usable. + const hydrateStrategies = ( + step: 'variable_transformer' | 'variable_scaler', + isStrategy: (value: string) => value is S + ): Record => { + const holder = preprocessing[step] as { strategies?: Record } | undefined; + const strategies: Record = {}; + Object.entries(holder?.strategies ?? {}).forEach(([code, strategy]) => { + if (isStrategy(String(strategy))) strategies[code] = String(strategy) as S; + }); + return strategies; + }; + this.appliedTransformStrategies = hydrateStrategies('variable_transformer', this.isTransformStrategy); + this.pendingTransformStrategies = { ...this.appliedTransformStrategies }; + this.appliedScalerStrategies = hydrateStrategies('variable_scaler', this.isScalerStrategy); + this.pendingScalerStrategies = { ...this.appliedScalerStrategies }; + const scalerHolder = preprocessing['variable_scaler'] as { feature_range?: unknown } | undefined; + const rawRange = Array.isArray(scalerHolder?.feature_range) ? scalerHolder!.feature_range as unknown[] : []; + const low = Number(rawRange[0]); + const high = Number(rawRange[1]); + this.appliedScalerFeatureRange = rawRange.length === 2 && Number.isFinite(low) && Number.isFinite(high) + ? [low, high] + : null; + this.pendingScalerFeatureRange = this.appliedScalerFeatureRange + ? [String(this.appliedScalerFeatureRange[0]), String(this.appliedScalerFeatureRange[1])] + : ['', '']; + this.hydrateAppliedTransformation(); } @@ -4504,7 +4883,15 @@ export class StatisticAnalysisPanelComponent implements OnDestroy { } private isMissingAction(value: string): value is MissingAction { - return ['drop', 'mean', 'median', 'constant'].includes(value); + return ['drop', 'mean', 'median', 'constant', 'most_frequent'].includes(value); + } + + private isTransformStrategy(value: string): value is TransformStrategy { + return ['log1p', 'sqrt', 'asinh', 'yeo_johnson', 'quantile_uniform', 'quantile_normal'].includes(value); + } + + private isScalerStrategy(value: string): value is ScalerStrategy { + return ['standard', 'robust', 'minmax', 'maxabs'].includes(value); } private isLongitudinalStrategy(value: string): value is LongitudinalStrategy { diff --git a/src/app/pages/experiment-studio/statistic-analysis-panel/statistic-analysis-panel.pipeline.spec.ts b/src/app/pages/experiment-studio/statistic-analysis-panel/statistic-analysis-panel.pipeline.spec.ts index d277390cb..e6b5366ab 100644 --- a/src/app/pages/experiment-studio/statistic-analysis-panel/statistic-analysis-panel.pipeline.spec.ts +++ b/src/app/pages/experiment-studio/statistic-analysis-panel/statistic-analysis-panel.pipeline.spec.ts @@ -114,7 +114,7 @@ describe('StatisticAnalysisPanelComponent pipeline presence', () => { 'kmeans_cluster_creator', ]); mockExpService.hasPreprocessingSpec.mockImplementation((name: string) => - ['categorical_column_creator', 'kmeans_cluster_creator'].includes(name), + ['categorical_column_creator', 'kmeans_cluster_creator', 'variable_transformer', 'variable_scaler'].includes(name), ); mockExpService.catalogLoaded.mockReturnValue(true); mockExpService.transformationDependents.mockReturnValue([]); @@ -338,10 +338,11 @@ describe('StatisticAnalysisPanelComponent pipeline presence', () => { expect(pipelineCard('analysis-preprocessing').querySelector('.pipeline-node-body')).toBeNull(); const rows = railRows(pipelineCard('analysis-preprocessing')); - expect(rows.ghost.map((row) => row.textContent?.trim())).toEqual([expect.stringMatching('Add outlier clipping')]); + const outlierGhost = rows.ghost.find((row) => row.textContent?.includes('Add outlier clipping')); + expect(outlierGhost).toBeTruthy(); mockExpService.setAppliedDescriptivePreprocessing.mockClear(); - (rows.ghost[0].querySelector('button') as HTMLButtonElement).click(); + (outlierGhost!.querySelector('button') as HTMLButtonElement).click(); fixture.detectChanges(); expect(component.sectionOpen().setup).toBe(true); @@ -350,7 +351,7 @@ describe('StatisticAnalysisPanelComponent pipeline presence', () => { expect(component.preprocessingStepOpen.missing).toBe(false); expect(pipelineCard('analysis-preprocessing').querySelector('.pipeline-node-body')).toBeTruthy(); expect(mockExpService.setAppliedDescriptivePreprocessing).not.toHaveBeenCalled(); - expect(railRows(pipelineCard('analysis-preprocessing')).ghost).toEqual([]); + expect(railRows(pipelineCard('analysis-preprocessing')).ghost.some((row) => row.textContent?.includes('Add outlier clipping'))).toBe(false); }); /** @@ -363,7 +364,8 @@ describe('StatisticAnalysisPanelComponent pipeline presence', () => { fixture.detectChanges(); const rail = railRows(pipelineCard('analysis-preprocessing')); - (rail.ghost[0].querySelector('button') as HTMLButtonElement).click(); + const outlierGhost = rail.ghost.find((row) => row.textContent?.includes('Add outlier clipping'))!; + (outlierGhost.querySelector('button') as HTMLButtonElement).click(); fixture.detectChanges(); expect(component.pendingChangeCount).toBe(0); @@ -381,9 +383,10 @@ describe('StatisticAnalysisPanelComponent pipeline presence', () => { fixture.detectChanges(); const rows = railRows(pipelineCard('analysis-preprocessing')); expect(rows.live.length).toBe(1); - expect(rows.ghost.map((row) => row.textContent?.trim())).toEqual([expect.stringMatching('Add outlier clipping')]); + expect(rows.ghost.some((row) => row.textContent?.includes('Add outlier clipping'))).toBe(true); - (rows.ghost[0].querySelector('button') as HTMLButtonElement).click(); + const outlierRow = rows.ghost.find((row) => row.textContent?.includes('Add outlier clipping'))!; + (outlierRow.querySelector('button') as HTMLButtonElement).click(); fixture.detectChanges(); expect(component.sectionOpen().setup).toBe(true); expect(component.preprocessingStepOpen.outlier).toBe(true); @@ -442,6 +445,69 @@ describe('StatisticAnalysisPanelComponent pipeline presence', () => { expect(component.preprocessingStatusLabel).toBe('2 steps applied ✓'); }); + it('lists the transform and scaler sub-nodes between outlier and longitudinal', () => { + (mockExpService.selectedVariables as any).set([age]); + seedAppliedConfig({ + missing_values_handler: { strategies: { age: 'drop' } }, + variable_transformer: { strategies: { age: 'log1p' } }, + variable_scaler: { strategies: { age: 'standard' } }, + }); + fixture.detectChanges(); + + const nodes = component.appliedPreprocessingSubNodes; + expect(nodes.map((n) => n.id)).toEqual(['missing', 'transform', 'scaler']); + expect(nodes[1].subtitle).toContain('log(1 + x)'); + expect(nodes[2].subtitle).toContain('Standard'); + }); + + it('offers transform and scaler ghost rows while they hold no strategies', () => { + (mockExpService.selectedVariables as any).set([age]); + fixture.detectChanges(); + + const card = pipelineCard('analysis-preprocessing'); + const { ghost } = railRows(card); + const texts = ghost.map((row) => row.textContent ?? ''); + expect(texts.some((t) => t.includes('Add variable transforms'))).toBe(true); + expect(texts.some((t) => t.includes('Add variable scaling'))).toBe(true); + }); + + it('restores the transform and scaler editors from a persisted config', () => { + (mockExpService.selectedVariables as any).set([age]); + seedAppliedConfig({ + variable_transformer: { strategies: { age: 'yeo_johnson' } }, + variable_scaler: { strategies: { age: 'minmax' }, feature_range: [-1, 1] }, + }); + fixture.detectChanges(); + + expect(component.isStepAdded('setup')).toBe(true); + expect(component.appliedTransformStrategies).toEqual({ age: 'yeo_johnson' }); + expect(component.pendingTransformStrategies).toEqual({ age: 'yeo_johnson' }); + expect(component.appliedScalerStrategies).toEqual({ age: 'minmax' }); + expect(component.appliedScalerFeatureRange).toEqual([-1, 1]); + expect(component.pendingScalerFeatureRange).toEqual(['-1', '1']); + }); + + it('hides the transform and scaler stations when the catalog lacks them', () => { + (mockExpService.selectedVariables as any).set([age]); + seedAppliedConfig({ + variable_transformer: { strategies: { age: 'log1p' } }, + variable_scaler: { strategies: { age: 'standard' } }, + }); + mockExpService.hasPreprocessingSpec.mockImplementation((name: string) => + ['categorical_column_creator', 'kmeans_cluster_creator'].includes(name)); + fixture.detectChanges(); + + // No rail node and no ghost row: the deployment cannot run the step, so the + // UI must not offer it even though a persisted config still carries it. + expect(component.appliedPreprocessingSubNodes.map((n) => n.id)).toEqual(['missing']); + const card = pipelineCard('analysis-preprocessing'); + const { ghost } = railRows(card); + expect(ghost.some((row) => row.textContent?.includes('Add variable transforms'))).toBe(false); + expect(ghost.some((row) => row.textContent?.includes('Add variable scaling'))).toBe(false); + expect(card.textContent).not.toContain('Variable Transforms'); + expect(card.textContent).not.toContain('Variable Scaling'); + }); + it('surfaces the applied derived column as a transformation sub-node', () => { seedAppliedConfig({ categorical_column_creator: [{ diff --git a/src/app/services/experiment-studio.service.spec.ts b/src/app/services/experiment-studio.service.spec.ts index eccee6d1f..5ad3d08e7 100644 --- a/src/app/services/experiment-studio.service.spec.ts +++ b/src/app/services/experiment-studio.service.spec.ts @@ -199,6 +199,16 @@ describe('ExperimentStudioService', () => { released: true, }; + const mockAlgorithmSpecs = [ + toAlgorithmSpec(mockRawAlgo), + toAlgorithmSpec(mockHistogramAlgo), + toAlgorithmSpec(mockDescribeAlgo), + toAlgorithmSpec(mockOutlierReportAlgo), + toAlgorithmSpec(mockLinearSvmAlgo), + toAlgorithmSpec(mockKmeansAlgo), + toAlgorithmSpec(mockPcaAlgo), + ]; + beforeEach(() => { sessionStorage.clear(); TestBed.configureTestingModule({ @@ -210,15 +220,7 @@ describe('ExperimentStudioService', () => { httpMock = TestBed.inject(HttpTestingController); flushSpecifications( - [ - toAlgorithmSpec(mockRawAlgo), - toAlgorithmSpec(mockHistogramAlgo), - toAlgorithmSpec(mockDescribeAlgo), - toAlgorithmSpec(mockOutlierReportAlgo), - toAlgorithmSpec(mockLinearSvmAlgo), - toAlgorithmSpec(mockKmeansAlgo), - toAlgorithmSpec(mockPcaAlgo), - ], + mockAlgorithmSpecs, [ { name: 'categorical_column_creator', @@ -281,6 +283,22 @@ describe('ExperimentStudioService', () => { ], }, }, + // The proposal names from the federated preprocessing spec; the gating test + // covers the case where the catalog ships without them. + { + name: 'variable_transformer', + label: 'Variable Transformer', + desc: '', + parameters: {}, + outputs: null, + }, + { + name: 'variable_scaler', + label: 'Variable Scaler', + desc: '', + parameters: {}, + outputs: null, + }, ], ); }); @@ -921,6 +939,104 @@ describe('ExperimentStudioService', () => { ); }); + it('summarizes most frequent imputation in plain language', () => { + const summary = service.formatPreprocessingConfig({ + missing_values_handler: { + strategies: { gender: 'most_frequent', age: 'mean' }, + }, + }, { gender: 'Gender', age: 'Age' }); + + expect(summary).toBe('Missing values: Gender: most frequent imputation, Age: mean imputation'); + }); + + it('summarizes transform and scaler steps with variable labels', () => { + const summary = service.formatPreprocessingConfig({ + missing_values_handler: { strategies: { age: 'drop' } }, + variable_transformer: { strategies: { age: 'log1p', bmi: 'quantile_uniform' } }, + variable_scaler: { strategies: { age: 'standard', bmi: 'minmax' }, feature_range: [0, 1] }, + }, { age: 'Age', bmi: 'BMI' }); + + expect(summary).toBe( + 'Missing values: Age: remove rows\n' + + 'Variable transforms: Age: log(1 + x), BMI: quantile to uniform\n' + + 'Variable scaling: Age: standard scaling, BMI: min-max scaling [0, 1]' + ); + }); + + it('drops transform and scaler steps the loaded catalog does not list', () => { + service.selectedDataModel.set(mockDataModel); + service.setSelectedDatasets(['ds1']); + const applied = { + missing_values_handler: { strategies: { age: 'drop' } }, + variable_transformer: { strategies: { age: 'log1p' } }, + variable_scaler: { strategies: { age: 'standard' } }, + }; + + // Simulate a deployment whose catalog ships without the new steps: reload the + // specifications with a reduced preprocessing catalog. + service.loadBackendAlgorithms().subscribe(); + httpMock.expectOne('/services/specifications/inputdata').flush(mockInputdataSpec); + httpMock.expectOne('/services/specifications/preprocessing').flush([]); + httpMock.expectOne('/services/specifications/algorithms').flush(mockAlgorithmSpecs); + + service.setAppliedDescriptivePreprocessing(applied); + const body = service.buildRequestBody('mock_algo', ['age']); + + expect(body.analysis.preprocessing).toEqual(preprocessingSteps({ + missing_values_handler: { strategies: { age: 'drop' } }, + })); + }); + + it('drops the scaler step when filtering leaves no strategy, not just feature_range', () => { + service.selectedDataModel.set(mockDataModel); + service.setSelectedDatasets(['ds1']); + const applied = { + variable_scaler: { strategies: { bmi: 'standard' }, feature_range: [0, 1] }, + }; + + const body = service.buildRequestBody('histogram', ['age'], null, null, null, null, applied); + + // The lone step-level `feature_range` must not keep the step in the request; the + // histogram fallback then supplies its usual drop default. + expect(body.analysis.preprocessing).toEqual(preprocessingSteps({ + missing_values_handler: { strategies: { age: 'drop' } }, + })); + }); + + it('emits applied transform and scaler singleton steps in rail order', () => { + service.selectedDataModel.set(mockDataModel); + service.setSelectedDatasets(['ds1']); + const applied = { + missing_values_handler: { strategies: { age: 'drop' } }, + outlier_winsorizer: { strategies: { age: 'iqr' }, tails: { age: 'both' }, folds: { age: 1.5 } }, + variable_transformer: { strategies: { age: 'yeo_johnson' } }, + variable_scaler: { strategies: { age: 'standard' } }, + }; + + service.setAppliedDescriptivePreprocessing(applied); + const body = service.buildRequestBody('mock_algo', ['age']); + + expect(body.analysis.preprocessing).toEqual(preprocessingSteps(applied)); + }); + + it('filters transform and scaler strategies to the histogram y variables', () => { + service.selectedDataModel.set(mockDataModel); + service.setSelectedDatasets(['ds1']); + const applied = { + variable_transformer: { strategies: { age: 'log1p', bmi: 'sqrt' } }, + variable_scaler: { strategies: { age: 'standard', bmi: 'maxabs' }, feature_range: [0, 1] }, + }; + + const body = service.buildRequestBody('histogram', ['age'], null, null, null, null, applied); + + // An override without a missing-values step carries no drop default: the filtered + // config is forwarded as stored, matching the outlier-only override behavior. + expect(body.analysis.preprocessing).toEqual(preprocessingSteps({ + variable_transformer: { strategies: { age: 'log1p' } }, + variable_scaler: { strategies: { age: 'standard' }, feature_range: [0, 1] }, + })); + }); + it('loads outlier report previews with outlier parameters and upstream missing preprocessing', () => new Promise((done) => { service.selectedDataModel.set(mockDataModel); service.setVariables([ diff --git a/src/app/services/experiment-studio.service.ts b/src/app/services/experiment-studio.service.ts index 82dded505..c47375b1a 100644 --- a/src/app/services/experiment-studio.service.ts +++ b/src/app/services/experiment-studio.service.ts @@ -63,6 +63,13 @@ const APPLIED_DESCRIPTIVE_PREPROCESSING = '__applied_descriptive_preprocessing__ /** Session-storage key for the ordered column-creating transformation steps. */ const APPLIED_TRANSFORMATIONS = '__applied_transformations__'; +/** + * Singleton steps gated on the preprocessing catalog: once the catalog has loaded without + * them, the deployment cannot run them, so they are hidden in the UI and dropped from every + * request (exaflow rejects unknown steps with `Transformer '' does not exist.`). + */ +const GATED_SINGLETON_STEPS = new Set(['variable_transformer', 'variable_scaler']); + /** Quick-preview / diagnostic algorithms hidden from the experiment algorithm picker. */ const ALGORITHM_PANEL_EXCLUDED = new Set([ AlgorithmNames.HISTOGRAM, @@ -996,6 +1003,14 @@ export class ExperimentStudioService { ); } + /** + * A gated singleton step is runnable only once the catalog loaded and still lists it — + * before that, empty specs mean "unknown", matching how creators are gated. + */ + private singletonStepRunnable(name: string): boolean { + return !this.preprocessingCatalogLoaded() || !!this.preprocessingSpecs()[name]; + } + private preprocessingConfigToSteps( config: PreprocessingConfig | null | undefined, ): AnalysisPreprocessingStep[] | null { @@ -1004,6 +1019,9 @@ export class ExperimentStudioService { // one ordered step per entry; the engine accepts repeated steps with the same name. const steps: AnalysisPreprocessingStep[] = []; for (const [name, parameters] of Object.entries(config)) { + // A gated step the deployment cannot run is dropped from every request, + // the same gate the column-creating creators go through. + if (GATED_SINGLETON_STEPS.has(name) && !this.singletonStepRunnable(name)) continue; if (Array.isArray(parameters)) { for (const item of parameters) { if (item && typeof item === 'object') { @@ -1417,6 +1435,30 @@ export class ExperimentStudioService { if (next) filtered[OUTLIER_WINSORIZER] = next; } + const transform = preprocessing['variable_transformer']; + if (transform && typeof transform === 'object' && !Array.isArray(transform)) { + const next = this.filterPreprocessingStepRecordMaps( + transform as Record, + allowed, + ['strategies'] + ); + if (next) filtered['variable_transformer'] = next; + } + + const scaler = preprocessing['variable_scaler']; + if (scaler && typeof scaler === 'object' && !Array.isArray(scaler)) { + const next = this.filterPreprocessingStepRecordMaps( + scaler as Record, + allowed, + ['strategies'] + ); + // A step whose strategies were all filtered out is dropped entirely: a lone + // step-level parameter such as `feature_range` must not keep it in the request. + if (next && Object.keys((next['strategies'] as Record) ?? {}).length > 0) { + filtered['variable_scaler'] = next; + } + } + const longitudinal = preprocessing['longitudinal_transformer']; if (longitudinal && typeof longitudinal === 'object' && !Array.isArray(longitudinal)) { const longitudinalRecord = longitudinal as Record; @@ -1548,6 +1590,34 @@ export class ExperimentStudioService { entries.push({ label: 'Outlier winsorizer', value: labels.join('; '), lines: labels }); } + const transform = preprocessing['variable_transformer'] as { + strategies?: Record; + } | undefined; + const transformEntries = Object.entries(transform?.strategies ?? {}); + if (transformEntries.length) { + const labels = transformEntries.map(([code, strategy]) => + `${this.preprocessingVariableLabel(code, labelMap)}: ${this.transformStrategyLabel(String(strategy))}` + ); + entries.push({ label: 'Variable transforms', value: labels.join(', '), lines: labels }); + } + + const scaler = preprocessing['variable_scaler'] as { + strategies?: Record; + feature_range?: unknown; + } | undefined; + const scalerEntries = Object.entries(scaler?.strategies ?? {}); + if (scalerEntries.length) { + // `feature_range` belongs to minmax only, so it rides on that strategy's lines. + const rawRange = Array.isArray(scaler?.feature_range) && scaler!.feature_range.length === 2 + ? `[${scaler!.feature_range.join(', ')}]` + : ''; + const labels = scalerEntries.map(([code, strategy]) => { + const rangeText = String(strategy) === 'minmax' && rawRange ? ` ${rawRange}` : ''; + return `${this.preprocessingVariableLabel(code, labelMap)}: ${this.scalerStrategyLabel(String(strategy))}${rangeText}`; + }); + entries.push({ label: 'Variable scaling', value: labels.join(', '), lines: labels }); + } + const longitudinal = preprocessing['longitudinal_transformer'] as Record | undefined; if (longitudinal) { const visit1 = longitudinal['visit1']; @@ -1565,7 +1635,13 @@ export class ExperimentStudioService { entries.push({ label: 'Longitudinal transformation', value: `${strategySummary}${visitLabel}` }); } - const knownKeys = new Set([MISSING_VALUES_HANDLER, OUTLIER_WINSORIZER, 'longitudinal_transformer']); + const knownKeys = new Set([ + MISSING_VALUES_HANDLER, + OUTLIER_WINSORIZER, + 'longitudinal_transformer', + 'variable_transformer', + 'variable_scaler', + ]); Object.keys(preprocessing) .filter((key) => !knownKeys.has(key)) .forEach((key) => { @@ -1634,6 +1710,40 @@ export class ExperimentStudioService { return labelMap[code] ?? code; } + transformStrategyLabel(strategy: string): string { + switch (strategy) { + case 'log1p': + return 'log(1 + x)'; + case 'sqrt': + return 'square root'; + case 'asinh': + return 'inverse hyperbolic sine'; + case 'yeo_johnson': + return 'Yeo-Johnson'; + case 'quantile_uniform': + return 'quantile to uniform'; + case 'quantile_normal': + return 'quantile to normal'; + default: + return strategy.replace(/_/g, ' '); + } + } + + scalerStrategyLabel(strategy: string): string { + switch (strategy) { + case 'standard': + return 'standard scaling'; + case 'robust': + return 'robust scaling'; + case 'minmax': + return 'min-max scaling'; + case 'maxabs': + return 'max absolute scaling'; + default: + return strategy.replace(/_/g, ' '); + } + } + private humanizePreprocessingValue(value: string): string { switch (value) { case 'drop': @@ -1642,6 +1752,8 @@ export class ExperimentStudioService { return 'mean imputation'; case 'median': return 'median imputation'; + case 'most_frequent': + return 'most frequent imputation'; case 'constant': return 'constant value'; default: