From f3c99b5f20417efa3ee654872188366a40312c78 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Tue, 24 Feb 2026 23:56:12 +0100 Subject: [PATCH 01/27] feat(purple_alien): align config_sweep with operational hyperparameters --- models/purple_alien/configs/config_sweep.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/models/purple_alien/configs/config_sweep.py b/models/purple_alien/configs/config_sweep.py index 401fc11f..ed034134 100644 --- a/models/purple_alien/configs/config_sweep.py +++ b/models/purple_alien/configs/config_sweep.py @@ -21,37 +21,36 @@ def get_sweep_config(): parameters_dict = { 'model' : {'value' :'HydraBNUNet06_LSTM4'}, - 'target_variable' : {'value': 'sb'}, # 'sb', 'ns', or 'os' for now - eval lib does not eval multiple targets yet!!!! 'weight_init' : {'value' : 'xavier_norm'}, # ['xavier_uni', 'xavier_norm', 'kaiming_uni', 'kaiming_normal'] 'clip_grad_norm' : {'value': True}, 'scheduler' : {'value': 'WarmupDecay'}, #CosineAnnealingLR004 'CosineAnnealingLR' 'OneCycleLR' 'total_hidden_channels': {'value': 32}, # you like need 32, it seems from qualitative results 'min_events': {'value': 5}, - 'samples': {'value': 600}, # 600 for run 10 for debug. should be a function of batches becaus batch 3 and sample 1000 = 3000.... + 'windows_per_lesson': {'value': 3}, + 'total_lessons': {'value': 150}, 'batch_size': {'value': 3}, # just speed running here.. "dropout_rate" : {'value' : 0.125}, 'learning_rate': {'value' : 0.001}, #0.001 default, but 0.005 might be better "weight_decay" : {'value' : 0.1}, "slope_ratio" : {'value' : 0.75}, "roof_ratio" : {'value' : 0.7}, + "max_ratio" : {'value' : 0.95}, + "min_ratio" : {'value' : 0.05}, 'input_channels' : {'value' : 3}, 'output_channels': {'value' : 1}, - 'targets' : {'value' : 6}, # 3 class and 3 reg for now. And for now this parameter is only used in utils, and changing it does not change the model - so don't. + 'classification_targets': {'value': ['by_sb_best', 'by_ns_best', 'by_os_best']}, + 'regression_targets': {'value': ['lr_sb_best', 'lr_ns_best', 'lr_os_best']}, 'loss_class' : { 'value' : 'b'}, # det nytter jo ikke noget at du køre over gamma og alpha for loss-class a... 'loss_class_gamma' : {'value' : 1.5}, 'loss_class_alpha' : {'value' : 0.75}, # should be between 0.5 and 0.95... 'loss_reg' : { 'value' : 'b'}, - 'loss_reg_a' : { 'value' : 256}, + 'loss_reg_a' : { 'value' : 258}, 'loss_reg_c' : { 'value' : 0.001}, - 'test_samples': { 'value' :128}, # 128 for actual testing, 10 for debug - 'np_seed' : {'values' : [4,8]}, - 'torch_seed' : {'values' : [4,8]}, + 'np_seed' : {'values' : [4, 8]}, + 'torch_seed' : {'values' : [4, 8]}, 'window_dim' : {'value' : 32}, 'h_init' : {'value' : 'abs_rand_exp-100'}, - 'un_log' : {'value' : False}, 'warmup_steps' : {'value' : 100}, - 'first_feature_idx' : {'value' : 5}, - 'norm_target' : {'value' : False}, 'freeze_h' : {'value' : "hl"}, 'time_steps' : {'value' : 36} } From 554f61a94b4a30b1350abd6ddc216b0206075bda Mon Sep 17 00:00:00 2001 From: Polichinl Date: Wed, 25 Feb 2026 00:03:30 +0100 Subject: [PATCH 02/27] fix(purple_alien): add missing 'name' key to sweep_config --- models/purple_alien/configs/config_sweep.py | 1 + 1 file changed, 1 insertion(+) diff --git a/models/purple_alien/configs/config_sweep.py b/models/purple_alien/configs/config_sweep.py index ed034134..2acdedaf 100644 --- a/models/purple_alien/configs/config_sweep.py +++ b/models/purple_alien/configs/config_sweep.py @@ -9,6 +9,7 @@ def get_sweep_config(): """ sweep_config = { + 'name': 'purple_alien_sweep', 'method': 'grid' } From 32e92155dc0579d4b73d68eb18981439c7b3a807 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Sun, 15 Mar 2026 00:28:56 +0100 Subject: [PATCH 03/27] =?UTF-8?q?fix(baselines):=20rename=20config=20key?= =?UTF-8?q?=20months=20=E2=86=92=20window=5Fmonths,=20add=20time=5Fsteps?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Companion change to views-baseline PR #3. Standardizes the config key for trailing training window from "months" to "window_months" and adds the explicit "time_steps" key (forecast horizon, previously a hidden default of 36 in views-baseline). Co-Authored-By: Claude Opus 4.6 (1M context) --- .../average_cmbaseline/configs/config_hyperparameters.py | 4 ++-- models/average_cmbaseline/configs/config_sweep.py | 8 ++++---- .../average_pgmbaseline/configs/config_hyperparameters.py | 4 ++-- models/average_pgmbaseline/configs/config_sweep.py | 8 ++++---- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/models/average_cmbaseline/configs/config_hyperparameters.py b/models/average_cmbaseline/configs/config_hyperparameters.py index 4ac09cbc..e60c819d 100644 --- a/models/average_cmbaseline/configs/config_hyperparameters.py +++ b/models/average_cmbaseline/configs/config_hyperparameters.py @@ -10,7 +10,7 @@ def get_hp_config(): hyperparameters = { 'steps': [*range(1, 36 + 1, 1)], - 'months':60, - # Add more hyperparameters as needed + 'time_steps': 36, + 'window_months': 60, } return hyperparameters diff --git a/models/average_cmbaseline/configs/config_sweep.py b/models/average_cmbaseline/configs/config_sweep.py index 2c76d425..db9e1bf3 100644 --- a/models/average_cmbaseline/configs/config_sweep.py +++ b/models/average_cmbaseline/configs/config_sweep.py @@ -10,19 +10,19 @@ def get_sweep_config(): sweep_config = { 'method': 'grid', - 'name': 'average_baseline' + 'name': 'average_cmbaseline' } - # Example metric setup: metric = { 'name': 'MSE', 'goal': 'minimize' } sweep_config['metric'] = metric - # Example parameters setup: parameters_dict = { - 'steps': {'values': [[*range(1, 36 + 1, 1)]]}, + 'steps': {'value': [*range(1, 36 + 1, 1)]}, + 'time_steps': {'value': 36}, + 'window_months': {'values': [6, 12, 18, 24, 36, 48, 60]}, } sweep_config['parameters'] = parameters_dict diff --git a/models/average_pgmbaseline/configs/config_hyperparameters.py b/models/average_pgmbaseline/configs/config_hyperparameters.py index af954a0e..ac7413b1 100644 --- a/models/average_pgmbaseline/configs/config_hyperparameters.py +++ b/models/average_pgmbaseline/configs/config_hyperparameters.py @@ -10,7 +10,7 @@ def get_hp_config(): hyperparameters = { 'steps': [*range(1, 36 + 1, 1)], - 'months':18, - # Add more hyperparameters as needed + 'time_steps': 36, + 'window_months': 18, } return hyperparameters diff --git a/models/average_pgmbaseline/configs/config_sweep.py b/models/average_pgmbaseline/configs/config_sweep.py index 57f0561f..b58a0efd 100644 --- a/models/average_pgmbaseline/configs/config_sweep.py +++ b/models/average_pgmbaseline/configs/config_sweep.py @@ -10,19 +10,19 @@ def get_sweep_config(): sweep_config = { 'method': 'grid', - 'name': 'averahe_pgmbaseline' + 'name': 'average_pgmbaseline' } - # Example metric setup: metric = { 'name': 'MSE', 'goal': 'minimize' } sweep_config['metric'] = metric - # Example parameters setup: parameters_dict = { - 'steps': {'values': [[*range(1, 36 + 1, 1)]]}, + 'steps': {'value': [*range(1, 36 + 1, 1)]}, + 'time_steps': {'value': 36}, + 'window_months': {'values': [6, 12, 18, 24, 36, 48, 60]}, } sweep_config['parameters'] = parameters_dict From 94aad1841ab410034e8c43c203cd6a3aa1990b63 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Sun, 15 Mar 2026 01:25:11 +0100 Subject: [PATCH 04/27] feat(configs): declare prediction_format in all model configs Add explicit `prediction_format: "dataframe"` to 54 models that previously relied on the implicit default. Prerequisite for making prediction_format a required key in pipeline-core (Phase 3A). Also includes baseline config updates and calibration log refreshes. Co-Authored-By: Claude Opus 4.6 --- models/adolecent_slob/configs/config_meta.py | 1 + .../average_cmbaseline/configs/config_meta.py | 12 +++---- models/average_cmbaseline/main.py | 4 ++- .../configs/config_meta.py | 12 +++---- models/average_pgmbaseline/main.py | 4 ++- models/bad_blood/configs/config_meta.py | 3 +- .../configs/config_meta.py | 3 +- models/blank_space/configs/config_meta.py | 3 +- models/bouncy_organ/configs/config_meta.py | 1 + models/brown_cheese/configs/config_meta.py | 3 +- models/car_radio/configs/config_meta.py | 3 +- models/caring_fish/configs/config_meta.py | 3 +- models/cheap_thrills/configs/config_meta.py | 1 + models/chunky_cat/configs/config_meta.py | 3 +- models/cool_cat/configs/config_meta.py | 1 + models/counting_stars/configs/config_meta.py | 3 +- models/dancing_queen/configs/config_meta.py | 1 + models/dark_paradise/configs/config_meta.py | 3 +- models/demon_days/configs/config_meta.py | 3 +- models/elastic_heart/configs/config_meta.py | 1 + .../configs/config_meta.py | 3 +- .../configs/config_meta.py | 1 + models/fancy_feline/configs/config_meta.py | 1 + .../data/generated/calibration_log.txt | 6 ++-- .../data/raw/calibration_data_fetch_log.txt | 2 +- models/fast_car/configs/config_meta.py | 3 +- .../configs/config_meta.py | 3 +- .../fourtieth_symphony/configs/config_meta.py | 1 + models/good_life/configs/config_meta.py | 1 + models/good_riddance/configs/config_meta.py | 3 +- models/green_squirrel/configs/config_meta.py | 3 +- models/heat_waves/configs/config_meta.py | 1 + models/heavy_rotation/configs/config_meta.py | 3 +- models/high_hopes/configs/config_meta.py | 3 +- models/hot_stream/configs/config_meta.py | 1 + .../invisible_string/configs/config_meta.py | 3 +- models/lavender_haze/configs/config_meta.py | 3 +- models/little_lies/configs/config_meta.py | 3 +- .../configs/config_hyperparameters.py | 2 +- models/locf_cmbaseline/configs/config_meta.py | 12 +++---- .../locf_cmbaseline/configs/config_sweep.py | 7 ++-- models/locf_cmbaseline/main.py | 4 ++- .../configs/config_hyperparameters.py | 2 +- .../locf_pgmbaseline/configs/config_meta.py | 12 +++---- .../locf_pgmbaseline/configs/config_sweep.py | 5 ++- models/locf_pgmbaseline/main.py | 4 ++- models/lovely_creature/configs/config_meta.py | 1 + models/midnight_rain/configs/config_meta.py | 3 +- models/national_anthem/configs/config_meta.py | 3 +- models/new_rules/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 2 ++ .../novel_heuristics/configs/config_meta.py | 3 +- .../data/generated/calibration_log.txt | 6 ++-- .../data/raw/calibration_data_fetch_log.txt | 2 +- models/old_money/configs/config_meta.py | 3 +- models/ominous_ox/configs/config_meta.py | 3 +- models/orange_pasta/configs/config_meta.py | 3 +- models/party_princess/configs/config_meta.py | 1 + models/plastic_beach/configs/config_meta.py | 3 +- models/popular_monster/configs/config_meta.py | 3 +- .../configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 23 ++++++------- models/purple_alien/configs/config_meta.py | 34 ++++++++++++++----- .../data/generated/calibration_log.txt | 6 ++-- .../data/raw/calibration_data_fetch_log.txt | 2 +- models/purple_haze/configs/config_meta.py | 1 + models/teen_spirit/configs/config_meta.py | 3 +- models/teenage_dirtbag/configs/config_meta.py | 1 + models/twin_flame/configs/config_meta.py | 3 +- models/wild_rose/configs/config_meta.py | 1 + models/wildest_dream/configs/config_meta.py | 3 +- .../wuthering_heights/configs/config_meta.py | 1 + models/yellow_pikachu/configs/config_meta.py | 3 +- .../yellow_submarine/configs/config_meta.py | 3 +- .../configs/config_hyperparameters.py | 2 +- models/zero_cmbaseline/configs/config_meta.py | 12 +++---- .../zero_cmbaseline/configs/config_sweep.py | 7 ++-- models/zero_cmbaseline/main.py | 4 ++- .../configs/config_hyperparameters.py | 2 +- .../zero_pgmbaseline/configs/config_meta.py | 12 +++---- .../zero_pgmbaseline/configs/config_sweep.py | 5 ++- models/zero_pgmbaseline/main.py | 4 ++- 82 files changed, 204 insertions(+), 126 deletions(-) diff --git a/models/adolecent_slob/configs/config_meta.py b/models/adolecent_slob/configs/config_meta.py index 40c5ada7..7f2c6acf 100644 --- a/models/adolecent_slob/configs/config_meta.py +++ b/models/adolecent_slob/configs/config_meta.py @@ -15,6 +15,7 @@ def get_meta_config(): # "queryset": "escwa001_cflong", "level": "cm", "creator": "Simon", + "prediction_format": "dataframe", "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], } return meta_config diff --git a/models/average_cmbaseline/configs/config_meta.py b/models/average_cmbaseline/configs/config_meta.py index 76644660..36e1edf8 100644 --- a/models/average_cmbaseline/configs/config_meta.py +++ b/models/average_cmbaseline/configs/config_meta.py @@ -6,15 +6,15 @@ def get_meta_config(): Returns: - meta_config (dict): A dictionary containing model meta configuration. """ - + meta_config = { - "name": "average_cmbaseline", + "name": "average_cmbaseline", "algorithm": "AverageModel", - # Uncomment and modify the following lines as needed for additional metadata: - "targets": ["lr_ged_sb"], - # "queryset": "escwa001_cflong", + "regression_targets": ["lr_ged_sb"], "level": "cm", "creator": "Sonja", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], + "prediction_format": "dataframe", + "rolling_origin_stride": 1, + "regression_point_metrics": ["MSE", "MSLE"], } return meta_config diff --git a/models/average_cmbaseline/main.py b/models/average_cmbaseline/main.py index 6cda3cb8..7770d7e9 100644 --- a/models/average_cmbaseline/main.py +++ b/models/average_cmbaseline/main.py @@ -18,7 +18,9 @@ validate_arguments(args) if args.sweep: - print("No Sweep Run for Baseline Models") + BaselineForecastingModelManager( + model_path=model_path, + ).execute_sweep_run(args) else: BaselineForecastingModelManager( model_path=model_path, diff --git a/models/average_pgmbaseline/configs/config_meta.py b/models/average_pgmbaseline/configs/config_meta.py index b09d46f5..b7c51db4 100644 --- a/models/average_pgmbaseline/configs/config_meta.py +++ b/models/average_pgmbaseline/configs/config_meta.py @@ -6,15 +6,15 @@ def get_meta_config(): Returns: - meta_config (dict): A dictionary containing model meta configuration. """ - + meta_config = { - "name": "average_pgmbaseline", + "name": "average_pgmbaseline", "algorithm": "AverageModel", - # Uncomment and modify the following lines as needed for additional metadata: - "targets": ["lr_ged_sb"], - # "queryset": "escwa001_cflong", + "regression_targets": ["lr_ged_sb"], "level": "pgm", "creator": "Sonja", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], + "prediction_format": "dataframe", + "rolling_origin_stride": 1, + "regression_point_metrics": ["MSE", "MSLE"], } return meta_config diff --git a/models/average_pgmbaseline/main.py b/models/average_pgmbaseline/main.py index 6cda3cb8..7770d7e9 100644 --- a/models/average_pgmbaseline/main.py +++ b/models/average_pgmbaseline/main.py @@ -18,7 +18,9 @@ validate_arguments(args) if args.sweep: - print("No Sweep Run for Baseline Models") + BaselineForecastingModelManager( + model_path=model_path, + ).execute_sweep_run(args) else: BaselineForecastingModelManager( model_path=model_path, diff --git a/models/bad_blood/configs/config_meta.py b/models/bad_blood/configs/config_meta.py index e26d7072..4680646d 100644 --- a/models/bad_blood/configs/config_meta.py +++ b/models/bad_blood/configs/config_meta.py @@ -14,6 +14,7 @@ def get_meta_config(): "targets": "lr_ged_sb", "queryset": "fatalities003_pgm_natsoc", "level": "pgm", - "creator": "Xiaolong" + "creator": "Xiaolong", + "prediction_format": "dataframe", } return meta_config diff --git a/models/bittersweet_symphony/configs/config_meta.py b/models/bittersweet_symphony/configs/config_meta.py index e949c680..fad0294f 100644 --- a/models/bittersweet_symphony/configs/config_meta.py +++ b/models/bittersweet_symphony/configs/config_meta.py @@ -14,6 +14,7 @@ def get_meta_config(): "targets": "lr_ged_sb", "queryset": " fatalities003_all_features", "level": "cm", - "creator": "Marina" + "creator": "Marina", + "prediction_format": "dataframe", } return meta_config diff --git a/models/blank_space/configs/config_meta.py b/models/blank_space/configs/config_meta.py index 24f71929..4cb0aae5 100644 --- a/models/blank_space/configs/config_meta.py +++ b/models/blank_space/configs/config_meta.py @@ -15,6 +15,7 @@ def get_meta_config(): "targets": "lr_ged_sb", "queryset": "fatalities003_pgm_natsoc", "level": "pgm", - "creator": "Xiaolong" + "creator": "Xiaolong", + "prediction_format": "dataframe", } return meta_config \ No newline at end of file diff --git a/models/bouncy_organ/configs/config_meta.py b/models/bouncy_organ/configs/config_meta.py index bc52c4fa..34ff8e65 100644 --- a/models/bouncy_organ/configs/config_meta.py +++ b/models/bouncy_organ/configs/config_meta.py @@ -15,6 +15,7 @@ def get_meta_config(): # "queryset": "escwa001_cflong", "level": "cm", "creator": "Simon", + "prediction_format": "dataframe", "regression_point_metrics": ["RMSLE","MSE", "MSLE", "y_hat_bar"], } return meta_config diff --git a/models/brown_cheese/configs/config_meta.py b/models/brown_cheese/configs/config_meta.py index d6844062..4165642e 100644 --- a/models/brown_cheese/configs/config_meta.py +++ b/models/brown_cheese/configs/config_meta.py @@ -14,6 +14,7 @@ def get_meta_config(): "targets": "lr_ged_sb", "queryset": "fatalities003_baseline", "level": "cm", - "creator": "Borbála" + "creator": "Borbála", + "prediction_format": "dataframe", } return meta_config diff --git a/models/car_radio/configs/config_meta.py b/models/car_radio/configs/config_meta.py index 9ce1980f..7c2182f9 100644 --- a/models/car_radio/configs/config_meta.py +++ b/models/car_radio/configs/config_meta.py @@ -14,6 +14,7 @@ def get_meta_config(): "targets": "lr_ged_sb", "queryset": "fatalities003_topics", "level": "cm", - "creator": "Borbála" + "creator": "Borbála", + "prediction_format": "dataframe", } return meta_config diff --git a/models/caring_fish/configs/config_meta.py b/models/caring_fish/configs/config_meta.py index 242025ce..14f1aeb8 100644 --- a/models/caring_fish/configs/config_meta.py +++ b/models/caring_fish/configs/config_meta.py @@ -14,6 +14,7 @@ def get_meta_config(): "targets": "lr_ged_sb", "queryset": "fatalities003_pgm_conflict_history", "level": "pgm", - "creator": "Xiaolong" + "creator": "Xiaolong", + "prediction_format": "dataframe", } return meta_config diff --git a/models/cheap_thrills/configs/config_meta.py b/models/cheap_thrills/configs/config_meta.py index c18d572f..8d5c5738 100644 --- a/models/cheap_thrills/configs/config_meta.py +++ b/models/cheap_thrills/configs/config_meta.py @@ -13,6 +13,7 @@ def get_meta_config(): "targets": "lr_sb_best", "level": "cm", "creator": "Håvard", + "prediction_format": "dataframe", "model_reg": "XGBRegressor", "model_clf": "XGBClassifier", "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], diff --git a/models/chunky_cat/configs/config_meta.py b/models/chunky_cat/configs/config_meta.py index 5b2a06a4..25252195 100644 --- a/models/chunky_cat/configs/config_meta.py +++ b/models/chunky_cat/configs/config_meta.py @@ -14,6 +14,7 @@ def get_meta_config(): "targets": "lr_ged_sb", "queryset": "fatalities003_pgm_conflictlong", "level": "pgm", - "creator": "Xiaolong" + "creator": "Xiaolong", + "prediction_format": "dataframe", } return meta_config diff --git a/models/cool_cat/configs/config_meta.py b/models/cool_cat/configs/config_meta.py index e1733cab..3448bc8b 100644 --- a/models/cool_cat/configs/config_meta.py +++ b/models/cool_cat/configs/config_meta.py @@ -15,6 +15,7 @@ def get_meta_config(): # "queryset": "escwa001_cflong", "level": "cm", "creator": "Dylan", + "prediction_format": "dataframe", "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], } return meta_config diff --git a/models/counting_stars/configs/config_meta.py b/models/counting_stars/configs/config_meta.py index ced385e0..2f4aa9e5 100644 --- a/models/counting_stars/configs/config_meta.py +++ b/models/counting_stars/configs/config_meta.py @@ -14,6 +14,7 @@ def get_meta_config(): "targets": "lr_ged_sb", "queryset": "fatalities003_conflict_history_long", "level": "cm", - "creator": "Borbála" + "creator": "Borbála", + "prediction_format": "dataframe", } return meta_config diff --git a/models/dancing_queen/configs/config_meta.py b/models/dancing_queen/configs/config_meta.py index 51b26fce..078ff483 100644 --- a/models/dancing_queen/configs/config_meta.py +++ b/models/dancing_queen/configs/config_meta.py @@ -15,6 +15,7 @@ def get_meta_config(): # "queryset": "escwa001_cflong", "level": "cm", "creator": "Dylan", + "prediction_format": "dataframe", "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], } return meta_config diff --git a/models/dark_paradise/configs/config_meta.py b/models/dark_paradise/configs/config_meta.py index c08887c8..96fc1ee9 100644 --- a/models/dark_paradise/configs/config_meta.py +++ b/models/dark_paradise/configs/config_meta.py @@ -16,6 +16,7 @@ def get_meta_config(): "targets": "lr_ged_sb", "queryset": "fatalities003_pgm_conflictlong", "level": "pgm", - "creator": "Xiaolong" + "creator": "Xiaolong", + "prediction_format": "dataframe", } return meta_config diff --git a/models/demon_days/configs/config_meta.py b/models/demon_days/configs/config_meta.py index 969d6eb4..7790b7bd 100644 --- a/models/demon_days/configs/config_meta.py +++ b/models/demon_days/configs/config_meta.py @@ -14,6 +14,7 @@ def get_meta_config(): "targets": "lr_ged_sb", "queryset": "fatalities003_faostat", "level": "cm", - "creator": "Marina" + "creator": "Marina", + "prediction_format": "dataframe", } return meta_config diff --git a/models/elastic_heart/configs/config_meta.py b/models/elastic_heart/configs/config_meta.py index 4c1e8559..06b4a7fd 100644 --- a/models/elastic_heart/configs/config_meta.py +++ b/models/elastic_heart/configs/config_meta.py @@ -15,6 +15,7 @@ def get_meta_config(): # "queryset": "escwa001_cflong", "level": "cm", "creator": "Dylan", + "prediction_format": "dataframe", "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], } return meta_config diff --git a/models/electric_relaxation/configs/config_meta.py b/models/electric_relaxation/configs/config_meta.py index 9afaabab..03ec566e 100644 --- a/models/electric_relaxation/configs/config_meta.py +++ b/models/electric_relaxation/configs/config_meta.py @@ -12,6 +12,7 @@ def get_meta_config(): "targets": "lr_ged_sb", "queryset": "escwa001_cflong", "level": "cm", - "creator": "Sara" + "creator": "Sara", + "prediction_format": "dataframe", } return model_config \ No newline at end of file diff --git a/models/emerging_principles/configs/config_meta.py b/models/emerging_principles/configs/config_meta.py index 6f4b09fc..53a4a41c 100644 --- a/models/emerging_principles/configs/config_meta.py +++ b/models/emerging_principles/configs/config_meta.py @@ -14,6 +14,7 @@ def get_meta_config(): # "queryset": "escwa001_cflong", "level": "cm", "creator": "Simon", + "prediction_format": "dataframe", "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], } return meta_config diff --git a/models/fancy_feline/configs/config_meta.py b/models/fancy_feline/configs/config_meta.py index e4ee6c10..8af7d794 100644 --- a/models/fancy_feline/configs/config_meta.py +++ b/models/fancy_feline/configs/config_meta.py @@ -15,6 +15,7 @@ def get_meta_config(): # "queryset": "escwa001_cflong", "level": "cm", "creator": "Simon", + "prediction_format": "dataframe", "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], } return meta_config diff --git a/models/fancy_feline/data/generated/calibration_log.txt b/models/fancy_feline/data/generated/calibration_log.txt index dd43d237..899ced4c 100644 --- a/models/fancy_feline/data/generated/calibration_log.txt +++ b/models/fancy_feline/data/generated/calibration_log.txt @@ -1,6 +1,6 @@ Single Model Name: fancy_feline -Single Model Timestamp: 20260224_134621 -Data Generation Timestamp: 20260224_135400 -Data Fetch Timestamp: 20260224_125948 +Single Model Timestamp: 20260224_200520 +Data Generation Timestamp: 20260224_201407 +Data Fetch Timestamp: 20260224_195334 Deployment Status: shadow diff --git a/models/fancy_feline/data/raw/calibration_data_fetch_log.txt b/models/fancy_feline/data/raw/calibration_data_fetch_log.txt index 936b3545..e08c4399 100644 --- a/models/fancy_feline/data/raw/calibration_data_fetch_log.txt +++ b/models/fancy_feline/data/raw/calibration_data_fetch_log.txt @@ -1,3 +1,3 @@ Single Model Name: fancy_feline -Data Fetch Timestamp: 20260224_141502 +Data Fetch Timestamp: 20260224_195334 diff --git a/models/fast_car/configs/config_meta.py b/models/fast_car/configs/config_meta.py index 2901652a..96ac3741 100644 --- a/models/fast_car/configs/config_meta.py +++ b/models/fast_car/configs/config_meta.py @@ -16,6 +16,7 @@ def get_meta_config(): "targets": "lr_ged_sb", "queryset": "fatalities003_vdem_short", "level": "cm", - "creator": "Borbála" + "creator": "Borbála", + "prediction_format": "dataframe", } return meta_config diff --git a/models/fluorescent_adolescent/configs/config_meta.py b/models/fluorescent_adolescent/configs/config_meta.py index 27b013d3..ae0f7b2d 100644 --- a/models/fluorescent_adolescent/configs/config_meta.py +++ b/models/fluorescent_adolescent/configs/config_meta.py @@ -16,6 +16,7 @@ def get_meta_config(): "targets": "lr_ged_sb", "queryset": "fatalities003_joint_narrow", "level": "cm", - "creator": "Marina" + "creator": "Marina", + "prediction_format": "dataframe", } return meta_config diff --git a/models/fourtieth_symphony/configs/config_meta.py b/models/fourtieth_symphony/configs/config_meta.py index 3f62bbaa..f96d6a4a 100644 --- a/models/fourtieth_symphony/configs/config_meta.py +++ b/models/fourtieth_symphony/configs/config_meta.py @@ -13,6 +13,7 @@ def get_meta_config(): "targets": ["lr_sb_best"], "level": "cm", "creator": "Håvard", + "prediction_format": "dataframe", "model_reg": "XGBRegressor", "model_clf": "XGBClassifier", "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], diff --git a/models/good_life/configs/config_meta.py b/models/good_life/configs/config_meta.py index dbb0fe86..b7b6cea0 100644 --- a/models/good_life/configs/config_meta.py +++ b/models/good_life/configs/config_meta.py @@ -15,6 +15,7 @@ def get_meta_config(): # "queryset": "escwa001_cflong", "level": "cm", "creator": "Dylan", + "prediction_format": "dataframe", "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], } return meta_config diff --git a/models/good_riddance/configs/config_meta.py b/models/good_riddance/configs/config_meta.py index caeab09d..ed04fd28 100644 --- a/models/good_riddance/configs/config_meta.py +++ b/models/good_riddance/configs/config_meta.py @@ -14,6 +14,7 @@ def get_meta_config(): "targets": "lr_ged_sb", "queryset": "fatalities003_joint_narrow", "level": "cm", - "creator": "Marina" + "creator": "Marina", + "prediction_format": "dataframe", } return meta_config diff --git a/models/green_squirrel/configs/config_meta.py b/models/green_squirrel/configs/config_meta.py index 7d0f1245..7b80e42e 100644 --- a/models/green_squirrel/configs/config_meta.py +++ b/models/green_squirrel/configs/config_meta.py @@ -16,6 +16,7 @@ def get_meta_config(): "targets": "lr_ged_sb", "queryset": "fatalities003_joint_broad", "level": "cm", - "creator": "Borbála" + "creator": "Borbála", + "prediction_format": "dataframe", } return meta_config diff --git a/models/heat_waves/configs/config_meta.py b/models/heat_waves/configs/config_meta.py index 34ff2e29..7e543259 100644 --- a/models/heat_waves/configs/config_meta.py +++ b/models/heat_waves/configs/config_meta.py @@ -15,6 +15,7 @@ def get_meta_config(): # "queryset": "escwa001_cflong", "level": "cm", "creator": "Dylan", + "prediction_format": "dataframe", "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], } return meta_config diff --git a/models/heavy_rotation/configs/config_meta.py b/models/heavy_rotation/configs/config_meta.py index 8ed5f759..dd4ec169 100644 --- a/models/heavy_rotation/configs/config_meta.py +++ b/models/heavy_rotation/configs/config_meta.py @@ -14,6 +14,7 @@ def get_meta_config(): "targets": "lr_ged_sb", "queryset": "fatalities003_joint_broad", "level": "cm", - "creator": "Borbála" + "creator": "Borbála", + "prediction_format": "dataframe", } return meta_config diff --git a/models/high_hopes/configs/config_meta.py b/models/high_hopes/configs/config_meta.py index 130cb31d..df369b98 100644 --- a/models/high_hopes/configs/config_meta.py +++ b/models/high_hopes/configs/config_meta.py @@ -16,6 +16,7 @@ def get_meta_config(): "targets": "lr_ged_sb", "queryset": "fatalities003_conflict_history", "level": "cm", - "creator": "Borbála" + "creator": "Borbála", + "prediction_format": "dataframe", } return meta_config diff --git a/models/hot_stream/configs/config_meta.py b/models/hot_stream/configs/config_meta.py index dab91a7c..c3389a26 100644 --- a/models/hot_stream/configs/config_meta.py +++ b/models/hot_stream/configs/config_meta.py @@ -15,6 +15,7 @@ def get_meta_config(): # "queryset": "escwa001_cflong", "level": "cm", "creator": "Simon", + "prediction_format": "dataframe", "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], } return meta_config diff --git a/models/invisible_string/configs/config_meta.py b/models/invisible_string/configs/config_meta.py index 395a8805..9ca59dc5 100644 --- a/models/invisible_string/configs/config_meta.py +++ b/models/invisible_string/configs/config_meta.py @@ -14,6 +14,7 @@ def get_meta_config(): "targets": "lr_ged_sb", "queryset": "fatalities003_pgm_broad", "level": "pgm", - "creator": "Xiaolong" + "creator": "Xiaolong", + "prediction_format": "dataframe", } return meta_config diff --git a/models/lavender_haze/configs/config_meta.py b/models/lavender_haze/configs/config_meta.py index 7998fcc2..b7f3760e 100644 --- a/models/lavender_haze/configs/config_meta.py +++ b/models/lavender_haze/configs/config_meta.py @@ -15,6 +15,7 @@ def get_meta_config(): "targets": "lr_ged_sb", "queryset": "fatalities003_pgm_broad", "level": "pgm", - "creator": "Xiaolong" + "creator": "Xiaolong", + "prediction_format": "dataframe", } return meta_config \ No newline at end of file diff --git a/models/little_lies/configs/config_meta.py b/models/little_lies/configs/config_meta.py index 2b0804a0..2abe73ef 100644 --- a/models/little_lies/configs/config_meta.py +++ b/models/little_lies/configs/config_meta.py @@ -16,6 +16,7 @@ def get_meta_config(): "targets": "lr_ged_sb", "queryset": "fatalities003_joint_narrow", "level": "cm", - "creator": "Marina" + "creator": "Marina", + "prediction_format": "dataframe", } return meta_config diff --git a/models/locf_cmbaseline/configs/config_hyperparameters.py b/models/locf_cmbaseline/configs/config_hyperparameters.py index c4f07553..0c146948 100644 --- a/models/locf_cmbaseline/configs/config_hyperparameters.py +++ b/models/locf_cmbaseline/configs/config_hyperparameters.py @@ -10,6 +10,6 @@ def get_hp_config(): hyperparameters = { 'steps': [*range(1, 36 + 1, 1)], - # Add more hyperparameters as needed + 'time_steps': 36, } return hyperparameters diff --git a/models/locf_cmbaseline/configs/config_meta.py b/models/locf_cmbaseline/configs/config_meta.py index c89ebbf4..3922dda0 100644 --- a/models/locf_cmbaseline/configs/config_meta.py +++ b/models/locf_cmbaseline/configs/config_meta.py @@ -6,15 +6,15 @@ def get_meta_config(): Returns: - meta_config (dict): A dictionary containing model meta configuration. """ - + meta_config = { - "name": "locf_cmbaseline", + "name": "locf_cmbaseline", "algorithm": "LocfModel", - # Uncomment and modify the following lines as needed for additional metadata: - "targets": ["lr_ged_sb"], - # "queryset": "escwa001_cflong", + "regression_targets": ["lr_ged_sb"], "level": "cm", "creator": "Sonja", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], + "prediction_format": "dataframe", + "rolling_origin_stride": 1, + "regression_point_metrics": ["MSE", "MSLE"], } return meta_config diff --git a/models/locf_cmbaseline/configs/config_sweep.py b/models/locf_cmbaseline/configs/config_sweep.py index 843c3f6b..6bde1602 100644 --- a/models/locf_cmbaseline/configs/config_sweep.py +++ b/models/locf_cmbaseline/configs/config_sweep.py @@ -10,19 +10,18 @@ def get_sweep_config(): sweep_config = { 'method': 'grid', - 'name': 'locf_baseline' + 'name': 'locf_cmbaseline' } - # Example metric setup: metric = { 'name': 'MSE', 'goal': 'minimize' } sweep_config['metric'] = metric - # Example parameters setup: parameters_dict = { - 'steps': {'values': [[*range(1, 36 + 1, 1)]]}, + 'steps': {'value': [*range(1, 36 + 1, 1)]}, + 'time_steps': {'value': 36}, } sweep_config['parameters'] = parameters_dict diff --git a/models/locf_cmbaseline/main.py b/models/locf_cmbaseline/main.py index 6cda3cb8..7770d7e9 100644 --- a/models/locf_cmbaseline/main.py +++ b/models/locf_cmbaseline/main.py @@ -18,7 +18,9 @@ validate_arguments(args) if args.sweep: - print("No Sweep Run for Baseline Models") + BaselineForecastingModelManager( + model_path=model_path, + ).execute_sweep_run(args) else: BaselineForecastingModelManager( model_path=model_path, diff --git a/models/locf_pgmbaseline/configs/config_hyperparameters.py b/models/locf_pgmbaseline/configs/config_hyperparameters.py index c4f07553..0c146948 100644 --- a/models/locf_pgmbaseline/configs/config_hyperparameters.py +++ b/models/locf_pgmbaseline/configs/config_hyperparameters.py @@ -10,6 +10,6 @@ def get_hp_config(): hyperparameters = { 'steps': [*range(1, 36 + 1, 1)], - # Add more hyperparameters as needed + 'time_steps': 36, } return hyperparameters diff --git a/models/locf_pgmbaseline/configs/config_meta.py b/models/locf_pgmbaseline/configs/config_meta.py index 5b076628..d53b50e0 100644 --- a/models/locf_pgmbaseline/configs/config_meta.py +++ b/models/locf_pgmbaseline/configs/config_meta.py @@ -6,15 +6,15 @@ def get_meta_config(): Returns: - meta_config (dict): A dictionary containing model meta configuration. """ - + meta_config = { - "name": "locf_pgmbaseline", + "name": "locf_pgmbaseline", "algorithm": "LocfModel", - # Uncomment and modify the following lines as needed for additional metadata: - "targets": ["lr_ged_sb"], - # "queryset": "escwa001_cflong", + "regression_targets": ["lr_ged_sb"], "level": "pgm", "creator": "Sonja", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], + "prediction_format": "dataframe", + "rolling_origin_stride": 1, + "regression_point_metrics": ["MSE", "MSLE"], } return meta_config diff --git a/models/locf_pgmbaseline/configs/config_sweep.py b/models/locf_pgmbaseline/configs/config_sweep.py index 63b4c5d8..c392b17b 100644 --- a/models/locf_pgmbaseline/configs/config_sweep.py +++ b/models/locf_pgmbaseline/configs/config_sweep.py @@ -13,16 +13,15 @@ def get_sweep_config(): 'name': 'locf_pgmbaseline' } - # Example metric setup: metric = { 'name': 'MSE', 'goal': 'minimize' } sweep_config['metric'] = metric - # Example parameters setup: parameters_dict = { - 'steps': {'values': [[*range(1, 36 + 1, 1)]]}, + 'steps': {'value': [*range(1, 36 + 1, 1)]}, + 'time_steps': {'value': 36}, } sweep_config['parameters'] = parameters_dict diff --git a/models/locf_pgmbaseline/main.py b/models/locf_pgmbaseline/main.py index 86972e5f..b48a90cc 100644 --- a/models/locf_pgmbaseline/main.py +++ b/models/locf_pgmbaseline/main.py @@ -18,7 +18,9 @@ validate_arguments(args) if args.sweep: - print("No Sweep Run for Baseline Models") + BaselineForecastingModelManager( + model_path=model_path, + ).execute_sweep_run(args) else: BaselineForecastingModelManager( model_path=model_path, diff --git a/models/lovely_creature/configs/config_meta.py b/models/lovely_creature/configs/config_meta.py index 6ea5b96d..647c3348 100644 --- a/models/lovely_creature/configs/config_meta.py +++ b/models/lovely_creature/configs/config_meta.py @@ -13,6 +13,7 @@ def get_meta_config(): "targets": ["lr_sb_best"], "level": "cm", "creator": "Håvard", + "prediction_format": "dataframe", "model_reg": "XGBRegressor", "model_clf": "XGBClassifier", "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], diff --git a/models/midnight_rain/configs/config_meta.py b/models/midnight_rain/configs/config_meta.py index d8c52179..77badcb2 100644 --- a/models/midnight_rain/configs/config_meta.py +++ b/models/midnight_rain/configs/config_meta.py @@ -14,6 +14,7 @@ def get_meta_config(): "targets": "lr_ged_sb", "queryset": "fatalities003_pgm_escwa_drought", "level": "pgm", - "creator": "Xiaolong" + "creator": "Xiaolong", + "prediction_format": "dataframe", } return meta_config diff --git a/models/national_anthem/configs/config_meta.py b/models/national_anthem/configs/config_meta.py index dcd611e5..50d2f47e 100644 --- a/models/national_anthem/configs/config_meta.py +++ b/models/national_anthem/configs/config_meta.py @@ -14,6 +14,7 @@ def get_meta_config(): "targets": "lr_ged_sb", "queryset": "fatalities003_wdi_short", "level": "cm", - "creator": "Borbála" + "creator": "Borbála", + "prediction_format": "dataframe", } return meta_config diff --git a/models/new_rules/configs/config_meta.py b/models/new_rules/configs/config_meta.py index cb4dfd52..29662c56 100644 --- a/models/new_rules/configs/config_meta.py +++ b/models/new_rules/configs/config_meta.py @@ -14,6 +14,7 @@ def get_meta_config(): # "queryset": "escwa001_cflong", "level": "cm", "creator": "Simon", + "prediction_format": "dataframe", "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], } return meta_config diff --git a/models/novel_heuristics/configs/config_hyperparameters.py b/models/novel_heuristics/configs/config_hyperparameters.py index c3759048..1153f645 100644 --- a/models/novel_heuristics/configs/config_hyperparameters.py +++ b/models/novel_heuristics/configs/config_hyperparameters.py @@ -19,6 +19,8 @@ def get_hp_config(): hyperparameters = { # --- Forecast horizon --- "steps": list(range(1, 37)), + "time_steps": 36, + "rolling_origin_stride": 1, # --- Architecture --- "activation": "LeakyReLU", diff --git a/models/novel_heuristics/configs/config_meta.py b/models/novel_heuristics/configs/config_meta.py index 2c0a1019..a16dcf52 100644 --- a/models/novel_heuristics/configs/config_meta.py +++ b/models/novel_heuristics/configs/config_meta.py @@ -10,10 +10,11 @@ def get_meta_config(): meta_config = { "name": "novel_heuristics", "algorithm": "NBEATSModel", - "regression_targets": ["lr_ged_sb_dep"], + "regression_targets": ["lr_ged_sb"], # "queryset": "escwa001_cflong", "level": "cm", "creator": "Simon", "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "prediction_format": "dataframe", } return meta_config diff --git a/models/novel_heuristics/data/generated/calibration_log.txt b/models/novel_heuristics/data/generated/calibration_log.txt index 6e1c8e32..54ddffdf 100644 --- a/models/novel_heuristics/data/generated/calibration_log.txt +++ b/models/novel_heuristics/data/generated/calibration_log.txt @@ -1,6 +1,6 @@ Single Model Name: novel_heuristics -Single Model Timestamp: 20260224_130433 -Data Generation Timestamp: 20260224_130514 -Data Fetch Timestamp: 20260224_125938 +Single Model Timestamp: 20260306_202134 +Data Generation Timestamp: 20260306_202158 +Data Fetch Timestamp: 20260306_201935 Deployment Status: shadow diff --git a/models/novel_heuristics/data/raw/calibration_data_fetch_log.txt b/models/novel_heuristics/data/raw/calibration_data_fetch_log.txt index 1c919abb..0b445bec 100644 --- a/models/novel_heuristics/data/raw/calibration_data_fetch_log.txt +++ b/models/novel_heuristics/data/raw/calibration_data_fetch_log.txt @@ -1,3 +1,3 @@ Single Model Name: novel_heuristics -Data Fetch Timestamp: 20260224_141451 +Data Fetch Timestamp: 20260314_180020 diff --git a/models/old_money/configs/config_meta.py b/models/old_money/configs/config_meta.py index b51ebe44..65533674 100644 --- a/models/old_money/configs/config_meta.py +++ b/models/old_money/configs/config_meta.py @@ -15,6 +15,7 @@ def get_meta_config(): "targets": "lr_ged_sb", "queryset": "fatalities003_pgm_escwa_drought", "level": "pgm", - "creator": "Xiaolong" + "creator": "Xiaolong", + "prediction_format": "dataframe", } return meta_config \ No newline at end of file diff --git a/models/ominous_ox/configs/config_meta.py b/models/ominous_ox/configs/config_meta.py index 8d972f98..41068aad 100644 --- a/models/ominous_ox/configs/config_meta.py +++ b/models/ominous_ox/configs/config_meta.py @@ -14,6 +14,7 @@ def get_meta_config(): "targets": "lr_ged_sb", "queryset": "fatalities003_conflict_history", "level": "cm", - "creator": "Borbála" + "creator": "Borbála", + "prediction_format": "dataframe", } return meta_config diff --git a/models/orange_pasta/configs/config_meta.py b/models/orange_pasta/configs/config_meta.py index fc368d71..55f98f76 100644 --- a/models/orange_pasta/configs/config_meta.py +++ b/models/orange_pasta/configs/config_meta.py @@ -13,6 +13,7 @@ def get_meta_config(): "targets": "lr_ged_sb", "queryset": "fatalities003_pgm_baseline", "level": "pgm", - "creator": "Xiaolong" + "creator": "Xiaolong", + "prediction_format": "dataframe", } return meta_config \ No newline at end of file diff --git a/models/party_princess/configs/config_meta.py b/models/party_princess/configs/config_meta.py index 3c921a95..3ff8f68f 100644 --- a/models/party_princess/configs/config_meta.py +++ b/models/party_princess/configs/config_meta.py @@ -15,6 +15,7 @@ def get_meta_config(): # "queryset": "escwa001_cflong", "level": "cm", "creator": "Simon", + "prediction_format": "dataframe", "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], } return meta_config diff --git a/models/plastic_beach/configs/config_meta.py b/models/plastic_beach/configs/config_meta.py index dbb54ece..4881ae3e 100644 --- a/models/plastic_beach/configs/config_meta.py +++ b/models/plastic_beach/configs/config_meta.py @@ -14,6 +14,7 @@ def get_meta_config(): "targets": "lr_ged_sb", "queryset": "fatalities003_aquastat", "level": "cm", - "creator": "Marina" + "creator": "Marina", + "prediction_format": "dataframe", } return meta_config diff --git a/models/popular_monster/configs/config_meta.py b/models/popular_monster/configs/config_meta.py index d0f81ff5..9315c86d 100644 --- a/models/popular_monster/configs/config_meta.py +++ b/models/popular_monster/configs/config_meta.py @@ -14,6 +14,7 @@ def get_meta_config(): "targets": "lr_ged_sb", "queryset": "fatalities003_topics", "level": "cm", - "creator": "Borbála" + "creator": "Borbála", + "prediction_format": "dataframe", } return meta_config diff --git a/models/preliminary_directives/configs/config_meta.py b/models/preliminary_directives/configs/config_meta.py index 920a431e..87c7c0ab 100644 --- a/models/preliminary_directives/configs/config_meta.py +++ b/models/preliminary_directives/configs/config_meta.py @@ -14,6 +14,7 @@ def get_meta_config(): # "queryset": "escwa001_cflong", "level": "cm", "creator": "Simon", + "prediction_format": "dataframe", "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], } return meta_config diff --git a/models/purple_alien/configs/config_hyperparameters.py b/models/purple_alien/configs/config_hyperparameters.py index 9e592895..8799a956 100644 --- a/models/purple_alien/configs/config_hyperparameters.py +++ b/models/purple_alien/configs/config_hyperparameters.py @@ -12,16 +12,6 @@ def get_hp_config(): hyperparameters = { - # ============================================================ - # diagnostic settings - # ============================================================ - "diagnostic_visualizations": True, - - # ============================================================ - # evaluation metric - # ============================================================ - "regression_metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], - "classification_metrics": ["AP"], # ============================================================ # Ledger / Topology (ADR 007 Compliance) @@ -110,10 +100,17 @@ def get_hp_config(): # Outbound / Evaluation # ============================================================ # Note: Internal Naming (pred_, _raw, _prob) is handled by VolumeHandler - 'n_posterior_samples': 3, - 'evalution_mode': "point", #'stochastic', + 'n_posterior_samples': 64, + #'evalution_mode': "point", #'stochastic', + 'evalution_mode': 'stochastic', 'aggregate_method': 'arithmetic_mean', - 'run_type': 'calibration', + # 'run_type': 'calibration', + + # Track B (list-in-cell parquet delivery) is suspended at pgm scale. + # to_prediction_df() creates 5.5M Python float objects per target per origin + # (~4.8–6.4 GB peak + 2.3 GB permanent fragmentation). Track A (.npy) is + # written per-origin for metrics. Re-enable once Track B has a PyArrow fix. + 'skip_predictions_delivery': False, #True, } return hyperparameters diff --git a/models/purple_alien/configs/config_meta.py b/models/purple_alien/configs/config_meta.py index 4fa9a204..aedeec73 100644 --- a/models/purple_alien/configs/config_meta.py +++ b/models/purple_alien/configs/config_meta.py @@ -7,16 +7,34 @@ def get_meta_config(): - meta_config (dict): A dictionary containing model meta configuration. """ meta_config = { + # ============================================================ + # General information + # ============================================================ "name": "purple_alien", "algorithm": "HydraNet", - # "regression_targets": ["lr_sb_best", "lr_ns_best", "lr_os_best"], #, "ln_sb_best_binarized", "ln_ns_best_binarized", "ln_os_best_binarized"], - # "queryset": "escwa001_cflong", - # "identity_cols" : ["priogrid_gid", "col", "row", "month_id", "c_id"], - # "index_names": ['month_id', 'priogrid_gid'], - # "features" : ["lr_sb_best", "lr_ns_best", "lr_os_best"], - "level": "pgm", "creator": "Simon", - "regression_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], - "classification_metrics": ["AP"], + "level": "pgm", + + # ============================================================ + # output format + # ============================================================ + + "prediction_format": "prediction_frame", #"dataframe", + # "prediction_format": "dataframe", + "skip_predictions_delivery": True, # Suspend Track B parquet delivery (OOM mitigation) + + + # ============================================================ + # diagnostic settings + # ============================================================ + "diagnostic_visualizations": False, #True, + + # ============================================================ + # evaluation settings + # ============================================================ + "regression_sample_metrics": ["twCRPS", "QIS", "MIS", "MCR_sample"], + "evaluation_profile": "hydranet_ucdp", + "classification_point_metrics": ["AP"], + "rolling_origin_stride": 1, } return meta_config diff --git a/models/purple_alien/data/generated/calibration_log.txt b/models/purple_alien/data/generated/calibration_log.txt index f7f05cd6..318f25d4 100644 --- a/models/purple_alien/data/generated/calibration_log.txt +++ b/models/purple_alien/data/generated/calibration_log.txt @@ -1,6 +1,6 @@ Single Model Name: purple_alien -Single Model Timestamp: 20260224_102247 -Data Generation Timestamp: 20260224_103136 -Data Fetch Timestamp: 20260224_091234 +Single Model Timestamp: 20260311_163540 +Data Generation Timestamp: 20260313_005219 +Data Fetch Timestamp: 20260311_160234 Deployment Status: shadow diff --git a/models/purple_alien/data/raw/calibration_data_fetch_log.txt b/models/purple_alien/data/raw/calibration_data_fetch_log.txt index fb375665..86ec2f82 100644 --- a/models/purple_alien/data/raw/calibration_data_fetch_log.txt +++ b/models/purple_alien/data/raw/calibration_data_fetch_log.txt @@ -1,3 +1,3 @@ Single Model Name: purple_alien -Data Fetch Timestamp: 20260224_141659 +Data Fetch Timestamp: 20260311_160234 diff --git a/models/purple_haze/configs/config_meta.py b/models/purple_haze/configs/config_meta.py index 2e456181..7e28ddcf 100644 --- a/models/purple_haze/configs/config_meta.py +++ b/models/purple_haze/configs/config_meta.py @@ -13,6 +13,7 @@ def get_meta_config(): "targets": ["lr_sb_best"], "level": "cm", "creator": "Håvard", + "prediction_format": "dataframe", "model_reg": "XGBRegressor", "model_clf": "XGBClassifier", "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], diff --git a/models/teen_spirit/configs/config_meta.py b/models/teen_spirit/configs/config_meta.py index 057bbbee..68ed1c09 100644 --- a/models/teen_spirit/configs/config_meta.py +++ b/models/teen_spirit/configs/config_meta.py @@ -14,6 +14,7 @@ def get_meta_config(): "targets": "lr_ged_sb", "queryset": "fatalities003_faoprices", "level": "cm", - "creator": "Marina" + "creator": "Marina", + "prediction_format": "dataframe", } return meta_config diff --git a/models/teenage_dirtbag/configs/config_meta.py b/models/teenage_dirtbag/configs/config_meta.py index ff8a2e37..1a9e8e19 100644 --- a/models/teenage_dirtbag/configs/config_meta.py +++ b/models/teenage_dirtbag/configs/config_meta.py @@ -15,6 +15,7 @@ def get_meta_config(): # "queryset": "escwa001_cflong", "level": "cm", "creator": "Dylan", + "prediction_format": "dataframe", "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], } return meta_config diff --git a/models/twin_flame/configs/config_meta.py b/models/twin_flame/configs/config_meta.py index 0ef8d76e..01c30606 100644 --- a/models/twin_flame/configs/config_meta.py +++ b/models/twin_flame/configs/config_meta.py @@ -16,6 +16,7 @@ def get_meta_config(): "targets": "lr_ged_sb", "queryset": "fatalities003_topics", "level": "cm", - "creator": "Borbála" + "creator": "Borbála", + "prediction_format": "dataframe", } return meta_config diff --git a/models/wild_rose/configs/config_meta.py b/models/wild_rose/configs/config_meta.py index 87552113..66cf0175 100644 --- a/models/wild_rose/configs/config_meta.py +++ b/models/wild_rose/configs/config_meta.py @@ -13,6 +13,7 @@ def get_meta_config(): "targets": ["lr_sb_best"], "level": "cm", "creator": "Håvard", + "prediction_format": "dataframe", "model_reg": "XGBRegressor", "model_clf": "XGBClassifier", "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], diff --git a/models/wildest_dream/configs/config_meta.py b/models/wildest_dream/configs/config_meta.py index 47e4a6ac..a1d46f6b 100644 --- a/models/wildest_dream/configs/config_meta.py +++ b/models/wildest_dream/configs/config_meta.py @@ -15,6 +15,7 @@ def get_meta_config(): "targets": "lr_ged_sb", "queryset": "fatalities003_pgm_conflict_sptime_dist", "level": "pgm", - "creator": "Xiaolong" + "creator": "Xiaolong", + "prediction_format": "dataframe", } return meta_config \ No newline at end of file diff --git a/models/wuthering_heights/configs/config_meta.py b/models/wuthering_heights/configs/config_meta.py index f5aea859..c3a52559 100644 --- a/models/wuthering_heights/configs/config_meta.py +++ b/models/wuthering_heights/configs/config_meta.py @@ -13,6 +13,7 @@ def get_meta_config(): "targets": ["lr_sb_best"], "level": "cm", "creator": "Håvard", + "prediction_format": "dataframe", "model_reg": "XGBRegressor", "model_clf": "XGBClassifier", "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], diff --git a/models/yellow_pikachu/configs/config_meta.py b/models/yellow_pikachu/configs/config_meta.py index e1a275cf..314f4626 100644 --- a/models/yellow_pikachu/configs/config_meta.py +++ b/models/yellow_pikachu/configs/config_meta.py @@ -15,6 +15,7 @@ def get_meta_config(): "targets": "lr_ged_sb", "queryset": "fatalities003_pgm_conflict_treelag", "level": "pgm", - "creator": "Xiaolong" + "creator": "Xiaolong", + "prediction_format": "dataframe", } return meta_config \ No newline at end of file diff --git a/models/yellow_submarine/configs/config_meta.py b/models/yellow_submarine/configs/config_meta.py index 9d02e65d..965c3ae1 100644 --- a/models/yellow_submarine/configs/config_meta.py +++ b/models/yellow_submarine/configs/config_meta.py @@ -14,6 +14,7 @@ def get_meta_config(): "targets": "lr_ged_sb", "queryset": "fatalities003_imfweo", "level": "cm", - "creator": "Marina" + "creator": "Marina", + "prediction_format": "dataframe", } return meta_config diff --git a/models/zero_cmbaseline/configs/config_hyperparameters.py b/models/zero_cmbaseline/configs/config_hyperparameters.py index c4f07553..0c146948 100644 --- a/models/zero_cmbaseline/configs/config_hyperparameters.py +++ b/models/zero_cmbaseline/configs/config_hyperparameters.py @@ -10,6 +10,6 @@ def get_hp_config(): hyperparameters = { 'steps': [*range(1, 36 + 1, 1)], - # Add more hyperparameters as needed + 'time_steps': 36, } return hyperparameters diff --git a/models/zero_cmbaseline/configs/config_meta.py b/models/zero_cmbaseline/configs/config_meta.py index dbf446ce..08108501 100644 --- a/models/zero_cmbaseline/configs/config_meta.py +++ b/models/zero_cmbaseline/configs/config_meta.py @@ -6,15 +6,15 @@ def get_meta_config(): Returns: - meta_config (dict): A dictionary containing model meta configuration. """ - + meta_config = { - "name": "zero_cmbaseline", + "name": "zero_cmbaseline", "algorithm": "ZeroModel", - # Uncomment and modify the following lines as needed for additional metadata: - "targets": ["lr_ged_sb"], - # "queryset": "escwa001_cflong", + "regression_targets": ["lr_ged_sb"], "level": "cm", "creator": "Sonja", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], + "prediction_format": "dataframe", + "rolling_origin_stride": 1, + "regression_point_metrics": ["MSE", "MSLE"], } return meta_config diff --git a/models/zero_cmbaseline/configs/config_sweep.py b/models/zero_cmbaseline/configs/config_sweep.py index 15a9a6e9..94788ef4 100644 --- a/models/zero_cmbaseline/configs/config_sweep.py +++ b/models/zero_cmbaseline/configs/config_sweep.py @@ -10,19 +10,18 @@ def get_sweep_config(): sweep_config = { 'method': 'grid', - 'name': 'zero_baseline' + 'name': 'zero_cmbaseline' } - # Example metric setup: metric = { 'name': 'MSE', 'goal': 'minimize' } sweep_config['metric'] = metric - # Example parameters setup: parameters_dict = { - 'steps': {'values': [[*range(1, 36 + 1, 1)]]}, + 'steps': {'value': [*range(1, 36 + 1, 1)]}, + 'time_steps': {'value': 36}, } sweep_config['parameters'] = parameters_dict diff --git a/models/zero_cmbaseline/main.py b/models/zero_cmbaseline/main.py index 86972e5f..b48a90cc 100644 --- a/models/zero_cmbaseline/main.py +++ b/models/zero_cmbaseline/main.py @@ -18,7 +18,9 @@ validate_arguments(args) if args.sweep: - print("No Sweep Run for Baseline Models") + BaselineForecastingModelManager( + model_path=model_path, + ).execute_sweep_run(args) else: BaselineForecastingModelManager( model_path=model_path, diff --git a/models/zero_pgmbaseline/configs/config_hyperparameters.py b/models/zero_pgmbaseline/configs/config_hyperparameters.py index c4f07553..0c146948 100644 --- a/models/zero_pgmbaseline/configs/config_hyperparameters.py +++ b/models/zero_pgmbaseline/configs/config_hyperparameters.py @@ -10,6 +10,6 @@ def get_hp_config(): hyperparameters = { 'steps': [*range(1, 36 + 1, 1)], - # Add more hyperparameters as needed + 'time_steps': 36, } return hyperparameters diff --git a/models/zero_pgmbaseline/configs/config_meta.py b/models/zero_pgmbaseline/configs/config_meta.py index 4e3fe1db..b21d99cf 100644 --- a/models/zero_pgmbaseline/configs/config_meta.py +++ b/models/zero_pgmbaseline/configs/config_meta.py @@ -6,15 +6,15 @@ def get_meta_config(): Returns: - meta_config (dict): A dictionary containing model meta configuration. """ - + meta_config = { - "name": "zero_pgmbaseline", + "name": "zero_pgmbaseline", "algorithm": "ZeroModel", - # Uncomment and modify the following lines as needed for additional metadata: - "targets": ["lr_ged_sb"], - # "queryset": "escwa001_cflong", + "regression_targets": ["lr_ged_sb"], "level": "pgm", "creator": "Sonja", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], + "prediction_format": "dataframe", + "rolling_origin_stride": 1, + "regression_point_metrics": ["MSE", "MSLE"], } return meta_config diff --git a/models/zero_pgmbaseline/configs/config_sweep.py b/models/zero_pgmbaseline/configs/config_sweep.py index b1df7acc..ad5a5488 100644 --- a/models/zero_pgmbaseline/configs/config_sweep.py +++ b/models/zero_pgmbaseline/configs/config_sweep.py @@ -13,16 +13,15 @@ def get_sweep_config(): 'name': 'zero_pgmbaseline' } - # Example metric setup: metric = { 'name': 'MSE', 'goal': 'minimize' } sweep_config['metric'] = metric - # Example parameters setup: parameters_dict = { - 'steps': {'values': [[*range(1, 36 + 1, 1)]]}, + 'steps': {'value': [*range(1, 36 + 1, 1)]}, + 'time_steps': {'value': 36}, } sweep_config['parameters'] = parameters_dict diff --git a/models/zero_pgmbaseline/main.py b/models/zero_pgmbaseline/main.py index 86972e5f..b48a90cc 100644 --- a/models/zero_pgmbaseline/main.py +++ b/models/zero_pgmbaseline/main.py @@ -18,7 +18,9 @@ validate_arguments(args) if args.sweep: - print("No Sweep Run for Baseline Models") + BaselineForecastingModelManager( + model_path=model_path, + ).execute_sweep_run(args) else: BaselineForecastingModelManager( model_path=model_path, From b04a97b02e5cc5c29f7451b55c203c5c11c2decc Mon Sep 17 00:00:00 2001 From: Polichinl Date: Sun, 15 Mar 2026 15:49:38 +0100 Subject: [PATCH 05/27] feat: add test suite, base_docs governance, fix config gaps and architectural risks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add mandatory config keys (time_steps, rolling_origin_stride) to 52+ models - Replace exec() with importlib.util in create_catalogs.py for safer config loading - Migrate 13 old-pattern models to ForecastingModelArgs CLI API - Fix heat_waves/hot_stream forecasting offset bug (-2 → -1) - Add comprehensive test suite (2029 tests): - Config completeness, structure, CLI pattern, partition consistency - Ensemble config validation and dependency checking - Red-team failure injection tests - Add base_docs governance: 9 ADRs, 3 CICs, contributor protocols, standards - Remove dead code from create_catalogs.py and purple_alien/main.py Co-Authored-By: Claude Opus 4.6 (1M context) --- base_docs/ADRs/000_use_of_adrs.md | 79 +++++++++++ base_docs/ADRs/001_ontology.md | 81 +++++++++++ base_docs/ADRs/002_topology.md | 76 ++++++++++ base_docs/ADRs/003_authority.md | 63 +++++++++ base_docs/ADRs/005_testing.md | 81 +++++++++++ base_docs/ADRs/006_intent_contracts.md | 64 +++++++++ base_docs/ADRs/007_silicon_agents.md | 74 ++++++++++ base_docs/ADRs/008_observability.md | 62 ++++++++ base_docs/ADRs/009_boundary_contracts.md | 82 +++++++++++ base_docs/ADRs/README.md | 62 ++++++++ base_docs/ADRs/adr_template.md | 66 +++++++++ base_docs/CICs/CatalogExtractor.md | 119 ++++++++++++++++ base_docs/CICs/EnsembleScaffoldBuilder.md | 108 ++++++++++++++ base_docs/CICs/ModelScaffoldBuilder.md | 127 +++++++++++++++++ base_docs/CICs/README.md | 30 ++++ base_docs/CICs/cic_template.md | 61 ++++++++ base_docs/INSTANTIATION_CHECKLIST.md | 57 ++++++++ .../carbon_based_agents.md | 107 ++++++++++++++ .../hardened_protocol.md | 77 ++++++++++ .../silicon_based_agents.md | 114 +++++++++++++++ .../logging_and_observability_standard.md | 106 ++++++++++++++ base_docs/validate_docs.sh | 94 +++++++++++++ create_catalogs.py | 38 ++--- .../configs/config_hyperparameters.py | 1 + models/adolecent_slob/configs/config_meta.py | 1 + models/average_cmbaseline/main.py | 25 ++-- models/average_pgmbaseline/main.py | 25 ++-- .../configs/config_hyperparameters.py | 1 + models/bad_blood/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + .../configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 18 +++ models/black_ranger/configs/config_meta.py | 20 +++ .../configs/config_hyperparameters.py | 1 + models/blank_space/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 18 +++ models/blue_ranger/configs/config_meta.py | 20 +++ .../configs/config_hyperparameters.py | 1 + models/bouncy_organ/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + models/brown_cheese/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + models/car_radio/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + models/caring_fish/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + models/cheap_thrills/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + models/chunky_cat/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + models/cool_cat/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + models/counting_stars/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + models/dancing_queen/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + models/dark_paradise/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + models/demon_days/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + models/elastic_heart/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + .../configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + .../configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + models/fancy_feline/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + models/fast_car/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + .../configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + .../fourtieth_symphony/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + models/good_life/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + models/good_riddance/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 18 +++ models/green_ranger/configs/config_meta.py | 20 +++ .../configs/config_hyperparameters.py | 1 + models/green_squirrel/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + models/heat_waves/configs/config_meta.py | 1 + .../heat_waves/configs/config_partitions.py | 4 +- .../configs/config_hyperparameters.py | 1 + models/heavy_rotation/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + models/high_hopes/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + models/hot_stream/configs/config_meta.py | 1 + .../hot_stream/configs/config_partitions.py | 4 +- .../configs/config_hyperparameters.py | 1 + .../invisible_string/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + models/lavender_haze/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + models/little_lies/configs/config_meta.py | 1 + models/locf_cmbaseline/main.py | 25 ++-- models/locf_pgmbaseline/main.py | 25 ++-- .../configs/config_hyperparameters.py | 1 + models/lovely_creature/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + models/midnight_rain/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + models/national_anthem/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + models/new_rules/configs/config_meta.py | 1 + .../novel_heuristics/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + models/old_money/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + models/ominous_ox/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + models/orange_pasta/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + models/party_princess/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 18 +++ models/pink_ranger/configs/config_meta.py | 20 +++ .../configs/config_hyperparameters.py | 1 + models/plastic_beach/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + models/popular_monster/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + .../configs/config_meta.py | 1 + models/purple_alien/main.py | 10 -- .../configs/config_hyperparameters.py | 1 + models/purple_haze/configs/config_meta.py | 1 + models/purple_haze/main.py | 23 +-- .../configs/config_hyperparameters.py | 18 +++ models/red_ranger/configs/config_meta.py | 20 +++ .../configs/config_hyperparameters.py | 1 + models/teen_spirit/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + models/teenage_dirtbag/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + models/twin_flame/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + models/wild_rose/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + models/wildest_dream/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + .../wuthering_heights/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 1 + models/yellow_pikachu/configs/config_meta.py | 1 + .../configs/config_hyperparameters.py | 18 +++ models/yellow_ranger/configs/config_meta.py | 20 +++ .../configs/config_hyperparameters.py | 1 + .../yellow_submarine/configs/config_meta.py | 1 + models/zero_cmbaseline/main.py | 25 ++-- models/zero_pgmbaseline/main.py | 25 ++-- tests/__init__.py | 0 tests/conftest.py | 57 ++++++++ tests/test_catalogs.py | 82 +++++++++++ tests/test_cli_pattern.py | 55 ++++++++ tests/test_config_completeness.py | 93 ++++++++++++ tests/test_config_partitions.py | 85 +++++++++++ tests/test_ensemble_configs.py | 132 ++++++++++++++++++ tests/test_failure_modes.py | 44 ++++++ tests/test_model_structure.py | 44 ++++++ 159 files changed, 2797 insertions(+), 147 deletions(-) create mode 100644 base_docs/ADRs/000_use_of_adrs.md create mode 100644 base_docs/ADRs/001_ontology.md create mode 100644 base_docs/ADRs/002_topology.md create mode 100644 base_docs/ADRs/003_authority.md create mode 100644 base_docs/ADRs/005_testing.md create mode 100644 base_docs/ADRs/006_intent_contracts.md create mode 100644 base_docs/ADRs/007_silicon_agents.md create mode 100644 base_docs/ADRs/008_observability.md create mode 100644 base_docs/ADRs/009_boundary_contracts.md create mode 100644 base_docs/ADRs/README.md create mode 100644 base_docs/ADRs/adr_template.md create mode 100644 base_docs/CICs/CatalogExtractor.md create mode 100644 base_docs/CICs/EnsembleScaffoldBuilder.md create mode 100644 base_docs/CICs/ModelScaffoldBuilder.md create mode 100644 base_docs/CICs/README.md create mode 100644 base_docs/CICs/cic_template.md create mode 100644 base_docs/INSTANTIATION_CHECKLIST.md create mode 100644 base_docs/contributor_protocols/carbon_based_agents.md create mode 100644 base_docs/contributor_protocols/hardened_protocol.md create mode 100644 base_docs/contributor_protocols/silicon_based_agents.md create mode 100644 base_docs/standards/logging_and_observability_standard.md create mode 100755 base_docs/validate_docs.sh create mode 100644 models/black_ranger/configs/config_hyperparameters.py create mode 100644 models/black_ranger/configs/config_meta.py create mode 100644 models/blue_ranger/configs/config_hyperparameters.py create mode 100644 models/blue_ranger/configs/config_meta.py create mode 100644 models/green_ranger/configs/config_hyperparameters.py create mode 100644 models/green_ranger/configs/config_meta.py create mode 100644 models/pink_ranger/configs/config_hyperparameters.py create mode 100644 models/pink_ranger/configs/config_meta.py create mode 100644 models/red_ranger/configs/config_hyperparameters.py create mode 100644 models/red_ranger/configs/config_meta.py create mode 100644 models/yellow_ranger/configs/config_hyperparameters.py create mode 100644 models/yellow_ranger/configs/config_meta.py create mode 100644 tests/__init__.py create mode 100644 tests/conftest.py create mode 100644 tests/test_catalogs.py create mode 100644 tests/test_cli_pattern.py create mode 100644 tests/test_config_completeness.py create mode 100644 tests/test_config_partitions.py create mode 100644 tests/test_ensemble_configs.py create mode 100644 tests/test_failure_modes.py create mode 100644 tests/test_model_structure.py diff --git a/base_docs/ADRs/000_use_of_adrs.md b/base_docs/ADRs/000_use_of_adrs.md new file mode 100644 index 00000000..246e4d93 --- /dev/null +++ b/base_docs/ADRs/000_use_of_adrs.md @@ -0,0 +1,79 @@ +# ADR-000: Use of Architecture Decision Records (ADRs) + +**Status:** Accepted +**Date:** 2026-03-15 +**Deciders:** Simon (project maintainer) +**Informed:** All contributors + +--- + +## Context + +views-models is a monorepo containing ~66 forecasting models, 5 ensembles, data extractors, postprocessors, and tooling for the VIEWS conflict prediction platform. The repository has multiple contributors, evolving conventions, and a history of implicit decisions that have led to architectural drift (e.g., two CLI patterns, duplicated partition configs, inconsistent config keys). + +Without a shared record of *why* decisions were made, the project risks: +- Re-litigating settled questions (e.g., why all models use the same partition boundaries) +- Accidental reversals of critical design choices +- Accumulating invisible technical debt +- Losing institutional memory as contributors change + +--- + +## Decision + +We will use **Architecture Decision Records (ADRs)** to document significant technical, architectural, and conceptual decisions in this project. + +ADRs are: +- Written in Markdown +- Stored in the repository under `base_docs/ADRs/` +- Numbered sequentially +- Treated as first-class project artifacts + +--- + +## When to Write an ADR + +Write an ADR when making a decision that: +- Affects model configuration conventions or required config keys +- Changes partition boundaries, training windows, or evaluation methodology +- Introduces new shared infrastructure (e.g., `common/`) +- Changes the CLI API pattern or model launcher conventions +- Modifies ensemble reconciliation logic or CM/PGM ordering +- Affects the CI/CD pipeline or catalog generation + +Do **not** write ADRs for: +- Adding a new model that follows existing conventions +- Routine hyperparameter changes within a single model +- Documentation-only changes + +--- + +## Lifecycle + +- **Proposed** — decision under consideration +- **Accepted** — decision is active and authoritative +- **Superseded** — replaced by a newer ADR +- **Deprecated** — decision remains but should no longer be used + +Decisions are never deleted. If a decision changes, it is **superseded**, not erased. + +--- + +## Consequences + +### Positive +- Clearer decision-making across a multi-contributor forecasting platform +- Fewer repeated debates about config conventions +- Easier onboarding for new model developers +- Better long-term coherence as the model zoo grows + +### Negative +- Small upfront cost in writing +- Requires discipline to maintain + +--- + +## References + +- `base_docs/ADRs/adr_template.md` +- `base_docs/ADRs/README.md` diff --git a/base_docs/ADRs/001_ontology.md b/base_docs/ADRs/001_ontology.md new file mode 100644 index 00000000..121b47e9 --- /dev/null +++ b/base_docs/ADRs/001_ontology.md @@ -0,0 +1,81 @@ +# ADR-001: Ontology of the Repository + +**Status:** Accepted +**Date:** 2026-03-15 +**Deciders:** Simon (project maintainer) +**Informed:** All contributors + +--- + +## Context + +views-models contains many different types of entities — model launchers, configuration files, ensembles, tooling scripts, shared infrastructure, and CI workflows. Without a clear taxonomy, contributors may place new code in the wrong location, conflate concerns, or introduce entities that don't fit the existing structure. + +--- + +## Decision + +The repository recognizes the following ontological categories: + +### Domain Entities +| Category | Location | Description | +|----------|----------|-------------| +| **Models** | `models/*/` | Individual forecasting model launchers (~66). Each is a thin `main.py` + config directory that delegates to an external architecture package. | +| **Ensembles** | `ensembles/*/` | Ensemble aggregation launchers (5). Aggregate predictions from constituent models. | + +### Configuration Entities +| Category | Location | Description | +|----------|----------|-------------| +| **Model Configs** | `models/*/configs/` | Six config files per model: `config_meta.py`, `config_deployment.py`, `config_hyperparameters.py`, `config_sweep.py`, `config_queryset.py`, `config_partitions.py` | +| **Ensemble Configs** | `ensembles/*/configs/` | Subset of config files per ensemble | + +### Infrastructure Entities +| Category | Location | Description | +|----------|----------|-------------| +| **Shared Infrastructure** | `common/` | Cross-model shared logic (e.g., `partitions.py`) | +| **CI/CD** | `.github/workflows/` | Automated catalog generation | +| **APIs** | `apis/` | External API integrations (e.g., UN FAO) | + +### Data Processing Entities +| Category | Location | Description | +|----------|----------|-------------| +| **Extractors** | `extractors/` | Data extraction modules (e.g., UCDP) | +| **Postprocessors** | `postprocessors/` | Output transformation modules | + +### Tooling Entities +| Category | Location | Description | +|----------|----------|-------------| +| **Scaffolding** | `build_model_scaffold.py`, `build_ensemble_scaffold.py`, `build_package_scaffold.py` | Interactive CLI tools for creating new model/ensemble directories | +| **Catalog Generation** | `create_catalogs.py`, `update_readme.py`, `generate_features_catalog.py` | Scripts that generate documentation from configs | +| **Validation** | `compare_configs.py`, `verify_architecture.py` | Ad-hoc validation/diagnostic scripts | + +### Testing Entities +| Category | Location | Description | +|----------|----------|-------------| +| **Tests** | `tests/` | Config completeness, structural conventions, CLI consistency, partition delegation, catalog safety | + +### Stability Levels + +| Level | Categories | Change Policy | +|-------|-----------|---------------| +| **Stable** | Shared Infrastructure (`common/`), config key requirements | Changes require ADR or team discussion | +| **Conventional** | Config structure, CLI pattern, naming conventions | Changes require updating all models + tests | +| **Volatile** | Individual model hyperparameters, querysets | Changed freely by model owners | + +--- + +## Consequences + +### Positive +- Clear guidance on where new code belongs +- Stability levels prevent accidental changes to load-bearing conventions + +### Negative +- Some entities (e.g., `verify_architecture.py`) are one-off scripts that don't fit cleanly into any category + +--- + +## References + +- ADR-002 (Topology) +- `tests/test_model_structure.py` — enforces structural conventions diff --git a/base_docs/ADRs/002_topology.md b/base_docs/ADRs/002_topology.md new file mode 100644 index 00000000..45341aee --- /dev/null +++ b/base_docs/ADRs/002_topology.md @@ -0,0 +1,76 @@ +# ADR-002: Topology and Dependency Rules + +**Status:** Accepted +**Date:** 2026-03-15 +**Deciders:** Simon (project maintainer) +**Informed:** All contributors + +--- + +## Context + +views-models is a configuration-and-orchestration layer. All ML logic lives in external packages (`views_pipeline_core`, `views_stepshifter`, `views_r2darts2`, `views_hydranet`, `views_baseline`). The repository's internal dependency structure must remain simple — models should not depend on each other, and shared infrastructure should be minimal. + +--- + +## Decision + +### Dependency Direction + +``` +External Packages (views_pipeline_core, views_stepshifter, etc.) + ↑ + models/*/main.py (each model imports ONE manager from ONE package) + ↑ + models/*/configs/ (config files import from ingester3, viewser) +``` + +### Self-Contained Config Files + +**Each model must have its own self-contained `config_partitions.py`.** The `views_pipeline_core` framework loads config files via `importlib` from each model's directory path. Config files cannot depend on repo-internal packages (like a hypothetical `common/` directory) because the repo root may not be on `sys.path` at runtime. + +This means partition logic is duplicated across ~66 models. This duplication is intentional — it ensures each model can run independently. Tests (`tests/test_config_partitions.py`) detect drift between models. + +### Allowed Dependencies + +| From | May Depend On | +|------|--------------| +| `models/*/main.py` | `views_pipeline_core`, one algorithm package, `pathlib` | +| `models/*/configs/config_partitions.py` | `ingester3` only (for `ViewsMonth`) | +| `models/*/configs/config_queryset.py` | `viewser`, `views_pipeline_core` | +| `models/*/configs/config_*.py` (others) | Nothing (pure dict-returning functions) | +| `ensembles/*/main.py` | `views_pipeline_core` | +| Tooling scripts (root) | `views_pipeline_core`, `importlib`, standard library | +| `tests/` | `conftest.py` helpers, `importlib`, standard library | + +### Forbidden Dependencies + +- **No cross-model imports** — `models/A/` must never import from `models/B/` +- **No model → tooling imports** — models must not import from root-level scripts +- **No repo-internal imports in config files** — config files must only import from installed packages (`ingester3`, `viewser`), not from repo-local modules +- **No config files with side effects** — config files must be pure functions returning dicts (exception: `config_queryset.py` which builds `Queryset` objects) + +--- + +## Known Deviations + +- `config_queryset.py` files import from `viewser` and `views_pipeline_core`, making them impossible to load without these packages installed. This is an accepted deviation — querysets require the VIEWS data layer. + +--- + +## Consequences + +### Positive +- Models remain independently deployable +- Tests can run without ML dependencies +- Adding a new model requires no changes to existing models + +### Negative +- Each model's `main.py` is a thin launcher with significant boilerplate duplication + +--- + +## References + +- ADR-001 (Ontology) +- `tests/test_cli_pattern.py` — enforces CLI import conventions diff --git a/base_docs/ADRs/003_authority.md b/base_docs/ADRs/003_authority.md new file mode 100644 index 00000000..b55cc8d4 --- /dev/null +++ b/base_docs/ADRs/003_authority.md @@ -0,0 +1,63 @@ +# ADR-003: Authority of Declarations Over Inference + +**Status:** Accepted +**Date:** 2026-03-15 +**Deciders:** Simon (project maintainer) +**Informed:** All contributors + +--- + +## Context + +In a monorepo with ~66 models, it is tempting to infer model properties from directory names, file patterns, or import statements. This leads to fragile conventions that break silently when someone deviates from the expected pattern. + +The system has already experienced this: `create_catalogs.py` originally used `exec()` to load configs, and the `heat_waves`/`hot_stream` models silently diverged from the standard forecasting offset without anyone noticing. + +--- + +## Decision + +**All meaningful model properties must be explicitly declared in configuration files, not inferred from structure.** + +Specifically: +- Model algorithm, level of analysis, targets, and creator are declared in `config_meta.py` +- Deployment status is declared in `config_deployment.py` +- Hyperparameters and temporal settings are declared in `config_hyperparameters.py` +- Partition boundaries are declared in `common/partitions.py` (single source of truth) +- Model name must match the directory name (enforced by `test_config_completeness.py`) + +### Fail-Loud Invariant + +When a required declaration is missing or invalid: +- The system must fail explicitly, not infer a default +- `config_meta.py` must contain all required keys: `name`, `algorithm`, `level`, `creator`, `prediction_format`, `rolling_origin_stride` +- `config_hyperparameters.py` must contain: `steps`, `time_steps` +- `config_deployment.py` must contain: `deployment_status` (one of: `shadow`, `deployed`, `baseline`, `deprecated`) + +### Forbidden Behaviors + +- Inferring model level (cm/pgm) from directory name or queryset +- Inferring algorithm type from import statements +- Using filename-based logic to determine model behavior +- Silently defaulting missing config keys + +--- + +## Consequences + +### Positive +- Config completeness is testable (see `tests/test_config_completeness.py`) +- Catalog generation reads from declarations, not heuristics +- New required keys can be added and enforced via tests + +### Negative +- Adding a new required key requires updating all ~66 models +- Config files contain some redundancy (e.g., `time_steps` duplicates `len(steps)`) + +--- + +## References + +- ADR-001 (Ontology) +- `tests/test_config_completeness.py` — enforces required keys +- `create_catalogs.py` — reads declarations to build catalogs diff --git a/base_docs/ADRs/005_testing.md b/base_docs/ADRs/005_testing.md new file mode 100644 index 00000000..37d1ff3a --- /dev/null +++ b/base_docs/ADRs/005_testing.md @@ -0,0 +1,81 @@ +# ADR-005: Testing as Mandatory Critical Infrastructure + +**Status:** Accepted +**Date:** 2026-03-15 +**Deciders:** Simon (project maintainer) +**Informed:** All contributors + +--- + +## Context + +views-models had zero test coverage until 2026-03-15. Configuration drift, partition divergence, and CLI pattern inconsistency accumulated undetected. The heat_waves/hot_stream offset bug and the 13 old-pattern CLI models were only discovered during a manual assimilation review. + +--- + +## Decision + +Tests are mandatory critical infrastructure, not optional documentation. + +### Test Categories + +We adopt a three-team testing taxonomy: + +| Team | Purpose | Examples in this repo | +|------|---------|----------------------| +| **Green** (Correctness) | Verify the system works as intended | `test_config_completeness.py` — required keys exist, values are valid | +| **Beige** (Convention) | Catch configuration drift and convention violations | `test_model_structure.py` — naming, file presence; `test_config_partitions.py` — delegation to shared module; `test_cli_pattern.py` — CLI import consistency | +| **Red** (Adversarial) | Expose failure modes by testing edge cases | Not yet implemented — future work | + +### Test Design Principles + +1. **Tests must run without ML dependencies** — Tests parse source code and use `importlib.util` to load config modules, avoiding dependency on `views_pipeline_core`, `ingester3`, or algorithm packages. +2. **Tests are parametrized over all models** — Every test runs against all ~66 models, catching drift immediately. +3. **Tests run fast** — The full suite completes in ~2 seconds. + +### Current Test Suite + +| File | Category | What it validates | +|------|----------|-------------------| +| `tests/test_config_completeness.py` | Green | Required config keys, valid values, `time_steps == len(steps)` | +| `tests/test_config_partitions.py` | Beige | Shared partition module correctness, model delegation | +| `tests/test_model_structure.py` | Beige | Naming convention, required files, config directory structure | +| `tests/test_cli_pattern.py` | Beige | New CLI import pattern, no explicit `wandb.login()` | +| `tests/test_catalogs.py` | Green | No `exec()` usage, markdown generation correctness | + +### Test Requirements for Changes + +- Adding a new required config key: add the key to `REQUIRED_META_KEYS` or `REQUIRED_HP_KEYS` in `test_config_completeness.py` +- Adding a new model: it must pass all existing tests (enforced by parametrization) +- Changing partition boundaries: update `common/partitions.py` and the expected values in `test_config_partitions.py` + +--- + +## Known Gaps + +- No red-team (adversarial) tests yet +- Catalog generation function tests require `views_pipeline_core` (skipped in most dev environments) +- No cross-validation between `config_meta.algorithm` and `main.py` manager import +- No ensemble config tests +- Tests are not wired into CI (`.github/workflows/`) + +--- + +## Consequences + +### Positive +- Config drift is now detectable before merge +- Partition consistency is enforced across all models +- New models automatically inherit all convention tests + +### Negative +- Source-based tests cannot validate runtime behavior +- Tests must be kept in sync with evolving config requirements + +--- + +## References + +- `tests/conftest.py` — test infrastructure and fixtures +- ADR-003 (Authority of Declarations) +- ADR-008 (Observability) diff --git a/base_docs/ADRs/006_intent_contracts.md b/base_docs/ADRs/006_intent_contracts.md new file mode 100644 index 00000000..573b1679 --- /dev/null +++ b/base_docs/ADRs/006_intent_contracts.md @@ -0,0 +1,64 @@ +# ADR-006: Intent Contracts for Non-Trivial Classes + +**Status:** Accepted +**Date:** 2026-03-15 +**Deciders:** Simon (project maintainer) +**Informed:** All contributors + +--- + +## Context + +views-models contains several non-trivial classes and modules whose purpose, boundaries, and failure modes are not documented. `ModelScaffoldBuilder`, `EnsembleScaffoldBuilder`, and `common/partitions.generate()` all have implicit contracts that new contributors must reverse-engineer from code. + +--- + +## Decision + +Non-trivial classes and modules in this repository must have **Intent Contracts** — human-readable documents that declare purpose, responsibilities, boundaries, and failure semantics. + +### When an Intent Contract is Required + +An Intent Contract is mandatory for: +- Classes with lifecycle logic (build → configure → assess) +- Shared infrastructure modules that all models depend on +- Boundary functions that load external data or configs +- Orchestration components + +### What an Intent Contract Must Define + +1. Purpose (what the class/module is for) +2. Non-Goals (what it is explicitly NOT responsible for) +3. Responsibilities and Guarantees +4. Inputs and Assumptions +5. Outputs and Side Effects +6. Failure Modes +7. Boundaries and Interactions +8. Correct and Incorrect Usage Examples +9. Test Alignment +10. Known Deviations from ideal patterns + +### Current Contracts + +See `base_docs/CICs/README.md` for the active contracts list. + +--- + +## Consequences + +### Positive +- New contributors can understand component purpose without reading implementation +- Silicon-based agents (ADR-007) can reference contracts to verify changes +- Tests can be derived from contracts + +### Negative +- Contracts must be maintained alongside code +- Contracts for simple config-returning functions may feel bureaucratic + +--- + +## References + +- `base_docs/CICs/` — Intent Contract directory +- ADR-003 (Authority of Declarations) +- ADR-005 (Testing) diff --git a/base_docs/ADRs/007_silicon_agents.md b/base_docs/ADRs/007_silicon_agents.md new file mode 100644 index 00000000..d01f27b7 --- /dev/null +++ b/base_docs/ADRs/007_silicon_agents.md @@ -0,0 +1,74 @@ +# ADR-007: Silicon-Based Agents as Untrusted Contributors + +**Status:** Accepted +**Date:** 2026-03-15 +**Deciders:** Simon (project maintainer) +**Informed:** All contributors + +--- + +## Context + +This repository is actively modified by AI-assisted tooling (Claude Code). Recent bulk changes include: migrating 13 model `main.py` files, adding config keys to 52+ models, and replacing 66 partition config files. These changes carry risks of silent semantic corruption — e.g., the `{{e}}` double-brace bug introduced during template-based CLI migration. + +--- + +## Decision + +Silicon-based agents (LLM assistants, code generators, refactoring tools) are treated as **untrusted contributors**. They may accelerate work but must not own intent. + +### Allowed Operations + +Silicon-based agents may: +- Perform scoped refactors within a single class or file +- Add or update tests that reflect declared intent +- Implement changes explicitly requested by a human contributor +- Make mechanical changes (formatting, renaming) with no semantic impact +- Perform bulk changes to config files when a clear template is provided + +### Forbidden Operations + +Silicon-based agents must not: +- Introduce or modify model semantics without updating intent contracts +- Infer model behavior from naming conventions or file structure +- Remove validation, fail-loud behavior, or tests +- Cross architectural boundaries (ADR-002) +- Perform full-file rewrites of existing files without reading first (anti-truncation rule) + +### Anti-Truncation Rule + +When modifying existing files, silicon-based agents must: +1. Read the file first +2. Apply targeted, minimal edits +3. Leave unrelated content untouched + +Full-file rewrites via template generation (as done during CLI migration) must be followed by verification against a known-good reference file. + +### Mandatory Artifacts + +Every silicon-based agent-assisted change must include: +- A summary of what was changed +- Reference to relevant ADRs +- Explicit declaration of uncertainty +- Verification that tests pass after the change + +--- + +## Consequences + +### Positive +- Prevents silent semantic corruption in bulk changes +- Forces verification of AI-generated code +- Makes human reviewers aware of heightened scrutiny needs + +### Negative +- Slows down bulk operations +- Requires discipline in AI-assisted workflows + +--- + +## References + +- `base_docs/contributor_protocols/silicon_based_agents.md` +- ADR-003 (Authority of Declarations) +- ADR-005 (Testing) diff --git a/base_docs/ADRs/008_observability.md b/base_docs/ADRs/008_observability.md new file mode 100644 index 00000000..b90c296f --- /dev/null +++ b/base_docs/ADRs/008_observability.md @@ -0,0 +1,62 @@ +# ADR-008: Observability and Explicit Failure + +**Status:** Accepted +**Date:** 2026-03-15 +**Deciders:** Simon (project maintainer) +**Informed:** All contributors + +--- + +## Context + +views-models delegates all ML logic to external packages, but the thin launcher layer (`main.py`, config files, tooling scripts) still has failure modes that must be handled explicitly. Silent failures in config loading, partition computation, or catalog generation can propagate undetected. + +--- + +## Decision + +### Fail-Loud Invariant + +Structural failures must be raised explicitly. Silent degradation is prohibited. + +Specifically: +- `main.py` files must wrap `ModelPathManager` initialization in try/except and raise `RuntimeError` with context +- Config loading must use `importlib.util` (not `exec()`), which naturally raises on syntax errors +- Missing required config keys must be caught by tests (ADR-005), not silently defaulted +- Catalog generation (`create_catalogs.py`) should log errors per-model rather than crashing entirely + +### Logging Standards + +This repository follows the logging standard defined in `base_docs/standards/logging_and_observability_standard.md`. + +Key rules: +- `ERROR` and `CRITICAL` must be both logged and raised +- Warnings must not mask invariant violations +- `print()` is not acceptable for structural diagnostics (use `logging`) + +### Current State (Known Deviations) + +- `create_catalogs.py` does not handle per-model errors — one broken config crashes the entire run +- Some models use `warnings.filterwarnings("ignore")` broadly, which could mask important warnings +- WandB experiment tracking provides some observability but is external to this repo +- No alerting is configured for CI failures + +--- + +## Consequences + +### Positive +- Config loading failures are visible at the point of error +- Test failures surface config drift before merge + +### Negative +- Broad `warnings.filterwarnings("ignore")` suppresses ML library warnings that could be informative +- No per-model resilience in catalog generation yet + +--- + +## References + +- `base_docs/standards/logging_and_observability_standard.md` +- ADR-003 (Authority of Declarations) +- ADR-005 (Testing) diff --git a/base_docs/ADRs/009_boundary_contracts.md b/base_docs/ADRs/009_boundary_contracts.md new file mode 100644 index 00000000..8fb578f8 --- /dev/null +++ b/base_docs/ADRs/009_boundary_contracts.md @@ -0,0 +1,82 @@ +# ADR-009: Boundary Contracts and Configuration Validation + +**Status:** Accepted +**Date:** 2026-03-15 +**Deciders:** Simon (project maintainer) +**Informed:** All contributors + +--- + +## Context + +views-models has several important boundaries where data enters or exits the system: +1. Config files → framework managers (config loading) +2. viewser querysets → data backend (data fetching) +3. Model predictions → ensemble aggregation → reconciliation → output +4. Config files → catalog generation scripts (documentation) + +At each boundary, assumptions about data format, required keys, and valid values must be enforced. + +--- + +## Decision + +### Configuration Boundaries + +Every config file boundary must enforce: + +| Config File | Required Keys | Validated By | +|------------|---------------|-------------| +| `config_meta.py` | `name`, `algorithm`, `level`, `creator`, `prediction_format`, `rolling_origin_stride` | `tests/test_config_completeness.py` | +| `config_deployment.py` | `deployment_status` (enum: shadow/deployed/baseline/deprecated) | `tests/test_config_completeness.py` | +| `config_hyperparameters.py` | `steps`, `time_steps` | `tests/test_config_completeness.py` | +| `config_partitions.py` | Self-contained `generate()` function; boundaries must match canonical values; offset must be `-1` | `tests/test_config_partitions.py` | + +### Structural Boundaries + +| Convention | Rule | Validated By | +|-----------|------|-------------| +| Model naming | `^[a-z]+_[a-z]+$` | `tests/test_model_structure.py` | +| Required files | `main.py`, `run.sh`, `configs/` with 6 config files | `tests/test_model_structure.py` | +| CLI pattern | Import from `views_pipeline_core.cli`, no `wandb.login()` | `tests/test_cli_pattern.py` | + +### Ensemble Boundaries + +| Convention | Rule | Validated By | +|-----------|------|-------------| +| Reconciliation | PGM ensembles declare `reconcile_with` referencing a CM ensemble | Not yet enforced by tests | +| Ordering | CM ensemble must complete before PGM ensemble | Documented in `docs/monthly_run_guide.md`, not programmatically enforced | + +### Catalog Generation Boundary + +- Config files are loaded via `importlib.util` (not `exec()`) +- Validated by `tests/test_catalogs.py::TestNoExecUsage` + +--- + +## Known Gaps + +- Ensemble `reconcile_with` targets are not validated against existing ensemble directories +- No boundary validation for `config_queryset.py` (requires external packages) +- No validation that `config_meta.algorithm` matches the manager import in `main.py` + +--- + +## Consequences + +### Positive +- Configuration errors are caught by tests before they reach production +- New required keys can be added to the test suite and enforced across all models +- Structural conventions are machine-verifiable + +### Negative +- Boundary validation requires keeping test expectations in sync with evolving config requirements +- Some boundaries (queryset, ensemble reconciliation) cannot be tested without external packages + +--- + +## References + +- ADR-003 (Authority of Declarations) +- ADR-005 (Testing) +- `tests/` — all boundary validation tests diff --git a/base_docs/ADRs/README.md b/base_docs/ADRs/README.md new file mode 100644 index 00000000..c3408dc7 --- /dev/null +++ b/base_docs/ADRs/README.md @@ -0,0 +1,62 @@ +# ADR README and Governance Map + +This repository uses Architectural Decision Records (ADRs) to govern +structural, semantic, and operational behavior. + +--- + +## Constitutional ADRs (000-009) + +These ADRs define system philosophy and governance: + +- **[ADR-000](000_use_of_adrs.md)** — Use of Architecture Decision Records +- **[ADR-001](001_ontology.md)** — Ontology of the Repository +- **[ADR-002](002_topology.md)** — Topology and Dependency Rules +- **[ADR-003](003_authority.md)** — Authority of Declarations Over Inference +- **ADR-004** — Evolution and Stability *(Deferred)* +- **[ADR-005](005_testing.md)** — Testing as Mandatory Critical Infrastructure +- **[ADR-006](006_intent_contracts.md)** — Intent Contracts for Non-Trivial Classes +- **[ADR-007](007_silicon_agents.md)** — Silicon-Based Agents as Untrusted Contributors +- **[ADR-008](008_observability.md)** — Observability and Explicit Failure +- **[ADR-009](009_boundary_contracts.md)** — Boundary Contracts and Configuration Validation + +--- + +## Governance Structure + +- **Ontology (001)** defines what exists. +- **Topology (002)** defines structural direction. +- **Authority (003)** defines who owns meaning. +- **Boundary Contracts (009)** define interaction rules. +- **Observability (008)** enforces failure semantics. +- **Testing (005)** verifies system integrity. +- **Intent Contracts (006)** bind class-level behavior. +- **Automation Governance (007)** constrains silicon-based agents. + +--- + +## Project-Specific ADRs (010+) + +No project-specific ADRs have been written yet. Candidates: + +- **ADR-010** — Partition Boundary Semantics (why 121-444/445-492/493-540) +- **ADR-011** — CM-before-PGM Ensemble Ordering +- **ADR-012** — Config Key Evolution Policy (how to add new required keys) +- **ADR-013** — Model Naming Convention and Governance +- **ADR-014** — Conda Environment Sharing via run.sh + +--- + +## Recommended Adoption Order + +### Phase 1 — Foundation (Done) +- ADR-000, ADR-003, ADR-008 + +### Phase 2 — Structure (Done) +- ADR-001, ADR-002 + +### Phase 3 — Testing & Intent (Done) +- ADR-005, ADR-006 + +### Phase 4 — Boundaries & Automation (Done) +- ADR-007, ADR-009 diff --git a/base_docs/ADRs/adr_template.md b/base_docs/ADRs/adr_template.md new file mode 100644 index 00000000..03e137c1 --- /dev/null +++ b/base_docs/ADRs/adr_template.md @@ -0,0 +1,66 @@ +# ADR-XXXX: + +**Status:** Proposed | Accepted | Superseded | Deprecated +**Date:** YYYY-MM-DD +**Deciders:** +**Consulted:** +**Informed:** + +--- + +## Context + +Describe the problem that motivated this decision. + +--- + +## Decision + +State the decision clearly and unambiguously. + +--- + +## Rationale + +Explain why this option was chosen over alternatives. + +--- + +## Considered Alternatives + +### Alternative A: +- **Pros:** +- **Cons:** +- **Reason for rejection:** + +--- + +## Consequences + +### Positive + +### Negative + +--- + +## Implementation Notes + +Concrete guidance for implementation. + +--- + +## Validation & Monitoring + +How will we know this decision was correct? + +--- + +## Open Questions + +List unresolved questions or known unknowns. + +--- + +## References + +Links to PRs, issues, design docs, or related ADRs. diff --git a/base_docs/CICs/CatalogExtractor.md b/base_docs/CICs/CatalogExtractor.md new file mode 100644 index 00000000..b8909760 --- /dev/null +++ b/base_docs/CICs/CatalogExtractor.md @@ -0,0 +1,119 @@ +# Class Intent Contract: create_catalogs.extract_models() + +**Status:** Active +**Owner:** Project maintainers +**Last reviewed:** 2026-03-15 +**Related ADRs:** ADR-003, ADR-008, ADR-009 + +--- + +## 1. Purpose + +> `extract_models()` loads metadata from a model's config files and produces a dictionary suitable for catalog/README generation. It is the boundary function between raw config files and documentation output. + +Located in: `create_catalogs.py:extract_models()` + +--- + +## 2. Non-Goals (Explicit Exclusions) + +- Does **not** validate config correctness (that's the test suite's job) +- Does **not** modify config files +- Does **not** run models or load data +- Does **not** handle ensemble-specific logic (uses same interface for both) + +--- + +## 3. Responsibilities and Guarantees + +- Loads `config_meta.py` via `importlib.util` and calls `get_meta_config()` +- Loads `config_deployment.py` via `importlib.util` and calls `get_deployment_config()` +- Creates GitHub markdown links for querysets and hyperparameters +- Returns a merged dictionary containing all catalog-relevant fields + +--- + +## 4. Inputs and Assumptions + +- Receives a `ModelPathManager` or `EnsemblePathManager` instance +- Assumes config files exist and define the expected functions +- Assumes config functions return dicts (no type enforcement) + +--- + +## 5. Outputs and Side Effects + +Returns a dict with keys from merged meta and deployment configs, plus: +- `queryset`: markdown link (or `'None'`) +- `hyperparameters`: markdown link to config_hyperparameters.py + +No side effects beyond logging. + +--- + +## 6. Failure Modes and Loudness + +- If a config file has a syntax error, `importlib` raises `SyntaxError` — currently crashes the entire catalog run +- If `get_meta_config()` or `get_deployment_config()` is missing, `AttributeError` is raised +- No per-model error isolation (known deviation — see ADR-008) + +--- + +## 7. Boundaries and Interactions + +- Depends on: `importlib.util`, `os`, `pathlib`, `views_pipeline_core.managers.model.ModelPathManager` +- Called by: `create_catalogs.py` main block +- Feeds into: `generate_markdown_table()`, `update_readme_with_tables()` + +--- + +## 8. Examples of Correct Usage + +```python +model_class = ModelPathManager("counting_stars", validate=True) +model_dict = extract_models(model_class) +# model_dict = {"name": "counting_stars", "algorithm": "XGBRegressor", ...} +``` + +--- + +## 9. Examples of Incorrect Usage + +```python +# Wrong: calling with a path string instead of a PathManager +model_dict = extract_models("models/counting_stars") # TypeError + +# Wrong: expecting runtime validation of config values +# extract_models does not check if deployment_status is valid +``` + +--- + +## 10. Test Alignment + +- `tests/test_catalogs.py::TestNoExecUsage` — validates this function uses importlib, not exec() +- `tests/test_catalogs.py::TestReplaceTableInSection` — validates downstream markdown generation (requires views_pipeline_core) +- No direct test of `extract_models()` return value (requires views_pipeline_core) + +--- + +## 11. Evolution Notes + +- Should add per-model try/except to prevent one broken config from crashing the entire catalog run +- Consider extracting the importlib loading pattern into a shared utility (currently duplicated from `conftest.py:load_config_module()`) + +--- + +## Known Deviations + +- No per-model error isolation — one broken config crashes all catalog generation +- The `tmp_dict` variable was a holdover from the `exec()` pattern and has been removed, but the function still lacks consistent error handling + +--- + +## End of Contract + +This document defines the **intended meaning** of `create_catalogs.extract_models()`. + +Changes to behavior that violate this intent are bugs. +Changes to intent must update this contract. diff --git a/base_docs/CICs/EnsembleScaffoldBuilder.md b/base_docs/CICs/EnsembleScaffoldBuilder.md new file mode 100644 index 00000000..a6a8ae6a --- /dev/null +++ b/base_docs/CICs/EnsembleScaffoldBuilder.md @@ -0,0 +1,108 @@ +# Class Intent Contract: EnsembleScaffoldBuilder + +**Status:** Active +**Owner:** Project maintainers +**Last reviewed:** 2026-03-15 +**Related ADRs:** ADR-001, ADR-002 + +--- + +## 1. Purpose + +> `EnsembleScaffoldBuilder` creates and validates the directory structure and scripts for a new ensemble model. It inherits from `ModelScaffoldBuilder` and overrides script generation to use ensemble-specific templates. + +Located in: `build_ensemble_scaffold.py` + +--- + +## 2. Non-Goals (Explicit Exclusions) + +- Does **not** aggregate model predictions or perform reconciliation +- Does **not** validate that constituent models exist +- Does **not** generate `config_queryset.py` or `config_sweep.py` (ensembles don't use these) + +--- + +## 3. Responsibilities and Guarantees + +- Creates an ensemble directory at the path determined by `EnsemblePathManager` +- Inherits directory creation and assessment from `ModelScaffoldBuilder` +- Generates ensemble-specific scripts: `config_deployment.py`, `config_hyperparameters.py`, `config_meta.py`, `main.py`, `run.sh`, `requirements.txt` +- Validates name uniqueness across both models and ensembles + +--- + +## 4. Inputs and Assumptions + +- Ensemble name must pass `ModelPathManager.validate_model_name()` (same convention as models) +- Must not collide with existing model or ensemble names +- `views_pipeline_core` must be installed + +--- + +## 5. Outputs and Side Effects + +- Creates filesystem directory tree under `ensembles/{ensemble_name}/` +- Creates config and script files from ensemble templates + +--- + +## 6. Failure Modes and Loudness + +- Same as `ModelScaffoldBuilder` — inherits `FileExistsError` and `FileNotFoundError` behavior + +--- + +## 7. Boundaries and Interactions + +- Inherits from: `ModelScaffoldBuilder` (`build_model_scaffold.py`) +- Depends on: `views_pipeline_core.managers.ensemble.EnsemblePathManager`, `views_pipeline_core.templates.ensemble.*` +- Must not depend on: individual ensemble code, model code + +--- + +## 8. Examples of Correct Usage + +```python +builder = EnsembleScaffoldBuilder("happy_ensemble") +builder.build_model_directory() +builder.build_model_scripts() +builder.update_gitkeep_empty_directories() +``` + +--- + +## 9. Examples of Incorrect Usage + +```python +# Wrong: using a model name that already exists +builder = EnsembleScaffoldBuilder("purple_alien") # Fails — model already exists +``` + +--- + +## 10. Test Alignment + +- No direct unit tests — relies on structural convention tests +- The output must conform to patterns validated by `tests/test_model_structure.py` (when extended to ensembles) + +--- + +## 11. Evolution Notes + +- Currently uses `self.requirements_path` from parent class, which is set during `build_model_directory()`. This coupling between directory creation and attribute state could be made more explicit. + +--- + +## Known Deviations + +- Inherits the `_model` attribute name from `ModelScaffoldBuilder` even though it represents an ensemble — naming is misleading + +--- + +## End of Contract + +This document defines the **intended meaning** of `EnsembleScaffoldBuilder`. + +Changes to behavior that violate this intent are bugs. +Changes to intent must update this contract. diff --git a/base_docs/CICs/ModelScaffoldBuilder.md b/base_docs/CICs/ModelScaffoldBuilder.md new file mode 100644 index 00000000..203c8273 --- /dev/null +++ b/base_docs/CICs/ModelScaffoldBuilder.md @@ -0,0 +1,127 @@ +# Class Intent Contract: ModelScaffoldBuilder + +**Status:** Active +**Owner:** Project maintainers +**Last reviewed:** 2026-03-15 +**Related ADRs:** ADR-001, ADR-002, ADR-009 + +--- + +## 1. Purpose + +> `ModelScaffoldBuilder` creates and validates the directory structure and configuration scripts for a new forecasting model. It ensures that new models conform to the repository's structural conventions. + +Located in: `build_model_scaffold.py` + +--- + +## 2. Non-Goals (Explicit Exclusions) + +- Does **not** train models or run inference +- Does **not** validate hyperparameter values (only creates the files) +- Does **not** modify existing model directories (only creates new ones) +- Does **not** manage conda environments or dependencies + +--- + +## 3. Responsibilities and Guarantees + +- Creates a model directory at the path determined by `ModelPathManager` +- Creates all required subdirectories (`configs/`, `data/`, `artifacts/`, etc.) +- Generates all required config files from templates: `config_meta.py`, `config_deployment.py`, `config_hyperparameters.py`, `config_queryset.py`, `config_sweep.py`, `config_partitions.py` +- Generates `main.py` and `run.sh` from templates +- Creates `README.md` with model name and creation date +- Assesses directory completeness via `assess_model_directory()` +- Assesses script completeness via `assess_model_scripts()` +- Manages `.gitkeep` files in empty directories + +--- + +## 4. Inputs and Assumptions + +- Model name must pass `ModelPathManager.validate_model_name()` (lowercase `adjective_noun` format) +- Model name must not already exist as a model or ensemble directory +- User must interactively provide: model algorithm name, architecture package name +- `views_pipeline_core` must be installed (for templates and path management) + +--- + +## 5. Outputs and Side Effects + +- Creates filesystem directory tree under `models/{model_name}/` +- Creates 8+ files from templates +- Fetches latest package release version from GitHub (network call) +- Prints reminder to update queryset file + +--- + +## 6. Failure Modes and Loudness + +- Raises `FileExistsError` if model directory already exists (in `build_model_directory()`) +- Raises `FileNotFoundError` if `build_model_scripts()` is called before `build_model_directory()` +- Logs errors for failed subdirectory creation but continues (partial creation possible) +- Invalid package name causes retry loop via user input + +--- + +## 7. Boundaries and Interactions + +- Depends on: `views_pipeline_core.managers.model.ModelPathManager`, `views_pipeline_core.templates.model.*` +- Used by: `build_ensemble_scaffold.py` (via inheritance) +- Must not depend on: individual model code, algorithm packages + +--- + +## 8. Examples of Correct Usage + +```python +builder = ModelScaffoldBuilder("happy_kitten") +builder.build_model_directory() +builder.build_model_scripts() +builder.update_gitkeep_empty_directories() +``` + +--- + +## 9. Examples of Incorrect Usage + +```python +# Wrong: calling build_model_scripts before build_model_directory +builder = ModelScaffoldBuilder("happy_kitten") +builder.build_model_scripts() # Raises FileNotFoundError + +# Wrong: using a name that doesn't match convention +builder = ModelScaffoldBuilder("HappyKitten") # Fails validation +``` + +--- + +## 10. Test Alignment + +- `tests/test_model_structure.py` validates the output conventions this builder must produce +- No direct unit tests for `ModelScaffoldBuilder` itself (relies on convention tests) +- Green tests: file existence, naming convention +- Beige tests: required config files present + +--- + +## 11. Evolution Notes + +- The builder currently prompts for user input interactively. A non-interactive mode would improve CI/scripting use cases. +- Template generation logic lives in `views_pipeline_core`, not in this repo. Changes to templates require coordinating across repos. + +--- + +## Known Deviations + +- `requirements.txt` creation is commented out (lines 134-139 in `build_model_scaffold.py`) — it's generated by `template_requirement_txt.generate()` instead +- Assessment methods return dicts, not structured types — no type enforcement on the return value + +--- + +## End of Contract + +This document defines the **intended meaning** of `ModelScaffoldBuilder`. + +Changes to behavior that violate this intent are bugs. +Changes to intent must update this contract. diff --git a/base_docs/CICs/README.md b/base_docs/CICs/README.md new file mode 100644 index 00000000..6e7c56ac --- /dev/null +++ b/base_docs/CICs/README.md @@ -0,0 +1,30 @@ +# Class Intent Contracts README + +This directory contains **Intent Contracts** as defined in ADR-006. + +An Intent Contract is a human-readable, unambiguous declaration of: + +- what a non-trivial class or module is meant to do, +- what it must never do, +- its invariants, +- and its failure semantics. + +--- + +## Active Contracts + +- `ModelScaffoldBuilder.md` +- `EnsembleScaffoldBuilder.md` +- `CatalogExtractor.md` + +--- + +## Governance Relationship + +Intent Contracts are governed by: + +- ADR-006 (Intent Contracts for Non-Trivial Classes) +- ADR-003 (Authority of Declarations) +- ADR-005 (Testing Doctrine) + +If a class changes meaning, its Intent Contract must be updated. diff --git a/base_docs/CICs/cic_template.md b/base_docs/CICs/cic_template.md new file mode 100644 index 00000000..9f091b26 --- /dev/null +++ b/base_docs/CICs/cic_template.md @@ -0,0 +1,61 @@ +# Class Intent Contract: + +**Status:** Draft | Active | Superseded +**Owner:** +**Last reviewed:** YYYY-MM-DD +**Related ADRs:** + +--- + +## 1. Purpose + +> **What is this class for?** + +--- + +## 2. Non-Goals (Explicit Exclusions) + +--- + +## 3. Responsibilities and Guarantees + +--- + +## 4. Inputs and Assumptions + +--- + +## 5. Outputs and Side Effects + +--- + +## 6. Failure Modes and Loudness + +--- + +## 7. Boundaries and Interactions + +--- + +## 8. Examples of Correct Usage + +--- + +## 9. Examples of Incorrect Usage + +--- + +## 10. Test Alignment + +--- + +## 11. Evolution Notes (Optional) + +--- + +## End of Contract + +This document defines the **intended meaning** of ``. + +Changes to behavior that violate this intent are bugs. +Changes to intent must update this contract. diff --git a/base_docs/INSTANTIATION_CHECKLIST.md b/base_docs/INSTANTIATION_CHECKLIST.md new file mode 100644 index 00000000..72d2d515 --- /dev/null +++ b/base_docs/INSTANTIATION_CHECKLIST.md @@ -0,0 +1,57 @@ +# Instantiation Checklist + +Completed during initial adoption of base_docs for views-models. + +--- + +## ADR Adaptation + +### All adopted ADRs +- [x] Update Status from template to Accepted +- [x] Fill in Date (2026-03-15), Deciders (Simon), Informed (All contributors) + +### Per-ADR adaptation notes +- [x] **ADR-000:** Path set to `base_docs/ADRs/` +- [x] **ADR-001:** Ontology categories mapped to views-models entities (models, configs, ensembles, tooling, etc.) +- [x] **ADR-002:** Dependency direction defined (models → external packages, configs → common/) +- [x] **ADR-003:** Fail-loud examples adapted to config validation and partition consistency +- [x] **ADR-004:** Deferred (evolution/stability rules not yet needed) +- [x] **ADR-005:** Testing taxonomy mapped to existing test suite (green/beige categories) +- [x] **ADR-006:** Intent contracts identified for 4 entities +- [x] **ADR-007:** Silicon agent constraints adapted for bulk config operations +- [x] **ADR-008:** Observability grounded in current logging patterns and known deviations +- [x] **ADR-009:** Boundary contracts mapped to config validation tests + +--- + +## CICs + +- [x] Replace placeholder active contracts list in `CICs/README.md` +- [x] Create `ModelScaffoldBuilder.md` +- [x] Create `EnsembleScaffoldBuilder.md` +- [x] Create `CommonPartitions.md` +- [x] Create `CatalogExtractor.md` + +--- + +## Contributor Protocols + +- [x] Adapt `carbon_based_agents.md` for views-models team and conventions +- [x] Adapt `silicon_based_agents.md` for bulk config operations and TDD workflow +- [x] Adapt hardened protocol for ML forecasting domain (partition integrity, ensemble ordering) + +--- + +## Standards + +- [x] Adapt `logging_and_observability_standard.md` to current patterns and known deviations +- [ ] Physical architecture standard — skipped (not applicable to this repo's architecture) + +--- + +## Final Verification + +- [x] No files still have Status `--template--` +- [x] No phantom references to non-existent files +- [x] All cross-ADR references resolve correctly +- [ ] Run `validate_docs.sh` to check internal consistency diff --git a/base_docs/contributor_protocols/carbon_based_agents.md b/base_docs/contributor_protocols/carbon_based_agents.md new file mode 100644 index 00000000..7fe66111 --- /dev/null +++ b/base_docs/contributor_protocols/carbon_based_agents.md @@ -0,0 +1,107 @@ + +# Carbon-Based Agent Protocol +*(For contributors composed primarily of carbon, caffeine, and responsibility)* + +**Status:** Active +**Applies to:** All human contributors +**Authority:** ADR-000 through ADR-009 + +--- + +## Purpose + +This protocol defines the responsibilities, expectations, and obligations +of **carbon-based agents** contributing to the views-models repository. + +Carbon-based agents are entrusted with: +- intent, +- judgment, +- and architectural authority. + +--- + +## Core Principle: Stewardship of Intent + +Carbon-based agents are **stewards of intent**, not merely authors of code. + +In views-models, stewardship means: +- preserving config conventions across ~66 models, +- enforcing partition boundary consistency, +- preventing silent divergence of model configurations, +- and ensuring new models follow established patterns. + +**Intent must not drift silently.** + +--- + +## Ownership of Intent and Semantics + +Carbon-based agents: +- own system intent and meaning, +- declare semantics explicitly in config files (ADR-003), +- and are accountable for their correctness. + +If a change alters config conventions: +- the relevant tests must be updated, +- an ADR must be written for significant changes, or +- the change must not be merged. + +--- + +## Fail-Loud Is a Moral Obligation + +Silent failure is unacceptable. + +Introducing: +- implicit defaults for missing config keys, +- fallback logic that hides config errors, +- or partition boundaries that differ from `common/partitions.py` + +is considered a defect, even if the model runs successfully. + +--- + +## Testing Is Part of the Change + +A change is incomplete if it: +- cannot be validated by `pytest tests/`, +- weakens existing config validation tests, +- or introduces a new convention without a corresponding test. + +When adding a new required config key: +1. Add it to `REQUIRED_META_KEYS` or `REQUIRED_HP_KEYS` in `tests/test_config_completeness.py` +2. Add the key to all existing models +3. Verify with `pytest tests/ -v` + +--- + +## Interaction with Silicon-Based Agents + +Using silicon-based agents does **not** reduce responsibility. + +When carbon-based agents use silicon-based agents for bulk changes, they must: +- verify the changes against a known-good reference, +- run the full test suite, +- check for template artifacts (e.g., `{{e}}` double-brace bugs), +- and take full responsibility for the result. + +--- + +## Non-Negotiable Expectations + +Carbon-based agents must not: +- merge changes they do not understand, +- add models that skip required config keys, +- bypass tests under time pressure, +- introduce model-specific partition boundaries, or +- use the old CLI pattern (`parse_args` + `wandb.login()`). + +--- + +## Final Note + +Carbon-based agents are the **last line of defense**. + +This protocol exists to ensure that, +even under pressure, +**the model zoo continues to mean what we think it means**. diff --git a/base_docs/contributor_protocols/hardened_protocol.md b/base_docs/contributor_protocols/hardened_protocol.md new file mode 100644 index 00000000..6237b71c --- /dev/null +++ b/base_docs/contributor_protocols/hardened_protocol.md @@ -0,0 +1,77 @@ +# The Hardened Protocol: ML Forecasting Governance + +This document defines mandatory engineering and scientific standards for the views-models repository. Adherence is required for all contributions to guarantee forecast integrity and reproducibility. + +--- + +## 1. Core Principles + +### A. The Authority of Declarations (ADR-003) +**"Never infer; only trust declarations."** +All meaningful model properties (algorithm, level, targets, partition boundaries) must be explicitly declared in config files. +- **Prohibited:** Inferring model level from queryset name, algorithm from import path, or partition boundaries from model type. +- **Requirement:** If a property affects model identity or evaluation, it must be a mandatory key in the config files. + +### B. The Fail-Loud Mandate (ADR-008) +**"A crash is a successful defense of scientific integrity."** +Silent failures, implicit fallbacks, and "best-effort" corrections are forbidden. +- **Requirement:** Missing config keys must cause test failures, not silent defaults. +- **Prohibited:** Using `warnings.filterwarnings("ignore")` to hide config-level problems (ML library warnings are acceptable to suppress). + +### C. Partition Integrity +**"All models evaluate on the same temporal windows."** +Partition boundaries are defined once in `common/partitions.py` and shared by all models. +- **Requirement:** Every model's `config_partitions.py` must be a one-line import from `common.partitions`. +- **Prohibited:** Model-specific partition overrides, custom forecasting offsets, or hardcoded boundary values in individual model configs. +- **Rationale:** Divergent partitions make model evaluation metrics incomparable. + +### D. Ensemble Ordering Discipline +**"CM before PGM."** +Country-month ensembles must complete before priogrid-month ensembles to enable reconciliation. +- **Requirement:** PGM ensembles declare `reconcile_with` referencing their CM counterpart. +- **Current enforcement:** Documented in `docs/monthly_run_guide.md` (not yet programmatic). + +--- + +## 2. Contributor Requirements + +### Adding a New Model +1. **Use the scaffold:** Run `python build_model_scaffold.py` to create the directory structure. +2. **Complete all configs:** Fill in all 6 config files with actual values — do not leave template defaults. +3. **Verify conventions:** Run `pytest tests/ -v` to confirm the new model passes all convention tests. +4. **Delegate partitions:** Ensure `config_partitions.py` contains only `from common.partitions import generate`. + +### Adding a New Required Config Key +1. **Add to test expectations:** Update `REQUIRED_META_KEYS` or `REQUIRED_HP_KEYS` in `tests/test_config_completeness.py`. +2. **Add to all models:** Add the key with the correct value to all ~66 models. +3. **Verify:** `pytest tests/ -v` — all models must pass. +4. **Document:** Write an ADR if the key represents a significant architectural decision. + +--- + +## 3. Mandatory Testing Taxonomy (ADR-005) + +### Green Team (Correctness) +- Config files contain all required keys with valid values +- `time_steps` matches `len(steps)` where applicable +- Catalog generation uses `importlib`, not `exec()` + +### Beige Team (Convention Drift) +- All models follow naming convention `^[a-z]+_[a-z]+$` +- All models have required files (`main.py`, `run.sh`, 6 config files) +- All models use the new CLI pattern (`ForecastingModelArgs`) +- All models delegate partitions to `common.partitions` + +### Red Team (Adversarial) — Future Work +- Invalid config values don't propagate silently +- Broken queryset files don't crash catalog generation for other models +- Ensemble dependency chains are validated + +--- + +## 4. Operational Invariants + +- **Partition Centralization:** All models share `common/partitions.py`. No exceptions. +- **CLI Uniformity:** All models use `ForecastingModelArgs.parse_args()`. No explicit `wandb.login()`. +- **Config Completeness:** All required keys are present. Tests enforce this. +- **Catalog Safety:** Config loading uses `importlib.util`, not `exec()`. diff --git a/base_docs/contributor_protocols/silicon_based_agents.md b/base_docs/contributor_protocols/silicon_based_agents.md new file mode 100644 index 00000000..2ce1fa26 --- /dev/null +++ b/base_docs/contributor_protocols/silicon_based_agents.md @@ -0,0 +1,114 @@ + +# Silicon-Based Agent Protocol +*(For contributors composed primarily of silicon, statistics, and confidence)* + +**Status:** Active +**Applies to:** All automated or AI-assisted code modification +**Authority:** ADR-007 (Silicon-Based Agents as Untrusted Contributors) + +--- + +## Purpose + +This document defines **mandatory operational constraints** under which +**silicon-based agents** (e.g., Claude Code, LLM assistants, code generators) +may interact with the views-models repository. + +This protocol exists to prevent: +- silent config corruption across 66+ models, +- architectural erosion of shared conventions, +- template artifacts (double-braces, escaped characters), +- and hard-to-detect partial failures in bulk operations. + +--- + +## Threat Model + +Silicon-based agents are assumed to: +- optimize for local plausibility, not global correctness, +- produce template artifacts when generating code via f-strings, +- silently truncate files during full-file rewrites, +- collapse abstractions for convenience, +- and produce outputs that *look valid* while being semantically incomplete. + +Silicon-based agents are therefore treated as **untrusted contributors**. + +--- + +## Allowed Operations + +Silicon-based agents **may**: +- Add required config keys to models (bulk operations with verification) +- Migrate `main.py` files to new CLI patterns +- Replace partition configs with shared module imports +- Add or update tests that reflect declared intent +- Create new documentation (ADRs, CICs) +- Perform scoped refactors within a single file + +All allowed operations remain subject to carbon-based agent review. + +--- + +## Forbidden Operations + +Silicon-based agents **must not**: +- Change partition boundaries without explicit authorization +- Modify model hyperparameters or querysets +- Infer model algorithm from directory names or import statements +- Remove config validation tests or weaken test assertions +- Introduce `exec()` or `eval()` for config loading +- Cross model boundaries (modifying model A based on model B) +- Perform full-file rewrites without reading first + +If a silicon-based agent cannot proceed without guessing, it must stop. + +--- + +## Mandatory Safety: Anti-Truncation Rule + +When modifying existing files: +1. **Read the file first** using the Read tool +2. **Apply targeted edits** to specific locations +3. **Never use full-file Write on existing files** unless explicitly confirmed + +When generating files from templates: +1. **Verify against a reference file** after generation +2. **Check for template artifacts** (double-braces `{{`, escaped characters) +3. **Run tests** to verify correctness + +--- + +## Mandatory Safety: Bulk Operations + +When modifying multiple models (bulk changes): +1. Write and run tests FIRST (TDD approach, ADR-005) +2. Apply changes via a script, not manual file-by-file edits +3. Verify with `pytest tests/ -v` after all changes +4. Spot-check 2-3 files manually against a known-good reference + +--- + +## Required Artifacts + +Every silicon-based agent-assisted change must include: +- Summary of what was changed and how many files were affected +- Reference to relevant ADRs +- Test results showing all tests pass +- Explicit note of any template-generated code that needs verification + +--- + +## Enforcement + +- Violations are treated as violations by the carbon-based agent who approved them +- Changes may be blocked solely on protocol grounds +- `pytest tests/` is the minimum verification bar + +--- + +## Final Note + +Silicon-based agents are tools, not collaborators. + +This protocol exists to ensure that +**automation never outruns understanding**. diff --git a/base_docs/standards/logging_and_observability_standard.md b/base_docs/standards/logging_and_observability_standard.md new file mode 100644 index 00000000..dfa37b65 --- /dev/null +++ b/base_docs/standards/logging_and_observability_standard.md @@ -0,0 +1,106 @@ +# Logging & Observability Standard + +**Status:** Active +**Governing ADRs:** ADR-003 (Authority of Declarations), ADR-005 (Testing), ADR-008 (Observability) + +--- + +## 1. Purpose + +This document defines operational standards for logging behavior in views-models. + +views-models is a thin orchestration layer — most ML logic lives in external packages. +Logging in this repo covers: +- Config loading and validation +- Model launcher initialization +- Scaffold building +- Catalog generation +- CI/CD pipeline output + +--- + +## 2. Core Principles + +### 2.1 Fail Loud and Persist + +- Config loading failures must raise exceptions (not log and continue) +- `ModelPathManager` initialization failures must raise `RuntimeError` with context +- Missing config keys are caught by tests, not by runtime logging + +### 2.2 Logs Must Support Understanding + +Logs must: +- identify which model is being processed +- provide sufficient context to reconstruct errors +- include the operation stage (scaffold building, catalog generation, model run) + +--- + +## 3. Log Levels + +### ERROR +- Config file cannot be loaded (syntax error, missing function) +- Model directory creation fails +- Catalog generation encounters a broken config + +### WARNING +- A model directory already exists during scaffolding +- A subdirectory already exists (skipped, not recreated) + +### INFO +- Model directory created successfully +- Config script generated +- Catalog generation completed + +### DEBUG +- Not used in current codebase + +--- + +## 4. Current Patterns + +### Scaffold builders (`build_model_scaffold.py`, `build_ensemble_scaffold.py`) +```python +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) +``` +Uses `logging.info()`, `logging.error()` for directory operations. + +### Catalog generation (`create_catalogs.py`) +```python +logging.basicConfig(level=logging.ERROR, ...) +logger = logging.getLogger(__name__) +``` +Uses `logging.info()` for config discovery. + +### Model launchers (`models/*/main.py`) +No logging — wraps `ModelPathManager` in try/except and raises `RuntimeError`. +External package managers handle their own logging. + +--- + +## 5. Known Deviations + +- `create_catalogs.py` sets logging to `ERROR` level, which suppresses its own `INFO` messages about config discovery. This is effectively silent unless something breaks. +- Model launchers use `warnings.filterwarnings("ignore")` broadly — this suppresses all Python warnings, including potentially useful ML library warnings. +- No structured logging (JSON/key-value) is used anywhere. +- No alerting is configured for CI failures. + +--- + +## 6. Anti-Patterns (Prohibited) + +- Swallowing exceptions without logging +- Logging and continuing after an invariant violation (ADR-003) +- Using `print()` for structural diagnostics in tooling scripts +- Downgrading errors to warnings to "keep catalog generation running" + +--- + +## 7. Evolution + +If logging becomes more important (e.g., operational monitoring of monthly runs), +consider: +- Adding a `LoggingManager` wrapper (already available in `views_pipeline_core`) +- Structured logging for catalog generation +- CI notification on catalog generation failures diff --git a/base_docs/validate_docs.sh b/base_docs/validate_docs.sh new file mode 100755 index 00000000..1347160c --- /dev/null +++ b/base_docs/validate_docs.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# Validates internal consistency of base_docs documentation set. +# Exit 0 if clean, exit 1 if issues found. + +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" + +errors=0 + +echo "=== base_docs validation ===" +echo "" + +# 1. Check for unfilled template placeholders in accepted/active files +echo "--- Checking for template placeholders in accepted/active files ---" +warnings=0 +while IFS= read -r file; do + [[ -z "$file" ]] && continue + [[ "$file" == *template* ]] && continue + if grep -q 'YYYY-MM-DD' "$file"; then + echo " WARN: Unfilled date placeholder in $file" + warnings=$((warnings + 1)) + fi + if grep -q '' "$file"; then + echo " WARN: Unfilled deciders placeholder in $file" + warnings=$((warnings + 1)) + fi + if grep -q '' "$file"; then + echo " WARN: Unfilled ClassName placeholder in $file" + warnings=$((warnings + 1)) + fi +done < <(grep -rl 'Status:.*\(Accepted\|Active\)' --include='*.md' . 2>/dev/null || true) +if [ "$warnings" -eq 0 ]; then + echo " OK" +fi + +# 2. Verify CIC active contracts exist +echo "--- Checking CIC active contract references ---" +if [ -f "CICs/README.md" ]; then + while IFS= read -r line; do + [[ -z "$line" ]] && continue + contract=$(echo "$line" | sed -n 's/^- `\(.*\.md\)`.*$/\1/p') + if [ -n "$contract" ] && [ ! -f "CICs/$contract" ]; then + echo " ERROR: CIC contract listed but missing: CICs/$contract" + errors=$((errors + 1)) + fi + done < <(grep -E '^- `[A-Z].*\.md`' CICs/README.md 2>/dev/null | grep -v '>' || true) +fi + +# 3. Cross-ADR reference integrity +echo "--- Checking cross-ADR references (constitutional: 000-009) ---" +while IFS= read -r ref; do + [[ -z "$ref" ]] && continue + file=$(echo "$ref" | cut -d: -f1) + adr_num=$(echo "$ref" | grep -oP 'ADR-00\K[0-9]' | head -1) + if [ -n "$adr_num" ]; then + # Skip ADR-004 (intentionally deferred) + if [ "$adr_num" = "4" ]; then + continue + fi + match_count=$(find ADRs -name "00${adr_num}_*.md" 2>/dev/null | wc -l) + if [ "$match_count" -eq 0 ]; then + echo " ERROR: $file references ADR-00${adr_num} but no matching file found" + errors=$((errors + 1)) + fi + fi +done < <(grep -rn 'ADR-00[0-9]' --include='*.md' . 2>/dev/null || true) + +# 4. Check that referenced protocol files exist +echo "--- Checking protocol file references ---" +while IFS= read -r ref; do + [[ -z "$ref" ]] && continue + file=$(echo "$ref" | cut -d: -f1) + proto=$(echo "$ref" | grep -oP 'contributor_protocols/[a-z_]+\.md' | head -1) + if [ -n "$proto" ] && [ ! -f "$proto" ]; then + echo " ERROR: $file references $proto but file does not exist" + errors=$((errors + 1)) + fi +done < <(grep -rn 'contributor_protocols/' --include='*.md' . 2>/dev/null || true) + +# 5. Report template status markers +echo "--- Checking template status markers ---" +template_count=$(grep -rl '\-\-template\-\-' --include='*.md' . 2>/dev/null | wc -l) +echo " INFO: $template_count files still have --template-- status" + +echo "" +if [ "$errors" -gt 0 ]; then + echo "=== FAILED: $errors issue(s) found ===" + exit 1 +else + echo "=== PASSED: no issues found ===" + exit 0 +fi diff --git a/create_catalogs.py b/create_catalogs.py index 94ed491b..9bab72a8 100644 --- a/create_catalogs.py +++ b/create_catalogs.py @@ -1,4 +1,5 @@ import os +import importlib.util import logging logging.basicConfig( level=logging.ERROR, format="%(asctime)s %(name)s - %(levelname)s - %(message)s" @@ -38,7 +39,6 @@ def extract_models(model_class): """ model_dict = {} - tmp_dict = {} config_meta = os.path.join(model_class.configs, 'config_meta.py') config_deployment = os.path.join(model_class.configs, 'config_deployment.py') config_hyperparameters = os.path.join(model_class.configs, 'config_hyperparameters.py') @@ -46,19 +46,19 @@ def extract_models(model_class): if os.path.exists(config_meta): logging.info(f"Found meta config: {config_meta}") - with open(config_meta, 'r') as file: - code = file.read() - exec(code, {}, tmp_dict) - model_dict.update(tmp_dict['get_meta_config']()) + spec = importlib.util.spec_from_file_location("config_meta", config_meta) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + model_dict.update(module.get_meta_config()) model_dict['queryset'] = create_link(model_dict['queryset'], model_class.queryset_path) if 'queryset' in model_dict else 'None' if os.path.exists(config_deployment): logging.info(f"Found deployment config: {config_deployment}") - with open(config_deployment, 'r') as file: - code = file.read() - exec(code, {}, tmp_dict) - model_dict.update(tmp_dict['get_deployment_config']()) + spec = importlib.util.spec_from_file_location("config_deployment", config_deployment) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + model_dict.update(module.get_deployment_config()) if os.path.exists(config_hyperparameters): logging.info(f"Found hyperparameters config: {config_hyperparameters}") @@ -192,9 +192,6 @@ def replace_table_in_section(content, section_name, new_table): if __name__ == "__main__": - #import time - #start_time = time.time() - models_list_cm = [] models_list_pgm = [] ensemble_list = [] @@ -224,20 +221,6 @@ def replace_table_in_section(content, section_name, new_table): - # markdown_table_pgm = generate_markdown_table(models_list_pgm) - # with open('pgm_model_catalog.md', 'w') as f: - # f.write(markdown_table_pgm) - - # markdown_table_cm = generate_markdown_table(models_list_cm) - # with open('cm_model_catalog.md', 'w') as f: - # f.write(markdown_table_cm) - - # markdown_table_ensembles = generate_markdown_table(ensemble_list) - # with open('ensembles_catalog.md', 'w') as f: - # f.write(markdown_table_ensembles) - - - markdown_table_cm = generate_markdown_table(models_list_cm) @@ -252,6 +235,3 @@ def replace_table_in_section(content, section_name, new_table): markdown_table_ensembles, ) - - #print("--- %s seconds ---" % (time.time() - start_time)) - diff --git a/models/adolecent_slob/configs/config_hyperparameters.py b/models/adolecent_slob/configs/config_hyperparameters.py index 0077e2ef..2c75534b 100644 --- a/models/adolecent_slob/configs/config_hyperparameters.py +++ b/models/adolecent_slob/configs/config_hyperparameters.py @@ -11,6 +11,7 @@ def get_hp_config(): hyperparameters = { # --- Forecast horizon --- 'steps': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36], + "time_steps": 36, # --- Input / output structure --- 'input_chunk_length': 48, diff --git a/models/adolecent_slob/configs/config_meta.py b/models/adolecent_slob/configs/config_meta.py index 7f2c6acf..cc2e7cc0 100644 --- a/models/adolecent_slob/configs/config_meta.py +++ b/models/adolecent_slob/configs/config_meta.py @@ -17,5 +17,6 @@ def get_meta_config(): "creator": "Simon", "prediction_format": "dataframe", "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "rolling_origin_stride": 1, } return meta_config diff --git a/models/average_cmbaseline/main.py b/models/average_cmbaseline/main.py index 7770d7e9..8ef12185 100644 --- a/models/average_cmbaseline/main.py +++ b/models/average_cmbaseline/main.py @@ -1,8 +1,7 @@ -import wandb import warnings from pathlib import Path -from views_pipeline_core.cli.utils import parse_args, validate_arguments -from views_pipeline_core.managers.model import ModelPathManager +from views_pipeline_core.cli import ForecastingModelArgs +from views_pipeline_core.managers import ModelPathManager from views_baseline.manager.baseline_manager import BaselineForecastingModelManager warnings.filterwarnings("ignore") @@ -13,17 +12,15 @@ raise RuntimeError(f"Unexpected error: {e}. Check the logs for details.") if __name__ == "__main__": - wandb.login() - args = parse_args() - validate_arguments(args) + args = ForecastingModelArgs.parse_args() + + manager = BaselineForecastingModelManager( + model_path=model_path, + wandb_notifications=args.wandb_notifications, + use_prediction_store=args.prediction_store, + ) if args.sweep: - BaselineForecastingModelManager( - model_path=model_path, - ).execute_sweep_run(args) + manager.execute_sweep_run(args) else: - BaselineForecastingModelManager( - model_path=model_path, - wandb_notifications=args.wandb_notifications, - use_prediction_store=args.prediction_store, - ).execute_single_run(args) + manager.execute_single_run(args) diff --git a/models/average_pgmbaseline/main.py b/models/average_pgmbaseline/main.py index 7770d7e9..8ef12185 100644 --- a/models/average_pgmbaseline/main.py +++ b/models/average_pgmbaseline/main.py @@ -1,8 +1,7 @@ -import wandb import warnings from pathlib import Path -from views_pipeline_core.cli.utils import parse_args, validate_arguments -from views_pipeline_core.managers.model import ModelPathManager +from views_pipeline_core.cli import ForecastingModelArgs +from views_pipeline_core.managers import ModelPathManager from views_baseline.manager.baseline_manager import BaselineForecastingModelManager warnings.filterwarnings("ignore") @@ -13,17 +12,15 @@ raise RuntimeError(f"Unexpected error: {e}. Check the logs for details.") if __name__ == "__main__": - wandb.login() - args = parse_args() - validate_arguments(args) + args = ForecastingModelArgs.parse_args() + + manager = BaselineForecastingModelManager( + model_path=model_path, + wandb_notifications=args.wandb_notifications, + use_prediction_store=args.prediction_store, + ) if args.sweep: - BaselineForecastingModelManager( - model_path=model_path, - ).execute_sweep_run(args) + manager.execute_sweep_run(args) else: - BaselineForecastingModelManager( - model_path=model_path, - wandb_notifications=args.wandb_notifications, - use_prediction_store=args.prediction_store, - ).execute_single_run(args) + manager.execute_single_run(args) diff --git a/models/bad_blood/configs/config_hyperparameters.py b/models/bad_blood/configs/config_hyperparameters.py index d7cad484..aff5e150 100644 --- a/models/bad_blood/configs/config_hyperparameters.py +++ b/models/bad_blood/configs/config_hyperparameters.py @@ -10,6 +10,7 @@ def get_hp_config(): hyperparameters = { 'steps': [*range(1, 36 + 1, 1)], + "time_steps": 36, 'parameters': { 'n_estimators': 200, } diff --git a/models/bad_blood/configs/config_meta.py b/models/bad_blood/configs/config_meta.py index 4680646d..8e6abf2b 100644 --- a/models/bad_blood/configs/config_meta.py +++ b/models/bad_blood/configs/config_meta.py @@ -16,5 +16,6 @@ def get_meta_config(): "level": "pgm", "creator": "Xiaolong", "prediction_format": "dataframe", + "rolling_origin_stride": 1, } return meta_config diff --git a/models/bittersweet_symphony/configs/config_hyperparameters.py b/models/bittersweet_symphony/configs/config_hyperparameters.py index 5bd4d6e9..92df168c 100644 --- a/models/bittersweet_symphony/configs/config_hyperparameters.py +++ b/models/bittersweet_symphony/configs/config_hyperparameters.py @@ -10,6 +10,7 @@ def get_hp_config(): hyperparameters = { "steps": [*range(1, 36 + 1, 1)], + "time_steps": 36, "parameters": { "n_estimators": 100, "n_jobs": 12, diff --git a/models/bittersweet_symphony/configs/config_meta.py b/models/bittersweet_symphony/configs/config_meta.py index fad0294f..ed05e0a9 100644 --- a/models/bittersweet_symphony/configs/config_meta.py +++ b/models/bittersweet_symphony/configs/config_meta.py @@ -16,5 +16,6 @@ def get_meta_config(): "level": "cm", "creator": "Marina", "prediction_format": "dataframe", + "rolling_origin_stride": 1, } return meta_config diff --git a/models/black_ranger/configs/config_hyperparameters.py b/models/black_ranger/configs/config_hyperparameters.py new file mode 100644 index 00000000..ab66b33c --- /dev/null +++ b/models/black_ranger/configs/config_hyperparameters.py @@ -0,0 +1,18 @@ + +def get_hp_config(): + """ + Contains the hyperparameter configurations for model training. + This configuration is "operational" so modifying these settings will impact the model's behavior during the training. + + Returns: + - hyperparameters (dict): A dictionary containing hyperparameters for training the model, which determine the model's behavior during the training phase. + """ + + hyperparameters = { + 'steps': [*range(1, 36 + 1, 1)], + 'time_steps': 36, + 'window_months': 18, + 'lambda_mix': 0.05, + 'n_samples': 256, + } + return hyperparameters diff --git a/models/black_ranger/configs/config_meta.py b/models/black_ranger/configs/config_meta.py new file mode 100644 index 00000000..0616399f --- /dev/null +++ b/models/black_ranger/configs/config_meta.py @@ -0,0 +1,20 @@ +def get_meta_config(): + """ + Contains the meta data for the model (model algorithm, name, target variable, and level of analysis). + This config is for documentation purposes only, and modifying it will not affect the model, the training, or the evaluation. + + Returns: + - meta_config (dict): A dictionary containing model meta configuration. + """ + + meta_config = { + "name": "black_ranger", + "algorithm": "MixtureBaseline", + "regression_targets": ["lr_os_best"], + "level": "pgm", + "creator": "Simon", + "prediction_format": "prediction_frame", + "rolling_origin_stride": 1, + "regression_sample_metrics": ["twCRPS", "QIS", "MIS", "MCR_sample"], + } + return meta_config diff --git a/models/blank_space/configs/config_hyperparameters.py b/models/blank_space/configs/config_hyperparameters.py index 45306450..dbebe94a 100644 --- a/models/blank_space/configs/config_hyperparameters.py +++ b/models/blank_space/configs/config_hyperparameters.py @@ -1,6 +1,7 @@ def get_hp_config(): hp_config = { "steps": [*range(1, 36 + 1, 1)], + "time_steps": 36, "parameters": { "clf":{ "n_estimators": 200, diff --git a/models/blank_space/configs/config_meta.py b/models/blank_space/configs/config_meta.py index 4cb0aae5..a3dbc4f5 100644 --- a/models/blank_space/configs/config_meta.py +++ b/models/blank_space/configs/config_meta.py @@ -17,5 +17,6 @@ def get_meta_config(): "level": "pgm", "creator": "Xiaolong", "prediction_format": "dataframe", + "rolling_origin_stride": 1, } return meta_config \ No newline at end of file diff --git a/models/blue_ranger/configs/config_hyperparameters.py b/models/blue_ranger/configs/config_hyperparameters.py new file mode 100644 index 00000000..ab66b33c --- /dev/null +++ b/models/blue_ranger/configs/config_hyperparameters.py @@ -0,0 +1,18 @@ + +def get_hp_config(): + """ + Contains the hyperparameter configurations for model training. + This configuration is "operational" so modifying these settings will impact the model's behavior during the training. + + Returns: + - hyperparameters (dict): A dictionary containing hyperparameters for training the model, which determine the model's behavior during the training phase. + """ + + hyperparameters = { + 'steps': [*range(1, 36 + 1, 1)], + 'time_steps': 36, + 'window_months': 18, + 'lambda_mix': 0.05, + 'n_samples': 256, + } + return hyperparameters diff --git a/models/blue_ranger/configs/config_meta.py b/models/blue_ranger/configs/config_meta.py new file mode 100644 index 00000000..a25d114a --- /dev/null +++ b/models/blue_ranger/configs/config_meta.py @@ -0,0 +1,20 @@ +def get_meta_config(): + """ + Contains the meta data for the model (model algorithm, name, target variable, and level of analysis). + This config is for documentation purposes only, and modifying it will not affect the model, the training, or the evaluation. + + Returns: + - meta_config (dict): A dictionary containing model meta configuration. + """ + + meta_config = { + "name": "blue_ranger", + "algorithm": "MixtureBaseline", + "regression_targets": ["lr_ged_sb"], + "level": "pgm", + "creator": "Simon", + "prediction_format": "prediction_frame", + "rolling_origin_stride": 1, + "regression_sample_metrics": ["twCRPS", "QIS", "MIS", "MCR_sample"], + } + return meta_config diff --git a/models/bouncy_organ/configs/config_hyperparameters.py b/models/bouncy_organ/configs/config_hyperparameters.py index c72452b4..1ab65bf6 100644 --- a/models/bouncy_organ/configs/config_hyperparameters.py +++ b/models/bouncy_organ/configs/config_hyperparameters.py @@ -12,6 +12,7 @@ def get_hp_config(): # --- Forecast horizon --- "steps": list(range(1, 36 + 1)), + "time_steps": 36, # --- Sampling --- "num_samples": 1, diff --git a/models/bouncy_organ/configs/config_meta.py b/models/bouncy_organ/configs/config_meta.py index 34ff8e65..5a31fba3 100644 --- a/models/bouncy_organ/configs/config_meta.py +++ b/models/bouncy_organ/configs/config_meta.py @@ -17,5 +17,6 @@ def get_meta_config(): "creator": "Simon", "prediction_format": "dataframe", "regression_point_metrics": ["RMSLE","MSE", "MSLE", "y_hat_bar"], + "rolling_origin_stride": 1, } return meta_config diff --git a/models/brown_cheese/configs/config_hyperparameters.py b/models/brown_cheese/configs/config_hyperparameters.py index b1eb0a5e..2a27d5d7 100644 --- a/models/brown_cheese/configs/config_hyperparameters.py +++ b/models/brown_cheese/configs/config_hyperparameters.py @@ -10,6 +10,7 @@ def get_hp_config(): hyperparameters = { "steps": [*range(1, 36 + 1, 1)], + "time_steps": 36, "parameters": { "n_estimators": 300, "n_jobs": 12 diff --git a/models/brown_cheese/configs/config_meta.py b/models/brown_cheese/configs/config_meta.py index 4165642e..d8aa548a 100644 --- a/models/brown_cheese/configs/config_meta.py +++ b/models/brown_cheese/configs/config_meta.py @@ -16,5 +16,6 @@ def get_meta_config(): "level": "cm", "creator": "Borbála", "prediction_format": "dataframe", + "rolling_origin_stride": 1, } return meta_config diff --git a/models/car_radio/configs/config_hyperparameters.py b/models/car_radio/configs/config_hyperparameters.py index e8d15390..f41e2609 100644 --- a/models/car_radio/configs/config_hyperparameters.py +++ b/models/car_radio/configs/config_hyperparameters.py @@ -10,6 +10,7 @@ def get_hp_config(): hyperparameters = { "steps": [*range(1, 36 + 1, 1)], + "time_steps": 36, "parameters": { "n_estimators": 80, "n_jobs": 12, diff --git a/models/car_radio/configs/config_meta.py b/models/car_radio/configs/config_meta.py index 7c2182f9..e7e82d7f 100644 --- a/models/car_radio/configs/config_meta.py +++ b/models/car_radio/configs/config_meta.py @@ -16,5 +16,6 @@ def get_meta_config(): "level": "cm", "creator": "Borbála", "prediction_format": "dataframe", + "rolling_origin_stride": 1, } return meta_config diff --git a/models/caring_fish/configs/config_hyperparameters.py b/models/caring_fish/configs/config_hyperparameters.py index b431df6b..bcf89305 100644 --- a/models/caring_fish/configs/config_hyperparameters.py +++ b/models/caring_fish/configs/config_hyperparameters.py @@ -10,6 +10,7 @@ def get_hp_config(): hyperparameters = { 'steps': [*range(1, 36 + 1, 1)], + "time_steps": 36, 'parameters': { 'n_estimators': 200, 'tree_method': 'hist', diff --git a/models/caring_fish/configs/config_meta.py b/models/caring_fish/configs/config_meta.py index 14f1aeb8..be1f2433 100644 --- a/models/caring_fish/configs/config_meta.py +++ b/models/caring_fish/configs/config_meta.py @@ -16,5 +16,6 @@ def get_meta_config(): "level": "pgm", "creator": "Xiaolong", "prediction_format": "dataframe", + "rolling_origin_stride": 1, } return meta_config diff --git a/models/cheap_thrills/configs/config_hyperparameters.py b/models/cheap_thrills/configs/config_hyperparameters.py index 0ffdb568..99191693 100644 --- a/models/cheap_thrills/configs/config_hyperparameters.py +++ b/models/cheap_thrills/configs/config_hyperparameters.py @@ -10,6 +10,7 @@ def get_hp_config(): hyperparameters = { 'steps': [*range(1, 36 + 1, 1)], + "time_steps": 36, 'submodels_to_train': 50, 'pred_samples': 10, 'log_target': False, diff --git a/models/cheap_thrills/configs/config_meta.py b/models/cheap_thrills/configs/config_meta.py index 8d5c5738..87107b2f 100644 --- a/models/cheap_thrills/configs/config_meta.py +++ b/models/cheap_thrills/configs/config_meta.py @@ -18,5 +18,6 @@ def get_meta_config(): "model_clf": "XGBClassifier", "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], "queryset": "structural_brief_nolog", + "rolling_origin_stride": 1, } return meta_config diff --git a/models/chunky_cat/configs/config_hyperparameters.py b/models/chunky_cat/configs/config_hyperparameters.py index d51f740a..888b9a8f 100644 --- a/models/chunky_cat/configs/config_hyperparameters.py +++ b/models/chunky_cat/configs/config_hyperparameters.py @@ -10,6 +10,7 @@ def get_hp_config(): hyperparameters = { 'steps': [*range(1, 36 + 1, 1)], + "time_steps": 36, 'parameters': { "n_estimators": 200, } diff --git a/models/chunky_cat/configs/config_meta.py b/models/chunky_cat/configs/config_meta.py index 25252195..3f5706f9 100644 --- a/models/chunky_cat/configs/config_meta.py +++ b/models/chunky_cat/configs/config_meta.py @@ -16,5 +16,6 @@ def get_meta_config(): "level": "pgm", "creator": "Xiaolong", "prediction_format": "dataframe", + "rolling_origin_stride": 1, } return meta_config diff --git a/models/cool_cat/configs/config_hyperparameters.py b/models/cool_cat/configs/config_hyperparameters.py index a1c5633b..81384551 100644 --- a/models/cool_cat/configs/config_hyperparameters.py +++ b/models/cool_cat/configs/config_hyperparameters.py @@ -10,6 +10,7 @@ def get_hp_config(): hyperparameters = { "steps": [*range(1, 36 + 1, 1)], + "time_steps": 36, "num_samples": 500, "mc_dropout": True, diff --git a/models/cool_cat/configs/config_meta.py b/models/cool_cat/configs/config_meta.py index 3448bc8b..e42dbe0a 100644 --- a/models/cool_cat/configs/config_meta.py +++ b/models/cool_cat/configs/config_meta.py @@ -17,5 +17,6 @@ def get_meta_config(): "creator": "Dylan", "prediction_format": "dataframe", "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], + "rolling_origin_stride": 1, } return meta_config diff --git a/models/counting_stars/configs/config_hyperparameters.py b/models/counting_stars/configs/config_hyperparameters.py index 5bd4d6e9..92df168c 100644 --- a/models/counting_stars/configs/config_hyperparameters.py +++ b/models/counting_stars/configs/config_hyperparameters.py @@ -10,6 +10,7 @@ def get_hp_config(): hyperparameters = { "steps": [*range(1, 36 + 1, 1)], + "time_steps": 36, "parameters": { "n_estimators": 100, "n_jobs": 12, diff --git a/models/counting_stars/configs/config_meta.py b/models/counting_stars/configs/config_meta.py index 2f4aa9e5..ea5aa0a7 100644 --- a/models/counting_stars/configs/config_meta.py +++ b/models/counting_stars/configs/config_meta.py @@ -16,5 +16,6 @@ def get_meta_config(): "level": "cm", "creator": "Borbála", "prediction_format": "dataframe", + "rolling_origin_stride": 1, } return meta_config diff --git a/models/dancing_queen/configs/config_hyperparameters.py b/models/dancing_queen/configs/config_hyperparameters.py index f31c5f0b..1a781a4c 100644 --- a/models/dancing_queen/configs/config_hyperparameters.py +++ b/models/dancing_queen/configs/config_hyperparameters.py @@ -9,6 +9,7 @@ def get_hp_config(): hyperparameters = { "steps": [*range(1, 36 + 1, 1)], + "time_steps": 36, "num_samples": 1, "mc_dropout": True, "activation": "ReLU", diff --git a/models/dancing_queen/configs/config_meta.py b/models/dancing_queen/configs/config_meta.py index 078ff483..97c6f9f0 100644 --- a/models/dancing_queen/configs/config_meta.py +++ b/models/dancing_queen/configs/config_meta.py @@ -17,5 +17,6 @@ def get_meta_config(): "creator": "Dylan", "prediction_format": "dataframe", "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], + "rolling_origin_stride": 1, } return meta_config diff --git a/models/dark_paradise/configs/config_hyperparameters.py b/models/dark_paradise/configs/config_hyperparameters.py index d74cf78e..db723eaa 100644 --- a/models/dark_paradise/configs/config_hyperparameters.py +++ b/models/dark_paradise/configs/config_hyperparameters.py @@ -10,6 +10,7 @@ def get_hp_config(): hyperparameters = { "steps": [*range(1, 36 + 1, 1)], + "time_steps": 36, "parameters": { "clf": { "n_estimators": 200, diff --git a/models/dark_paradise/configs/config_meta.py b/models/dark_paradise/configs/config_meta.py index 96fc1ee9..784cc02a 100644 --- a/models/dark_paradise/configs/config_meta.py +++ b/models/dark_paradise/configs/config_meta.py @@ -18,5 +18,6 @@ def get_meta_config(): "level": "pgm", "creator": "Xiaolong", "prediction_format": "dataframe", + "rolling_origin_stride": 1, } return meta_config diff --git a/models/demon_days/configs/config_hyperparameters.py b/models/demon_days/configs/config_hyperparameters.py index 73900504..4033eec8 100644 --- a/models/demon_days/configs/config_hyperparameters.py +++ b/models/demon_days/configs/config_hyperparameters.py @@ -10,6 +10,7 @@ def get_hp_config(): hyperparameters = { "steps": [*range(1, 36 + 1, 1)], + "time_steps": 36, "parameters": { "n_estimators": 300, "n_jobs": 12, diff --git a/models/demon_days/configs/config_meta.py b/models/demon_days/configs/config_meta.py index 7790b7bd..84a411f2 100644 --- a/models/demon_days/configs/config_meta.py +++ b/models/demon_days/configs/config_meta.py @@ -16,5 +16,6 @@ def get_meta_config(): "level": "cm", "creator": "Marina", "prediction_format": "dataframe", + "rolling_origin_stride": 1, } return meta_config diff --git a/models/elastic_heart/configs/config_hyperparameters.py b/models/elastic_heart/configs/config_hyperparameters.py index 40deab9f..f66269c1 100644 --- a/models/elastic_heart/configs/config_hyperparameters.py +++ b/models/elastic_heart/configs/config_hyperparameters.py @@ -10,6 +10,7 @@ def get_hp_config(): hyperparameters = { 'steps': [*range(1, 36 + 1, 1)], + "time_steps": 36, # classic-sweep-120 "activation": "Tanh", diff --git a/models/elastic_heart/configs/config_meta.py b/models/elastic_heart/configs/config_meta.py index 06b4a7fd..2af6bb78 100644 --- a/models/elastic_heart/configs/config_meta.py +++ b/models/elastic_heart/configs/config_meta.py @@ -17,5 +17,6 @@ def get_meta_config(): "creator": "Dylan", "prediction_format": "dataframe", "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], + "rolling_origin_stride": 1, } return meta_config diff --git a/models/electric_relaxation/configs/config_hyperparameters.py b/models/electric_relaxation/configs/config_hyperparameters.py index feeee797..f41f0997 100644 --- a/models/electric_relaxation/configs/config_hyperparameters.py +++ b/models/electric_relaxation/configs/config_hyperparameters.py @@ -7,6 +7,7 @@ def get_hp_config(): """ hp_config = { "steps": [*range(1, 36 + 1, 1)], + "time_steps": 36, "parameters": { "n_estimators": 100, "n_jobs": 2 diff --git a/models/electric_relaxation/configs/config_meta.py b/models/electric_relaxation/configs/config_meta.py index 03ec566e..49c11530 100644 --- a/models/electric_relaxation/configs/config_meta.py +++ b/models/electric_relaxation/configs/config_meta.py @@ -14,5 +14,6 @@ def get_meta_config(): "level": "cm", "creator": "Sara", "prediction_format": "dataframe", + "rolling_origin_stride": 1, } return model_config \ No newline at end of file diff --git a/models/emerging_principles/configs/config_hyperparameters.py b/models/emerging_principles/configs/config_hyperparameters.py index bf70d64a..8c6c7903 100644 --- a/models/emerging_principles/configs/config_hyperparameters.py +++ b/models/emerging_principles/configs/config_hyperparameters.py @@ -19,6 +19,7 @@ def get_hp_config(): hyperparameters = { # --- Forecast horizon --- "steps": list(range(1, 37)), + "time_steps": 36, # --- Architecture --- "activation": "LeakyReLU", diff --git a/models/emerging_principles/configs/config_meta.py b/models/emerging_principles/configs/config_meta.py index 53a4a41c..0fec8db1 100644 --- a/models/emerging_principles/configs/config_meta.py +++ b/models/emerging_principles/configs/config_meta.py @@ -16,5 +16,6 @@ def get_meta_config(): "creator": "Simon", "prediction_format": "dataframe", "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "rolling_origin_stride": 1, } return meta_config diff --git a/models/fancy_feline/configs/config_hyperparameters.py b/models/fancy_feline/configs/config_hyperparameters.py index 0acda48f..cbcc5d83 100644 --- a/models/fancy_feline/configs/config_hyperparameters.py +++ b/models/fancy_feline/configs/config_hyperparameters.py @@ -13,6 +13,7 @@ def get_hp_config(): # --- Forecast horizon --- "steps": list(range(1, 36 + 1)), + "time_steps": 36, # --- Sampling --- "num_samples": 1, diff --git a/models/fancy_feline/configs/config_meta.py b/models/fancy_feline/configs/config_meta.py index 8af7d794..3014337d 100644 --- a/models/fancy_feline/configs/config_meta.py +++ b/models/fancy_feline/configs/config_meta.py @@ -17,5 +17,6 @@ def get_meta_config(): "creator": "Simon", "prediction_format": "dataframe", "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "rolling_origin_stride": 1, } return meta_config diff --git a/models/fast_car/configs/config_hyperparameters.py b/models/fast_car/configs/config_hyperparameters.py index 80916d98..16aba50a 100644 --- a/models/fast_car/configs/config_hyperparameters.py +++ b/models/fast_car/configs/config_hyperparameters.py @@ -10,6 +10,7 @@ def get_hp_config(): hyperparameters = { "steps": [*range(1, 36 + 1, 1)], + "time_steps": 36, "parameters": { "clf": { "n_estimators": 100, diff --git a/models/fast_car/configs/config_meta.py b/models/fast_car/configs/config_meta.py index 96ac3741..4fa14726 100644 --- a/models/fast_car/configs/config_meta.py +++ b/models/fast_car/configs/config_meta.py @@ -18,5 +18,6 @@ def get_meta_config(): "level": "cm", "creator": "Borbála", "prediction_format": "dataframe", + "rolling_origin_stride": 1, } return meta_config diff --git a/models/fluorescent_adolescent/configs/config_hyperparameters.py b/models/fluorescent_adolescent/configs/config_hyperparameters.py index 5dfd0c53..c935bf45 100644 --- a/models/fluorescent_adolescent/configs/config_hyperparameters.py +++ b/models/fluorescent_adolescent/configs/config_hyperparameters.py @@ -10,6 +10,7 @@ def get_hp_config(): hyperparameters = { "steps": [*range(1, 36 + 1, 1)], + "time_steps": 36, "parameters": { "clf": { "n_estimators": 100, diff --git a/models/fluorescent_adolescent/configs/config_meta.py b/models/fluorescent_adolescent/configs/config_meta.py index ae0f7b2d..9eab7760 100644 --- a/models/fluorescent_adolescent/configs/config_meta.py +++ b/models/fluorescent_adolescent/configs/config_meta.py @@ -18,5 +18,6 @@ def get_meta_config(): "level": "cm", "creator": "Marina", "prediction_format": "dataframe", + "rolling_origin_stride": 1, } return meta_config diff --git a/models/fourtieth_symphony/configs/config_hyperparameters.py b/models/fourtieth_symphony/configs/config_hyperparameters.py index 2eb3888d..bee887d6 100644 --- a/models/fourtieth_symphony/configs/config_hyperparameters.py +++ b/models/fourtieth_symphony/configs/config_hyperparameters.py @@ -10,6 +10,7 @@ def get_hp_config(): hyperparameters = { 'steps': [*range(1, 36 + 1, 1)], + "time_steps": 36, 'submodels_to_train': 20, 'pred_samples': 10, 'log_target': True, diff --git a/models/fourtieth_symphony/configs/config_meta.py b/models/fourtieth_symphony/configs/config_meta.py index f96d6a4a..c80abe43 100644 --- a/models/fourtieth_symphony/configs/config_meta.py +++ b/models/fourtieth_symphony/configs/config_meta.py @@ -18,5 +18,6 @@ def get_meta_config(): "model_clf": "XGBClassifier", "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], "queryset": "uncertainty_broad_nolog", + "rolling_origin_stride": 1, } return meta_config diff --git a/models/good_life/configs/config_hyperparameters.py b/models/good_life/configs/config_hyperparameters.py index ae66cbe1..04bbb902 100644 --- a/models/good_life/configs/config_hyperparameters.py +++ b/models/good_life/configs/config_hyperparameters.py @@ -10,6 +10,7 @@ def get_hp_config(): hyperparameters = { "steps": [*range(1, 36 + 1, 1)], + "time_steps": 36, # Good! silvery-sweep-1 # "batch_size": 64, # "decoder_output_dim": 8, diff --git a/models/good_life/configs/config_meta.py b/models/good_life/configs/config_meta.py index b7b6cea0..b72253d8 100644 --- a/models/good_life/configs/config_meta.py +++ b/models/good_life/configs/config_meta.py @@ -17,5 +17,6 @@ def get_meta_config(): "creator": "Dylan", "prediction_format": "dataframe", "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], + "rolling_origin_stride": 1, } return meta_config diff --git a/models/good_riddance/configs/config_hyperparameters.py b/models/good_riddance/configs/config_hyperparameters.py index da785a64..3c614e1d 100644 --- a/models/good_riddance/configs/config_hyperparameters.py +++ b/models/good_riddance/configs/config_hyperparameters.py @@ -10,6 +10,7 @@ def get_hp_config(): hyperparameters = { "steps": [*range(1, 36 + 1, 1)], + "time_steps": 36, "parameters": { "n_estimators": 250, "n_jobs": 12, diff --git a/models/good_riddance/configs/config_meta.py b/models/good_riddance/configs/config_meta.py index ed04fd28..aabf1f1d 100644 --- a/models/good_riddance/configs/config_meta.py +++ b/models/good_riddance/configs/config_meta.py @@ -16,5 +16,6 @@ def get_meta_config(): "level": "cm", "creator": "Marina", "prediction_format": "dataframe", + "rolling_origin_stride": 1, } return meta_config diff --git a/models/green_ranger/configs/config_hyperparameters.py b/models/green_ranger/configs/config_hyperparameters.py new file mode 100644 index 00000000..ab66b33c --- /dev/null +++ b/models/green_ranger/configs/config_hyperparameters.py @@ -0,0 +1,18 @@ + +def get_hp_config(): + """ + Contains the hyperparameter configurations for model training. + This configuration is "operational" so modifying these settings will impact the model's behavior during the training. + + Returns: + - hyperparameters (dict): A dictionary containing hyperparameters for training the model, which determine the model's behavior during the training phase. + """ + + hyperparameters = { + 'steps': [*range(1, 36 + 1, 1)], + 'time_steps': 36, + 'window_months': 18, + 'lambda_mix': 0.05, + 'n_samples': 256, + } + return hyperparameters diff --git a/models/green_ranger/configs/config_meta.py b/models/green_ranger/configs/config_meta.py new file mode 100644 index 00000000..169591e2 --- /dev/null +++ b/models/green_ranger/configs/config_meta.py @@ -0,0 +1,20 @@ +def get_meta_config(): + """ + Contains the meta data for the model (model algorithm, name, target variable, and level of analysis). + This config is for documentation purposes only, and modifying it will not affect the model, the training, or the evaluation. + + Returns: + - meta_config (dict): A dictionary containing model meta configuration. + """ + + meta_config = { + "name": "green_ranger", + "algorithm": "MixtureBaseline", + "regression_targets": ["lr_ns_best"], + "level": "cm", + "creator": "Simon", + "prediction_format": "prediction_frame", + "rolling_origin_stride": 1, + "regression_sample_metrics": ["twCRPS", "QIS", "MIS", "MCR_sample"], + } + return meta_config diff --git a/models/green_squirrel/configs/config_hyperparameters.py b/models/green_squirrel/configs/config_hyperparameters.py index 175a66f1..4db46061 100644 --- a/models/green_squirrel/configs/config_hyperparameters.py +++ b/models/green_squirrel/configs/config_hyperparameters.py @@ -10,6 +10,7 @@ def get_hp_config(): hyperparameters = { "steps": [*range(1, 36 + 1, 1)], + "time_steps": 36, "parameters": { "clf": { "n_estimators": 250, diff --git a/models/green_squirrel/configs/config_meta.py b/models/green_squirrel/configs/config_meta.py index 7b80e42e..e51932d3 100644 --- a/models/green_squirrel/configs/config_meta.py +++ b/models/green_squirrel/configs/config_meta.py @@ -18,5 +18,6 @@ def get_meta_config(): "level": "cm", "creator": "Borbála", "prediction_format": "dataframe", + "rolling_origin_stride": 1, } return meta_config diff --git a/models/heat_waves/configs/config_hyperparameters.py b/models/heat_waves/configs/config_hyperparameters.py index 61a853af..047b4cf5 100644 --- a/models/heat_waves/configs/config_hyperparameters.py +++ b/models/heat_waves/configs/config_hyperparameters.py @@ -9,6 +9,7 @@ def get_hp_config(): hyperparameters = { 'steps': [*range(1, 36 + 1, 1)], + "time_steps": 36, # royal-sweep-63 diff --git a/models/heat_waves/configs/config_meta.py b/models/heat_waves/configs/config_meta.py index 7e543259..13ed2409 100644 --- a/models/heat_waves/configs/config_meta.py +++ b/models/heat_waves/configs/config_meta.py @@ -17,5 +17,6 @@ def get_meta_config(): "creator": "Dylan", "prediction_format": "dataframe", "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], + "rolling_origin_stride": 1, } return meta_config diff --git a/models/heat_waves/configs/config_partitions.py b/models/heat_waves/configs/config_partitions.py index 289a93b7..27314c19 100644 --- a/models/heat_waves/configs/config_partitions.py +++ b/models/heat_waves/configs/config_partitions.py @@ -21,11 +21,11 @@ def generate(steps: int = 36) -> dict: """ def forecasting_train_range(): - month_last = ViewsMonth.now().id - 2 + month_last = ViewsMonth.now().id - 1 return (121, month_last) def forecasting_test_range(steps): - month_last = ViewsMonth.now().id - 2 + month_last = ViewsMonth.now().id - 1 return (month_last + 1, month_last + 1 + steps) return { diff --git a/models/heavy_rotation/configs/config_hyperparameters.py b/models/heavy_rotation/configs/config_hyperparameters.py index 47c85547..1bf57562 100644 --- a/models/heavy_rotation/configs/config_hyperparameters.py +++ b/models/heavy_rotation/configs/config_hyperparameters.py @@ -10,6 +10,7 @@ def get_hp_config(): hyperparameters = { "steps": [*range(1, 36 + 1, 1)], + "time_steps": 36, "parameters": { "n_estimators": 250, "n_jobs": 12 diff --git a/models/heavy_rotation/configs/config_meta.py b/models/heavy_rotation/configs/config_meta.py index dd4ec169..38004218 100644 --- a/models/heavy_rotation/configs/config_meta.py +++ b/models/heavy_rotation/configs/config_meta.py @@ -16,5 +16,6 @@ def get_meta_config(): "level": "cm", "creator": "Borbála", "prediction_format": "dataframe", + "rolling_origin_stride": 1, } return meta_config diff --git a/models/high_hopes/configs/config_hyperparameters.py b/models/high_hopes/configs/config_hyperparameters.py index 5cbe5a6d..983fc26e 100644 --- a/models/high_hopes/configs/config_hyperparameters.py +++ b/models/high_hopes/configs/config_hyperparameters.py @@ -10,6 +10,7 @@ def get_hp_config(): hyperparameters = { 'steps': [*range(1, 36 + 1, 1)], + "time_steps": 36, "parameters": { "clf": { "n_estimators": 250, diff --git a/models/high_hopes/configs/config_meta.py b/models/high_hopes/configs/config_meta.py index df369b98..b5ed6211 100644 --- a/models/high_hopes/configs/config_meta.py +++ b/models/high_hopes/configs/config_meta.py @@ -18,5 +18,6 @@ def get_meta_config(): "level": "cm", "creator": "Borbála", "prediction_format": "dataframe", + "rolling_origin_stride": 1, } return meta_config diff --git a/models/hot_stream/configs/config_hyperparameters.py b/models/hot_stream/configs/config_hyperparameters.py index 7a74a29f..4a6d763f 100644 --- a/models/hot_stream/configs/config_hyperparameters.py +++ b/models/hot_stream/configs/config_hyperparameters.py @@ -10,6 +10,7 @@ def get_hp_config(): hyperparameters = { # --- Forecast horizon --- 'steps': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36], + "time_steps": 36, # --- Input / output structure --- 'input_chunk_length': 24, diff --git a/models/hot_stream/configs/config_meta.py b/models/hot_stream/configs/config_meta.py index c3389a26..19c2e50f 100644 --- a/models/hot_stream/configs/config_meta.py +++ b/models/hot_stream/configs/config_meta.py @@ -17,5 +17,6 @@ def get_meta_config(): "creator": "Simon", "prediction_format": "dataframe", "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "rolling_origin_stride": 1, } return meta_config diff --git a/models/hot_stream/configs/config_partitions.py b/models/hot_stream/configs/config_partitions.py index 289a93b7..27314c19 100644 --- a/models/hot_stream/configs/config_partitions.py +++ b/models/hot_stream/configs/config_partitions.py @@ -21,11 +21,11 @@ def generate(steps: int = 36) -> dict: """ def forecasting_train_range(): - month_last = ViewsMonth.now().id - 2 + month_last = ViewsMonth.now().id - 1 return (121, month_last) def forecasting_test_range(steps): - month_last = ViewsMonth.now().id - 2 + month_last = ViewsMonth.now().id - 1 return (month_last + 1, month_last + 1 + steps) return { diff --git a/models/invisible_string/configs/config_hyperparameters.py b/models/invisible_string/configs/config_hyperparameters.py index d7cad484..aff5e150 100644 --- a/models/invisible_string/configs/config_hyperparameters.py +++ b/models/invisible_string/configs/config_hyperparameters.py @@ -10,6 +10,7 @@ def get_hp_config(): hyperparameters = { 'steps': [*range(1, 36 + 1, 1)], + "time_steps": 36, 'parameters': { 'n_estimators': 200, } diff --git a/models/invisible_string/configs/config_meta.py b/models/invisible_string/configs/config_meta.py index 9ca59dc5..eba5d3d6 100644 --- a/models/invisible_string/configs/config_meta.py +++ b/models/invisible_string/configs/config_meta.py @@ -16,5 +16,6 @@ def get_meta_config(): "level": "pgm", "creator": "Xiaolong", "prediction_format": "dataframe", + "rolling_origin_stride": 1, } return meta_config diff --git a/models/lavender_haze/configs/config_hyperparameters.py b/models/lavender_haze/configs/config_hyperparameters.py index 45306450..dbebe94a 100644 --- a/models/lavender_haze/configs/config_hyperparameters.py +++ b/models/lavender_haze/configs/config_hyperparameters.py @@ -1,6 +1,7 @@ def get_hp_config(): hp_config = { "steps": [*range(1, 36 + 1, 1)], + "time_steps": 36, "parameters": { "clf":{ "n_estimators": 200, diff --git a/models/lavender_haze/configs/config_meta.py b/models/lavender_haze/configs/config_meta.py index b7f3760e..514a53f9 100644 --- a/models/lavender_haze/configs/config_meta.py +++ b/models/lavender_haze/configs/config_meta.py @@ -17,5 +17,6 @@ def get_meta_config(): "level": "pgm", "creator": "Xiaolong", "prediction_format": "dataframe", + "rolling_origin_stride": 1, } return meta_config \ No newline at end of file diff --git a/models/little_lies/configs/config_hyperparameters.py b/models/little_lies/configs/config_hyperparameters.py index 1fd2b86c..1ef3ed12 100644 --- a/models/little_lies/configs/config_hyperparameters.py +++ b/models/little_lies/configs/config_hyperparameters.py @@ -10,6 +10,7 @@ def get_hp_config(): hyperparameters = { "steps": [*range(1, 36 + 1, 1)], + "time_steps": 36, "parameters": { "clf": { "n_estimators": 250, diff --git a/models/little_lies/configs/config_meta.py b/models/little_lies/configs/config_meta.py index 2abe73ef..de51aa35 100644 --- a/models/little_lies/configs/config_meta.py +++ b/models/little_lies/configs/config_meta.py @@ -18,5 +18,6 @@ def get_meta_config(): "level": "cm", "creator": "Marina", "prediction_format": "dataframe", + "rolling_origin_stride": 1, } return meta_config diff --git a/models/locf_cmbaseline/main.py b/models/locf_cmbaseline/main.py index 7770d7e9..8ef12185 100644 --- a/models/locf_cmbaseline/main.py +++ b/models/locf_cmbaseline/main.py @@ -1,8 +1,7 @@ -import wandb import warnings from pathlib import Path -from views_pipeline_core.cli.utils import parse_args, validate_arguments -from views_pipeline_core.managers.model import ModelPathManager +from views_pipeline_core.cli import ForecastingModelArgs +from views_pipeline_core.managers import ModelPathManager from views_baseline.manager.baseline_manager import BaselineForecastingModelManager warnings.filterwarnings("ignore") @@ -13,17 +12,15 @@ raise RuntimeError(f"Unexpected error: {e}. Check the logs for details.") if __name__ == "__main__": - wandb.login() - args = parse_args() - validate_arguments(args) + args = ForecastingModelArgs.parse_args() + + manager = BaselineForecastingModelManager( + model_path=model_path, + wandb_notifications=args.wandb_notifications, + use_prediction_store=args.prediction_store, + ) if args.sweep: - BaselineForecastingModelManager( - model_path=model_path, - ).execute_sweep_run(args) + manager.execute_sweep_run(args) else: - BaselineForecastingModelManager( - model_path=model_path, - wandb_notifications=args.wandb_notifications, - use_prediction_store=args.prediction_store, - ).execute_single_run(args) + manager.execute_single_run(args) diff --git a/models/locf_pgmbaseline/main.py b/models/locf_pgmbaseline/main.py index b48a90cc..8ef12185 100644 --- a/models/locf_pgmbaseline/main.py +++ b/models/locf_pgmbaseline/main.py @@ -1,8 +1,7 @@ -import wandb import warnings from pathlib import Path -from views_pipeline_core.cli.utils import parse_args, validate_arguments -from views_pipeline_core.managers.model import ModelPathManager +from views_pipeline_core.cli import ForecastingModelArgs +from views_pipeline_core.managers import ModelPathManager from views_baseline.manager.baseline_manager import BaselineForecastingModelManager warnings.filterwarnings("ignore") @@ -13,17 +12,15 @@ raise RuntimeError(f"Unexpected error: {e}. Check the logs for details.") if __name__ == "__main__": - wandb.login() - args = parse_args() - validate_arguments(args) + args = ForecastingModelArgs.parse_args() + + manager = BaselineForecastingModelManager( + model_path=model_path, + wandb_notifications=args.wandb_notifications, + use_prediction_store=args.prediction_store, + ) if args.sweep: - BaselineForecastingModelManager( - model_path=model_path, - ).execute_sweep_run(args) + manager.execute_sweep_run(args) else: - BaselineForecastingModelManager( - model_path=model_path, - wandb_notifications=args.wandb_notifications, - use_prediction_store=args.prediction_store, - ).execute_single_run(args) \ No newline at end of file + manager.execute_single_run(args) diff --git a/models/lovely_creature/configs/config_hyperparameters.py b/models/lovely_creature/configs/config_hyperparameters.py index 0ffdb568..99191693 100644 --- a/models/lovely_creature/configs/config_hyperparameters.py +++ b/models/lovely_creature/configs/config_hyperparameters.py @@ -10,6 +10,7 @@ def get_hp_config(): hyperparameters = { 'steps': [*range(1, 36 + 1, 1)], + "time_steps": 36, 'submodels_to_train': 50, 'pred_samples': 10, 'log_target': False, diff --git a/models/lovely_creature/configs/config_meta.py b/models/lovely_creature/configs/config_meta.py index 647c3348..1ca2708d 100644 --- a/models/lovely_creature/configs/config_meta.py +++ b/models/lovely_creature/configs/config_meta.py @@ -18,5 +18,6 @@ def get_meta_config(): "model_clf": "XGBClassifier", "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], "queryset": "uncertainty_broad_nolog", + "rolling_origin_stride": 1, } return meta_config diff --git a/models/midnight_rain/configs/config_hyperparameters.py b/models/midnight_rain/configs/config_hyperparameters.py index f51225b5..8952ae48 100644 --- a/models/midnight_rain/configs/config_hyperparameters.py +++ b/models/midnight_rain/configs/config_hyperparameters.py @@ -10,6 +10,7 @@ def get_hp_config(): hyperparameters = { 'steps': [*range(1, 36 + 1, 1)], + "time_steps": 36, 'parameters': { 'n_estimators': 200, } diff --git a/models/midnight_rain/configs/config_meta.py b/models/midnight_rain/configs/config_meta.py index 77badcb2..33249303 100644 --- a/models/midnight_rain/configs/config_meta.py +++ b/models/midnight_rain/configs/config_meta.py @@ -16,5 +16,6 @@ def get_meta_config(): "level": "pgm", "creator": "Xiaolong", "prediction_format": "dataframe", + "rolling_origin_stride": 1, } return meta_config diff --git a/models/national_anthem/configs/config_hyperparameters.py b/models/national_anthem/configs/config_hyperparameters.py index b1eb0a5e..2a27d5d7 100644 --- a/models/national_anthem/configs/config_hyperparameters.py +++ b/models/national_anthem/configs/config_hyperparameters.py @@ -10,6 +10,7 @@ def get_hp_config(): hyperparameters = { "steps": [*range(1, 36 + 1, 1)], + "time_steps": 36, "parameters": { "n_estimators": 300, "n_jobs": 12 diff --git a/models/national_anthem/configs/config_meta.py b/models/national_anthem/configs/config_meta.py index 50d2f47e..7d232ff2 100644 --- a/models/national_anthem/configs/config_meta.py +++ b/models/national_anthem/configs/config_meta.py @@ -16,5 +16,6 @@ def get_meta_config(): "level": "cm", "creator": "Borbála", "prediction_format": "dataframe", + "rolling_origin_stride": 1, } return meta_config diff --git a/models/new_rules/configs/config_hyperparameters.py b/models/new_rules/configs/config_hyperparameters.py index a90811ac..89cd9cd1 100644 --- a/models/new_rules/configs/config_hyperparameters.py +++ b/models/new_rules/configs/config_hyperparameters.py @@ -9,6 +9,7 @@ def get_hp_config(): hyperparameters = { # --- Forecast horizon --- "steps": list(range(1, 37)), + "time_steps": 36, # --- Architecture --- "activation": "LeakyReLU", diff --git a/models/new_rules/configs/config_meta.py b/models/new_rules/configs/config_meta.py index 29662c56..2364ca45 100644 --- a/models/new_rules/configs/config_meta.py +++ b/models/new_rules/configs/config_meta.py @@ -16,5 +16,6 @@ def get_meta_config(): "creator": "Simon", "prediction_format": "dataframe", "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "rolling_origin_stride": 1, } return meta_config diff --git a/models/novel_heuristics/configs/config_meta.py b/models/novel_heuristics/configs/config_meta.py index a16dcf52..bed47774 100644 --- a/models/novel_heuristics/configs/config_meta.py +++ b/models/novel_heuristics/configs/config_meta.py @@ -16,5 +16,6 @@ def get_meta_config(): "creator": "Simon", "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], "prediction_format": "dataframe", + "rolling_origin_stride": 1, } return meta_config diff --git a/models/old_money/configs/config_hyperparameters.py b/models/old_money/configs/config_hyperparameters.py index e75f36db..1c72abb4 100644 --- a/models/old_money/configs/config_hyperparameters.py +++ b/models/old_money/configs/config_hyperparameters.py @@ -1,6 +1,7 @@ def get_hp_config(): hp_config = { "steps": [*range(1, 36 + 1, 1)], + "time_steps": 36, "parameters": { "clf": { "n_estimators": 200, diff --git a/models/old_money/configs/config_meta.py b/models/old_money/configs/config_meta.py index 65533674..fe9b28d0 100644 --- a/models/old_money/configs/config_meta.py +++ b/models/old_money/configs/config_meta.py @@ -17,5 +17,6 @@ def get_meta_config(): "level": "pgm", "creator": "Xiaolong", "prediction_format": "dataframe", + "rolling_origin_stride": 1, } return meta_config \ No newline at end of file diff --git a/models/ominous_ox/configs/config_hyperparameters.py b/models/ominous_ox/configs/config_hyperparameters.py index 73eab15d..77280431 100644 --- a/models/ominous_ox/configs/config_hyperparameters.py +++ b/models/ominous_ox/configs/config_hyperparameters.py @@ -10,6 +10,7 @@ def get_hp_config(): hyperparameters = { 'steps': [*range(1, 36 + 1, 1)], + "time_steps": 36, "parameters": { "n_estimators": 250, "n_jobs": 12 diff --git a/models/ominous_ox/configs/config_meta.py b/models/ominous_ox/configs/config_meta.py index 41068aad..ab8d1cd3 100644 --- a/models/ominous_ox/configs/config_meta.py +++ b/models/ominous_ox/configs/config_meta.py @@ -16,5 +16,6 @@ def get_meta_config(): "level": "cm", "creator": "Borbála", "prediction_format": "dataframe", + "rolling_origin_stride": 1, } return meta_config diff --git a/models/orange_pasta/configs/config_hyperparameters.py b/models/orange_pasta/configs/config_hyperparameters.py index 8fee48ad..7ca4995c 100644 --- a/models/orange_pasta/configs/config_hyperparameters.py +++ b/models/orange_pasta/configs/config_hyperparameters.py @@ -1,6 +1,7 @@ def get_hp_config(): hp_config = { "steps": [*range(1, 36 + 1, 1)], + "time_steps": 36, "parameters": { "n_estimators": 200, } diff --git a/models/orange_pasta/configs/config_meta.py b/models/orange_pasta/configs/config_meta.py index 55f98f76..e4a38475 100644 --- a/models/orange_pasta/configs/config_meta.py +++ b/models/orange_pasta/configs/config_meta.py @@ -15,5 +15,6 @@ def get_meta_config(): "level": "pgm", "creator": "Xiaolong", "prediction_format": "dataframe", + "rolling_origin_stride": 1, } return meta_config \ No newline at end of file diff --git a/models/party_princess/configs/config_hyperparameters.py b/models/party_princess/configs/config_hyperparameters.py index 240ee5b5..3c07f92d 100644 --- a/models/party_princess/configs/config_hyperparameters.py +++ b/models/party_princess/configs/config_hyperparameters.py @@ -10,6 +10,7 @@ def get_hp_config(): hyperparameters = { # --- Forecast horizon --- "steps": list(range(1, 36 + 1)), + "time_steps": 36, # --- Sampling --- "num_samples": 1, diff --git a/models/party_princess/configs/config_meta.py b/models/party_princess/configs/config_meta.py index 3ff8f68f..8e33a92a 100644 --- a/models/party_princess/configs/config_meta.py +++ b/models/party_princess/configs/config_meta.py @@ -17,5 +17,6 @@ def get_meta_config(): "creator": "Simon", "prediction_format": "dataframe", "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "rolling_origin_stride": 1, } return meta_config diff --git a/models/pink_ranger/configs/config_hyperparameters.py b/models/pink_ranger/configs/config_hyperparameters.py new file mode 100644 index 00000000..ab66b33c --- /dev/null +++ b/models/pink_ranger/configs/config_hyperparameters.py @@ -0,0 +1,18 @@ + +def get_hp_config(): + """ + Contains the hyperparameter configurations for model training. + This configuration is "operational" so modifying these settings will impact the model's behavior during the training. + + Returns: + - hyperparameters (dict): A dictionary containing hyperparameters for training the model, which determine the model's behavior during the training phase. + """ + + hyperparameters = { + 'steps': [*range(1, 36 + 1, 1)], + 'time_steps': 36, + 'window_months': 18, + 'lambda_mix': 0.05, + 'n_samples': 256, + } + return hyperparameters diff --git a/models/pink_ranger/configs/config_meta.py b/models/pink_ranger/configs/config_meta.py new file mode 100644 index 00000000..c72bad86 --- /dev/null +++ b/models/pink_ranger/configs/config_meta.py @@ -0,0 +1,20 @@ +def get_meta_config(): + """ + Contains the meta data for the model (model algorithm, name, target variable, and level of analysis). + This config is for documentation purposes only, and modifying it will not affect the model, the training, or the evaluation. + + Returns: + - meta_config (dict): A dictionary containing model meta configuration. + """ + + meta_config = { + "name": "pink_ranger", + "algorithm": "MixtureBaseline", + "regression_targets": ["lr_ns_best"], + "level": "pgm", + "creator": "Simon", + "prediction_format": "prediction_frame", + "rolling_origin_stride": 1, + "regression_sample_metrics": ["twCRPS", "QIS", "MIS", "MCR_sample"], + } + return meta_config diff --git a/models/plastic_beach/configs/config_hyperparameters.py b/models/plastic_beach/configs/config_hyperparameters.py index 73900504..4033eec8 100644 --- a/models/plastic_beach/configs/config_hyperparameters.py +++ b/models/plastic_beach/configs/config_hyperparameters.py @@ -10,6 +10,7 @@ def get_hp_config(): hyperparameters = { "steps": [*range(1, 36 + 1, 1)], + "time_steps": 36, "parameters": { "n_estimators": 300, "n_jobs": 12, diff --git a/models/plastic_beach/configs/config_meta.py b/models/plastic_beach/configs/config_meta.py index 4881ae3e..ced78c08 100644 --- a/models/plastic_beach/configs/config_meta.py +++ b/models/plastic_beach/configs/config_meta.py @@ -16,5 +16,6 @@ def get_meta_config(): "level": "cm", "creator": "Marina", "prediction_format": "dataframe", + "rolling_origin_stride": 1, } return meta_config diff --git a/models/popular_monster/configs/config_hyperparameters.py b/models/popular_monster/configs/config_hyperparameters.py index 47c85547..1bf57562 100644 --- a/models/popular_monster/configs/config_hyperparameters.py +++ b/models/popular_monster/configs/config_hyperparameters.py @@ -10,6 +10,7 @@ def get_hp_config(): hyperparameters = { "steps": [*range(1, 36 + 1, 1)], + "time_steps": 36, "parameters": { "n_estimators": 250, "n_jobs": 12 diff --git a/models/popular_monster/configs/config_meta.py b/models/popular_monster/configs/config_meta.py index 9315c86d..2ddabc5c 100644 --- a/models/popular_monster/configs/config_meta.py +++ b/models/popular_monster/configs/config_meta.py @@ -16,5 +16,6 @@ def get_meta_config(): "level": "cm", "creator": "Borbála", "prediction_format": "dataframe", + "rolling_origin_stride": 1, } return meta_config diff --git a/models/preliminary_directives/configs/config_hyperparameters.py b/models/preliminary_directives/configs/config_hyperparameters.py index e5282bfc..ad22f61e 100644 --- a/models/preliminary_directives/configs/config_hyperparameters.py +++ b/models/preliminary_directives/configs/config_hyperparameters.py @@ -19,6 +19,7 @@ def get_hp_config(): hyperparameters = { # --- Forecast horizon --- "steps": list(range(1, 37)), + "time_steps": 36, # --- Architecture --- "activation": "LeakyReLU", diff --git a/models/preliminary_directives/configs/config_meta.py b/models/preliminary_directives/configs/config_meta.py index 87c7c0ab..006201e2 100644 --- a/models/preliminary_directives/configs/config_meta.py +++ b/models/preliminary_directives/configs/config_meta.py @@ -16,5 +16,6 @@ def get_meta_config(): "creator": "Simon", "prediction_format": "dataframe", "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "rolling_origin_stride": 1, } return meta_config diff --git a/models/purple_alien/main.py b/models/purple_alien/main.py index b5af1b2a..2945ea84 100644 --- a/models/purple_alien/main.py +++ b/models/purple_alien/main.py @@ -29,13 +29,3 @@ HydranetManager(model_path=model_path).execute_sweep_run(args) else: HydranetManager(model_path=model_path).execute_single_run(args) - -# -#if __name__ == "__main__": -# wandb.login() -# args = parse_args() -# validate_arguments(args) -# if args.sweep: -# HydranetManager(model_path=model_path).execute_sweep_run(args) -# else: -# HydranetManager(model_path=model_path).execute_single_run(args) diff --git a/models/purple_haze/configs/config_hyperparameters.py b/models/purple_haze/configs/config_hyperparameters.py index 687b5641..90e3ca76 100644 --- a/models/purple_haze/configs/config_hyperparameters.py +++ b/models/purple_haze/configs/config_hyperparameters.py @@ -10,6 +10,7 @@ def get_hp_config(): hyperparameters = { 'steps': [*range(1, 36 + 1, 1)], + "time_steps": 36, 'submodels_to_train': 50, 'pred_samples': 10, 'log_target': False, diff --git a/models/purple_haze/configs/config_meta.py b/models/purple_haze/configs/config_meta.py index 7e28ddcf..4c69c996 100644 --- a/models/purple_haze/configs/config_meta.py +++ b/models/purple_haze/configs/config_meta.py @@ -18,5 +18,6 @@ def get_meta_config(): "model_clf": "XGBClassifier", "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], "queryset": "uncertainty_broad_nolog", + "rolling_origin_stride": 1, } return meta_config diff --git a/models/purple_haze/main.py b/models/purple_haze/main.py index 379f0fd2..63621928 100644 --- a/models/purple_haze/main.py +++ b/models/purple_haze/main.py @@ -1,35 +1,18 @@ -import wandb import warnings from pathlib import Path -from views_pipeline_core.cli.utils import parse_args, validate_arguments -from views_pipeline_core.managers.log import LoggingManager -from views_pipeline_core.managers.model import ModelPathManager +from views_pipeline_core.cli import ForecastingModelArgs +from views_pipeline_core.managers import ModelPathManager from views_stepshifter.manager.stepshifter_manager import StepshifterManager - -# Import your model manager class here -# E.g. from views_stepshifter.manager.stepshifter_manager import StepshifterManager - warnings.filterwarnings("ignore") try: model_path = ModelPathManager(Path(__file__)) - logger = LoggingManager(model_path).get_logger() -except FileNotFoundError as fnf_error: - raise RuntimeError( - f"File not found: {fnf_error}. Check the file path and try again." - ) -except PermissionError as perm_error: - raise RuntimeError( - f"Permission denied: {perm_error}. Check your permissions and try again." - ) except Exception as e: raise RuntimeError(f"Unexpected error: {e}. Check the logs for details.") if __name__ == "__main__": - wandb.login() - args = parse_args() - validate_arguments(args) + args = ForecastingModelArgs.parse_args() manager = StepshifterManager( model_path=model_path, diff --git a/models/red_ranger/configs/config_hyperparameters.py b/models/red_ranger/configs/config_hyperparameters.py new file mode 100644 index 00000000..ab66b33c --- /dev/null +++ b/models/red_ranger/configs/config_hyperparameters.py @@ -0,0 +1,18 @@ + +def get_hp_config(): + """ + Contains the hyperparameter configurations for model training. + This configuration is "operational" so modifying these settings will impact the model's behavior during the training. + + Returns: + - hyperparameters (dict): A dictionary containing hyperparameters for training the model, which determine the model's behavior during the training phase. + """ + + hyperparameters = { + 'steps': [*range(1, 36 + 1, 1)], + 'time_steps': 36, + 'window_months': 18, + 'lambda_mix': 0.05, + 'n_samples': 256, + } + return hyperparameters diff --git a/models/red_ranger/configs/config_meta.py b/models/red_ranger/configs/config_meta.py new file mode 100644 index 00000000..d98a0b75 --- /dev/null +++ b/models/red_ranger/configs/config_meta.py @@ -0,0 +1,20 @@ +def get_meta_config(): + """ + Contains the meta data for the model (model algorithm, name, target variable, and level of analysis). + This config is for documentation purposes only, and modifying it will not affect the model, the training, or the evaluation. + + Returns: + - meta_config (dict): A dictionary containing model meta configuration. + """ + + meta_config = { + "name": "red_ranger", + "algorithm": "MixtureBaseline", + "regression_targets": ["lr_ged_sb"], + "level": "cm", + "creator": "Simon", + "prediction_format": "prediction_frame", + "rolling_origin_stride": 1, + "regression_sample_metrics": ["twCRPS", "QIS", "MIS", "MCR_sample"], + } + return meta_config diff --git a/models/teen_spirit/configs/config_hyperparameters.py b/models/teen_spirit/configs/config_hyperparameters.py index 73900504..4033eec8 100644 --- a/models/teen_spirit/configs/config_hyperparameters.py +++ b/models/teen_spirit/configs/config_hyperparameters.py @@ -10,6 +10,7 @@ def get_hp_config(): hyperparameters = { "steps": [*range(1, 36 + 1, 1)], + "time_steps": 36, "parameters": { "n_estimators": 300, "n_jobs": 12, diff --git a/models/teen_spirit/configs/config_meta.py b/models/teen_spirit/configs/config_meta.py index 68ed1c09..d9f33d48 100644 --- a/models/teen_spirit/configs/config_meta.py +++ b/models/teen_spirit/configs/config_meta.py @@ -16,5 +16,6 @@ def get_meta_config(): "level": "cm", "creator": "Marina", "prediction_format": "dataframe", + "rolling_origin_stride": 1, } return meta_config diff --git a/models/teenage_dirtbag/configs/config_hyperparameters.py b/models/teenage_dirtbag/configs/config_hyperparameters.py index d596bfb6..5227f81d 100644 --- a/models/teenage_dirtbag/configs/config_hyperparameters.py +++ b/models/teenage_dirtbag/configs/config_hyperparameters.py @@ -10,6 +10,7 @@ def get_hp_config(): hyperparameters = { 'steps': [*range(1, 36 + 1, 1)], + "time_steps": 36, # "batch_size": 1024, # "delta": 4.296149099765397, diff --git a/models/teenage_dirtbag/configs/config_meta.py b/models/teenage_dirtbag/configs/config_meta.py index 1a9e8e19..bdc09fa5 100644 --- a/models/teenage_dirtbag/configs/config_meta.py +++ b/models/teenage_dirtbag/configs/config_meta.py @@ -17,5 +17,6 @@ def get_meta_config(): "creator": "Dylan", "prediction_format": "dataframe", "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], + "rolling_origin_stride": 1, } return meta_config diff --git a/models/twin_flame/configs/config_hyperparameters.py b/models/twin_flame/configs/config_hyperparameters.py index ccc84e2a..86f58aab 100644 --- a/models/twin_flame/configs/config_hyperparameters.py +++ b/models/twin_flame/configs/config_hyperparameters.py @@ -10,6 +10,7 @@ def get_hp_config(): hyperparameters = { "steps": [*range(1, 36 + 1, 1)], + "time_steps": 36, "parameters": { "clf": { "n_estimators": 250, diff --git a/models/twin_flame/configs/config_meta.py b/models/twin_flame/configs/config_meta.py index 01c30606..90992c2a 100644 --- a/models/twin_flame/configs/config_meta.py +++ b/models/twin_flame/configs/config_meta.py @@ -18,5 +18,6 @@ def get_meta_config(): "level": "cm", "creator": "Borbála", "prediction_format": "dataframe", + "rolling_origin_stride": 1, } return meta_config diff --git a/models/wild_rose/configs/config_hyperparameters.py b/models/wild_rose/configs/config_hyperparameters.py index 0ffdb568..99191693 100644 --- a/models/wild_rose/configs/config_hyperparameters.py +++ b/models/wild_rose/configs/config_hyperparameters.py @@ -10,6 +10,7 @@ def get_hp_config(): hyperparameters = { 'steps': [*range(1, 36 + 1, 1)], + "time_steps": 36, 'submodels_to_train': 50, 'pred_samples': 10, 'log_target': False, diff --git a/models/wild_rose/configs/config_meta.py b/models/wild_rose/configs/config_meta.py index 66cf0175..3cb7d6b3 100644 --- a/models/wild_rose/configs/config_meta.py +++ b/models/wild_rose/configs/config_meta.py @@ -18,5 +18,6 @@ def get_meta_config(): "model_clf": "XGBClassifier", "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], "queryset": "uncertainty_conflict_nolog", + "rolling_origin_stride": 1, } return meta_config diff --git a/models/wildest_dream/configs/config_hyperparameters.py b/models/wildest_dream/configs/config_hyperparameters.py index 6093620b..97f89e4c 100644 --- a/models/wildest_dream/configs/config_hyperparameters.py +++ b/models/wildest_dream/configs/config_hyperparameters.py @@ -1,6 +1,7 @@ def get_hp_config(): hp_config = { "steps": [*range(1, 36 + 1, 1)], + "time_steps": 36, "parameters": { "clf":{ "n_estimators": 200, diff --git a/models/wildest_dream/configs/config_meta.py b/models/wildest_dream/configs/config_meta.py index a1d46f6b..f8e058a9 100644 --- a/models/wildest_dream/configs/config_meta.py +++ b/models/wildest_dream/configs/config_meta.py @@ -17,5 +17,6 @@ def get_meta_config(): "level": "pgm", "creator": "Xiaolong", "prediction_format": "dataframe", + "rolling_origin_stride": 1, } return meta_config \ No newline at end of file diff --git a/models/wuthering_heights/configs/config_hyperparameters.py b/models/wuthering_heights/configs/config_hyperparameters.py index 0ffdb568..99191693 100644 --- a/models/wuthering_heights/configs/config_hyperparameters.py +++ b/models/wuthering_heights/configs/config_hyperparameters.py @@ -10,6 +10,7 @@ def get_hp_config(): hyperparameters = { 'steps': [*range(1, 36 + 1, 1)], + "time_steps": 36, 'submodels_to_train': 50, 'pred_samples': 10, 'log_target': False, diff --git a/models/wuthering_heights/configs/config_meta.py b/models/wuthering_heights/configs/config_meta.py index c3a52559..cfe2dc4b 100644 --- a/models/wuthering_heights/configs/config_meta.py +++ b/models/wuthering_heights/configs/config_meta.py @@ -18,5 +18,6 @@ def get_meta_config(): "model_clf": "XGBClassifier", "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], "queryset": "uncertainty_deep_conflict_nolog", + "rolling_origin_stride": 1, } return meta_config diff --git a/models/yellow_pikachu/configs/config_hyperparameters.py b/models/yellow_pikachu/configs/config_hyperparameters.py index 6093620b..97f89e4c 100644 --- a/models/yellow_pikachu/configs/config_hyperparameters.py +++ b/models/yellow_pikachu/configs/config_hyperparameters.py @@ -1,6 +1,7 @@ def get_hp_config(): hp_config = { "steps": [*range(1, 36 + 1, 1)], + "time_steps": 36, "parameters": { "clf":{ "n_estimators": 200, diff --git a/models/yellow_pikachu/configs/config_meta.py b/models/yellow_pikachu/configs/config_meta.py index 314f4626..d22bbc3b 100644 --- a/models/yellow_pikachu/configs/config_meta.py +++ b/models/yellow_pikachu/configs/config_meta.py @@ -17,5 +17,6 @@ def get_meta_config(): "level": "pgm", "creator": "Xiaolong", "prediction_format": "dataframe", + "rolling_origin_stride": 1, } return meta_config \ No newline at end of file diff --git a/models/yellow_ranger/configs/config_hyperparameters.py b/models/yellow_ranger/configs/config_hyperparameters.py new file mode 100644 index 00000000..ab66b33c --- /dev/null +++ b/models/yellow_ranger/configs/config_hyperparameters.py @@ -0,0 +1,18 @@ + +def get_hp_config(): + """ + Contains the hyperparameter configurations for model training. + This configuration is "operational" so modifying these settings will impact the model's behavior during the training. + + Returns: + - hyperparameters (dict): A dictionary containing hyperparameters for training the model, which determine the model's behavior during the training phase. + """ + + hyperparameters = { + 'steps': [*range(1, 36 + 1, 1)], + 'time_steps': 36, + 'window_months': 18, + 'lambda_mix': 0.05, + 'n_samples': 256, + } + return hyperparameters diff --git a/models/yellow_ranger/configs/config_meta.py b/models/yellow_ranger/configs/config_meta.py new file mode 100644 index 00000000..c4b6af61 --- /dev/null +++ b/models/yellow_ranger/configs/config_meta.py @@ -0,0 +1,20 @@ +def get_meta_config(): + """ + Contains the meta data for the model (model algorithm, name, target variable, and level of analysis). + This config is for documentation purposes only, and modifying it will not affect the model, the training, or the evaluation. + + Returns: + - meta_config (dict): A dictionary containing model meta configuration. + """ + + meta_config = { + "name": "yellow_ranger", + "algorithm": "MixtureBaseline", + "regression_targets": ["lr_os_best"], + "level": "cm", + "creator": "Simon", + "prediction_format": "prediction_frame", + "rolling_origin_stride": 1, + "regression_sample_metrics": ["twCRPS", "QIS", "MIS", "MCR_sample"], + } + return meta_config diff --git a/models/yellow_submarine/configs/config_hyperparameters.py b/models/yellow_submarine/configs/config_hyperparameters.py index b9336ae6..b22a9fb3 100644 --- a/models/yellow_submarine/configs/config_hyperparameters.py +++ b/models/yellow_submarine/configs/config_hyperparameters.py @@ -10,6 +10,7 @@ def get_hp_config(): hyperparameters = { 'steps': [*range(1, 36 + 1, 1)], + "time_steps": 36, "parameters": { "n_estimators": 300, "n_jobs": 12, diff --git a/models/yellow_submarine/configs/config_meta.py b/models/yellow_submarine/configs/config_meta.py index 965c3ae1..a68a1737 100644 --- a/models/yellow_submarine/configs/config_meta.py +++ b/models/yellow_submarine/configs/config_meta.py @@ -16,5 +16,6 @@ def get_meta_config(): "level": "cm", "creator": "Marina", "prediction_format": "dataframe", + "rolling_origin_stride": 1, } return meta_config diff --git a/models/zero_cmbaseline/main.py b/models/zero_cmbaseline/main.py index b48a90cc..8ef12185 100644 --- a/models/zero_cmbaseline/main.py +++ b/models/zero_cmbaseline/main.py @@ -1,8 +1,7 @@ -import wandb import warnings from pathlib import Path -from views_pipeline_core.cli.utils import parse_args, validate_arguments -from views_pipeline_core.managers.model import ModelPathManager +from views_pipeline_core.cli import ForecastingModelArgs +from views_pipeline_core.managers import ModelPathManager from views_baseline.manager.baseline_manager import BaselineForecastingModelManager warnings.filterwarnings("ignore") @@ -13,17 +12,15 @@ raise RuntimeError(f"Unexpected error: {e}. Check the logs for details.") if __name__ == "__main__": - wandb.login() - args = parse_args() - validate_arguments(args) + args = ForecastingModelArgs.parse_args() + + manager = BaselineForecastingModelManager( + model_path=model_path, + wandb_notifications=args.wandb_notifications, + use_prediction_store=args.prediction_store, + ) if args.sweep: - BaselineForecastingModelManager( - model_path=model_path, - ).execute_sweep_run(args) + manager.execute_sweep_run(args) else: - BaselineForecastingModelManager( - model_path=model_path, - wandb_notifications=args.wandb_notifications, - use_prediction_store=args.prediction_store, - ).execute_single_run(args) \ No newline at end of file + manager.execute_single_run(args) diff --git a/models/zero_pgmbaseline/main.py b/models/zero_pgmbaseline/main.py index b48a90cc..8ef12185 100644 --- a/models/zero_pgmbaseline/main.py +++ b/models/zero_pgmbaseline/main.py @@ -1,8 +1,7 @@ -import wandb import warnings from pathlib import Path -from views_pipeline_core.cli.utils import parse_args, validate_arguments -from views_pipeline_core.managers.model import ModelPathManager +from views_pipeline_core.cli import ForecastingModelArgs +from views_pipeline_core.managers import ModelPathManager from views_baseline.manager.baseline_manager import BaselineForecastingModelManager warnings.filterwarnings("ignore") @@ -13,17 +12,15 @@ raise RuntimeError(f"Unexpected error: {e}. Check the logs for details.") if __name__ == "__main__": - wandb.login() - args = parse_args() - validate_arguments(args) + args = ForecastingModelArgs.parse_args() + + manager = BaselineForecastingModelManager( + model_path=model_path, + wandb_notifications=args.wandb_notifications, + use_prediction_store=args.prediction_store, + ) if args.sweep: - BaselineForecastingModelManager( - model_path=model_path, - ).execute_sweep_run(args) + manager.execute_sweep_run(args) else: - BaselineForecastingModelManager( - model_path=model_path, - wandb_notifications=args.wandb_notifications, - use_prediction_store=args.prediction_store, - ).execute_single_run(args) \ No newline at end of file + manager.execute_single_run(args) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..479d6f75 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,57 @@ +import importlib.util +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parent.parent +MODELS_DIR = REPO_ROOT / "models" +ENSEMBLES_DIR = REPO_ROOT / "ensembles" + + +def _collect_model_dirs(base_dir: Path) -> list[Path]: + """Return sorted list of model directories (dirs containing main.py).""" + if not base_dir.exists(): + return [] + return sorted( + d for d in base_dir.iterdir() + if d.is_dir() and (d / "main.py").exists() + ) + + +ALL_MODEL_DIRS = _collect_model_dirs(MODELS_DIR) +ALL_ENSEMBLE_DIRS = _collect_model_dirs(ENSEMBLES_DIR) + +MODEL_NAMES = [d.name for d in ALL_MODEL_DIRS] +ENSEMBLE_NAMES = [d.name for d in ALL_ENSEMBLE_DIRS] + + +def load_config_module(config_path: Path, module_name: str = None): + """Load a Python config file as a module using importlib (not exec).""" + if module_name is None: + module_name = config_path.stem + # Use a unique module name to avoid collisions across models + unique_name = f"_cfg_{config_path.parent.parent.name}_{module_name}" + spec = importlib.util.spec_from_file_location(unique_name, config_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture(params=ALL_MODEL_DIRS, ids=MODEL_NAMES) +def model_dir(request): + """Parametrized fixture yielding each model directory.""" + return request.param + + +@pytest.fixture(params=ALL_ENSEMBLE_DIRS, ids=ENSEMBLE_NAMES) +def ensemble_dir(request): + """Parametrized fixture yielding each ensemble directory.""" + return request.param + + +@pytest.fixture(params=ALL_MODEL_DIRS + ALL_ENSEMBLE_DIRS, + ids=MODEL_NAMES + ENSEMBLE_NAMES) +def any_model_dir(request): + """Parametrized fixture yielding each model and ensemble directory.""" + return request.param diff --git a/tests/test_catalogs.py b/tests/test_catalogs.py new file mode 100644 index 00000000..5ef89b1f --- /dev/null +++ b/tests/test_catalogs.py @@ -0,0 +1,82 @@ +"""Tests for create_catalogs.py — catalog generation utilities.""" +import ast +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT)) + +try: + from create_catalogs import replace_table_in_section, generate_markdown_table + _HAS_PIPELINE_CORE = True +except (ImportError, ModuleNotFoundError): + _HAS_PIPELINE_CORE = False + +_skip_no_pipeline = pytest.mark.skipif( + not _HAS_PIPELINE_CORE, + reason="views_pipeline_core not installed" +) + + +class TestNoExecUsage: + def test_create_catalogs_does_not_use_exec(self): + """create_catalogs.py should use importlib, not raw exec().""" + source = (REPO_ROOT / "create_catalogs.py").read_text() + tree = ast.parse(source) + exec_calls = [ + node for node in ast.walk(tree) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "exec" + ] + assert len(exec_calls) == 0, ( + f"create_catalogs.py uses exec() {len(exec_calls)} time(s). " + "Replace with importlib.util for safer config loading." + ) + + +@_skip_no_pipeline +class TestReplaceTableInSection: + def test_replaces_content_between_markers(self): + content = ( + "before\n" + "\n" + "old table\n" + "\n" + "after" + ) + result = replace_table_in_section(content, "FOO", "new table") + assert "new table" in result + assert "old table" not in result + assert "before" in result + assert "after" in result + + def test_preserves_markers(self): + content = "old" + result = replace_table_in_section(content, "X", "new") + assert "" in result + assert "" in result + + +@_skip_no_pipeline +class TestGenerateMarkdownTable: + def test_produces_valid_markdown_table(self): + models_list = [ + { + "name": "test_model", + "algorithm": "XGBRegressor", + "targets": "lr_ged_sb", + "queryset": "test_qs", + "hyperparameters": "test_hp", + "deployment_status": "shadow", + "creator": "Test", + } + ] + table = generate_markdown_table(models_list) + assert "test_model" in table + assert "XGBRegressor" in table + assert "|" in table + lines = [l for l in table.strip().split("\n") if l.strip()] + assert len(lines) >= 3 diff --git a/tests/test_cli_pattern.py b/tests/test_cli_pattern.py new file mode 100644 index 00000000..224ac118 --- /dev/null +++ b/tests/test_cli_pattern.py @@ -0,0 +1,55 @@ +"""Tests that all model main.py files use the current CLI API pattern. + +Uses AST-based detection rather than string matching for robustness +against false positives from comments, docstrings, or multi-line strings. +""" +import ast + +import pytest + +from tests.conftest import ALL_MODEL_DIRS, MODEL_NAMES + + +def _find_imports_from(tree: ast.AST, module: str) -> list[ast.ImportFrom]: + """Find all 'from import ...' nodes in an AST.""" + return [ + node for node in ast.walk(tree) + if isinstance(node, ast.ImportFrom) + and node.module == module + ] + + +def _find_attribute_calls(tree: ast.AST, obj: str, method: str) -> list[ast.Call]: + """Find all '.()' call nodes in an AST.""" + return [ + node for node in ast.walk(tree) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and isinstance(node.func.value, ast.Name) + and node.func.value.id == obj + and node.func.attr == method + ] + + +class TestCLIPattern: + @pytest.mark.parametrize("model_dir", ALL_MODEL_DIRS, ids=MODEL_NAMES) + def test_uses_new_cli_import(self, model_dir): + """All models should import from views_pipeline_core.cli, not cli.utils.""" + source = (model_dir / "main.py").read_text() + tree = ast.parse(source) + old_imports = _find_imports_from(tree, "views_pipeline_core.cli.utils") + assert len(old_imports) == 0, ( + f"{model_dir.name}/main.py uses old CLI pattern " + "(views_pipeline_core.cli.utils). Migrate to ForecastingModelArgs." + ) + + @pytest.mark.parametrize("model_dir", ALL_MODEL_DIRS, ids=MODEL_NAMES) + def test_no_explicit_wandb_login(self, model_dir): + """Models should not call wandb.login() directly — the manager handles it.""" + source = (model_dir / "main.py").read_text() + tree = ast.parse(source) + wandb_login_calls = _find_attribute_calls(tree, "wandb", "login") + assert len(wandb_login_calls) == 0, ( + f"{model_dir.name}/main.py calls wandb.login() explicitly. " + "Remove it — the manager handles authentication." + ) diff --git a/tests/test_config_completeness.py b/tests/test_config_completeness.py new file mode 100644 index 00000000..c89b1abc --- /dev/null +++ b/tests/test_config_completeness.py @@ -0,0 +1,93 @@ +"""Tests that every model has complete and consistent config files.""" +import pytest + +from tests.conftest import load_config_module + + +# ── Required keys ────────────────────────────────────────────────────── + +REQUIRED_META_KEYS = { + "name", "algorithm", "level", "creator", + "prediction_format", "rolling_origin_stride", +} + +REQUIRED_HP_KEYS = {"steps", "time_steps"} + +VALID_DEPLOYMENT_STATUSES = {"shadow", "deployed", "baseline", "deprecated"} + + +# ── Fixtures for pre-loaded configs ──────────────────────────────────── + +@pytest.fixture +def meta_config(model_dir): + module = load_config_module(model_dir / "configs" / "config_meta.py") + return module.get_meta_config() + + +@pytest.fixture +def deployment_config(model_dir): + module = load_config_module(model_dir / "configs" / "config_deployment.py") + return module.get_deployment_config() + + +@pytest.fixture +def hp_config(model_dir): + module = load_config_module(model_dir / "configs" / "config_hyperparameters.py") + return module.get_hp_config() + + +# ── config_meta.py ───────────────────────────────────────────────────── + +class TestConfigMeta: + def test_meta_config_exists(self, model_dir): + assert (model_dir / "configs" / "config_meta.py").exists() + + def test_meta_config_has_required_keys(self, model_dir, meta_config): + missing = REQUIRED_META_KEYS - set(meta_config.keys()) + assert not missing, f"{model_dir.name} config_meta missing keys: {missing}" + + def test_meta_name_matches_directory(self, model_dir, meta_config): + assert meta_config["name"] == model_dir.name, ( + f"config_meta name '{meta_config['name']}' does not match " + f"directory '{model_dir.name}'" + ) + + def test_meta_level_is_valid(self, model_dir, meta_config): + assert meta_config["level"] in ("cm", "pgm"), ( + f"{model_dir.name} has invalid level: {meta_config['level']}" + ) + + +# ── config_deployment.py ─────────────────────────────────────────────── + +class TestConfigDeployment: + def test_deployment_config_exists(self, model_dir): + assert (model_dir / "configs" / "config_deployment.py").exists() + + def test_deployment_has_status(self, model_dir, deployment_config): + assert "deployment_status" in deployment_config + + def test_deployment_status_is_valid(self, model_dir, deployment_config): + status = deployment_config["deployment_status"] + assert status in VALID_DEPLOYMENT_STATUSES, ( + f"{model_dir.name} has invalid deployment_status: '{status}'" + ) + + +# ── config_hyperparameters.py ────────────────────────────────────────── + +class TestConfigHyperparameters: + def test_hp_config_exists(self, model_dir): + assert (model_dir / "configs" / "config_hyperparameters.py").exists() + + def test_hp_config_has_required_keys(self, model_dir, hp_config): + missing = REQUIRED_HP_KEYS - set(hp_config.keys()) + assert not missing, f"{model_dir.name} config_hp missing keys: {missing}" + + def test_time_steps_matches_steps_length(self, model_dir, hp_config): + steps = hp_config.get("steps") + time_steps = hp_config.get("time_steps") + if isinstance(steps, list) and time_steps is not None: + assert time_steps == len(steps), ( + f"{model_dir.name}: time_steps={time_steps} but len(steps)={len(steps)}" + ) diff --git a/tests/test_config_partitions.py b/tests/test_config_partitions.py new file mode 100644 index 00000000..e9458874 --- /dev/null +++ b/tests/test_config_partitions.py @@ -0,0 +1,85 @@ +"""Tests that partition configs are consistent across all models and ensembles. + +Each model/ensemble has its own self-contained config_partitions.py (required +by the framework's importlib-based loading). These tests verify that all use +the same canonical partition boundaries and forecasting offset. +""" +import re + +import pytest + + +def _extract_partition_tuples(source: str) -> dict: + """Extract calibration/validation train/test tuples from source code.""" + result = {} + for section in ("calibration", "validation"): + section_match = re.search( + rf'"{section}":\s*\{{([^}}]+)\}}', source, re.DOTALL + ) + if section_match: + block = section_match.group(1) + for key in ("train", "test"): + tuple_match = re.search( + rf'"{key}":\s*\((\d+),\s*(\d+)\)', block + ) + if tuple_match: + result[f"{section}_{key}"] = ( + int(tuple_match.group(1)), + int(tuple_match.group(2)), + ) + return result + + +class TestPartitionConsistency: + """Runs against all models AND ensembles via any_model_dir fixture.""" + + def test_partition_config_exists(self, any_model_dir): + assert (any_model_dir / "configs" / "config_partitions.py").exists() + + def test_has_generate_function(self, any_model_dir): + source = (any_model_dir / "configs" / "config_partitions.py").read_text() + assert "def generate" in source, ( + f"{any_model_dir.name} config_partitions.py has no generate() function" + ) + + def test_calibration_train(self, any_model_dir): + source = (any_model_dir / "configs" / "config_partitions.py").read_text() + tuples = _extract_partition_tuples(source) + assert tuples.get("calibration_train") == (121, 444), ( + f"{any_model_dir.name} calibration train mismatch: " + f"{tuples.get('calibration_train')}" + ) + + def test_calibration_test(self, any_model_dir): + source = (any_model_dir / "configs" / "config_partitions.py").read_text() + tuples = _extract_partition_tuples(source) + assert tuples.get("calibration_test") == (445, 492), ( + f"{any_model_dir.name} calibration test mismatch: " + f"{tuples.get('calibration_test')}" + ) + + def test_validation_train(self, any_model_dir): + source = (any_model_dir / "configs" / "config_partitions.py").read_text() + tuples = _extract_partition_tuples(source) + assert tuples.get("validation_train") == (121, 492), ( + f"{any_model_dir.name} validation train mismatch: " + f"{tuples.get('validation_train')}" + ) + + def test_validation_test(self, any_model_dir): + source = (any_model_dir / "configs" / "config_partitions.py").read_text() + tuples = _extract_partition_tuples(source) + assert tuples.get("validation_test") == (493, 540), ( + f"{any_model_dir.name} validation test mismatch: " + f"{tuples.get('validation_test')}" + ) + + def test_forecasting_offset_is_minus_one(self, any_model_dir): + """All models/ensembles should use ViewsMonth.now().id - 1.""" + source = (any_model_dir / "configs" / "config_partitions.py").read_text() + offsets = re.findall(r'ViewsMonth\.now\(\)\.id\s*-\s*(\d+)', source) + for offset in offsets: + assert offset == "1", ( + f"{any_model_dir.name} uses forecasting offset -{offset} " + f"instead of -1" + ) diff --git a/tests/test_ensemble_configs.py b/tests/test_ensemble_configs.py new file mode 100644 index 00000000..7810f5fb --- /dev/null +++ b/tests/test_ensemble_configs.py @@ -0,0 +1,132 @@ +"""Tests for ensemble configuration completeness and dependency validation. + +Ensembles have different required config keys than individual models: +- config_meta.py: name, models, targets, level, aggregation +- config_deployment.py: deployment_status +- config_hyperparameters.py: steps +- config_partitions.py: generate() function + +Ensembles do NOT have config_sweep.py or config_queryset.py. +""" +import re + +import pytest + +from tests.conftest import ( + load_config_module, + MODELS_DIR, + ENSEMBLES_DIR, + REPO_ROOT, +) + + +REQUIRED_ENSEMBLE_META_KEYS = {"name", "models", "targets", "level", "aggregation"} + +REQUIRED_ENSEMBLE_CONFIG_FILES = [ + "config_meta.py", + "config_deployment.py", + "config_hyperparameters.py", + "config_partitions.py", +] + +VALID_DEPLOYMENT_STATUSES = {"shadow", "deployed", "baseline", "deprecated"} + + +# ── Structure ────────────────────────────────────────────────────────── + +class TestEnsembleStructure: + def test_main_py_exists(self, ensemble_dir): + assert (ensemble_dir / "main.py").exists() + + def test_run_sh_exists(self, ensemble_dir): + assert (ensemble_dir / "run.sh").exists() + + def test_configs_directory_exists(self, ensemble_dir): + assert (ensemble_dir / "configs").is_dir() + + @pytest.mark.parametrize("config_file", REQUIRED_ENSEMBLE_CONFIG_FILES) + def test_required_config_file_exists(self, ensemble_dir, config_file): + cfg_path = ensemble_dir / "configs" / config_file + assert cfg_path.exists(), ( + f"{ensemble_dir.name} missing config file: {config_file}" + ) + + +# ── Config Completeness ─────────────────────────────────────────────── + +class TestEnsembleConfigMeta: + def test_meta_has_required_keys(self, ensemble_dir): + cfg_path = ensemble_dir / "configs" / "config_meta.py" + module = load_config_module(cfg_path) + meta = module.get_meta_config() + missing = REQUIRED_ENSEMBLE_META_KEYS - set(meta.keys()) + assert not missing, ( + f"{ensemble_dir.name} config_meta missing keys: {missing}" + ) + + def test_meta_name_matches_directory(self, ensemble_dir): + cfg_path = ensemble_dir / "configs" / "config_meta.py" + module = load_config_module(cfg_path) + meta = module.get_meta_config() + assert meta["name"] == ensemble_dir.name + + def test_meta_level_is_valid(self, ensemble_dir): + cfg_path = ensemble_dir / "configs" / "config_meta.py" + module = load_config_module(cfg_path) + meta = module.get_meta_config() + assert meta["level"] in ("cm", "pgm") + + def test_meta_models_is_nonempty_list(self, ensemble_dir): + cfg_path = ensemble_dir / "configs" / "config_meta.py" + module = load_config_module(cfg_path) + meta = module.get_meta_config() + assert isinstance(meta["models"], list) and len(meta["models"]) > 0, ( + f"{ensemble_dir.name} config_meta.models must be a non-empty list" + ) + + +class TestEnsembleConfigDeployment: + def test_deployment_has_valid_status(self, ensemble_dir): + cfg_path = ensemble_dir / "configs" / "config_deployment.py" + module = load_config_module(cfg_path) + dep = module.get_deployment_config() + assert dep.get("deployment_status") in VALID_DEPLOYMENT_STATUSES, ( + f"{ensemble_dir.name} has invalid deployment_status" + ) + + +class TestEnsembleConfigHyperparameters: + def test_hp_has_steps(self, ensemble_dir): + cfg_path = ensemble_dir / "configs" / "config_hyperparameters.py" + module = load_config_module(cfg_path) + hp = module.get_hp_config() + assert "steps" in hp, f"{ensemble_dir.name} config_hp missing 'steps'" + + +# ── Dependency Validation ───────────────────────────────────────────── + +class TestEnsembleDependencies: + def test_all_constituent_models_exist(self, ensemble_dir): + """Every model listed in config_meta.models must exist as a model directory.""" + cfg_path = ensemble_dir / "configs" / "config_meta.py" + module = load_config_module(cfg_path) + meta = module.get_meta_config() + missing = [ + m for m in meta["models"] + if not (MODELS_DIR / m).is_dir() + ] + assert not missing, ( + f"{ensemble_dir.name} references non-existent models: {missing}" + ) + + def test_reconcile_with_target_exists(self, ensemble_dir): + """If reconcile_with is declared, the target must exist as an ensemble.""" + cfg_path = ensemble_dir / "configs" / "config_meta.py" + module = load_config_module(cfg_path) + meta = module.get_meta_config() + target = meta.get("reconcile_with") + if target is not None: + assert (ENSEMBLES_DIR / target).is_dir(), ( + f"{ensemble_dir.name} declares reconcile_with='{target}' " + f"but no ensemble directory '{target}' exists" + ) diff --git a/tests/test_failure_modes.py b/tests/test_failure_modes.py new file mode 100644 index 00000000..f1ad4712 --- /dev/null +++ b/tests/test_failure_modes.py @@ -0,0 +1,44 @@ +"""Red-team tests: verify correct error behavior for malformed configs. + +These tests use temporary files to inject failures and verify that +the config loading infrastructure fails loudly rather than silently. +""" +import pytest + +from tests.conftest import load_config_module + + +class TestConfigLoadingSyntaxError: + def test_syntax_error_raises(self, tmp_path): + """A config file with a syntax error must raise SyntaxError.""" + bad_config = tmp_path / "config_meta.py" + bad_config.write_text("def get_meta_config(\n # missing closing paren") + with pytest.raises(SyntaxError): + load_config_module(bad_config) + + def test_missing_function_is_detectable(self, tmp_path): + """A config file that loads but lacks the expected function.""" + empty_config = tmp_path / "config_meta.py" + empty_config.write_text("x = 42\n") + module = load_config_module(empty_config) + assert not hasattr(module, "get_meta_config"), ( + "Module should not have get_meta_config if the file doesn't define it" + ) + + def test_nonexistent_file_raises(self, tmp_path): + """Loading a non-existent config file must raise an error.""" + missing = tmp_path / "does_not_exist.py" + with pytest.raises((FileNotFoundError, OSError)): + load_config_module(missing) + + def test_function_returning_non_dict_is_loadable(self, tmp_path): + """A config that returns a non-dict loads successfully — + the caller is responsible for type checking.""" + bad_return = tmp_path / "config_meta.py" + bad_return.write_text( + 'def get_meta_config():\n' + ' return "not a dict"\n' + ) + module = load_config_module(bad_return) + result = module.get_meta_config() + assert not isinstance(result, dict) diff --git a/tests/test_model_structure.py b/tests/test_model_structure.py new file mode 100644 index 00000000..402dc360 --- /dev/null +++ b/tests/test_model_structure.py @@ -0,0 +1,44 @@ +"""Tests that model directories follow the required structure and naming conventions.""" +import re + +import pytest + +from tests.conftest import ALL_MODEL_DIRS, MODEL_NAMES, REPO_ROOT + +MODEL_NAME_PATTERN = re.compile(r'^[a-z]+_[a-z]+$') + +REQUIRED_CONFIG_FILES = [ + "config_meta.py", + "config_deployment.py", + "config_hyperparameters.py", + "config_partitions.py", + "config_sweep.py", + "config_queryset.py", +] + + +class TestModelNaming: + @pytest.mark.parametrize("name", MODEL_NAMES) + def test_model_name_follows_convention(self, name): + """Model directory names must be adjective_noun in lowercase.""" + assert MODEL_NAME_PATTERN.match(name), ( + f"Model name '{name}' does not match required pattern 'adjective_noun'" + ) + + +class TestModelFiles: + def test_main_py_exists(self, model_dir): + assert (model_dir / "main.py").exists() + + def test_run_sh_exists(self, model_dir): + assert (model_dir / "run.sh").exists() + + def test_configs_directory_exists(self, model_dir): + assert (model_dir / "configs").is_dir() + + @pytest.mark.parametrize("config_file", REQUIRED_CONFIG_FILES) + def test_required_config_file_exists(self, model_dir, config_file): + cfg_path = model_dir / "configs" / config_file + assert cfg_path.exists(), ( + f"{model_dir.name} missing config file: {config_file}" + ) From 66cab0b8c5fba04ca7e255e915c14d80e63d2a21 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Sun, 15 Mar 2026 16:16:13 +0100 Subject: [PATCH 06/27] chore: remove obsolete debug scripts and archived logs Remove files superseded by the test suite or no longer referenced: - reports/archived/ (15 old training/sweep logs from Feb 2025) - verify_architecture.py (one-off NBEATS debug investigation) - compare_configs.py (hardcoded 3-model check, replaced by test suite) Also add .ruff_cache/ to .gitignore and update ADR-001 ontology. Co-Authored-By: Claude Opus 4.6 (1M context) --- .gitignore | 1 + base_docs/ADRs/001_ontology.md | 3 +- compare_configs.py | 85 -- reports/archived/single_run_config_log.txt | 482 ----------- reports/archived/single_run_config_log_v2.txt | 482 ----------- reports/archived/single_run_config_log_v3.txt | 19 - .../archived/single_run_extracted_config.txt | 0 .../single_run_extracted_config_v2.txt | 0 .../single_run_extracted_config_v3.txt | 1 - .../single_run_extracted_random_state.txt | 0 reports/archived/single_run_module_path.txt | 11 - .../single_run_random_state_log_v1.txt | 20 - .../single_run_random_state_log_v2.txt | 756 ------------------ reports/archived/sweep_run_config_log.txt | 623 --------------- reports/archived/sweep_run_config_log_v2.txt | 623 --------------- reports/archived/sweep_run_config_log_v3.txt | 22 - .../sweep_run_extracted_config_v3.txt | 1 - .../sweep_run_random_state_log_v1.txt | 22 - verify_architecture.py | 78 -- 19 files changed, 2 insertions(+), 3227 deletions(-) delete mode 100644 compare_configs.py delete mode 100644 reports/archived/single_run_config_log.txt delete mode 100644 reports/archived/single_run_config_log_v2.txt delete mode 100644 reports/archived/single_run_config_log_v3.txt delete mode 100644 reports/archived/single_run_extracted_config.txt delete mode 100644 reports/archived/single_run_extracted_config_v2.txt delete mode 100644 reports/archived/single_run_extracted_config_v3.txt delete mode 100644 reports/archived/single_run_extracted_random_state.txt delete mode 100644 reports/archived/single_run_module_path.txt delete mode 100644 reports/archived/single_run_random_state_log_v1.txt delete mode 100644 reports/archived/single_run_random_state_log_v2.txt delete mode 100644 reports/archived/sweep_run_config_log.txt delete mode 100644 reports/archived/sweep_run_config_log_v2.txt delete mode 100644 reports/archived/sweep_run_config_log_v3.txt delete mode 100644 reports/archived/sweep_run_extracted_config_v3.txt delete mode 100644 reports/archived/sweep_run_random_state_log_v1.txt delete mode 100644 verify_architecture.py diff --git a/.gitignore b/.gitignore index a4fbb82f..321d0f90 100644 --- a/.gitignore +++ b/.gitignore @@ -86,6 +86,7 @@ coverage.xml *.py,cover .hypothesis/ .pytest_cache/ +.ruff_cache/ cover/ # Translations diff --git a/base_docs/ADRs/001_ontology.md b/base_docs/ADRs/001_ontology.md index 121b47e9..03d5d12f 100644 --- a/base_docs/ADRs/001_ontology.md +++ b/base_docs/ADRs/001_ontology.md @@ -47,7 +47,6 @@ The repository recognizes the following ontological categories: |----------|----------|-------------| | **Scaffolding** | `build_model_scaffold.py`, `build_ensemble_scaffold.py`, `build_package_scaffold.py` | Interactive CLI tools for creating new model/ensemble directories | | **Catalog Generation** | `create_catalogs.py`, `update_readme.py`, `generate_features_catalog.py` | Scripts that generate documentation from configs | -| **Validation** | `compare_configs.py`, `verify_architecture.py` | Ad-hoc validation/diagnostic scripts | ### Testing Entities | Category | Location | Description | @@ -58,7 +57,7 @@ The repository recognizes the following ontological categories: | Level | Categories | Change Policy | |-------|-----------|---------------| -| **Stable** | Shared Infrastructure (`common/`), config key requirements | Changes require ADR or team discussion | +| **Stable** | Config key requirements, partition boundaries | Changes require ADR or team discussion | | **Conventional** | Config structure, CLI pattern, naming conventions | Changes require updating all models + tests | | **Volatile** | Individual model hyperparameters, querysets | Changed freely by model owners | diff --git a/compare_configs.py b/compare_configs.py deleted file mode 100644 index 1c7262bd..00000000 --- a/compare_configs.py +++ /dev/null @@ -1,85 +0,0 @@ -import importlib.util -import os -import sys - -def load_module(file_path): - module_name = os.path.splitext(os.path.basename(file_path))[0] - spec = importlib.util.spec_from_file_location(module_name, file_path) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - -models = ["novel_heuristics", "emerging_principles", "preliminary_directives"] - -for model in models: - print(f"\n--- Checking Model: {model} ---") - hp_path = f"models/{model}/configs/config_hyperparameters.py" - sweep_path = f"models/{model}/configs/config_sweep.py" - - if not os.path.exists(hp_path) or not os.path.exists(sweep_path): - print(f"Skipping {model}: files not found.") - continue - - hp_module = load_module(hp_path) - sweep_module = load_module(sweep_path) - - hp_config = hp_module.get_hp_config() - sweep_config_full = sweep_module.get_sweep_config() - sweep_params = sweep_config_full.get('parameters', {}) - - hp_keys = set(hp_config.keys()) - sweep_keys = set(sweep_params.keys()) - - # Check for use_static_covariates specifically - if "use_static_covariates" not in hp_keys: - print(" MISSING: 'use_static_covariates' in config_hyperparameters") - elif hp_config["use_static_covariates"] is not True: - print(f" WRONG VALUE: 'use_static_covariates' is {hp_config['use_static_covariates']} in config_hyperparameters") - - if "use_static_covariates" not in sweep_keys: - print(" MISSING: 'use_static_covariates' in config_sweep") - else: - sweep_val = sweep_params["use_static_covariates"].get("values", []) - if sweep_val != [True]: - print(f" WRONG VALUE: 'use_static_covariates' is {sweep_val} in config_sweep") - - only_hp = hp_keys - sweep_keys - only_sweep = sweep_keys - hp_keys - - if only_hp: - print(f" Keys only in config_hyperparameters: {only_hp}") - if only_sweep: - print(f" Keys only in config_sweep: {only_sweep}") - - common_keys = hp_keys & sweep_keys - mismatches = [] - - for key in common_keys: - hp_val = hp_config[key] - sweep_val_struct = sweep_params[key] - - # Sweep values are usually in a list under 'values' - if 'values' in sweep_val_struct: - sweep_vals = sweep_val_struct['values'] - if len(sweep_vals) == 1: - # If there's only one value in sweep, it should match hp_val - if sweep_vals[0] != hp_val: - mismatches.append((key, hp_val, sweep_vals[0])) - else: - # If multiple values, check if hp_val is one of them - if hp_val not in sweep_vals: - mismatches.append((key, hp_val, f"NOT IN {sweep_vals}")) - else: - # Could be other sweep types (distribution etc), but here it seems to be mostly grid/list - print(f" Key '{key}' has non-standard sweep structure: {sweep_val_struct}") - - if mismatches: - print(" Value mismatches (Key, HP Value, Sweep Value[0] or List):") - for m in mismatches: - print(f" - {m[0]}: HP={m[1]}, Sweep={m[2]}") - else: - if not only_hp and not only_sweep: - print(" All keys and single-value parameters match exactly.") - else: - print(" Common keys match in values.") - diff --git a/reports/archived/single_run_config_log.txt b/reports/archived/single_run_config_log.txt deleted file mode 100644 index c7f8104e..00000000 --- a/reports/archived/single_run_config_log.txt +++ /dev/null @@ -1,482 +0,0 @@ - - ##### ##### ###### ### ##### ## ## ##### ## ## ###### ##### ## ## # ##### ##### ##### ###### ###### ###### ##### ## # ###### #### - ## ## ## ## ### ## ### ## ###### ## ### ## ### ## ## ## ## ## ## ## ## ## ## ## ### ## ### ## ## ## ## ## ### ## ## ## - ## ### ## ### #### ## ## ####### ## ###### ## ## ## ### ### ####### ## ## ## ## ### #### ## ## ## ## ## #### ### - ###### ###### ## ### ## ## ## ## ## ### ####### ###### ## ####### ### ## ## ###### ## ## ## ## ## ## ## ### -## ## ## ### ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ### ## ## ## ## ## ##### ### ## ## ## -## ## ### ####### ####### ##### ## ## ##### ## ## ## ## ## ### ## # ###### ##### ## ### ####### ###### ## ##### ### ####### ##### - -views-pipeline-core v - -2026-01-28 11:49:08,176 /home/simon/Documents/scripts/views_platform/views-r2darts2/views_r2darts2/manager/model.py [model.py:99] [80917] [MainThread] - INFO - Current model architecture: NBEATSModel -2026-01-28 11:49:11,537 /home/simon/anaconda3/envs/views_pipeline/lib/python3.11/site-packages/views_pipeline_core/modules/dataloaders/dataloaders.py [dataloaders.py:1348] [80917] [MainThread] - INFO - Fetching data from viewser... -2026-01-28 11:49:11,537 /home/simon/anaconda3/envs/views_pipeline/lib/python3.11/site-packages/views_pipeline_core/modules/dataloaders/dataloaders.py [dataloaders.py:1021] [80917] [MainThread] - INFO - Beginning file download through viewser with month range 121,492 -Adding conflict history features... -2026-01-28 11:49:11,538 /home/simon/anaconda3/envs/views_pipeline/lib/python3.11/site-packages/views_pipeline_core/modules/dataloaders/dataloaders.py [dataloaders.py:1030] [80917] [MainThread] - INFO - Found queryset for preliminary_directives -2026-01-28 11:49:11,538 /home/simon/anaconda3/envs/views_pipeline/lib/python3.11/site-packages/viewser/commands/queryset/models/queryset.py [queryset.py:208] [80917] [MainThread] - INFO - Publishing queryset preliminary_directives -2026-01-28 11:49:11,807 /home/simon/anaconda3/envs/views_pipeline/lib/python3.11/site-packages/viewser/commands/queryset/models/queryset.py [queryset.py:238] [80917] [MainThread] - INFO - Fetching queryset preliminary_directives -Queryset preliminary_directives read successfully -2026-01-28 11:49:17,647 /home/simon/anaconda3/envs/views_pipeline/lib/python3.11/site-packages/views_pipeline_core/data/utils.py [utils.py:19] [80917] [MainThread] - WARNING - DataFrame contains non-np.float64 numeric columns. Converting the following columns: lr_ged_sb_dep, lr_ged_sb -2026-01-28 11:49:17,650 /home/simon/anaconda3/envs/views_pipeline/lib/python3.11/site-packages/views_pipeline_core/files/utils.py [utils.py:58] [80917] [MainThread] - INFO - Data fetch log file created at /home/simon/Documents/scripts/views_platform/views-models/models/preliminary_directives/data/raw/calibration_data_fetch_log.txt -2026-01-28 11:49:17,650 /home/simon/anaconda3/envs/views_pipeline/lib/python3.11/site-packages/views_pipeline_core/modules/dataloaders/dataloaders.py [dataloaders.py:1354] [80917] [MainThread] - INFO - Saving data to /home/simon/Documents/scripts/views_platform/views-models/models/preliminary_directives/data/raw/calibration_viewser_df.parquet -2026-01-28 11:49:22,691 /home/simon/anaconda3/envs/views_pipeline/lib/python3.11/site-packages/views_pipeline_core/managers/model/model.py [model.py:1951] [80917] [MainThread] - INFO - Training model preliminary_directives... -2026-01-28 11:49:22,709 /home/simon/Documents/scripts/views_platform/views-r2darts2/views_r2darts2/utils/loss.py [loss.py:410] [80917] [MainThread] - INFO - -zero_threshold 0.01 -delta 0.025 -non_zero_weight 7.0 -false_positive_weight 1.0 -false_negative_weight 10.0 -DEBUG: N-BEATS dropout value: 0.3 -2026-01-28 11:49:22,842 /home/simon/Documents/scripts/views_platform/views-r2darts2/views_r2darts2/model/forecaster.py [forecaster.py:102] [80917] [MainThread] - INFO - Using feature scaler: MinMaxScaler -2026-01-28 11:49:22,842 /home/simon/Documents/scripts/views_platform/views-r2darts2/views_r2darts2/model/forecaster.py [forecaster.py:103] [80917] [MainThread] - INFO - Using target scaler: MinMaxScaler -2026-01-28 11:49:22,842 /home/simon/Documents/scripts/views_platform/views-r2darts2/views_r2darts2/model/forecaster.py [forecaster.py:106] [80917] [MainThread] - INFO - Using device: cuda -2026-01-28 11:49:22,861 /home/simon/anaconda3/envs/views_pipeline/lib/python3.11/site-packages/darts/timeseries.py [timeseries.py:987] [80917] [MainThread] - WARNING - UserWarning: The (time) index from `df` is monotonically increasing. This may result in time series groups with non-overlapping (time) index. You can ignore this warning if the index represents the actual index of each individual time series group. -2026-01-28 11:49:24,028 /home/simon/Documents/scripts/views_platform/views-r2darts2/views_r2darts2/model/forecaster.py [forecaster.py:148] [80917] [MainThread] - INFO - Applying vectorized log1p transform to target series... -2026-01-28 11:49:24,041 /home/simon/Documents/scripts/views_platform/views-r2darts2/views_r2darts2/model/forecaster.py [forecaster.py:284] [80917] [MainThread] - INFO - Fitting scalers for training data... -2026-01-28 11:49:24,188 /home/simon/anaconda3/envs/views_pipeline/lib/python3.11/site-packages/darts/models/forecasting/torch_forecasting_model.py [torch_forecasting_model.py:1065] [80917] [MainThread] - INFO - Train dataset contains 43548 samples. -2026-01-28 11:49:24,198 /home/simon/anaconda3/envs/views_pipeline/lib/python3.11/site-packages/darts/models/forecasting/torch_forecasting_model.py [torch_forecasting_model.py:462] [80917] [MainThread] - INFO - Time series values are 32-bits; casting model to float32. - Training: | | 0/? [00:00 - DartsForecastingModelManager( - File "/home/simon/Documents/scripts/views_platform/views-models/models/emerging_principles/temp-views-r2darts2/views_r2darts2/manager/model.py", line 97, in __init__ - print(f"DartsForecastingModelManager.__init__ - self.config:\n{pprint.pformat(self.config)}") - ^^^^^^ -NameError: name 'pprint' is not defined. Did you mean: 'print'? - -ERROR conda.cli.main_run:execute(47): `conda run python models/preliminary_directives/main.py -r calibration -t -e` failed. (See above for error) - _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ - _( )__ _( )__ _( )__ _( )__ _( )__ _( )__ _( )__ _( )__ _( )__ _( )__ _( )__ _( )__ _( )__ _( )__ _( )__ _( )__ _( )__ _( )__ _( )__ _( )__ _( )__ _( )__ - _| _| _| _| _| _| _| _| _| _| _| _| _| _| _| _| _| _| _| _| _| _| _| _| _| _| _| _| _| _| _| _| _| _| _| _| _| _| _| _| _| _| _| _| -(_ P _ (_ (_ R _ (_ (_ E _ (_ (_ L _ (_ (_ I _ (_ (_ M _ (_ (_ I _ (_ (_ N _ (_ (_ A _ (_ (_ R _ (_ (_ Y _ (_ (_ _ _ (_ (_ D _ (_ (_ I _ (_ (_ R _ (_ (_ E _ (_ (_ C _ (_ (_ T _ (_ (_ I _ (_ (_ V _ (_ (_ E _ (_ (_ S _ (_ - |_( )__| |_( )__| |_( )__| |_( )__| |_( )__| |_( )__| |_( )__| |_( )__| |_( )__| |_( )__| |_( )__| |_( )__| |_( )__| |_( )__| |_( )__| |_( )__| |_( )__| |_( )__| |_( )__| |_( )__| |_( )__| |_( )__| -views-pipeline-core v - - diff --git a/reports/archived/single_run_extracted_config.txt b/reports/archived/single_run_extracted_config.txt deleted file mode 100644 index e69de29b..00000000 diff --git a/reports/archived/single_run_extracted_config_v2.txt b/reports/archived/single_run_extracted_config_v2.txt deleted file mode 100644 index e69de29b..00000000 diff --git a/reports/archived/single_run_extracted_config_v3.txt b/reports/archived/single_run_extracted_config_v3.txt deleted file mode 100644 index 9fc3d2b2..00000000 --- a/reports/archived/single_run_extracted_config_v3.txt +++ /dev/null @@ -1 +0,0 @@ - print(f"DartsForecastingModelManager.__init__ - self.config:\n{pprint.pformat(self.config)}") diff --git a/reports/archived/single_run_extracted_random_state.txt b/reports/archived/single_run_extracted_random_state.txt deleted file mode 100644 index e69de29b..00000000 diff --git a/reports/archived/single_run_module_path.txt b/reports/archived/single_run_module_path.txt deleted file mode 100644 index 944fed2d..00000000 --- a/reports/archived/single_run_module_path.txt +++ /dev/null @@ -1,11 +0,0 @@ -/home/simon/anaconda3/envs/views_pipeline/lib/python3.11/site-packages/fs/__init__.py:4: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81. - __import__("pkg_resources").declare_namespace(__name__) # type: ignore -Traceback (most recent call last): - File "/home/simon/Documents/scripts/views_platform/views-models/models/preliminary_directives/main.py", line 8, in - print(f"Loaded DartsForecastingModelManager from file: {DartsForecastingModelManager.__file__}") - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -AttributeError: type object 'DartsForecastingModelManager' has no attribute '__file__'. Did you mean: '__le__'? - -ERROR conda.cli.main_run:execute(47): `conda run python models/preliminary_directives/main.py -r calibration -t -e` failed. (See above for error) -Loaded DartsForecastingModelManager from module: views_r2darts2.manager.model - diff --git a/reports/archived/single_run_random_state_log_v1.txt b/reports/archived/single_run_random_state_log_v1.txt deleted file mode 100644 index 1b9df8dd..00000000 --- a/reports/archived/single_run_random_state_log_v1.txt +++ /dev/null @@ -1,20 +0,0 @@ -/home/simon/anaconda3/envs/views_pipeline/lib/python3.11/site-packages/fs/__init__.py:4: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81. - __import__("pkg_resources").declare_namespace(__name__) # type: ignore -Traceback (most recent call last): - File "/home/simon/Documents/scripts/views_platform/views-models/models/preliminary_directives/main.py", line 27, in - DartsForecastingModelManager( - File "/home/simon/Documents/scripts/views_platform/views-models/models/emerging_principles/temp-views-r2darts2/views_r2darts2/manager/model.py", line 97, in __init__ - print(f"DartsForecastingModelManager.__init__ - self.config:\n{pprint.pformat(self.config)}") - ^^^^^^ -NameError: name 'pprint' is not defined. Did you mean: 'print'? - -ERROR conda.cli.main_run:execute(47): `conda run python models/preliminary_directives/main.py -r calibration -t -e` failed. (See above for error) - dBBBBBb dBBBBBb dBBBP dBP dBP dBBBBBBb dBP dBBBBb dBBBBBb dBBBBBb dBP dBP dBBBBb dBP dBBBBBb dBBBP dBBBP dBBBBBBP dBP dBP dP dBBBP .dBBBBP - dB' dBP dBP dBP BB dBP dBP dBP dBP BP - dBBBP' dBBBBK dBBP dBP dBP dBPdBPdBP dBP dBP dBP dBP BB dBBBBK dBP dBP dBP dBP dBBBBK dBBP dBP dBP dBP dB .BP dBBP `BBBBb - dBP dBP BB dBP dBP dBP dBPdBPdBP dBP dBP dBP dBP BB dBP BB dBP dBP dBP dBP dBP BB dBP dBP dBP dBP BB.BP dBP dBP - dBP dBP dB' dBBBBP dBBBBP dBP dBPdBPdBP dBP dBP dBP dBBBBBBB dBP dB' dBP dBBBBBP dBP dBP dB' dBBBBP dBBBBP dBP dBP BBBP dBBBBP dBBBBP' - dBBBBBP -views-pipeline-core v - - diff --git a/reports/archived/single_run_random_state_log_v2.txt b/reports/archived/single_run_random_state_log_v2.txt deleted file mode 100644 index 395186cd..00000000 --- a/reports/archived/single_run_random_state_log_v2.txt +++ /dev/null @@ -1,756 +0,0 @@ - ### ### ### # # ## # # ## ## ## ### ## ## ### # ### ### ## ###### # ## ## ### ## - ## ## ## ## ## # # ## ## # ## ## ### ## ## ## ## # #### # ## ## ## ## # ## ## # ## ## ## ## # - ## ## ## ## ## ## ## ###### ## ### ## #### ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## - ## ## ##### ##### ## ## #### # ## ### ## ## ## ##### #### ####### ## ## ## ##### ##### ## ## ## ## ## ##### #### - ##### #### ## ### ### ## # ## ### ## # ## ###### #### ### ####### ## ## ### #### ## ## ## ### ## ## ## ## - ## ## ## ### ### ### ## ## ### ## ### ## ## ## ## ## ## ## ## ### ## ## ### ### # ### ### #### ### ### ## - ## ## ## ##### ####### ### ## ## ### ## ## ## # ## ## ## # ###### ### ## ## ##### #### ### ### ## ##### ##### - -views-pipeline-core v - -DartsForecastingModelManager.__init__ - self.configs: -{'activation': 'LeakyReLU', - 'algorithm': 'NBEATSModel', - 'batch_norm': False, - 'batch_size': 8, - 'calibration': {'test': (445, 492), 'train': (121, 444)}, - 'creator': 'Dylan', - 'delta': 0.025, - 'deployment_status': 'shadow', - 'dropout': 0.3, - 'early_stopping_min_delta': 0.01, - 'early_stopping_patience': 1, - 'false_negative_weight': 10.0, - 'false_positive_weight': 1.0, - 'feature_scaler': 'MinMaxScaler', - 'force_reset': True, - 'forecasting': {'test': (553, 589), 'train': (121, 552)}, - 'generic_architecture': True, - 'gradient_clip_val': 1.0, - 'input_chunk_length': 24, - 'layer_widths': 64, - 'level': 'cm', - 'log_features': None, - 'log_targets': True, - 'loss_function': 'WeightedPenaltyHuberLoss', - 'lr': 0.0003, - 'lr_scheduler_cls': 'ReduceLROnPlateau', - 'lr_scheduler_factor': 0.46, - 'lr_scheduler_min_lr': 1e-05, - 'lr_scheduler_patience': 7, - 'mc_dropout': True, - 'metrics': ['RMSLE', 'CRPS', 'MSE', 'MSLE', 'y_hat_bar'], - 'n_epochs': 1, - 'name': 'preliminary_directives', - 'non_zero_weight': 7.0, - 'num_blocks': 4, - 'num_layers': 3, - 'num_samples': 1, - 'num_stacks': 2, - 'optimizer_cls': 'Adam', - 'output_chunk_length': 36, - 'output_chunk_shift': 0, - 'random_state': 1, - 'steps': [1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - 23, - 24, - 25, - 26, - 27, - 28, - 29, - 30, - 31, - 32, - 33, - 34, - 35, - 36], - 'target_scaler': 'MinMaxScaler', - 'targets': ['lr_ged_sb_dep'], - 'timestamp': '20260128_134912', - 'validation': {'test': (493, 540), 'train': (121, 492)}, - 'weight_decay': 0.0003, - 'zero_threshold': 0.01} -DEBUG_RANDOM: Random state - Python: (3, (456564148, 2611905626, 592316382, 787855846, 641386653, 436260381, 1131431592, 611485174, 4184208554, 1507531188, 1414416195, 2818044443, 1451567435, 1108953984, 2194331947, 763830546, 1310896731, 1283513936, 199225566, 2746292213, 1546858833, 2290313291, 3974682621, 1311214517, 1252710614, 1049573926, 1205724794, 3226910825, 2438410871, 2385315097, 1763884408, 3892600307, 3081563684, 1123850291, 2187278271, 689880982, 274054850, 545519776, 949147096, 2330700447, 2619468161, 2084306526, 3203592226, 303666462, 3133234466, 1830932316, 719641770, 2150223135, 1613349056, 2281482638, 3996455193, 2593233812, 2541039782, 2196535855, 3861169802, 1502435058, 1490853549, 2797465322, 2352218757, 2557218482, 3511934739, 3040919819, 3336418430, 412093602, 1325850758, 551038936, 4135464466, 895803124, 1810728810, 398238446, 1272512872, 1285139615, 3170869585, 1336172300, 380632129, 2402080119, 467356405, 4191277210, 2492997107, 85574276, 2765175054, 1936944656, 2747395653, 1091843737, 2242586646, 1175309312, 3928062921, 4194604980, 1847319884, 1124462116, 4029214586, 96557441, 1947513514, 2723937067, 828465518, 3114015567, 2723405760, 968525647, 3789484945, 4276419949, 2052948776, 2114205390, 1497290354, 711334162, 2447890724, 952152732, 4174176394, 3998606692, 958231765, 1862390971, 2989326209, 841367892, 2886212561, 290476824, 2333993013, 3405007977, 3902216862, 2033268322, 2005557232, 4237329544, 746290568, 3420502125, 1169223372, 2930094504, 3742508109, 326288352, 1607836906, 4212591683, 1863758496, 3009067498, 4178637313, 3969358674, 3928281091, 752519622, 3241198552, 2672485564, 1533627654, 4182498985, 3837805428, 3636898565, 730443940, 1331034810, 552836728, 1751504307, 3458594797, 2680232215, 2534923419, 1410360961, 757627297, 1156233302, 2744110953, 907203410, 2897952766, 2047650074, 1911847661, 4102013153, 2091913545, 2997421218, 3370413058, 873944876, 2933566878, 3326988209, 2399329211, 14761894, 3285354216, 3371515002, 4111697575, 1581128841, 3043600708, 940892155, 1486074264, 2206262641, 4276243055, 519759776, 3955564139, 2761578900, 1495157986, 4160699609, 3037871743, 2463101912, 402183466, 1762947167, 2012129992, 3801567959, 1436104084, 1472917009, 3669845711, 3585862029, 2193399541, 320189339, 1732575725, 3994167881, 4221010676, 779641721, 2748753013, 2413107515, 2612732524, 2736729978, 3302336750, 2514827415, 3735683532, 666868400, 2839841142, 1422324578, 771349672, 1698409083, 2462141075, 1541034506, 531578355, 2685962295, 3745433567, 3617371540, 2790144062, 884224608, 1544690955, 1158949451, 2478891997, 2936932842, 2092555912, 348762330, 793878729, 393165335, 3250285461, 3147069459, 1316050335, 1557474137, 2139609576, 709165938, 468543035, 3307483517, 45785328, 3554573287, 4128726499, 3967628255, 1966754838, 920689438, 1822374597, 2196225156, 3811870125, 3340360805, 2158934083, 1247306175, 2813566473, 1626831600, 3679084378, 560399654, 2542935195, 3305967734, 3464358031, 360215734, 3005316369, 2616360743, 3263651473, 434880581, 3364767514, 2615499798, 4247959324, 171010190, 3356690207, 1244993508, 2853576153, 2591273176, 4003016036, 3235117222, 52470602, 3677678435, 170342381, 1583562361, 2544376148, 4045193326, 1438535222, 2308648535, 2061450685, 1239106562, 3737877496, 2538221558, 3228309897, 1648231599, 2675032086, 569412397, 2595229828, 3639721668, 277429164, 3159105045, 2684607928, 258125810, 3976086787, 949377550, 1008928825, 1740076144, 3072010336, 1482577144, 2559051177, 2742081140, 3585525598, 944105200, 1703032404, 2000898031, 1851338480, 106693473, 784395044, 2699931889, 1444152517, 669051715, 820561732, 3427160916, 165398750, 106782685, 873114980, 3736292235, 3354619705, 3798635493, 2677758390, 4024915810, 27473589, 1766899311, 3915040678, 2553891650, 1540833934, 282079013, 1747816768, 1901697909, 3527556353, 3186877425, 3230489081, 2464516600, 668330819, 748114230, 3384229696, 1125287868, 447654857, 4293581670, 1490193591, 1550737148, 1249136464, 2043908134, 4034683237, 3267772330, 547160554, 3514941464, 1435125169, 1899789525, 2918250213, 1340590423, 3375143470, 3801447265, 268443009, 2642816250, 364617325, 994449619, 2866413787, 3347807057, 70504842, 1019704301, 3982151746, 687728046, 2862127604, 941700464, 72289207, 712785379, 1705399043, 3322956689, 3201508826, 2006768738, 4091536475, 1281642454, 2694608689, 672224243, 2424325539, 3116542287, 1681233304, 3407603215, 3294714865, 1552611654, 3851461017, 703829416, 2928209683, 2464573241, 1887967519, 1347724910, 2478841893, 3684079987, 298721552, 1156222604, 1223388534, 250594198, 3837125903, 3027678681, 1142815396, 2800751983, 688726187, 3944849466, 1620535262, 1815651152, 459850449, 691265604, 3574653309, 2264168388, 399903633, 2014975199, 3671104122, 2803092614, 1038193384, 501376349, 2717515982, 2558046826, 4154044794, 4267333613, 597263793, 1219554544, 2184346180, 3905066004, 301386107, 3620441918, 2559803181, 3680359830, 3247218304, 3975090885, 520856224, 1504402331, 2202036090, 2637791829, 1720202944, 3862792504, 2887067844, 1662033168, 3051551923, 4270394846, 367999844, 1315806494, 3681835522, 2503363283, 776667375, 3180495661, 3850303911, 3429373735, 331124683, 2728448321, 3145453755, 488458593, 994028424, 644986741, 3960509148, 2176263759, 736442016, 516977460, 4281243409, 299314821, 4209791307, 3381266161, 1336746632, 2052817668, 3036084934, 1056378265, 3829158605, 1787155443, 3615005090, 3906676528, 1014196379, 4096861339, 3416517801, 811006184, 4236543516, 2261316690, 2975841189, 677449934, 209077256, 174549845, 1955478821, 3381945616, 4107032890, 3907895285, 1187364798, 383253925, 1819678977, 1318287961, 3589529399, 2947300184, 377688945, 2126818104, 289020565, 1746981157, 1676022064, 2423406876, 1556710227, 2633381768, 4227360471, 1086707346, 32680910, 3024771002, 2102616991, 1976412811, 4246081994, 113534032, 109768698, 1403776103, 402385734, 337090127, 1985342731, 3864243588, 501223413, 297133635, 1974127119, 1329255943, 1927512242, 1363362043, 1611842253, 961279717, 1233333415, 3558287526, 1716085788, 2962821702, 3652331458, 873771381, 2983583000, 266228391, 858392569, 3414057846, 2563396760, 2257604774, 3214976125, 2868410235, 212428659, 3012320297, 13885816, 491442508, 4190881043, 2880008522, 1367485243, 766548337, 1735096073, 102605620, 3041484525, 2527374457, 2068324088, 4009714748, 2901304420, 1521500940, 1510988686, 3487288203, 3162725127, 3327614662, 2854500750, 1736024530, 2211043737, 2717763763, 2990008215, 2093025610, 4220870178, 325110216, 411048713, 3576612386, 3675242092, 3239499530, 3560270487, 2207462979, 897231212, 3931293388, 3200437702, 4086159445, 2767436117, 2006296793, 1169265972, 1971448230, 4104765154, 4276629362, 4078738646, 1449596374, 169249608, 454707672, 249762916, 490502561, 2505765287, 3322581916, 3621665343, 553492900, 2686098776, 3828288445, 125133367, 3928084929, 865873774, 1644336435, 3324716652, 3223888472, 264209089, 3530062329, 4129511485, 3999053531, 792213595, 3864987956, 2268113284, 1913913222, 2240437257, 4052205172, 4010800026, 2032602046, 3932356498, 268463209, 2688063532, 3096535287, 866347726, 1866844083, 3960311568, 3370725122, 2287563544, 3768409382, 3691453248, 512810403, 2927625004, 3020067408, 3247531699, 4171151950, 2676196655, 544775557, 3872113579, 1729560701, 1248397269, 326032463, 1319653023, 4281029053, 846732470, 3342008662, 3130255302, 2559291303, 305), None) -DEBUG_RANDOM: Random state - NumPy: ('MT19937', array([2147483648, 753920679, 2941639623, 2523706338, 2814097119, - 2690624423, 1875406890, 3805014005, 1836555523, 2645315844, - 3642991761, 2765935922, 4190460033, 52658683, 1499886436, - 515781920, 3919163085, 876987853, 729689388, 4059543865, - 1969209467, 4044593707, 3072182207, 121569088, 1310644010, - 1562055411, 1353139925, 1737866939, 3108277056, 2226577070, - 2431214379, 3411646855, 209962903, 3904504946, 2662262204, - 2271497322, 1825878320, 1504056286, 2103631616, 1352836636, - 2860400266, 654050772, 3563090002, 3085401225, 2440424625, - 1778473817, 1263630291, 2837047303, 4092626980, 1302525354, - 276955216, 1072836028, 2058614551, 4060527550, 731129014, - 1579538395, 4076501838, 2026225325, 1875246903, 1885479628, - 1515189451, 1248944873, 515718399, 2287627268, 2202574044, - 3928799588, 2704228541, 2204415978, 1379481693, 504354551, - 3127286404, 927846259, 2767498624, 1481073135, 3871808629, - 1217289312, 398234104, 2291013462, 545909643, 1099051804, - 1885826099, 3159360216, 1736921301, 786392337, 3519704046, - 825645326, 248987509, 1639470790, 646916283, 433677617, - 2353228593, 3574048041, 1403730095, 262224057, 386205639, - 691752165, 2884857831, 1485906265, 1124511508, 218333965, - 879498888, 1392596089, 220123772, 1015275837, 1810777626, - 2611907462, 3013902998, 2718406761, 619038241, 2515983278, - 1773072428, 2556628093, 956754386, 1638065322, 85801167, - 1726280618, 1851640057, 2971596710, 4023402482, 347593801, - 1186763784, 666588371, 402748915, 2588574692, 4045873827, - 29066275, 2145451170, 1278819175, 3032001580, 1451898562, - 2846944034, 1720127753, 548147144, 1067072098, 4127501286, - 2320172405, 1217816290, 2442147640, 1113115114, 766520926, - 296939254, 1090747867, 139440205, 3275294375, 3615101486, - 2352514992, 3367185633, 2082268882, 1422002150, 3149454976, - 3852978949, 148365993, 4217875539, 1728499438, 3950333703, - 3929979152, 585446113, 2171934061, 1744865730, 334495537, - 311876482, 94667628, 3636277679, 1116620672, 266433541, - 976944706, 4251655433, 1799633914, 571949283, 923716134, - 260591034, 1869394434, 2641295018, 4049541639, 4115611597, - 3870168496, 135197146, 3571648235, 26570567, 2099779942, - 2385880465, 2431603543, 938052299, 3590303233, 602170448, - 316682888, 3949745566, 1635294678, 1268369181, 4151272951, - 3763466259, 246980838, 1017647420, 3538173868, 3347343382, - 320834961, 2277347289, 834892603, 2594133117, 3062543540, - 1746349923, 2139712545, 2978093360, 1846397155, 1547155571, - 611089844, 2062885879, 3944828863, 1708457856, 2162186344, - 1319103437, 3518647412, 2952230867, 2041998222, 1215403987, - 1407783575, 2634436914, 2564967946, 169115494, 4133680204, - 1800057422, 3001127929, 135657175, 3805224209, 1501862800, - 2325322065, 3222805002, 1544197124, 1964586493, 802401963, - 4262327151, 432232061, 3370165012, 1541321206, 312487314, - 1133324259, 3030742113, 2504531216, 1374123621, 1216056933, - 1477777089, 2709601929, 1245888912, 433127070, 3298132017, - 517271148, 2252929130, 2098946021, 2534377403, 3241348187, - 2571735739, 964629201, 3222073736, 819093322, 2790709012, - 759524379, 2168494036, 1835046696, 3183615312, 560513254, - 1129844284, 2244737587, 1359993581, 1501583550, 1983711817, - 816885784, 3953480215, 2668061417, 2012781244, 739531468, - 4011251988, 869319758, 2236532250, 2837875886, 3083265428, - 2450769629, 280531356, 88836723, 1971595558, 3824949122, - 1611449656, 1584291206, 2184400961, 1315187530, 4251489132, - 3480760672, 28761716, 3560917729, 942632763, 3943353282, - 1570892467, 3625226010, 2681407031, 2190490856, 668740030, - 1805330725, 1543502603, 4212993077, 2607205204, 1453806025, - 429674584, 1886410024, 1692609633, 924727808, 3434204005, - 1168157096, 2640060814, 217137855, 665808395, 2740244204, - 3128213846, 2667825917, 2337902317, 4122382302, 548170846, - 742564588, 2081621655, 2378776913, 3287801269, 2566158506, - 979767794, 3923340945, 1236096063, 2685194370, 1423499964, - 4252549504, 99223816, 1080046686, 2555714849, 1040634798, - 3370847833, 2858191538, 2328873099, 4122699363, 3380481660, - 3049347935, 3162547412, 4083941860, 3053663569, 871851065, - 3165506185, 3671742178, 283741515, 1277460314, 891404053, - 2666049024, 1424324503, 3692130367, 379789735, 4002362775, - 4098488459, 2329754371, 1868672885, 756707538, 440316460, - 333384145, 3188154595, 4062677343, 3426850595, 1736883768, - 182428459, 2328579811, 2536733185, 3642928972, 2898264580, - 3798048825, 3792107396, 3826648072, 2365931985, 1512452910, - 3914567809, 3691973953, 3358636692, 1311941770, 3329133132, - 4210814469, 1670462757, 11020224, 2379387936, 2091309320, - 3299789610, 2290131224, 570318524, 1589587425, 1079268544, - 1812539457, 3888998751, 1577840005, 4105379799, 71204294, - 512804722, 1599603814, 2528259270, 51141028, 4014123248, - 2256603503, 1771217622, 1890336884, 2990085691, 3485568607, - 1136693908, 2065962156, 439630438, 3473959390, 3239534131, - 927514224, 1075276469, 2179983249, 2630877179, 210850169, - 3840709790, 3126605170, 2354663517, 1833981256, 3404408617, - 19425782, 3549595270, 1064031651, 1787133847, 1963635716, - 1866803361, 3541953381, 3235028144, 29019396, 3815685530, - 1747507456, 1696308009, 28416223, 3244320420, 3448115934, - 946081510, 1801081924, 1066678848, 550737627, 3677319677, - 2863827708, 1172808259, 2526376237, 2582958294, 4237310830, - 3434614540, 1414037536, 849467475, 4104322060, 2756967639, - 3031773741, 3996054124, 4255942642, 4188590140, 516935559, - 834263595, 941265556, 280413394, 1622410828, 1494025199, - 1292010300, 388027476, 2747294824, 1544718496, 172734629, - 3847968031, 122157727, 2131966481, 3531731487, 2911026134, - 3347479310, 3717401793, 1445036015, 1213000808, 194350509, - 3544103068, 645081869, 369380912, 2204699009, 1778210959, - 28163971, 4078233104, 3496330621, 179433587, 1349790634, - 2606376270, 2411296881, 3655913466, 3125493317, 536077079, - 693799970, 3324796750, 1383995840, 3502090057, 1784161392, - 2494472092, 854147719, 1055601199, 3048905186, 3482741780, - 1652256869, 3891117369, 4095571733, 2137565370, 1462563938, - 1206179648, 3350373130, 380050565, 899885917, 3437219947, - 1346920736, 872150025, 3882537156, 708728762, 2811544588, - 1009537390, 4049468419, 4208810057, 3585543167, 1165761006, - 1261840056, 1807479519, 4096284624, 435428886, 1990605720, - 3982759184, 551535735, 2109282315, 431562839, 1946967368, - 3076304260, 1847024661, 2234840746, 3935610304, 2313814114, - 4198397036, 1243306709, 4241408490, 4129813791, 860089313, - 219827556, 1244490521, 1085810512, 2085519932, 1240131780, - 3743442212, 1765317408, 2903930580, 4261471454, 942992160, - 237603553, 1500076667, 1172494060, 2436199633, 2461987014, - 261785080, 2672337398, 1324864003, 1170439915, 1775396320, - 2360483804, 3395637153, 17680469, 3126230740, 2539331141, - 1683416962, 2724754016, 4274914844, 2937681239, 1123833688, - 3563926997, 3239917119, 3735818066, 804830796, 37149790, - 1897753071, 2512228115, 2469504437, 4115084767, 2899748500, - 63254427, 63841370, 215771028, 1273889404, 641063611, - 1054756026, 1073815801, 2742168464, 3021206573, 3300971956, - 1034054666, 3902892563, 3257671362, 3903579887, 70890465, - 3799134576, 4272931889, 2062114493, 4171902498, 836767410, - 2354814869, 3440709425, 4126340554, 4251837647, 1344745305, - 1196446052, 3923672374, 4091755665, 2404136802, 780885391, - 3452633811, 3179889995, 311148373, 312270472, 2211779101, - 1494854923, 1121522903, 2918647571, 3137741950, 4107776949, - 2946206245, 265468454, 2178787829, 753776095, 1653235940, - 101051585, 4264171634, 3737976654, 1241257442], dtype=uint32), 623, 0, 0.0) -DEBUG_RANDOM: Random state - PyTorch CPU: tensor([219, 177, 41, ..., 0, 0, 0], dtype=torch.uint8) -DEBUG_RANDOM: Random state - PyTorch CUDA: tensor([ 77, 66, 99, 206, 73, 97, 11, 0, 0, 0, 0, 0, 0, 0, - 0, 0], dtype=torch.uint8) -DEBUG_RANDOM: torch.backends.cudnn.deterministic: False -DEBUG_RANDOM: torch.backends.cudnn.benchmark: False -2026-01-28 13:49:12,070 /home/simon/Documents/scripts/views_platform/views-models/models/emerging_principles/temp-views-r2darts2/views_r2darts2/manager/model.py [model.py:110] [116185] [MainThread] - INFO - Current model architecture: NBEATSModel -2026-01-28 13:49:13,523 /home/simon/anaconda3/envs/views_pipeline/lib/python3.11/site-packages/views_pipeline_core/modules/dataloaders/dataloaders.py [dataloaders.py:1348] [116185] [MainThread] - INFO - Fetching data from viewser... -2026-01-28 13:49:13,523 /home/simon/anaconda3/envs/views_pipeline/lib/python3.11/site-packages/views_pipeline_core/modules/dataloaders/dataloaders.py [dataloaders.py:1021] [116185] [MainThread] - INFO - Beginning file download through viewser with month range 121,492 -Adding conflict history features... -2026-01-28 13:49:13,534 /home/simon/anaconda3/envs/views_pipeline/lib/python3.11/site-packages/views_pipeline_core/modules/dataloaders/dataloaders.py [dataloaders.py:1030] [116185] [MainThread] - INFO - Found queryset for preliminary_directives -2026-01-28 13:49:13,534 /home/simon/anaconda3/envs/views_pipeline/lib/python3.11/site-packages/viewser/commands/queryset/models/queryset.py [queryset.py:208] [116185] [MainThread] - INFO - Publishing queryset preliminary_directives -2026-01-28 13:49:13,799 /home/simon/anaconda3/envs/views_pipeline/lib/python3.11/site-packages/viewser/commands/queryset/models/queryset.py [queryset.py:238] [116185] [MainThread] - INFO - Fetching queryset preliminary_directives -Queryset preliminary_directives read successfully -2026-01-28 13:49:19,625 /home/simon/anaconda3/envs/views_pipeline/lib/python3.11/site-packages/views_pipeline_core/data/utils.py [utils.py:19] [116185] [MainThread] - WARNING - DataFrame contains non-np.float64 numeric columns. Converting the following columns: lr_ged_sb_dep, lr_ged_sb -2026-01-28 13:49:19,635 /home/simon/anaconda3/envs/views_pipeline/lib/python3.11/site-packages/views_pipeline_core/files/utils.py [utils.py:58] [116185] [MainThread] - INFO - Data fetch log file created at /home/simon/Documents/scripts/views_platform/views-models/models/preliminary_directives/data/raw/calibration_data_fetch_log.txt -2026-01-28 13:49:19,635 /home/simon/anaconda3/envs/views_pipeline/lib/python3.11/site-packages/views_pipeline_core/modules/dataloaders/dataloaders.py [dataloaders.py:1354] [116185] [MainThread] - INFO - Saving data to /home/simon/Documents/scripts/views_platform/views-models/models/preliminary_directives/data/raw/calibration_viewser_df.parquet -2026-01-28 13:49:22,694 /home/simon/anaconda3/envs/views_pipeline/lib/python3.11/site-packages/views_pipeline_core/managers/model/model.py [model.py:1951] [116185] [MainThread] - INFO - Training model preliminary_directives... -2026-01-28 13:49:22,708 /home/simon/Documents/scripts/views_platform/views-models/models/emerging_principles/temp-views-r2darts2/views_r2darts2/utils/loss.py [loss.py:410] [116185] [MainThread] - INFO - -zero_threshold 0.01 -delta 0.025 -non_zero_weight 7.0 -false_positive_weight 1.0 -false_negative_weight 10.0 -DEBUG: N-BEATS dropout value: 0.3 -2026-01-28 13:49:22,709 /home/simon/Documents/scripts/views_platform/views-models/models/emerging_principles/temp-views-r2darts2/views_r2darts2/model/catalog.py [catalog.py:242] [116185] [MainThread] - INFO - NBEATSModel kwargs: {'activation': 'LeakyReLU', - 'batch_size': 8, - 'dropout': 0.3, - 'force_reset': True, - 'generic_architecture': True, - 'input_chunk_length': 24, - 'layer_widths': 64, - 'loss_fn': WeightedPenaltyHuberLoss(), - 'lr_scheduler_cls': , - 'lr_scheduler_kwargs': {'factor': 0.46, - 'min_lr': 1e-05, - 'mode': 'min', - 'monitor': 'train_loss', - 'patience': 7}, - 'model_name': 'preliminary_directives', - 'n_epochs': 1, - 'num_blocks': 4, - 'num_layers': 3, - 'num_stacks': 2, - 'optimizer_kwargs': {'lr': 0.0003, 'weight_decay': 0.0003}, - 'output_chunk_length': 36, - 'output_chunk_shift': 0, - 'pl_trainer_kwargs': {'accelerator': 'gpu', - 'callbacks': [, - ], - 'enable_progress_bar': True, - 'gradient_clip_val': 1.0, - 'logger': }, - 'random_state': 1} -2026-01-28 13:49:22,820 /home/simon/Documents/scripts/views_platform/views-models/models/emerging_principles/temp-views-r2darts2/views_r2darts2/model/forecaster.py [forecaster.py:102] [116185] [MainThread] - INFO - Using feature scaler: MinMaxScaler -2026-01-28 13:49:22,820 /home/simon/Documents/scripts/views_platform/views-models/models/emerging_principles/temp-views-r2darts2/views_r2darts2/model/forecaster.py [forecaster.py:103] [116185] [MainThread] - INFO - Using target scaler: MinMaxScaler -2026-01-28 13:49:22,821 /home/simon/Documents/scripts/views_platform/views-models/models/emerging_principles/temp-views-r2darts2/views_r2darts2/model/forecaster.py [forecaster.py:106] [116185] [MainThread] - INFO - Using device: cuda -2026-01-28 13:49:22,859 /home/simon/anaconda3/envs/views_pipeline/lib/python3.11/site-packages/darts/timeseries.py [timeseries.py:987] [116185] [MainThread] - WARNING - UserWarning: The (time) index from `df` is monotonically increasing. This may result in time series groups with non-overlapping (time) index. You can ignore this warning if the index represents the actual index of each individual time series group. -2026-01-28 13:49:24,575 /home/simon/Documents/scripts/views_platform/views-models/models/emerging_principles/temp-views-r2darts2/views_r2darts2/model/forecaster.py [forecaster.py:148] [116185] [MainThread] - INFO - Applying vectorized log1p transform to target series... -2026-01-28 13:49:24,594 /home/simon/Documents/scripts/views_platform/views-models/models/emerging_principles/temp-views-r2darts2/views_r2darts2/model/forecaster.py [forecaster.py:284] [116185] [MainThread] - INFO - Fitting scalers for training data... -2026-01-28 13:49:24,789 /home/simon/anaconda3/envs/views_pipeline/lib/python3.11/site-packages/darts/models/forecasting/torch_forecasting_model.py [torch_forecasting_model.py:1065] [116185] [MainThread] - INFO - Train dataset contains 43548 samples. -2026-01-28 13:49:24,798 /home/simon/anaconda3/envs/views_pipeline/lib/python3.11/site-packages/darts/models/forecasting/torch_forecasting_model.py [torch_forecasting_model.py:462] [116185] [MainThread] - INFO - Time series values are 32-bits; casting model to float32. - Training: | | 0/? [00:00, - 'lr_scheduler_kwargs': {'factor': 0.46, - 'min_lr': 1e-05, - 'mode': 'min', - 'monitor': 'train_loss', - 'patience': 7}, - 'model_name': 'preliminary_directives', - 'n_epochs': 1, - 'num_blocks': 4, - 'num_layers': 3, - 'num_stacks': 2, - 'optimizer_kwargs': {'lr': 0.0003, 'weight_decay': 0.0003}, - 'output_chunk_length': 36, - 'output_chunk_shift': 0, - 'pl_trainer_kwargs': {'accelerator': 'gpu', - 'callbacks': [, - ], - 'enable_progress_bar': True, - 'gradient_clip_val': 1.0, - 'logger': }, - 'random_state': 1} -2026-01-28 13:50:28,889 /home/simon/Documents/scripts/views_platform/views-models/models/emerging_principles/temp-views-r2darts2/views_r2darts2/model/forecaster.py [forecaster.py:102] [116185] [MainThread] - INFO - Using feature scaler: MinMaxScaler -2026-01-28 13:50:28,890 /home/simon/Documents/scripts/views_platform/views-models/models/emerging_principles/temp-views-r2darts2/views_r2darts2/model/forecaster.py [forecaster.py:103] [116185] [MainThread] - INFO - Using target scaler: MinMaxScaler -2026-01-28 13:50:28,890 /home/simon/Documents/scripts/views_platform/views-models/models/emerging_principles/temp-views-r2darts2/views_r2darts2/model/forecaster.py [forecaster.py:106] [116185] [MainThread] - INFO - Using device: cuda -2026-01-28 13:50:29,135 /home/simon/Documents/scripts/views_platform/views-models/models/emerging_principles/temp-views-r2darts2/views_r2darts2/model/forecaster.py [forecaster.py:473] [116185] [MainThread] - INFO - Model loaded and moved to device: cuda -2026-01-28 13:50:29,135 /home/simon/Documents/scripts/views_platform/views-models/models/emerging_principles/temp-views-r2darts2/views_r2darts2/manager/model.py [model.py:252] [116185] [MainThread] - INFO - Starting parallel prediction with None workers for 12 sequences -2026-01-28 13:50:29,135 /home/simon/Documents/scripts/views_platform/views-models/models/emerging_principles/temp-views-r2darts2/views_r2darts2/manager/model.py [model.py:238] [116185] [ThreadPoolExecutor-1_0] - INFO - Starting prediction for sequence 1/12 -2026-01-28 13:50:29,148 /home/simon/anaconda3/envs/views_pipeline/lib/python3.11/site-packages/darts/timeseries.py [timeseries.py:987] [116185] [ThreadPoolExecutor-1_0] - WARNING - UserWarning: The (time) index from `df` is monotonically increasing. This may result in time series groups with non-overlapping (time) index. You can ignore this warning if the index represents the actual index of each individual time series group. -2026-01-28 13:50:30,473 /home/simon/Documents/scripts/views_platform/views-models/models/emerging_principles/temp-views-r2darts2/views_r2darts2/model/forecaster.py [forecaster.py:148] [116185] [ThreadPoolExecutor-1_0] - INFO - Applying vectorized log1p transform to target series... -2026-01-28 13:50:30,485 /home/simon/Documents/scripts/views_platform/views-models/models/emerging_principles/temp-views-r2darts2/views_r2darts2/model/forecaster.py [forecaster.py:291] [116185] [ThreadPoolExecutor-1_0] - INFO - Transforming scalers for prediction data... - Predicting: | | 0/? [00:00 - DartsForecastingModelManager( - File "/home/simon/Documents/scripts/views_platform/views-models/models/emerging_principles/temp-views-r2darts2/views_r2darts2/manager/model.py", line 97, in __init__ - print(f"DartsForecastingModelManager.__init__ - self.config:\n{pprint.pformat(self.config)}") - ^^^^^^ -NameError: name 'pprint' is not defined. Did you mean: 'print'? - -ERROR conda.cli.main_run:execute(47): `conda run python models/preliminary_directives/main.py -r calibration -s` failed. (See above for error) -###### ###### ####### ### ##### ## ## ##### ## ### ## ###### ## ## ## ##### ##### ###### ####### #### ####### ##### ## ## ####### ###### -## ### ## ### ## ### ### ### ### ### ### ### ### #### ## ### ## ## ## ## ### ### ## ### ## ### ## ### ####### ### ## ## ## ### ##### -### ## ### ## ## ## ## ## ####### ## ####### ## ## ### ## ### ## ## ## ### ## ### ## ## ## ## ### ## ## ## ## ## ## -###### ###### ### ## ## ## # ## ## ## #### ####### ###### ##### ## # ## ## ###### ### ### ### ## ## ## ### ##### -## ## ## #### ## ### ## ## ### ## ## ## ### ## ## #### ## ## ## ### ## ## #### #### ## ### ### ## ## #### ### -### ### ## ## ## ## ## ### ### ## ### ### ## ### ### ### ## ### ## ### ### ### ### ## ## ## ###### #### ### ##### ## ## ### ## -#### ### ## ####### ###### #### ### ## #### ### ## ### ### ### ## ### ## ###### #### ### ## ####### #### #### #### ### ####### ###### - -views-pipeline-core v - - diff --git a/reports/archived/sweep_run_extracted_config_v3.txt b/reports/archived/sweep_run_extracted_config_v3.txt deleted file mode 100644 index 9fc3d2b2..00000000 --- a/reports/archived/sweep_run_extracted_config_v3.txt +++ /dev/null @@ -1 +0,0 @@ - print(f"DartsForecastingModelManager.__init__ - self.config:\n{pprint.pformat(self.config)}") diff --git a/reports/archived/sweep_run_random_state_log_v1.txt b/reports/archived/sweep_run_random_state_log_v1.txt deleted file mode 100644 index e7c178d7..00000000 --- a/reports/archived/sweep_run_random_state_log_v1.txt +++ /dev/null @@ -1,22 +0,0 @@ -/home/simon/anaconda3/envs/views_pipeline/lib/python3.11/site-packages/fs/__init__.py:4: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81. - __import__("pkg_resources").declare_namespace(__name__) # type: ignore -Traceback (most recent call last): - File "/home/simon/Documents/scripts/views_platform/views-models/models/preliminary_directives/main.py", line 22, in - DartsForecastingModelManager( - File "/home/simon/Documents/scripts/views_platform/views-models/models/emerging_principles/temp-views-r2darts2/views_r2darts2/manager/model.py", line 97, in __init__ - print(f"DartsForecastingModelManager.__init__ - self.config:\n{pprint.pformat(self.config)}") - ^^^^^^ -NameError: name 'pprint' is not defined. Did you mean: 'print'? - -ERROR conda.cli.main_run:execute(47): `conda run python models/preliminary_directives/main.py -r calibration -s` failed. (See above for error) - -###### ###### ####### ##### ####### ## ## ####### ## ## ##### ###### ## ## # ###### ####### ###### ####### ##### ###### ####### ## ## ####### ##### - ###### ###### ### ### ### ###### ### ### ## ##### ###### ## ## ## ### ## ### ###### ### ### ## ###### ### ## ## ### ### # - ## ## ## ## ##### ### ### ###### ### ###### ## ## ## ## ###### ####### ### ## ### ## ## ##### ### # ## # ### ## ## ##### ##### - ##### ##### ### ### ### ###### ### ###### ##### ##### #### ####### ### ## ### ##### ### ### ## ### ###### ### ##### - ### ##### ### ### ## ### ## ## ### ## ### ## ## ##### ## ## ### ## ### ##### ### ### ## ## ### #### ### # ### -##### ## ## ####### ####### ####### ## ## ####### ## ## ### ### ## ## #### # ###### ####### ## ## ####### ##### #### ####### ## ####### ##### - -views-pipeline-core v - - diff --git a/verify_architecture.py b/verify_architecture.py deleted file mode 100644 index 6898a418..00000000 --- a/verify_architecture.py +++ /dev/null @@ -1,78 +0,0 @@ - -import logging -import torch -from darts.models import NBEATSModel -from darts.timeseries import TimeSeries -import pandas as pd - -# Known Pitfall: A local logging.py can shadow the standard library. -# This import helps avoid AttributeError. -from importlib import reload -reload(logging) - -def verify_nbeats_architecture(): - """ - Instantiates and prints the architecture of two NBEATS models - with different num_blocks to verify Darts library behavior. - """ - print("--- Verifying Darts NBEATSModel architecture ---") - - # Create a minimal dummy TimeSeries for fitting - dummy_data = pd.DataFrame({ - 'time': pd.to_datetime(pd.date_range('2023-01-01', periods=30, freq='D')), - 'value': range(30) - }) - dummy_ts = TimeSeries.from_dataframe(dummy_data, 'time', 'value') - - # --- Model 1: num_blocks = 1 --- - print("\n--- Architecture for num_blocks=1 ---") - model_1 = NBEATSModel( - input_chunk_length=24, - output_chunk_length=1, - generic_architecture=True, - num_blocks=1, - num_stacks=2, - num_layers=2, - layer_widths=64, - n_epochs=1, - random_state=42, - model_name="nbeats_blocks_1", - ) - # Fitting the model triggers the internal PyTorch model creation - model_1.fit(dummy_ts, verbose=False) - print(model_1.model) - - - # --- Model 2: num_blocks = 2 --- - print("\n--- Architecture for num_blocks=2 ---") - model_2 = NBEATSModel( - input_chunk_length=24, - output_chunk_length=1, - generic_architecture=True, - num_blocks=2, - num_stacks=2, - num_layers=2, - layer_widths=64, - n_epochs=1, - random_state=42, - model_name="nbeats_blocks_2", - ) - # Fitting the model triggers the internal PyTorch model creation - model_2.fit(dummy_ts, verbose=False) - print(model_2.model) - - # --- Comparison --- - arch1_str = str(model_1.model) - arch2_str = str(model_2.model) - - print("\n--- Conclusion ---") - if arch1_str == arch2_str: - print("Architectures are IDENTICAL. This suggests a potential issue within the Darts library itself.") - else: - print("Architectures are DIFFERENT. The Darts library is functioning correctly.") - print("The root cause is very likely in the views-r2darts2 wrapper code, which is probably not passing the 'num_blocks' parameter correctly to the model.") - -if __name__ == "__main__": - # Set a higher log level to suppress unnecessary Darts/PyTorch Lightning info - logging.basicConfig(level=logging.ERROR) - verify_nbeats_architecture() From 773c9f24724a05a3d764335194d93b6bf78b041a Mon Sep 17 00:00:00 2001 From: Polichinl Date: Sun, 15 Mar 2026 16:31:28 +0100 Subject: [PATCH 07/27] fix(tests): resolve ruff lint errors in test suite Remove unused imports (sys, re, pytest, REPO_ROOT, ALL_MODEL_DIRS) and rename ambiguous variable `l` to `line` in test_catalogs.py. Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/conftest.py | 1 - tests/test_catalogs.py | 2 +- tests/test_config_partitions.py | 2 -- tests/test_ensemble_configs.py | 3 --- tests/test_model_structure.py | 2 +- 5 files changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 479d6f75..5646c827 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,4 @@ import importlib.util -import sys from pathlib import Path import pytest diff --git a/tests/test_catalogs.py b/tests/test_catalogs.py index 5ef89b1f..9839077a 100644 --- a/tests/test_catalogs.py +++ b/tests/test_catalogs.py @@ -78,5 +78,5 @@ def test_produces_valid_markdown_table(self): assert "test_model" in table assert "XGBRegressor" in table assert "|" in table - lines = [l for l in table.strip().split("\n") if l.strip()] + lines = [line for line in table.strip().split("\n") if line.strip()] assert len(lines) >= 3 diff --git a/tests/test_config_partitions.py b/tests/test_config_partitions.py index e9458874..45364ea0 100644 --- a/tests/test_config_partitions.py +++ b/tests/test_config_partitions.py @@ -6,8 +6,6 @@ """ import re -import pytest - def _extract_partition_tuples(source: str) -> dict: """Extract calibration/validation train/test tuples from source code.""" diff --git a/tests/test_ensemble_configs.py b/tests/test_ensemble_configs.py index 7810f5fb..7aebea5d 100644 --- a/tests/test_ensemble_configs.py +++ b/tests/test_ensemble_configs.py @@ -8,15 +8,12 @@ Ensembles do NOT have config_sweep.py or config_queryset.py. """ -import re - import pytest from tests.conftest import ( load_config_module, MODELS_DIR, ENSEMBLES_DIR, - REPO_ROOT, ) diff --git a/tests/test_model_structure.py b/tests/test_model_structure.py index 402dc360..07faa6b5 100644 --- a/tests/test_model_structure.py +++ b/tests/test_model_structure.py @@ -3,7 +3,7 @@ import pytest -from tests.conftest import ALL_MODEL_DIRS, MODEL_NAMES, REPO_ROOT +from tests.conftest import MODEL_NAMES MODEL_NAME_PATTERN = re.compile(r'^[a-z]+_[a-z]+$') From 654b56eecf2d3254ffd29a26186418cb467b7a35 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Sun, 15 Mar 2026 20:42:17 +0100 Subject: [PATCH 08/27] feat: add integration test runner script Shell script that trains and evaluates each model on calibration and validation partitions, logging results per model without aborting on failure. Supports --models, --partitions, and --timeout flags. Co-Authored-By: Claude Opus 4.6 (1M context) --- .gitignore | 3 + run_integration_tests.sh | 244 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 247 insertions(+) create mode 100755 run_integration_tests.sh diff --git a/.gitignore b/.gitignore index 321d0f90..64a3619d 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,9 @@ # But please, take a second to consult with the team before doing so anyways. +# Integration test logs +logs/ + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/run_integration_tests.sh b/run_integration_tests.sh new file mode 100755 index 00000000..732edb29 --- /dev/null +++ b/run_integration_tests.sh @@ -0,0 +1,244 @@ +#!/bin/bash +# +# Integration test runner for views-models +# +# Trains and evaluates each model on calibration and validation partitions. +# Logs results per model — never aborts on failure. +# +# Usage: +# bash run_integration_tests.sh # all models, calibration + validation +# bash run_integration_tests.sh --models "counting_stars purple_alien" # subset +# bash run_integration_tests.sh --partitions "calibration" # one partition only +# bash run_integration_tests.sh --timeout 3600 # 60-minute timeout per run +# + +set -uo pipefail + +# ── Defaults ────────────────────────────────────────────────────────── + +TIMEOUT=1800 # 30 minutes per model+partition run +PARTITIONS="calibration validation" +FILTER_MODELS="" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +MODELS_DIR="$SCRIPT_DIR/models" +TIMESTAMP=$(date +%Y-%m-%d_%H%M%S) +LOG_DIR="$SCRIPT_DIR/logs/integration_test_$TIMESTAMP" + +# ── Colors ──────────────────────────────────────────────────────────── + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[0;33m' +BOLD='\033[1m' +NC='\033[0m' + +# ── Parse arguments ─────────────────────────────────────────────────── + +while [[ $# -gt 0 ]]; do + case "$1" in + --models) + FILTER_MODELS="$2" + shift 2 + ;; + --partitions) + PARTITIONS="$2" + shift 2 + ;; + --timeout) + TIMEOUT="$2" + shift 2 + ;; + --help|-h) + echo "Usage: bash run_integration_tests.sh [OPTIONS]" + echo "" + echo "Options:" + echo " --models \"model1 model2\" Run only these models (space-separated)" + echo " --partitions \"cal val\" Partitions to test (default: calibration validation)" + echo " --timeout SECONDS Timeout per run (default: 1800)" + echo " --help Show this help" + exit 0 + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac +done + +# ── Discover models ────────────────────────────────────────────────── + +if [ -n "$FILTER_MODELS" ]; then + MODELS=() + for m in $FILTER_MODELS; do + if [ -f "$MODELS_DIR/$m/run.sh" ]; then + MODELS+=("$m") + else + echo -e "${YELLOW}WARNING: Model '$m' not found or has no run.sh — skipping${NC}" + fi + done +else + MODELS=() + for dir in "$MODELS_DIR"/*/; do + model_name=$(basename "$dir") + if [ -f "$dir/run.sh" ]; then + MODELS+=("$model_name") + fi + done + IFS=$'\n' MODELS=($(sort <<<"${MODELS[*]}")); unset IFS +fi + +TOTAL_MODELS=${#MODELS[@]} +if [ "$TOTAL_MODELS" -eq 0 ]; then + echo "No models found to test." + exit 1 +fi + +# ── Create log directories ─────────────────────────────────────────── + +for partition in $PARTITIONS; do + mkdir -p "$LOG_DIR/$partition" +done + +# ── Header ─────────────────────────────────────────────────────────── + +echo -e "${BOLD}═══════════════════════════════════════════════════════════${NC}" +echo -e "${BOLD} views-models integration test${NC}" +echo -e "${BOLD}═══════════════════════════════════════════════════════════${NC}" +echo " Models: $TOTAL_MODELS" +echo " Partitions: $PARTITIONS" +echo " Timeout: ${TIMEOUT}s per run" +echo " Logs: $LOG_DIR" +echo -e "${BOLD}═══════════════════════════════════════════════════════════${NC}" +echo "" + +# ── Run models ─────────────────────────────────────────────────────── + +declare -A RESULTS +PASS_COUNT=0 +FAIL_COUNT=0 +TIMEOUT_COUNT=0 +RUN_INDEX=0 +TOTAL_RUNS=$(( TOTAL_MODELS * $(echo $PARTITIONS | wc -w) )) + +for model in "${MODELS[@]}"; do + for partition in $PARTITIONS; do + RUN_INDEX=$((RUN_INDEX + 1)) + model_log="$LOG_DIR/$partition/${model}.log" + result_key="${model}__${partition}" + + echo -ne "[${RUN_INDEX}/${TOTAL_RUNS}] ${BOLD}${model}${NC} (${partition})... " + + # Record start time + start_time=$(date +%s) + + # Run the model via its own run.sh, capturing all output + # The run.sh handles conda env activation internally + timeout "$TIMEOUT" bash -c " + cd '$MODELS_DIR/$model' && \ + bash run.sh -r '$partition' -t -e + " > "$model_log" 2>&1 + exit_code=$? + + # Record duration + end_time=$(date +%s) + duration=$((end_time - start_time)) + duration_str="${duration}s" + + if [ "$exit_code" -eq 0 ]; then + echo -e "${GREEN}PASS${NC} (${duration_str})" + RESULTS[$result_key]="PASS" + PASS_COUNT=$((PASS_COUNT + 1)) + elif [ "$exit_code" -eq 124 ]; then + echo -e "${RED}TIMEOUT${NC} (>${TIMEOUT}s)" + RESULTS[$result_key]="TIMEOUT" + TIMEOUT_COUNT=$((TIMEOUT_COUNT + 1)) + echo "=== TIMEOUT after ${TIMEOUT}s ===" >> "$model_log" + else + echo -e "${RED}FAIL${NC} (exit ${exit_code}, ${duration_str})" + RESULTS[$result_key]="FAIL(${exit_code})" + FAIL_COUNT=$((FAIL_COUNT + 1)) + fi + done +done + +# ── Summary ────────────────────────────────────────────────────────── + +echo "" +echo -e "${BOLD}═══════════════════════════════════════════════════════════${NC}" +echo -e "${BOLD} Summary${NC}" +echo -e "${BOLD}═══════════════════════════════════════════════════════════${NC}" +echo "" + +# Print table header +printf "%-30s" "Model" +for partition in $PARTITIONS; do + printf "%-15s" "$partition" +done +echo "" +printf "%-30s" "-----" +for partition in $PARTITIONS; do + printf "%-15s" "----------" +done +echo "" + +# Print results per model +for model in "${MODELS[@]}"; do + printf "%-30s" "$model" + for partition in $PARTITIONS; do + result_key="${model}__${partition}" + result="${RESULTS[$result_key]:-SKIPPED}" + if [ "$result" = "PASS" ]; then + printf "${GREEN}%-15s${NC}" "$result" + else + printf "${RED}%-15s${NC}" "$result" + fi + done + echo "" +done + +echo "" +echo -e " ${GREEN}Passed:${NC} $PASS_COUNT" +echo -e " ${RED}Failed:${NC} $FAIL_COUNT" +if [ "$TIMEOUT_COUNT" -gt 0 ]; then + echo -e " ${RED}Timeout:${NC} $TIMEOUT_COUNT" +fi +echo " Total: $TOTAL_RUNS" +echo "" + +# ── Write summary log ──────────────────────────────────────────────── + +{ + echo "Integration Test Summary — $TIMESTAMP" + echo "Models: $TOTAL_MODELS | Partitions: $PARTITIONS | Timeout: ${TIMEOUT}s" + echo "" + printf "%-30s" "Model" + for partition in $PARTITIONS; do + printf "%-15s" "$partition" + done + echo "" + printf "%-30s" "-----" + for partition in $PARTITIONS; do + printf "%-15s" "----------" + done + echo "" + for model in "${MODELS[@]}"; do + printf "%-30s" "$model" + for partition in $PARTITIONS; do + result_key="${model}__${partition}" + result="${RESULTS[$result_key]:-SKIPPED}" + printf "%-15s" "$result" + done + echo "" + done + echo "" + echo "Passed: $PASS_COUNT | Failed: $FAIL_COUNT | Timeout: $TIMEOUT_COUNT | Total: $TOTAL_RUNS" +} > "$LOG_DIR/summary.log" + +echo "Full summary: $LOG_DIR/summary.log" +echo "Per-model logs: $LOG_DIR/{partition}/{model}.log" + +# Exit with non-zero if any failures +if [ "$FAIL_COUNT" -gt 0 ] || [ "$TIMEOUT_COUNT" -gt 0 ]; then + exit 1 +fi +exit 0 From df2c6ee584a08bc9c6d9b0384b0dfd96adc5d7fc Mon Sep 17 00:00:00 2001 From: Polichinl Date: Sun, 15 Mar 2026 21:17:02 +0100 Subject: [PATCH 09/27] fix(models): add missing requirements.txt for 33 models Models without requirements.txt fail to install packages when run.sh creates their conda environment. Generated from main.py imports: - 15 models: views-r2darts2>=1.0.0,<2.0.0 - 12 models: views-baseline>=1.0.0,<2.0.0 - 6 models: views-stepshifter>=1.0.0,<2.0.0 Co-Authored-By: Claude Opus 4.6 (1M context) --- models/adolecent_slob/requirements.txt | 1 + models/average_cmbaseline/requirements.txt | 1 + models/average_pgmbaseline/requirements.txt | 1 + models/black_ranger/requirements.txt | 1 + models/blue_ranger/requirements.txt | 1 + models/bouncy_organ/requirements.txt | 1 + models/cheap_thrills/requirements.txt | 1 + models/cool_cat/requirements.txt | 1 + models/dancing_queen/requirements.txt | 1 + models/elastic_heart/requirements.txt | 1 + models/emerging_principles/requirements.txt | 1 + models/fancy_feline/requirements.txt | 1 + models/fourtieth_symphony/requirements.txt | 1 + models/good_life/requirements.txt | 1 + models/green_ranger/requirements.txt | 1 + models/heat_waves/requirements.txt | 1 + models/hot_stream/requirements.txt | 1 + models/locf_cmbaseline/requirements.txt | 1 + models/locf_pgmbaseline/requirements.txt | 1 + models/lovely_creature/requirements.txt | 1 + models/new_rules/requirements.txt | 1 + models/novel_heuristics/requirements.txt | 1 + models/party_princess/requirements.txt | 1 + models/pink_ranger/requirements.txt | 1 + models/preliminary_directives/requirements.txt | 1 + models/purple_haze/requirements.txt | 1 + models/red_ranger/requirements.txt | 1 + models/teenage_dirtbag/requirements.txt | 1 + models/wild_rose/requirements.txt | 1 + models/wuthering_heights/requirements.txt | 1 + models/yellow_ranger/requirements.txt | 1 + models/zero_cmbaseline/requirements.txt | 1 + models/zero_pgmbaseline/requirements.txt | 1 + 33 files changed, 33 insertions(+) create mode 100644 models/adolecent_slob/requirements.txt create mode 100644 models/average_cmbaseline/requirements.txt create mode 100644 models/average_pgmbaseline/requirements.txt create mode 100644 models/black_ranger/requirements.txt create mode 100644 models/blue_ranger/requirements.txt create mode 100644 models/bouncy_organ/requirements.txt create mode 100644 models/cheap_thrills/requirements.txt create mode 100644 models/cool_cat/requirements.txt create mode 100644 models/dancing_queen/requirements.txt create mode 100644 models/elastic_heart/requirements.txt create mode 100644 models/emerging_principles/requirements.txt create mode 100644 models/fancy_feline/requirements.txt create mode 100644 models/fourtieth_symphony/requirements.txt create mode 100644 models/good_life/requirements.txt create mode 100644 models/green_ranger/requirements.txt create mode 100644 models/heat_waves/requirements.txt create mode 100644 models/hot_stream/requirements.txt create mode 100644 models/locf_cmbaseline/requirements.txt create mode 100644 models/locf_pgmbaseline/requirements.txt create mode 100644 models/lovely_creature/requirements.txt create mode 100644 models/new_rules/requirements.txt create mode 100644 models/novel_heuristics/requirements.txt create mode 100644 models/party_princess/requirements.txt create mode 100644 models/pink_ranger/requirements.txt create mode 100644 models/preliminary_directives/requirements.txt create mode 100644 models/purple_haze/requirements.txt create mode 100644 models/red_ranger/requirements.txt create mode 100644 models/teenage_dirtbag/requirements.txt create mode 100644 models/wild_rose/requirements.txt create mode 100644 models/wuthering_heights/requirements.txt create mode 100644 models/yellow_ranger/requirements.txt create mode 100644 models/zero_cmbaseline/requirements.txt create mode 100644 models/zero_pgmbaseline/requirements.txt diff --git a/models/adolecent_slob/requirements.txt b/models/adolecent_slob/requirements.txt new file mode 100644 index 00000000..a223a7c3 --- /dev/null +++ b/models/adolecent_slob/requirements.txt @@ -0,0 +1 @@ +views-r2darts2>=1.0.0,<2.0.0 diff --git a/models/average_cmbaseline/requirements.txt b/models/average_cmbaseline/requirements.txt new file mode 100644 index 00000000..876dbf67 --- /dev/null +++ b/models/average_cmbaseline/requirements.txt @@ -0,0 +1 @@ +views-baseline>=1.0.0,<2.0.0 diff --git a/models/average_pgmbaseline/requirements.txt b/models/average_pgmbaseline/requirements.txt new file mode 100644 index 00000000..876dbf67 --- /dev/null +++ b/models/average_pgmbaseline/requirements.txt @@ -0,0 +1 @@ +views-baseline>=1.0.0,<2.0.0 diff --git a/models/black_ranger/requirements.txt b/models/black_ranger/requirements.txt new file mode 100644 index 00000000..876dbf67 --- /dev/null +++ b/models/black_ranger/requirements.txt @@ -0,0 +1 @@ +views-baseline>=1.0.0,<2.0.0 diff --git a/models/blue_ranger/requirements.txt b/models/blue_ranger/requirements.txt new file mode 100644 index 00000000..876dbf67 --- /dev/null +++ b/models/blue_ranger/requirements.txt @@ -0,0 +1 @@ +views-baseline>=1.0.0,<2.0.0 diff --git a/models/bouncy_organ/requirements.txt b/models/bouncy_organ/requirements.txt new file mode 100644 index 00000000..a223a7c3 --- /dev/null +++ b/models/bouncy_organ/requirements.txt @@ -0,0 +1 @@ +views-r2darts2>=1.0.0,<2.0.0 diff --git a/models/cheap_thrills/requirements.txt b/models/cheap_thrills/requirements.txt new file mode 100644 index 00000000..48bf1e92 --- /dev/null +++ b/models/cheap_thrills/requirements.txt @@ -0,0 +1 @@ +views-stepshifter>=1.0.0,<2.0.0 diff --git a/models/cool_cat/requirements.txt b/models/cool_cat/requirements.txt new file mode 100644 index 00000000..a223a7c3 --- /dev/null +++ b/models/cool_cat/requirements.txt @@ -0,0 +1 @@ +views-r2darts2>=1.0.0,<2.0.0 diff --git a/models/dancing_queen/requirements.txt b/models/dancing_queen/requirements.txt new file mode 100644 index 00000000..a223a7c3 --- /dev/null +++ b/models/dancing_queen/requirements.txt @@ -0,0 +1 @@ +views-r2darts2>=1.0.0,<2.0.0 diff --git a/models/elastic_heart/requirements.txt b/models/elastic_heart/requirements.txt new file mode 100644 index 00000000..a223a7c3 --- /dev/null +++ b/models/elastic_heart/requirements.txt @@ -0,0 +1 @@ +views-r2darts2>=1.0.0,<2.0.0 diff --git a/models/emerging_principles/requirements.txt b/models/emerging_principles/requirements.txt new file mode 100644 index 00000000..a223a7c3 --- /dev/null +++ b/models/emerging_principles/requirements.txt @@ -0,0 +1 @@ +views-r2darts2>=1.0.0,<2.0.0 diff --git a/models/fancy_feline/requirements.txt b/models/fancy_feline/requirements.txt new file mode 100644 index 00000000..a223a7c3 --- /dev/null +++ b/models/fancy_feline/requirements.txt @@ -0,0 +1 @@ +views-r2darts2>=1.0.0,<2.0.0 diff --git a/models/fourtieth_symphony/requirements.txt b/models/fourtieth_symphony/requirements.txt new file mode 100644 index 00000000..48bf1e92 --- /dev/null +++ b/models/fourtieth_symphony/requirements.txt @@ -0,0 +1 @@ +views-stepshifter>=1.0.0,<2.0.0 diff --git a/models/good_life/requirements.txt b/models/good_life/requirements.txt new file mode 100644 index 00000000..a223a7c3 --- /dev/null +++ b/models/good_life/requirements.txt @@ -0,0 +1 @@ +views-r2darts2>=1.0.0,<2.0.0 diff --git a/models/green_ranger/requirements.txt b/models/green_ranger/requirements.txt new file mode 100644 index 00000000..876dbf67 --- /dev/null +++ b/models/green_ranger/requirements.txt @@ -0,0 +1 @@ +views-baseline>=1.0.0,<2.0.0 diff --git a/models/heat_waves/requirements.txt b/models/heat_waves/requirements.txt new file mode 100644 index 00000000..a223a7c3 --- /dev/null +++ b/models/heat_waves/requirements.txt @@ -0,0 +1 @@ +views-r2darts2>=1.0.0,<2.0.0 diff --git a/models/hot_stream/requirements.txt b/models/hot_stream/requirements.txt new file mode 100644 index 00000000..a223a7c3 --- /dev/null +++ b/models/hot_stream/requirements.txt @@ -0,0 +1 @@ +views-r2darts2>=1.0.0,<2.0.0 diff --git a/models/locf_cmbaseline/requirements.txt b/models/locf_cmbaseline/requirements.txt new file mode 100644 index 00000000..876dbf67 --- /dev/null +++ b/models/locf_cmbaseline/requirements.txt @@ -0,0 +1 @@ +views-baseline>=1.0.0,<2.0.0 diff --git a/models/locf_pgmbaseline/requirements.txt b/models/locf_pgmbaseline/requirements.txt new file mode 100644 index 00000000..876dbf67 --- /dev/null +++ b/models/locf_pgmbaseline/requirements.txt @@ -0,0 +1 @@ +views-baseline>=1.0.0,<2.0.0 diff --git a/models/lovely_creature/requirements.txt b/models/lovely_creature/requirements.txt new file mode 100644 index 00000000..48bf1e92 --- /dev/null +++ b/models/lovely_creature/requirements.txt @@ -0,0 +1 @@ +views-stepshifter>=1.0.0,<2.0.0 diff --git a/models/new_rules/requirements.txt b/models/new_rules/requirements.txt new file mode 100644 index 00000000..a223a7c3 --- /dev/null +++ b/models/new_rules/requirements.txt @@ -0,0 +1 @@ +views-r2darts2>=1.0.0,<2.0.0 diff --git a/models/novel_heuristics/requirements.txt b/models/novel_heuristics/requirements.txt new file mode 100644 index 00000000..a223a7c3 --- /dev/null +++ b/models/novel_heuristics/requirements.txt @@ -0,0 +1 @@ +views-r2darts2>=1.0.0,<2.0.0 diff --git a/models/party_princess/requirements.txt b/models/party_princess/requirements.txt new file mode 100644 index 00000000..a223a7c3 --- /dev/null +++ b/models/party_princess/requirements.txt @@ -0,0 +1 @@ +views-r2darts2>=1.0.0,<2.0.0 diff --git a/models/pink_ranger/requirements.txt b/models/pink_ranger/requirements.txt new file mode 100644 index 00000000..876dbf67 --- /dev/null +++ b/models/pink_ranger/requirements.txt @@ -0,0 +1 @@ +views-baseline>=1.0.0,<2.0.0 diff --git a/models/preliminary_directives/requirements.txt b/models/preliminary_directives/requirements.txt new file mode 100644 index 00000000..a223a7c3 --- /dev/null +++ b/models/preliminary_directives/requirements.txt @@ -0,0 +1 @@ +views-r2darts2>=1.0.0,<2.0.0 diff --git a/models/purple_haze/requirements.txt b/models/purple_haze/requirements.txt new file mode 100644 index 00000000..48bf1e92 --- /dev/null +++ b/models/purple_haze/requirements.txt @@ -0,0 +1 @@ +views-stepshifter>=1.0.0,<2.0.0 diff --git a/models/red_ranger/requirements.txt b/models/red_ranger/requirements.txt new file mode 100644 index 00000000..876dbf67 --- /dev/null +++ b/models/red_ranger/requirements.txt @@ -0,0 +1 @@ +views-baseline>=1.0.0,<2.0.0 diff --git a/models/teenage_dirtbag/requirements.txt b/models/teenage_dirtbag/requirements.txt new file mode 100644 index 00000000..a223a7c3 --- /dev/null +++ b/models/teenage_dirtbag/requirements.txt @@ -0,0 +1 @@ +views-r2darts2>=1.0.0,<2.0.0 diff --git a/models/wild_rose/requirements.txt b/models/wild_rose/requirements.txt new file mode 100644 index 00000000..48bf1e92 --- /dev/null +++ b/models/wild_rose/requirements.txt @@ -0,0 +1 @@ +views-stepshifter>=1.0.0,<2.0.0 diff --git a/models/wuthering_heights/requirements.txt b/models/wuthering_heights/requirements.txt new file mode 100644 index 00000000..48bf1e92 --- /dev/null +++ b/models/wuthering_heights/requirements.txt @@ -0,0 +1 @@ +views-stepshifter>=1.0.0,<2.0.0 diff --git a/models/yellow_ranger/requirements.txt b/models/yellow_ranger/requirements.txt new file mode 100644 index 00000000..876dbf67 --- /dev/null +++ b/models/yellow_ranger/requirements.txt @@ -0,0 +1 @@ +views-baseline>=1.0.0,<2.0.0 diff --git a/models/zero_cmbaseline/requirements.txt b/models/zero_cmbaseline/requirements.txt new file mode 100644 index 00000000..876dbf67 --- /dev/null +++ b/models/zero_cmbaseline/requirements.txt @@ -0,0 +1 @@ +views-baseline>=1.0.0,<2.0.0 diff --git a/models/zero_pgmbaseline/requirements.txt b/models/zero_pgmbaseline/requirements.txt new file mode 100644 index 00000000..876dbf67 --- /dev/null +++ b/models/zero_pgmbaseline/requirements.txt @@ -0,0 +1 @@ +views-baseline>=1.0.0,<2.0.0 From 5a2fd2e6a05bd8aa1c90e9d72bd5f64baaf5d622 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Sun, 15 Mar 2026 21:59:01 +0100 Subject: [PATCH 10/27] fix(integration-tests): use single conda env, exclude purple_alien Simplify integration test runner to activate a single conda env (default: views_pipeline) instead of trying to use per-model envs via run.sh. Adds --env and --exclude flags. Excludes purple_alien by default (needs views-hydranet env). Co-Authored-By: Claude Opus 4.6 (1M context) --- run_integration_tests.sh | 116 ++++++++++++++++++--------------------- 1 file changed, 53 insertions(+), 63 deletions(-) diff --git a/run_integration_tests.sh b/run_integration_tests.sh index 732edb29..6083bfb7 100755 --- a/run_integration_tests.sh +++ b/run_integration_tests.sh @@ -2,23 +2,27 @@ # # Integration test runner for views-models # -# Trains and evaluates each model on calibration and validation partitions. -# Logs results per model — never aborts on failure. +# Trains and evaluates each model on calibration and validation partitions +# using a single conda environment. Logs results per model — never aborts on failure. # # Usage: -# bash run_integration_tests.sh # all models, calibration + validation -# bash run_integration_tests.sh --models "counting_stars purple_alien" # subset -# bash run_integration_tests.sh --partitions "calibration" # one partition only -# bash run_integration_tests.sh --timeout 3600 # 60-minute timeout per run +# bash run_integration_tests.sh # all models +# bash run_integration_tests.sh --models "counting_stars bad_blood" # subset +# bash run_integration_tests.sh --partitions "calibration" # one partition +# bash run_integration_tests.sh --exclude "purple_alien novel_heuristics" # skip models +# bash run_integration_tests.sh --env my_conda_env # different env +# bash run_integration_tests.sh --timeout 3600 # 60-min timeout # set -uo pipefail # ── Defaults ────────────────────────────────────────────────────────── -TIMEOUT=1800 # 30 minutes per model+partition run +CONDA_ENV="views_pipeline" +TIMEOUT=1800 PARTITIONS="calibration validation" FILTER_MODELS="" +EXCLUDE_MODELS="purple_alien" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" MODELS_DIR="$SCRIPT_DIR/models" TIMESTAMP=$(date +%Y-%m-%d_%H%M%S) @@ -36,51 +40,53 @@ NC='\033[0m' while [[ $# -gt 0 ]]; do case "$1" in - --models) - FILTER_MODELS="$2" - shift 2 - ;; - --partitions) - PARTITIONS="$2" - shift 2 - ;; - --timeout) - TIMEOUT="$2" - shift 2 - ;; + --models) FILTER_MODELS="$2"; shift 2 ;; + --exclude) EXCLUDE_MODELS="$2"; shift 2 ;; + --partitions) PARTITIONS="$2"; shift 2 ;; + --timeout) TIMEOUT="$2"; shift 2 ;; + --env) CONDA_ENV="$2"; shift 2 ;; --help|-h) echo "Usage: bash run_integration_tests.sh [OPTIONS]" echo "" echo "Options:" - echo " --models \"model1 model2\" Run only these models (space-separated)" + echo " --env NAME Conda env to activate (default: views_pipeline)" + echo " --models \"m1 m2\" Run only these models" + echo " --exclude \"m1 m2\" Skip these models (default: purple_alien)" echo " --partitions \"cal val\" Partitions to test (default: calibration validation)" echo " --timeout SECONDS Timeout per run (default: 1800)" - echo " --help Show this help" exit 0 ;; - *) - echo "Unknown option: $1" - exit 1 - ;; + *) echo "Unknown option: $1"; exit 1 ;; esac done +# ── Build exclusion set ────────────────────────────────────────────── + +declare -A EXCLUDED +for m in $EXCLUDE_MODELS; do + EXCLUDED[$m]=1 +done + # ── Discover models ────────────────────────────────────────────────── +MODELS=() if [ -n "$FILTER_MODELS" ]; then - MODELS=() for m in $FILTER_MODELS; do - if [ -f "$MODELS_DIR/$m/run.sh" ]; then + if [[ -n "${EXCLUDED[$m]:-}" ]]; then + echo -e "${YELLOW}Excluding: $m${NC}" + elif [ -f "$MODELS_DIR/$m/main.py" ]; then MODELS+=("$m") else - echo -e "${YELLOW}WARNING: Model '$m' not found or has no run.sh — skipping${NC}" + echo -e "${YELLOW}WARNING: '$m' not found — skipping${NC}" fi done else - MODELS=() for dir in "$MODELS_DIR"/*/; do model_name=$(basename "$dir") - if [ -f "$dir/run.sh" ]; then + if [[ -n "${EXCLUDED[$model_name]:-}" ]]; then + continue + fi + if [ -f "$dir/main.py" ]; then MODELS+=("$model_name") fi done @@ -104,7 +110,9 @@ done echo -e "${BOLD}═══════════════════════════════════════════════════════════${NC}" echo -e "${BOLD} views-models integration test${NC}" echo -e "${BOLD}═══════════════════════════════════════════════════════════${NC}" +echo " Conda env: $CONDA_ENV" echo " Models: $TOTAL_MODELS" +echo " Excluded: $EXCLUDE_MODELS" echo " Partitions: $PARTITIONS" echo " Timeout: ${TIMEOUT}s per run" echo " Logs: $LOG_DIR" @@ -128,24 +136,21 @@ for model in "${MODELS[@]}"; do echo -ne "[${RUN_INDEX}/${TOTAL_RUNS}] ${BOLD}${model}${NC} (${partition})... " - # Record start time start_time=$(date +%s) - # Run the model via its own run.sh, capturing all output - # The run.sh handles conda env activation internally timeout "$TIMEOUT" bash -c " - cd '$MODELS_DIR/$model' && \ - bash run.sh -r '$partition' -t -e + eval \"\$(conda shell.bash hook)\" + conda activate '$CONDA_ENV' + cd '$MODELS_DIR/$model' + python main.py -r '$partition' -t -e " > "$model_log" 2>&1 exit_code=$? - # Record duration end_time=$(date +%s) duration=$((end_time - start_time)) - duration_str="${duration}s" if [ "$exit_code" -eq 0 ]; then - echo -e "${GREEN}PASS${NC} (${duration_str})" + echo -e "${GREEN}PASS${NC} (${duration}s)" RESULTS[$result_key]="PASS" PASS_COUNT=$((PASS_COUNT + 1)) elif [ "$exit_code" -eq 124 ]; then @@ -154,7 +159,7 @@ for model in "${MODELS[@]}"; do TIMEOUT_COUNT=$((TIMEOUT_COUNT + 1)) echo "=== TIMEOUT after ${TIMEOUT}s ===" >> "$model_log" else - echo -e "${RED}FAIL${NC} (exit ${exit_code}, ${duration_str})" + echo -e "${RED}FAIL${NC} (exit ${exit_code}, ${duration}s)" RESULTS[$result_key]="FAIL(${exit_code})" FAIL_COUNT=$((FAIL_COUNT + 1)) fi @@ -169,19 +174,13 @@ echo -e "${BOLD} Summary${NC}" echo -e "${BOLD}═══════════════════════════════════════════════════════════${NC}" echo "" -# Print table header printf "%-30s" "Model" -for partition in $PARTITIONS; do - printf "%-15s" "$partition" -done +for partition in $PARTITIONS; do printf "%-15s" "$partition"; done echo "" printf "%-30s" "-----" -for partition in $PARTITIONS; do - printf "%-15s" "----------" -done +for partition in $PARTITIONS; do printf "%-15s" "----------"; done echo "" -# Print results per model for model in "${MODELS[@]}"; do printf "%-30s" "$model" for partition in $PARTITIONS; do @@ -199,9 +198,7 @@ done echo "" echo -e " ${GREEN}Passed:${NC} $PASS_COUNT" echo -e " ${RED}Failed:${NC} $FAIL_COUNT" -if [ "$TIMEOUT_COUNT" -gt 0 ]; then - echo -e " ${RED}Timeout:${NC} $TIMEOUT_COUNT" -fi +[ "$TIMEOUT_COUNT" -gt 0 ] && echo -e " ${RED}Timeout:${NC} $TIMEOUT_COUNT" echo " Total: $TOTAL_RUNS" echo "" @@ -209,23 +206,19 @@ echo "" { echo "Integration Test Summary — $TIMESTAMP" - echo "Models: $TOTAL_MODELS | Partitions: $PARTITIONS | Timeout: ${TIMEOUT}s" + echo "Env: $CONDA_ENV | Models: $TOTAL_MODELS | Excluded: $EXCLUDE_MODELS" + echo "Partitions: $PARTITIONS | Timeout: ${TIMEOUT}s" echo "" printf "%-30s" "Model" - for partition in $PARTITIONS; do - printf "%-15s" "$partition" - done + for partition in $PARTITIONS; do printf "%-15s" "$partition"; done echo "" printf "%-30s" "-----" - for partition in $PARTITIONS; do - printf "%-15s" "----------" - done + for partition in $PARTITIONS; do printf "%-15s" "----------"; done echo "" for model in "${MODELS[@]}"; do printf "%-30s" "$model" for partition in $PARTITIONS; do - result_key="${model}__${partition}" - result="${RESULTS[$result_key]:-SKIPPED}" + result="${RESULTS[${model}__${partition}]:-SKIPPED}" printf "%-15s" "$result" done echo "" @@ -237,8 +230,5 @@ echo "" echo "Full summary: $LOG_DIR/summary.log" echo "Per-model logs: $LOG_DIR/{partition}/{model}.log" -# Exit with non-zero if any failures -if [ "$FAIL_COUNT" -gt 0 ] || [ "$TIMEOUT_COUNT" -gt 0 ]; then - exit 1 -fi +[ "$FAIL_COUNT" -gt 0 ] || [ "$TIMEOUT_COUNT" -gt 0 ] && exit 1 exit 0 From be496558a826756cd7b5d79f118b72f7f8000e06 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 16 Mar 2026 01:22:48 +0100 Subject: [PATCH 11/27] =?UTF-8?q?fix(configs):=20rename=20targets=E2=86=92?= =?UTF-8?q?regression=5Ftargets,=20metrics=E2=86=92regression=5Fpoint=5Fme?= =?UTF-8?q?trics?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CoreConfigSniffer in views-pipeline-core v2.2.0 requires the new key names. Updates 44 models from the old generic keys to the type-specific format. Also removes CRPS from regression_point_metrics (not meaningful for point estimates). Co-Authored-By: Claude Opus 4.6 (1M context) --- models/bad_blood/configs/config_meta.py | 4 ++-- models/bittersweet_symphony/configs/config_meta.py | 4 ++-- models/blank_space/configs/config_meta.py | 4 ++-- models/brown_cheese/configs/config_meta.py | 4 ++-- models/car_radio/configs/config_meta.py | 4 ++-- models/caring_fish/configs/config_meta.py | 4 ++-- models/cheap_thrills/configs/config_meta.py | 4 ++-- models/chunky_cat/configs/config_meta.py | 4 ++-- models/cool_cat/configs/config_meta.py | 4 ++-- models/counting_stars/configs/config_meta.py | 4 ++-- models/dancing_queen/configs/config_meta.py | 4 ++-- models/dark_paradise/configs/config_meta.py | 4 ++-- models/demon_days/configs/config_meta.py | 4 ++-- models/elastic_heart/configs/config_meta.py | 4 ++-- models/electric_relaxation/configs/config_meta.py | 4 ++-- models/fast_car/configs/config_meta.py | 4 ++-- models/fluorescent_adolescent/configs/config_meta.py | 4 ++-- models/fourtieth_symphony/configs/config_meta.py | 4 ++-- models/good_life/configs/config_meta.py | 4 ++-- models/good_riddance/configs/config_meta.py | 4 ++-- models/green_squirrel/configs/config_meta.py | 4 ++-- models/heat_waves/configs/config_meta.py | 4 ++-- models/heavy_rotation/configs/config_meta.py | 4 ++-- models/high_hopes/configs/config_meta.py | 4 ++-- models/invisible_string/configs/config_meta.py | 4 ++-- models/lavender_haze/configs/config_meta.py | 4 ++-- models/little_lies/configs/config_meta.py | 4 ++-- models/lovely_creature/configs/config_meta.py | 4 ++-- models/midnight_rain/configs/config_meta.py | 4 ++-- models/national_anthem/configs/config_meta.py | 4 ++-- models/old_money/configs/config_meta.py | 4 ++-- models/ominous_ox/configs/config_meta.py | 4 ++-- models/orange_pasta/configs/config_meta.py | 4 ++-- models/plastic_beach/configs/config_meta.py | 4 ++-- models/popular_monster/configs/config_meta.py | 4 ++-- models/purple_haze/configs/config_meta.py | 4 ++-- models/teen_spirit/configs/config_meta.py | 4 ++-- models/teenage_dirtbag/configs/config_meta.py | 4 ++-- models/twin_flame/configs/config_meta.py | 4 ++-- models/wild_rose/configs/config_meta.py | 4 ++-- models/wildest_dream/configs/config_meta.py | 4 ++-- models/wuthering_heights/configs/config_meta.py | 4 ++-- models/yellow_pikachu/configs/config_meta.py | 4 ++-- models/yellow_submarine/configs/config_meta.py | 4 ++-- 44 files changed, 88 insertions(+), 88 deletions(-) diff --git a/models/bad_blood/configs/config_meta.py b/models/bad_blood/configs/config_meta.py index 8e6abf2b..438bb805 100644 --- a/models/bad_blood/configs/config_meta.py +++ b/models/bad_blood/configs/config_meta.py @@ -10,8 +10,8 @@ def get_meta_config(): meta_config = { "name": "bad_blood", "algorithm": "LGBMRegressor", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], - "targets": "lr_ged_sb", + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "regression_targets": ["lr_ged_sb"], "queryset": "fatalities003_pgm_natsoc", "level": "pgm", "creator": "Xiaolong", diff --git a/models/bittersweet_symphony/configs/config_meta.py b/models/bittersweet_symphony/configs/config_meta.py index ed05e0a9..d5c76c40 100644 --- a/models/bittersweet_symphony/configs/config_meta.py +++ b/models/bittersweet_symphony/configs/config_meta.py @@ -10,8 +10,8 @@ def get_meta_config(): meta_config = { "name": "bittersweet_symphony", "algorithm": "XGBRegressor", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], - "targets": "lr_ged_sb", + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "regression_targets": ["lr_ged_sb"], "queryset": " fatalities003_all_features", "level": "cm", "creator": "Marina", diff --git a/models/blank_space/configs/config_meta.py b/models/blank_space/configs/config_meta.py index a3dbc4f5..13e763ed 100644 --- a/models/blank_space/configs/config_meta.py +++ b/models/blank_space/configs/config_meta.py @@ -11,8 +11,8 @@ def get_meta_config(): "algorithm": "HurdleModel", "model_clf": "LGBMClassifier", "model_reg": "LGBMRegressor", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], - "targets": "lr_ged_sb", + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "regression_targets": ["lr_ged_sb"], "queryset": "fatalities003_pgm_natsoc", "level": "pgm", "creator": "Xiaolong", diff --git a/models/brown_cheese/configs/config_meta.py b/models/brown_cheese/configs/config_meta.py index d8aa548a..a984b6b2 100644 --- a/models/brown_cheese/configs/config_meta.py +++ b/models/brown_cheese/configs/config_meta.py @@ -10,8 +10,8 @@ def get_meta_config(): meta_config = { "name": "brown_cheese", "algorithm": "XGBRFRegressor", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], - "targets": "lr_ged_sb", + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "regression_targets": ["lr_ged_sb"], "queryset": "fatalities003_baseline", "level": "cm", "creator": "Borbála", diff --git a/models/car_radio/configs/config_meta.py b/models/car_radio/configs/config_meta.py index e7e82d7f..7feb20d2 100644 --- a/models/car_radio/configs/config_meta.py +++ b/models/car_radio/configs/config_meta.py @@ -10,8 +10,8 @@ def get_meta_config(): meta_config = { "name": "car_radio", "algorithm": "XGBRegressor", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], - "targets": "lr_ged_sb", + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "regression_targets": ["lr_ged_sb"], "queryset": "fatalities003_topics", "level": "cm", "creator": "Borbála", diff --git a/models/caring_fish/configs/config_meta.py b/models/caring_fish/configs/config_meta.py index be1f2433..38855ef2 100644 --- a/models/caring_fish/configs/config_meta.py +++ b/models/caring_fish/configs/config_meta.py @@ -10,8 +10,8 @@ def get_meta_config(): meta_config = { "name": "caring_fish", "algorithm": "XGBRegressor", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], - "targets": "lr_ged_sb", + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "regression_targets": ["lr_ged_sb"], "queryset": "fatalities003_pgm_conflict_history", "level": "pgm", "creator": "Xiaolong", diff --git a/models/cheap_thrills/configs/config_meta.py b/models/cheap_thrills/configs/config_meta.py index 87107b2f..1784ff41 100644 --- a/models/cheap_thrills/configs/config_meta.py +++ b/models/cheap_thrills/configs/config_meta.py @@ -10,13 +10,13 @@ def get_meta_config(): meta_config = { "name": "cheap_thrills", "algorithm": "ShurfModel", - "targets": "lr_sb_best", + "regression_targets": ["lr_sb_best"], "level": "cm", "creator": "Håvard", "prediction_format": "dataframe", "model_reg": "XGBRegressor", "model_clf": "XGBClassifier", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], "queryset": "structural_brief_nolog", "rolling_origin_stride": 1, } diff --git a/models/chunky_cat/configs/config_meta.py b/models/chunky_cat/configs/config_meta.py index 3f5706f9..f6ea10c3 100644 --- a/models/chunky_cat/configs/config_meta.py +++ b/models/chunky_cat/configs/config_meta.py @@ -10,8 +10,8 @@ def get_meta_config(): meta_config = { "name": "chunky_cat", "algorithm": "LGBMRegressor", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], - "targets": "lr_ged_sb", + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "regression_targets": ["lr_ged_sb"], "queryset": "fatalities003_pgm_conflictlong", "level": "pgm", "creator": "Xiaolong", diff --git a/models/cool_cat/configs/config_meta.py b/models/cool_cat/configs/config_meta.py index e42dbe0a..7c8a9003 100644 --- a/models/cool_cat/configs/config_meta.py +++ b/models/cool_cat/configs/config_meta.py @@ -11,12 +11,12 @@ def get_meta_config(): "name": "cool_cat", "algorithm": "TiDEModel", # Uncomment and modify the following lines as needed for additional metadata: - "targets": ["lr_ged_sb_dep"], + "regression_targets": ["lr_ged_sb_dep"], # "queryset": "escwa001_cflong", "level": "cm", "creator": "Dylan", "prediction_format": "dataframe", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], "rolling_origin_stride": 1, } return meta_config diff --git a/models/counting_stars/configs/config_meta.py b/models/counting_stars/configs/config_meta.py index ea5aa0a7..c5437628 100644 --- a/models/counting_stars/configs/config_meta.py +++ b/models/counting_stars/configs/config_meta.py @@ -10,8 +10,8 @@ def get_meta_config(): meta_config = { "name": "counting_stars", "algorithm": "XGBRegressor", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], - "targets": "lr_ged_sb", + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "regression_targets": ["lr_ged_sb"], "queryset": "fatalities003_conflict_history_long", "level": "cm", "creator": "Borbála", diff --git a/models/dancing_queen/configs/config_meta.py b/models/dancing_queen/configs/config_meta.py index 97c6f9f0..6f5453a6 100644 --- a/models/dancing_queen/configs/config_meta.py +++ b/models/dancing_queen/configs/config_meta.py @@ -11,12 +11,12 @@ def get_meta_config(): "name": "dancing_queen", "algorithm": "BlockRNNModel", # Uncomment and modify the following lines as needed for additional metadata: - "targets": ["lr_ged_sb_dep"], + "regression_targets": ["lr_ged_sb_dep"], # "queryset": "escwa001_cflong", "level": "cm", "creator": "Dylan", "prediction_format": "dataframe", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], "rolling_origin_stride": 1, } return meta_config diff --git a/models/dark_paradise/configs/config_meta.py b/models/dark_paradise/configs/config_meta.py index 784cc02a..b4a4ed07 100644 --- a/models/dark_paradise/configs/config_meta.py +++ b/models/dark_paradise/configs/config_meta.py @@ -12,8 +12,8 @@ def get_meta_config(): "algorithm": "HurdleModel", "model_clf": "LGBMClassifier", "model_reg": "LGBMRegressor", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], - "targets": "lr_ged_sb", + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "regression_targets": ["lr_ged_sb"], "queryset": "fatalities003_pgm_conflictlong", "level": "pgm", "creator": "Xiaolong", diff --git a/models/demon_days/configs/config_meta.py b/models/demon_days/configs/config_meta.py index 84a411f2..54d6880e 100644 --- a/models/demon_days/configs/config_meta.py +++ b/models/demon_days/configs/config_meta.py @@ -10,8 +10,8 @@ def get_meta_config(): meta_config = { "name": "demon_days", "algorithm": "XGBRFRegressor", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], - "targets": "lr_ged_sb", + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "regression_targets": ["lr_ged_sb"], "queryset": "fatalities003_faostat", "level": "cm", "creator": "Marina", diff --git a/models/elastic_heart/configs/config_meta.py b/models/elastic_heart/configs/config_meta.py index 2af6bb78..c2def11a 100644 --- a/models/elastic_heart/configs/config_meta.py +++ b/models/elastic_heart/configs/config_meta.py @@ -11,12 +11,12 @@ def get_meta_config(): "name": "elastic_heart", "algorithm": "TSMixerModel", # Uncomment and modify the following lines as needed for additional metadata: - "targets": ["lr_ged_sb_dep"], + "regression_targets": ["lr_ged_sb_dep"], # "queryset": "escwa001_cflong", "level": "cm", "creator": "Dylan", "prediction_format": "dataframe", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], "rolling_origin_stride": 1, } return meta_config diff --git a/models/electric_relaxation/configs/config_meta.py b/models/electric_relaxation/configs/config_meta.py index 49c11530..68985877 100644 --- a/models/electric_relaxation/configs/config_meta.py +++ b/models/electric_relaxation/configs/config_meta.py @@ -8,8 +8,8 @@ def get_meta_config(): model_config = { "name": "electric_relaxation", "algorithm": "RandomForestRegressor", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], - "targets": "lr_ged_sb", + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "regression_targets": ["lr_ged_sb"], "queryset": "escwa001_cflong", "level": "cm", "creator": "Sara", diff --git a/models/fast_car/configs/config_meta.py b/models/fast_car/configs/config_meta.py index 4fa14726..525eb08b 100644 --- a/models/fast_car/configs/config_meta.py +++ b/models/fast_car/configs/config_meta.py @@ -12,8 +12,8 @@ def get_meta_config(): "algorithm": "HurdleModel", "model_clf": "XGBClassifier", "model_reg": "XGBRegressor", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], - "targets": "lr_ged_sb", + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "regression_targets": ["lr_ged_sb"], "queryset": "fatalities003_vdem_short", "level": "cm", "creator": "Borbála", diff --git a/models/fluorescent_adolescent/configs/config_meta.py b/models/fluorescent_adolescent/configs/config_meta.py index 9eab7760..b6ed1501 100644 --- a/models/fluorescent_adolescent/configs/config_meta.py +++ b/models/fluorescent_adolescent/configs/config_meta.py @@ -12,8 +12,8 @@ def get_meta_config(): "algorithm": "HurdleModel", "model_clf": "XGBClassifier", "model_reg": "XGBRegressor", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], - "targets": "lr_ged_sb", + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "regression_targets": ["lr_ged_sb"], "queryset": "fatalities003_joint_narrow", "level": "cm", "creator": "Marina", diff --git a/models/fourtieth_symphony/configs/config_meta.py b/models/fourtieth_symphony/configs/config_meta.py index c80abe43..2b943c7f 100644 --- a/models/fourtieth_symphony/configs/config_meta.py +++ b/models/fourtieth_symphony/configs/config_meta.py @@ -10,13 +10,13 @@ def get_meta_config(): meta_config = { "name": "fourtieth_symphony", "algorithm": "ShurfModel", - "targets": ["lr_sb_best"], + "regression_targets": ["lr_sb_best"], "level": "cm", "creator": "Håvard", "prediction_format": "dataframe", "model_reg": "XGBRegressor", "model_clf": "XGBClassifier", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], "queryset": "uncertainty_broad_nolog", "rolling_origin_stride": 1, } diff --git a/models/good_life/configs/config_meta.py b/models/good_life/configs/config_meta.py index b72253d8..2857010a 100644 --- a/models/good_life/configs/config_meta.py +++ b/models/good_life/configs/config_meta.py @@ -11,12 +11,12 @@ def get_meta_config(): "name": "good_life", "algorithm": "TransformerModel", # Uncomment and modify the following lines as needed for additional metadata: - "targets": ["lr_ged_sb_dep"], + "regression_targets": ["lr_ged_sb_dep"], # "queryset": "escwa001_cflong", "level": "cm", "creator": "Dylan", "prediction_format": "dataframe", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], "rolling_origin_stride": 1, } return meta_config diff --git a/models/good_riddance/configs/config_meta.py b/models/good_riddance/configs/config_meta.py index aabf1f1d..7f66c430 100644 --- a/models/good_riddance/configs/config_meta.py +++ b/models/good_riddance/configs/config_meta.py @@ -10,8 +10,8 @@ def get_meta_config(): meta_config = { "name": "good_riddance", "algorithm": "XGBRFRegressor", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], - "targets": "lr_ged_sb", + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "regression_targets": ["lr_ged_sb"], "queryset": "fatalities003_joint_narrow", "level": "cm", "creator": "Marina", diff --git a/models/green_squirrel/configs/config_meta.py b/models/green_squirrel/configs/config_meta.py index e51932d3..99b77df9 100644 --- a/models/green_squirrel/configs/config_meta.py +++ b/models/green_squirrel/configs/config_meta.py @@ -12,8 +12,8 @@ def get_meta_config(): "algorithm": "HurdleModel", "model_clf": "XGBRFClassifier", "model_reg": "XGBRFRegressor", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], - "targets": "lr_ged_sb", + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "regression_targets": ["lr_ged_sb"], "queryset": "fatalities003_joint_broad", "level": "cm", "creator": "Borbála", diff --git a/models/heat_waves/configs/config_meta.py b/models/heat_waves/configs/config_meta.py index 13ed2409..c7af8ff7 100644 --- a/models/heat_waves/configs/config_meta.py +++ b/models/heat_waves/configs/config_meta.py @@ -11,12 +11,12 @@ def get_meta_config(): "name": "heat_waves", "algorithm": "TFTModel", # Uncomment and modify the following lines as needed for additional metadata: - "targets": ["lr_ged_sb_dep"], + "regression_targets": ["lr_ged_sb_dep"], # "queryset": "escwa001_cflong", "level": "cm", "creator": "Dylan", "prediction_format": "dataframe", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], "rolling_origin_stride": 1, } return meta_config diff --git a/models/heavy_rotation/configs/config_meta.py b/models/heavy_rotation/configs/config_meta.py index 38004218..c8949806 100644 --- a/models/heavy_rotation/configs/config_meta.py +++ b/models/heavy_rotation/configs/config_meta.py @@ -10,8 +10,8 @@ def get_meta_config(): meta_config = { "name": "heavy_rotation", "algorithm": "XGBRFRegressor", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], - "targets": "lr_ged_sb", + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "regression_targets": ["lr_ged_sb"], "queryset": "fatalities003_joint_broad", "level": "cm", "creator": "Borbála", diff --git a/models/high_hopes/configs/config_meta.py b/models/high_hopes/configs/config_meta.py index b5ed6211..30449372 100644 --- a/models/high_hopes/configs/config_meta.py +++ b/models/high_hopes/configs/config_meta.py @@ -12,8 +12,8 @@ def get_meta_config(): "algorithm": "HurdleModel", "model_clf": "LGBMClassifier", "model_reg": "LGBMRegressor", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], - "targets": "lr_ged_sb", + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "regression_targets": ["lr_ged_sb"], "queryset": "fatalities003_conflict_history", "level": "cm", "creator": "Borbála", diff --git a/models/invisible_string/configs/config_meta.py b/models/invisible_string/configs/config_meta.py index eba5d3d6..7e8c4892 100644 --- a/models/invisible_string/configs/config_meta.py +++ b/models/invisible_string/configs/config_meta.py @@ -10,8 +10,8 @@ def get_meta_config(): meta_config = { "name": "invisible_string", "algorithm": "LGBMRegressor", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], - "targets": "lr_ged_sb", + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "regression_targets": ["lr_ged_sb"], "queryset": "fatalities003_pgm_broad", "level": "pgm", "creator": "Xiaolong", diff --git a/models/lavender_haze/configs/config_meta.py b/models/lavender_haze/configs/config_meta.py index 514a53f9..9f598ab1 100644 --- a/models/lavender_haze/configs/config_meta.py +++ b/models/lavender_haze/configs/config_meta.py @@ -11,8 +11,8 @@ def get_meta_config(): "algorithm": "HurdleModel", "model_clf": "XGBClassifier", "model_reg": "XGBRegressor", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], - "targets": "lr_ged_sb", + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "regression_targets": ["lr_ged_sb"], "queryset": "fatalities003_pgm_broad", "level": "pgm", "creator": "Xiaolong", diff --git a/models/little_lies/configs/config_meta.py b/models/little_lies/configs/config_meta.py index de51aa35..d131df6f 100644 --- a/models/little_lies/configs/config_meta.py +++ b/models/little_lies/configs/config_meta.py @@ -12,8 +12,8 @@ def get_meta_config(): "algorithm": "HurdleModel", "model_clf": "LGBMClassifier", "model_reg": "LGBMRegressor", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], - "targets": "lr_ged_sb", + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "regression_targets": ["lr_ged_sb"], "queryset": "fatalities003_joint_narrow", "level": "cm", "creator": "Marina", diff --git a/models/lovely_creature/configs/config_meta.py b/models/lovely_creature/configs/config_meta.py index 1ca2708d..0a9b54f2 100644 --- a/models/lovely_creature/configs/config_meta.py +++ b/models/lovely_creature/configs/config_meta.py @@ -10,13 +10,13 @@ def get_meta_config(): meta_config = { "name": "lovely_creature", "algorithm": "ShurfModel", - "targets": ["lr_sb_best"], + "regression_targets": ["lr_sb_best"], "level": "cm", "creator": "Håvard", "prediction_format": "dataframe", "model_reg": "XGBRegressor", "model_clf": "XGBClassifier", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], "queryset": "uncertainty_broad_nolog", "rolling_origin_stride": 1, } diff --git a/models/midnight_rain/configs/config_meta.py b/models/midnight_rain/configs/config_meta.py index 33249303..ac64cb8d 100644 --- a/models/midnight_rain/configs/config_meta.py +++ b/models/midnight_rain/configs/config_meta.py @@ -10,8 +10,8 @@ def get_meta_config(): meta_config = { "name": "midnight_rain", "algorithm": "LGBMRegressor", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], - "targets": "lr_ged_sb", + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "regression_targets": ["lr_ged_sb"], "queryset": "fatalities003_pgm_escwa_drought", "level": "pgm", "creator": "Xiaolong", diff --git a/models/national_anthem/configs/config_meta.py b/models/national_anthem/configs/config_meta.py index 7d232ff2..12c550fb 100644 --- a/models/national_anthem/configs/config_meta.py +++ b/models/national_anthem/configs/config_meta.py @@ -10,8 +10,8 @@ def get_meta_config(): meta_config = { "name": "national_anthem", "algorithm": "XGBRFRegressor", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], - "targets": "lr_ged_sb", + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "regression_targets": ["lr_ged_sb"], "queryset": "fatalities003_wdi_short", "level": "cm", "creator": "Borbála", diff --git a/models/old_money/configs/config_meta.py b/models/old_money/configs/config_meta.py index fe9b28d0..59c9b61a 100644 --- a/models/old_money/configs/config_meta.py +++ b/models/old_money/configs/config_meta.py @@ -11,8 +11,8 @@ def get_meta_config(): "algorithm": "HurdleModel", "model_clf": "LGBMClassifier", "model_reg": "LGBMRegressor", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], - "targets": "lr_ged_sb", + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "regression_targets": ["lr_ged_sb"], "queryset": "fatalities003_pgm_escwa_drought", "level": "pgm", "creator": "Xiaolong", diff --git a/models/ominous_ox/configs/config_meta.py b/models/ominous_ox/configs/config_meta.py index ab8d1cd3..a39a3c42 100644 --- a/models/ominous_ox/configs/config_meta.py +++ b/models/ominous_ox/configs/config_meta.py @@ -10,8 +10,8 @@ def get_meta_config(): meta_config = { "name": "ominous_ox", "algorithm": "XGBRFRegressor", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], - "targets": "lr_ged_sb", + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "regression_targets": ["lr_ged_sb"], "queryset": "fatalities003_conflict_history", "level": "cm", "creator": "Borbála", diff --git a/models/orange_pasta/configs/config_meta.py b/models/orange_pasta/configs/config_meta.py index e4a38475..e9159820 100644 --- a/models/orange_pasta/configs/config_meta.py +++ b/models/orange_pasta/configs/config_meta.py @@ -9,8 +9,8 @@ def get_meta_config(): meta_config = { "name": "orange_pasta", "algorithm": "LGBMRegressor", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], - "targets": "lr_ged_sb", + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "regression_targets": ["lr_ged_sb"], "queryset": "fatalities003_pgm_baseline", "level": "pgm", "creator": "Xiaolong", diff --git a/models/plastic_beach/configs/config_meta.py b/models/plastic_beach/configs/config_meta.py index ced78c08..cd745d9d 100644 --- a/models/plastic_beach/configs/config_meta.py +++ b/models/plastic_beach/configs/config_meta.py @@ -10,8 +10,8 @@ def get_meta_config(): meta_config = { "name": "plastic_beach", "algorithm": "XGBRFRegressor", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], - "targets": "lr_ged_sb", + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "regression_targets": ["lr_ged_sb"], "queryset": "fatalities003_aquastat", "level": "cm", "creator": "Marina", diff --git a/models/popular_monster/configs/config_meta.py b/models/popular_monster/configs/config_meta.py index 2ddabc5c..0cb62d77 100644 --- a/models/popular_monster/configs/config_meta.py +++ b/models/popular_monster/configs/config_meta.py @@ -10,8 +10,8 @@ def get_meta_config(): meta_config = { "name": "popular_monster", "algorithm": "XGBRFRegressor", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], - "targets": "lr_ged_sb", + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "regression_targets": ["lr_ged_sb"], "queryset": "fatalities003_topics", "level": "cm", "creator": "Borbála", diff --git a/models/purple_haze/configs/config_meta.py b/models/purple_haze/configs/config_meta.py index 4c69c996..17f343e0 100644 --- a/models/purple_haze/configs/config_meta.py +++ b/models/purple_haze/configs/config_meta.py @@ -10,13 +10,13 @@ def get_meta_config(): meta_config = { "name": "purple_haze", "algorithm": "ShurfModel", - "targets": ["lr_sb_best"], + "regression_targets": ["lr_sb_best"], "level": "cm", "creator": "Håvard", "prediction_format": "dataframe", "model_reg": "XGBRegressor", "model_clf": "XGBClassifier", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], "queryset": "uncertainty_broad_nolog", "rolling_origin_stride": 1, } diff --git a/models/teen_spirit/configs/config_meta.py b/models/teen_spirit/configs/config_meta.py index d9f33d48..fd39fc50 100644 --- a/models/teen_spirit/configs/config_meta.py +++ b/models/teen_spirit/configs/config_meta.py @@ -10,8 +10,8 @@ def get_meta_config(): meta_config = { "name": "teen_spirit", "algorithm": "XGBRFRegressor", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], - "targets": "lr_ged_sb", + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "regression_targets": ["lr_ged_sb"], "queryset": "fatalities003_faoprices", "level": "cm", "creator": "Marina", diff --git a/models/teenage_dirtbag/configs/config_meta.py b/models/teenage_dirtbag/configs/config_meta.py index bdc09fa5..0b5ad0fb 100644 --- a/models/teenage_dirtbag/configs/config_meta.py +++ b/models/teenage_dirtbag/configs/config_meta.py @@ -11,12 +11,12 @@ def get_meta_config(): "name": "teenage_dirtbag", "algorithm": "TCNModel", # Uncomment and modify the following lines as needed for additional metadata: - "targets": ["lr_ged_sb_dep"], + "regression_targets": ["lr_ged_sb_dep"], # "queryset": "escwa001_cflong", "level": "cm", "creator": "Dylan", "prediction_format": "dataframe", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], "rolling_origin_stride": 1, } return meta_config diff --git a/models/twin_flame/configs/config_meta.py b/models/twin_flame/configs/config_meta.py index 90992c2a..49a670ae 100644 --- a/models/twin_flame/configs/config_meta.py +++ b/models/twin_flame/configs/config_meta.py @@ -12,8 +12,8 @@ def get_meta_config(): "algorithm": "HurdleModel", "model_clf": "LGBMClassifier", "model_reg": "LGBMRegressor", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], - "targets": "lr_ged_sb", + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "regression_targets": ["lr_ged_sb"], "queryset": "fatalities003_topics", "level": "cm", "creator": "Borbála", diff --git a/models/wild_rose/configs/config_meta.py b/models/wild_rose/configs/config_meta.py index 3cb7d6b3..fa16618c 100644 --- a/models/wild_rose/configs/config_meta.py +++ b/models/wild_rose/configs/config_meta.py @@ -10,13 +10,13 @@ def get_meta_config(): meta_config = { "name": "wild_rose", "algorithm": "ShurfModel", - "targets": ["lr_sb_best"], + "regression_targets": ["lr_sb_best"], "level": "cm", "creator": "Håvard", "prediction_format": "dataframe", "model_reg": "XGBRegressor", "model_clf": "XGBClassifier", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], "queryset": "uncertainty_conflict_nolog", "rolling_origin_stride": 1, } diff --git a/models/wildest_dream/configs/config_meta.py b/models/wildest_dream/configs/config_meta.py index f8e058a9..6731c721 100644 --- a/models/wildest_dream/configs/config_meta.py +++ b/models/wildest_dream/configs/config_meta.py @@ -11,8 +11,8 @@ def get_meta_config(): "algorithm": "HurdleModel", "model_clf": "XGBClassifier", "model_reg": "XGBRegressor", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], - "targets": "lr_ged_sb", + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "regression_targets": ["lr_ged_sb"], "queryset": "fatalities003_pgm_conflict_sptime_dist", "level": "pgm", "creator": "Xiaolong", diff --git a/models/wuthering_heights/configs/config_meta.py b/models/wuthering_heights/configs/config_meta.py index cfe2dc4b..213519b0 100644 --- a/models/wuthering_heights/configs/config_meta.py +++ b/models/wuthering_heights/configs/config_meta.py @@ -10,13 +10,13 @@ def get_meta_config(): meta_config = { "name": "wuthering_heights", "algorithm": "ShurfModel", - "targets": ["lr_sb_best"], + "regression_targets": ["lr_sb_best"], "level": "cm", "creator": "Håvard", "prediction_format": "dataframe", "model_reg": "XGBRegressor", "model_clf": "XGBClassifier", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], "queryset": "uncertainty_deep_conflict_nolog", "rolling_origin_stride": 1, } diff --git a/models/yellow_pikachu/configs/config_meta.py b/models/yellow_pikachu/configs/config_meta.py index d22bbc3b..06eaaae1 100644 --- a/models/yellow_pikachu/configs/config_meta.py +++ b/models/yellow_pikachu/configs/config_meta.py @@ -11,8 +11,8 @@ def get_meta_config(): "algorithm": "HurdleModel", "model_clf": "XGBClassifier", "model_reg": "XGBRegressor", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], - "targets": "lr_ged_sb", + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "regression_targets": ["lr_ged_sb"], "queryset": "fatalities003_pgm_conflict_treelag", "level": "pgm", "creator": "Xiaolong", diff --git a/models/yellow_submarine/configs/config_meta.py b/models/yellow_submarine/configs/config_meta.py index a68a1737..df774dcf 100644 --- a/models/yellow_submarine/configs/config_meta.py +++ b/models/yellow_submarine/configs/config_meta.py @@ -10,8 +10,8 @@ def get_meta_config(): meta_config = { "name": "yellow_submarine", "algorithm": "XGBRFRegressor", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], - "targets": "lr_ged_sb", + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "regression_targets": ["lr_ged_sb"], "queryset": "fatalities003_imfweo", "level": "cm", "creator": "Marina", From b75097b73bc716c280399caa2ba90e061c37e31e Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 16 Mar 2026 09:38:32 +0100 Subject: [PATCH 12/27] fix(darts): add missing ReproducibilityGate parameters to 6 models The views-r2darts2 ReproducibilityGate requires core parameters (random_state, optimizer_cls, lr_scheduler_*, early_stopping_min_delta, gradient_clip_val, output_chunk_length/shift) that these models were missing. Values taken from working reference models of the same arch. Fixed: heat_waves, good_life, elastic_heart, teenage_dirtbag, dancing_queen, cool_cat. Also cleaned up 60 lines of commented-out sweep results in good_life. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../configs/config_hyperparameters.py | 6 +- .../configs/config_hyperparameters.py | 4 + .../configs/config_hyperparameters.py | 11 +++ .../configs/config_hyperparameters.py | 84 +++---------------- .../configs/config_hyperparameters.py | 11 +++ .../configs/config_hyperparameters.py | 10 +++ 6 files changed, 50 insertions(+), 76 deletions(-) diff --git a/models/cool_cat/configs/config_hyperparameters.py b/models/cool_cat/configs/config_hyperparameters.py index 81384551..ea004bbd 100644 --- a/models/cool_cat/configs/config_hyperparameters.py +++ b/models/cool_cat/configs/config_hyperparameters.py @@ -63,8 +63,10 @@ def get_hp_config(): "use_layer_norm": True, "use_static_covariates": False, "weight_decay": 0.0007039914716229751, - "zero_threshold": 0.28929505832987634 - } + "zero_threshold": 0.28929505832987634, + "output_chunk_length": 36, + "optimizer_cls": "Adam", + } return hyperparameters \ No newline at end of file diff --git a/models/dancing_queen/configs/config_hyperparameters.py b/models/dancing_queen/configs/config_hyperparameters.py index 1a781a4c..a213bc98 100644 --- a/models/dancing_queen/configs/config_hyperparameters.py +++ b/models/dancing_queen/configs/config_hyperparameters.py @@ -55,6 +55,10 @@ def get_hp_config(): "use_reversible_instance_norm": False, "weight_decay": 0.0007293652167062485, "zero_threshold": 0.15954413640606746, + + "output_chunk_length": 36, + "random_state": 1, + "optimizer_cls": "Adam", } return hyperparameters diff --git a/models/elastic_heart/configs/config_hyperparameters.py b/models/elastic_heart/configs/config_hyperparameters.py index f66269c1..9f2988c6 100644 --- a/models/elastic_heart/configs/config_hyperparameters.py +++ b/models/elastic_heart/configs/config_hyperparameters.py @@ -35,6 +35,17 @@ def get_hp_config(): "weight_decay": 0.005893888408985461, "zero_threshold": 0.2163136317477652, + "output_chunk_length": 36, + "output_chunk_shift": 0, + + "random_state": 1, + "optimizer_cls": "Adam", + "lr_scheduler_factor": 0.46, + "lr_scheduler_patience": 7, + "lr_scheduler_min_lr": 1e-05, + "early_stopping_min_delta": 0.01, + "gradient_clip_val": 1, + "num_samples": 1, "mc_dropout": True, } diff --git a/models/good_life/configs/config_hyperparameters.py b/models/good_life/configs/config_hyperparameters.py index 04bbb902..2e9e5602 100644 --- a/models/good_life/configs/config_hyperparameters.py +++ b/models/good_life/configs/config_hyperparameters.py @@ -11,78 +11,6 @@ def get_hp_config(): hyperparameters = { "steps": [*range(1, 36 + 1, 1)], "time_steps": 36, - # Good! silvery-sweep-1 - # "batch_size": 64, - # "decoder_output_dim": 8, - # "delta": 3.696974087786076, - # "dropout": 0.2, - # "early_stopping_patience": 2, - # "false_negative_weight": 5.0977656184083395, - # "false_positive_weight": 4.507821247953932, - # "feature_scaler": None, - # "gradient_clip_val": 0.2, - # "hidden_size": 256, - # "input_chunk_length": 72, - # "loss_function": "WeightedPenaltyHuberLoss", - # "lr": 0.00009976652946422868, - # "n_epochs": 5, - # "non_zero_weight": 1.7773296987133582, - # "num_decoder_layers": 4, - # "num_encoder_layers": 1, - # "target_scaler": None, - # "temporal_decoder_hidden": 32, - # "use_layer_norm": True, - # "weight_decay": 0.00533531735917749, - # "zero_threshold": 0.17066172076836716, - - #crimson-sweep-169. Psychotic. - # "batch_size": 128, - # "decoder_output_dim": 16, - # "delta": 1.7234705394805443, - # "dropout": 0.1, - # "early_stopping_patience": 6, - # "false_negative_weight": 9.3313776622516, - # "false_positive_weight": 10.78492842081859, - # "feature_scaler": "MaxAbsScaler", - # "gradient_clip_val": 0.8, - # "hidden_size": 128, - # "input_chunk_length": 36, - # "loss_function": "WeightedPenaltyHuberLoss", - # "lr": 0.00001471988186818606, - # "n_epochs": 300, - # "non_zero_weight": 13.23535161231906, - # "num_decoder_layers": 2, - # "num_encoder_layers": 5, - # "target_scaler": "MaxAbsScaler", - # "temporal_decoder_hidden": 16, - # "use_layer_norm": False, - # "weight_decay": 0.0000214973309074918, - # "zero_threshold": 0.26369001329592373, - - #peachy-sweep-64 - # "batch_size": 32, - # "decoder_output_dim": 32, - # "delta": 3.402036667021411, - # "dropout": 0.4, - # # "early_stopping_patience": 6, - # "false_negative_weight": 0.4094813655878722, - # "false_positive_weight": 5.344567094760235, - # "feature_scaler": None, - # "gradient_clip_val": 0.2, - # "hidden_size": 32, - # "input_chunk_length": 48, - # "loss_function": "WeightedPenaltyHuberLoss", - # "lr": 0.0008657902172163073, - # "n_epochs": 300, - # "non_zero_weight": 13.833818931287857, - # "num_decoder_layers": 2, - # "num_encoder_layers": 2, - # "target_scaler": "MinMaxScaler", - # "temporal_decoder_hidden": 16, - # "use_layer_norm": False, - # "weight_decay": 0.00010196612489718342, - # "zero_threshold": 0.0965370826949934, - "num_samples": 1, "mc_dropout": True, @@ -110,8 +38,16 @@ def get_hp_config(): 'temporal_width_past': 2, 'use_layer_norm': True, 'weight_decay': 0.00000253889071290023, - 'zero_threshold': 0.6323834242080557 + 'zero_threshold': 0.6323834242080557, + + "output_chunk_length": 36, + "output_chunk_shift": 0, + "random_state": 1, + "optimizer_cls": "Adam", + "lr_scheduler_factor": 0.46, + "lr_scheduler_patience": 7, + "lr_scheduler_min_lr": 1e-05, + "early_stopping_min_delta": 0.01, } - return hyperparameters \ No newline at end of file diff --git a/models/heat_waves/configs/config_hyperparameters.py b/models/heat_waves/configs/config_hyperparameters.py index 047b4cf5..86d26357 100644 --- a/models/heat_waves/configs/config_hyperparameters.py +++ b/models/heat_waves/configs/config_hyperparameters.py @@ -58,6 +58,17 @@ def get_hp_config(): "weight_decay": 0.00002389357463410524, "zero_threshold": 0.0881212976698389, + "output_chunk_length": 36, + "output_chunk_shift": 0, + + "random_state": 1, + "optimizer_cls": "Adam", + "lr_scheduler_factor": 0.46, + "lr_scheduler_patience": 7, + "lr_scheduler_min_lr": 1e-05, + "early_stopping_min_delta": 0.01, + "gradient_clip_val": 1, + "num_samples": 1, "mc_dropout": True, } diff --git a/models/teenage_dirtbag/configs/config_hyperparameters.py b/models/teenage_dirtbag/configs/config_hyperparameters.py index 5227f81d..37f23a36 100644 --- a/models/teenage_dirtbag/configs/config_hyperparameters.py +++ b/models/teenage_dirtbag/configs/config_hyperparameters.py @@ -57,6 +57,16 @@ def get_hp_config(): "weight_norm": False, "zero_threshold": 0.05638024262912267, + "output_chunk_length": 36, + "output_chunk_shift": 0, + + "random_state": 1, + "optimizer_cls": "Adam", + "lr_scheduler_factor": 0.46, + "lr_scheduler_patience": 7, + "lr_scheduler_min_lr": 1e-05, + "early_stopping_min_delta": 0.01, + "num_samples": 1, "mc_dropout": True, } From 90e3753e6877415ad82ab648930ddaf357159002 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 16 Mar 2026 09:55:44 +0100 Subject: [PATCH 13/27] fix(darts): add missing architecture-specific params to 4 models The ReproducibilityGate checks both core and architecture-specific parameters. After fixing core params, 4 models still had missing arch-specific params: - cool_cat (TiDEModel): use_reversible_instance_norm - good_life (TransformerModel): d_model, nhead, dim_feedforward, activation, norm_type, use_reversible_instance_norm, detect_anomaly - heat_waves (TFTModel): dropout, add_relative_index, use_static_covariates, norm_type, skip_interpolation, hidden_continuous_size - elastic_heart (TSMixerModel): use_static_covariates, use_reversible_instance_norm Co-Authored-By: Claude Opus 4.6 (1M context) --- models/cool_cat/configs/config_hyperparameters.py | 1 + models/elastic_heart/configs/config_hyperparameters.py | 2 ++ models/good_life/configs/config_hyperparameters.py | 7 +++++++ models/heat_waves/configs/config_hyperparameters.py | 6 ++++++ 4 files changed, 16 insertions(+) diff --git a/models/cool_cat/configs/config_hyperparameters.py b/models/cool_cat/configs/config_hyperparameters.py index ea004bbd..8dd509e4 100644 --- a/models/cool_cat/configs/config_hyperparameters.py +++ b/models/cool_cat/configs/config_hyperparameters.py @@ -67,6 +67,7 @@ def get_hp_config(): "output_chunk_length": 36, "optimizer_cls": "Adam", + "use_reversible_instance_norm": False, } return hyperparameters \ No newline at end of file diff --git a/models/elastic_heart/configs/config_hyperparameters.py b/models/elastic_heart/configs/config_hyperparameters.py index 9f2988c6..272e2582 100644 --- a/models/elastic_heart/configs/config_hyperparameters.py +++ b/models/elastic_heart/configs/config_hyperparameters.py @@ -37,6 +37,8 @@ def get_hp_config(): "output_chunk_length": 36, "output_chunk_shift": 0, + "use_static_covariates": True, + "use_reversible_instance_norm": False, "random_state": 1, "optimizer_cls": "Adam", diff --git a/models/good_life/configs/config_hyperparameters.py b/models/good_life/configs/config_hyperparameters.py index 2e9e5602..e0d4e8ae 100644 --- a/models/good_life/configs/config_hyperparameters.py +++ b/models/good_life/configs/config_hyperparameters.py @@ -43,6 +43,13 @@ def get_hp_config(): "output_chunk_length": 36, "output_chunk_shift": 0, "random_state": 1, + "d_model": 512, + "nhead": 8, + "dim_feedforward": 2048, + "activation": "relu", + "norm_type": "LayerNorm", + "use_reversible_instance_norm": False, + "detect_anomaly": False, "optimizer_cls": "Adam", "lr_scheduler_factor": 0.46, "lr_scheduler_patience": 7, diff --git a/models/heat_waves/configs/config_hyperparameters.py b/models/heat_waves/configs/config_hyperparameters.py index 86d26357..bd0f4040 100644 --- a/models/heat_waves/configs/config_hyperparameters.py +++ b/models/heat_waves/configs/config_hyperparameters.py @@ -60,6 +60,12 @@ def get_hp_config(): "output_chunk_length": 36, "output_chunk_shift": 0, + "dropout": 0.3, + "add_relative_index": True, + "use_static_covariates": True, + "norm_type": "LayerNorm", + "skip_interpolation": False, + "hidden_continuous_size": 8, "random_state": 1, "optimizer_cls": "Adam", From f2f7cbcb3517256ddede9f5d0f00fdb67a5c7bd2 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 16 Mar 2026 10:29:01 +0100 Subject: [PATCH 14/27] fix(configs): fix queryset typo, migrate ensemble target/metrics keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix leading space in bittersweet_symphony queryset name - Rename targets→regression_targets in all 5 ensembles (wrap strings in lists) - Rename metrics→regression_point_metrics in all 5 ensembles (remove CRPS — not meaningful for point estimates) - Update ensemble test to expect new key names Co-Authored-By: Claude Opus 4.6 (1M context) --- ensembles/cruel_summer/configs/config_meta.py | 4 ++-- ensembles/pink_ponyclub/configs/config_meta.py | 4 ++-- ensembles/rude_boy/configs/config_meta.py | 4 ++-- ensembles/skinny_love/configs/config_meta.py | 4 ++-- ensembles/white_mustang/configs/config_meta.py | 4 ++-- models/bittersweet_symphony/configs/config_meta.py | 2 +- tests/test_ensemble_configs.py | 4 ++-- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/ensembles/cruel_summer/configs/config_meta.py b/ensembles/cruel_summer/configs/config_meta.py index acbba9c0..8e7b7a3a 100644 --- a/ensembles/cruel_summer/configs/config_meta.py +++ b/ensembles/cruel_summer/configs/config_meta.py @@ -9,8 +9,8 @@ def get_meta_config(): meta_config = { "name": "cruel_summer", "models": ["bittersweet_symphony", "brown_cheese"], - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], - "targets": "lr_ged_sb", + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "regression_targets": ["lr_ged_sb"], "level": "cm", "aggregation": "median", "creator": "Xiaolong", diff --git a/ensembles/pink_ponyclub/configs/config_meta.py b/ensembles/pink_ponyclub/configs/config_meta.py index e209828d..db43f9ca 100644 --- a/ensembles/pink_ponyclub/configs/config_meta.py +++ b/ensembles/pink_ponyclub/configs/config_meta.py @@ -29,10 +29,10 @@ def get_meta_config(): "twin_flame", "yellow_submarine", ], - "targets": "lr_ged_sb", + "regression_targets": ["lr_ged_sb"], "level": "cm", "aggregation": "mean", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], "creator": "Xiaolong", "reconciliation": None, } diff --git a/ensembles/rude_boy/configs/config_meta.py b/ensembles/rude_boy/configs/config_meta.py index 64d61feb..46473d06 100644 --- a/ensembles/rude_boy/configs/config_meta.py +++ b/ensembles/rude_boy/configs/config_meta.py @@ -9,10 +9,10 @@ def get_meta_config(): meta_config = { "name": "rude_boy", "models": ["cool_cat", "dancing_queen", "elastic_heart", "good_life", "heat_waves", "new_rules", "teenage_dirtbag"], - "targets": ["lr_ged_sb_dep"], + "regression_targets": ["lr_ged_sb_dep"], "level": "cm", "aggregation": "mean", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], "creator": "Dylan" } return meta_config diff --git a/ensembles/skinny_love/configs/config_meta.py b/ensembles/skinny_love/configs/config_meta.py index 1f4b2b8d..59a8e99d 100644 --- a/ensembles/skinny_love/configs/config_meta.py +++ b/ensembles/skinny_love/configs/config_meta.py @@ -22,10 +22,10 @@ def get_meta_config(): "wildest_dream", "yellow_pikachu", ], - "targets": "lr_ged_sb", + "regression_targets": ["lr_ged_sb"], "level": "pgm", "aggregation": "mean", - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], "creator": "Xiaolong", "reconciliation": "pgm_cm_point", "reconcile_with": "pink_ponyclub", diff --git a/ensembles/white_mustang/configs/config_meta.py b/ensembles/white_mustang/configs/config_meta.py index ea6397d0..9a4bf17e 100644 --- a/ensembles/white_mustang/configs/config_meta.py +++ b/ensembles/white_mustang/configs/config_meta.py @@ -9,8 +9,8 @@ def get_meta_config(): meta_config = { "name": "white_mustang", "models": ["lavender_haze", "blank_space"], - "metrics": ["RMSLE", "CRPS", "MSE", "MSLE", "y_hat_bar"], - "targets": "lr_ged_sb", # Double-check the target variables of each model + "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], + "regression_targets": ["lr_ged_sb"], # Double-check the target variables of each model "level": "pgm", "aggregation": "mean", "creator": "Xiaolong", diff --git a/models/bittersweet_symphony/configs/config_meta.py b/models/bittersweet_symphony/configs/config_meta.py index d5c76c40..448072a7 100644 --- a/models/bittersweet_symphony/configs/config_meta.py +++ b/models/bittersweet_symphony/configs/config_meta.py @@ -12,7 +12,7 @@ def get_meta_config(): "algorithm": "XGBRegressor", "regression_point_metrics": ["RMSLE", "MSE", "MSLE", "y_hat_bar"], "regression_targets": ["lr_ged_sb"], - "queryset": " fatalities003_all_features", + "queryset": "fatalities003_all_features", "level": "cm", "creator": "Marina", "prediction_format": "dataframe", diff --git a/tests/test_ensemble_configs.py b/tests/test_ensemble_configs.py index 7aebea5d..8cc44fbd 100644 --- a/tests/test_ensemble_configs.py +++ b/tests/test_ensemble_configs.py @@ -1,7 +1,7 @@ """Tests for ensemble configuration completeness and dependency validation. Ensembles have different required config keys than individual models: -- config_meta.py: name, models, targets, level, aggregation +- config_meta.py: name, models, regression_targets, level, aggregation - config_deployment.py: deployment_status - config_hyperparameters.py: steps - config_partitions.py: generate() function @@ -17,7 +17,7 @@ ) -REQUIRED_ENSEMBLE_META_KEYS = {"name", "models", "targets", "level", "aggregation"} +REQUIRED_ENSEMBLE_META_KEYS = {"name", "models", "regression_targets", "level", "aggregation"} REQUIRED_ENSEMBLE_CONFIG_FILES = [ "config_meta.py", From e2d53e7b052248521db392d0426c16b55ad35b7c Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 16 Mar 2026 13:41:40 +0100 Subject: [PATCH 15/27] refactor(docs): merge base_docs/ into docs/, add reports/, remove scratchpad Align documentation structure with platform convention (views-r2darts2, views-hydranet pattern): - Move ADRs, CICs, contributor_protocols, standards from base_docs/ to docs/ - Delete docs/internal/ (leftover scratchpad from prior session) - Delete docs/model_catalog_old_pipeline.md (superseded by README catalogs) - Create reports/ directory for future operational outputs - Update all internal cross-references from base_docs/ to docs/ Co-Authored-By: Claude Opus 4.6 (1M context) --- {base_docs => docs}/ADRs/000_use_of_adrs.md | 6 ++-- {base_docs => docs}/ADRs/001_ontology.md | 0 {base_docs => docs}/ADRs/002_topology.md | 0 {base_docs => docs}/ADRs/003_authority.md | 0 {base_docs => docs}/ADRs/005_testing.md | 0 .../ADRs/006_intent_contracts.md | 4 +-- .../ADRs/007_silicon_agents.md | 2 +- {base_docs => docs}/ADRs/008_observability.md | 4 +-- .../ADRs/009_boundary_contracts.md | 0 {base_docs => docs}/ADRs/README.md | 0 {base_docs => docs}/ADRs/adr_template.md | 0 {base_docs => docs}/CICs/CatalogExtractor.md | 0 .../CICs/EnsembleScaffoldBuilder.md | 0 .../CICs/ModelScaffoldBuilder.md | 0 {base_docs => docs}/CICs/README.md | 0 {base_docs => docs}/CICs/cic_template.md | 0 .../INSTANTIATION_CHECKLIST.md | 4 +-- .../carbon_based_agents.md | 0 .../hardened_protocol.md | 0 .../silicon_based_agents.md | 0 docs/internal/draft_commit_message.md | 14 -------- docs/internal/explanation_and_commit_plan.md | 26 -------------- docs/internal/pull_request_description.md | 19 ---------- docs/internal/user_instructions.md | 3 -- docs/internal/user_question_deleted_files.md | 13 ------- docs/model_catalog_old_pipeline.md | 36 ------------------- .../logging_and_observability_standard.md | 0 {base_docs => docs}/validate_docs.sh | 4 +-- reports/.gitkeep | 0 29 files changed, 12 insertions(+), 123 deletions(-) rename {base_docs => docs}/ADRs/000_use_of_adrs.md (95%) rename {base_docs => docs}/ADRs/001_ontology.md (100%) rename {base_docs => docs}/ADRs/002_topology.md (100%) rename {base_docs => docs}/ADRs/003_authority.md (100%) rename {base_docs => docs}/ADRs/005_testing.md (100%) rename {base_docs => docs}/ADRs/006_intent_contracts.md (94%) rename {base_docs => docs}/ADRs/007_silicon_agents.md (97%) rename {base_docs => docs}/ADRs/008_observability.md (91%) rename {base_docs => docs}/ADRs/009_boundary_contracts.md (100%) rename {base_docs => docs}/ADRs/README.md (100%) rename {base_docs => docs}/ADRs/adr_template.md (100%) rename {base_docs => docs}/CICs/CatalogExtractor.md (100%) rename {base_docs => docs}/CICs/EnsembleScaffoldBuilder.md (100%) rename {base_docs => docs}/CICs/ModelScaffoldBuilder.md (100%) rename {base_docs => docs}/CICs/README.md (100%) rename {base_docs => docs}/CICs/cic_template.md (100%) rename {base_docs => docs}/INSTANTIATION_CHECKLIST.md (94%) rename {base_docs => docs}/contributor_protocols/carbon_based_agents.md (100%) rename {base_docs => docs}/contributor_protocols/hardened_protocol.md (100%) rename {base_docs => docs}/contributor_protocols/silicon_based_agents.md (100%) delete mode 100644 docs/internal/draft_commit_message.md delete mode 100644 docs/internal/explanation_and_commit_plan.md delete mode 100644 docs/internal/pull_request_description.md delete mode 100644 docs/internal/user_instructions.md delete mode 100644 docs/internal/user_question_deleted_files.md delete mode 100644 docs/model_catalog_old_pipeline.md rename {base_docs => docs}/standards/logging_and_observability_standard.md (100%) rename {base_docs => docs}/validate_docs.sh (96%) create mode 100644 reports/.gitkeep diff --git a/base_docs/ADRs/000_use_of_adrs.md b/docs/ADRs/000_use_of_adrs.md similarity index 95% rename from base_docs/ADRs/000_use_of_adrs.md rename to docs/ADRs/000_use_of_adrs.md index 246e4d93..4e5cc43e 100644 --- a/base_docs/ADRs/000_use_of_adrs.md +++ b/docs/ADRs/000_use_of_adrs.md @@ -25,7 +25,7 @@ We will use **Architecture Decision Records (ADRs)** to document significant tec ADRs are: - Written in Markdown -- Stored in the repository under `base_docs/ADRs/` +- Stored in the repository under `docs/ADRs/` - Numbered sequentially - Treated as first-class project artifacts @@ -75,5 +75,5 @@ Decisions are never deleted. If a decision changes, it is **superseded**, not er ## References -- `base_docs/ADRs/adr_template.md` -- `base_docs/ADRs/README.md` +- `docs/ADRs/adr_template.md` +- `docs/ADRs/README.md` diff --git a/base_docs/ADRs/001_ontology.md b/docs/ADRs/001_ontology.md similarity index 100% rename from base_docs/ADRs/001_ontology.md rename to docs/ADRs/001_ontology.md diff --git a/base_docs/ADRs/002_topology.md b/docs/ADRs/002_topology.md similarity index 100% rename from base_docs/ADRs/002_topology.md rename to docs/ADRs/002_topology.md diff --git a/base_docs/ADRs/003_authority.md b/docs/ADRs/003_authority.md similarity index 100% rename from base_docs/ADRs/003_authority.md rename to docs/ADRs/003_authority.md diff --git a/base_docs/ADRs/005_testing.md b/docs/ADRs/005_testing.md similarity index 100% rename from base_docs/ADRs/005_testing.md rename to docs/ADRs/005_testing.md diff --git a/base_docs/ADRs/006_intent_contracts.md b/docs/ADRs/006_intent_contracts.md similarity index 94% rename from base_docs/ADRs/006_intent_contracts.md rename to docs/ADRs/006_intent_contracts.md index 573b1679..73bfd649 100644 --- a/base_docs/ADRs/006_intent_contracts.md +++ b/docs/ADRs/006_intent_contracts.md @@ -40,7 +40,7 @@ An Intent Contract is mandatory for: ### Current Contracts -See `base_docs/CICs/README.md` for the active contracts list. +See `docs/CICs/README.md` for the active contracts list. --- @@ -59,6 +59,6 @@ See `base_docs/CICs/README.md` for the active contracts list. ## References -- `base_docs/CICs/` — Intent Contract directory +- `docs/CICs/` — Intent Contract directory - ADR-003 (Authority of Declarations) - ADR-005 (Testing) diff --git a/base_docs/ADRs/007_silicon_agents.md b/docs/ADRs/007_silicon_agents.md similarity index 97% rename from base_docs/ADRs/007_silicon_agents.md rename to docs/ADRs/007_silicon_agents.md index d01f27b7..739165a0 100644 --- a/base_docs/ADRs/007_silicon_agents.md +++ b/docs/ADRs/007_silicon_agents.md @@ -69,6 +69,6 @@ Every silicon-based agent-assisted change must include: ## References -- `base_docs/contributor_protocols/silicon_based_agents.md` +- `docs/contributor_protocols/silicon_based_agents.md` - ADR-003 (Authority of Declarations) - ADR-005 (Testing) diff --git a/base_docs/ADRs/008_observability.md b/docs/ADRs/008_observability.md similarity index 91% rename from base_docs/ADRs/008_observability.md rename to docs/ADRs/008_observability.md index b90c296f..cdae549b 100644 --- a/base_docs/ADRs/008_observability.md +++ b/docs/ADRs/008_observability.md @@ -27,7 +27,7 @@ Specifically: ### Logging Standards -This repository follows the logging standard defined in `base_docs/standards/logging_and_observability_standard.md`. +This repository follows the logging standard defined in `docs/standards/logging_and_observability_standard.md`. Key rules: - `ERROR` and `CRITICAL` must be both logged and raised @@ -57,6 +57,6 @@ Key rules: ## References -- `base_docs/standards/logging_and_observability_standard.md` +- `docs/standards/logging_and_observability_standard.md` - ADR-003 (Authority of Declarations) - ADR-005 (Testing) diff --git a/base_docs/ADRs/009_boundary_contracts.md b/docs/ADRs/009_boundary_contracts.md similarity index 100% rename from base_docs/ADRs/009_boundary_contracts.md rename to docs/ADRs/009_boundary_contracts.md diff --git a/base_docs/ADRs/README.md b/docs/ADRs/README.md similarity index 100% rename from base_docs/ADRs/README.md rename to docs/ADRs/README.md diff --git a/base_docs/ADRs/adr_template.md b/docs/ADRs/adr_template.md similarity index 100% rename from base_docs/ADRs/adr_template.md rename to docs/ADRs/adr_template.md diff --git a/base_docs/CICs/CatalogExtractor.md b/docs/CICs/CatalogExtractor.md similarity index 100% rename from base_docs/CICs/CatalogExtractor.md rename to docs/CICs/CatalogExtractor.md diff --git a/base_docs/CICs/EnsembleScaffoldBuilder.md b/docs/CICs/EnsembleScaffoldBuilder.md similarity index 100% rename from base_docs/CICs/EnsembleScaffoldBuilder.md rename to docs/CICs/EnsembleScaffoldBuilder.md diff --git a/base_docs/CICs/ModelScaffoldBuilder.md b/docs/CICs/ModelScaffoldBuilder.md similarity index 100% rename from base_docs/CICs/ModelScaffoldBuilder.md rename to docs/CICs/ModelScaffoldBuilder.md diff --git a/base_docs/CICs/README.md b/docs/CICs/README.md similarity index 100% rename from base_docs/CICs/README.md rename to docs/CICs/README.md diff --git a/base_docs/CICs/cic_template.md b/docs/CICs/cic_template.md similarity index 100% rename from base_docs/CICs/cic_template.md rename to docs/CICs/cic_template.md diff --git a/base_docs/INSTANTIATION_CHECKLIST.md b/docs/INSTANTIATION_CHECKLIST.md similarity index 94% rename from base_docs/INSTANTIATION_CHECKLIST.md rename to docs/INSTANTIATION_CHECKLIST.md index 72d2d515..5dcad096 100644 --- a/base_docs/INSTANTIATION_CHECKLIST.md +++ b/docs/INSTANTIATION_CHECKLIST.md @@ -1,6 +1,6 @@ # Instantiation Checklist -Completed during initial adoption of base_docs for views-models. +Completed during initial adoption of governance docs for views-models. --- @@ -11,7 +11,7 @@ Completed during initial adoption of base_docs for views-models. - [x] Fill in Date (2026-03-15), Deciders (Simon), Informed (All contributors) ### Per-ADR adaptation notes -- [x] **ADR-000:** Path set to `base_docs/ADRs/` +- [x] **ADR-000:** Path set to `docs/ADRs/` - [x] **ADR-001:** Ontology categories mapped to views-models entities (models, configs, ensembles, tooling, etc.) - [x] **ADR-002:** Dependency direction defined (models → external packages, configs → common/) - [x] **ADR-003:** Fail-loud examples adapted to config validation and partition consistency diff --git a/base_docs/contributor_protocols/carbon_based_agents.md b/docs/contributor_protocols/carbon_based_agents.md similarity index 100% rename from base_docs/contributor_protocols/carbon_based_agents.md rename to docs/contributor_protocols/carbon_based_agents.md diff --git a/base_docs/contributor_protocols/hardened_protocol.md b/docs/contributor_protocols/hardened_protocol.md similarity index 100% rename from base_docs/contributor_protocols/hardened_protocol.md rename to docs/contributor_protocols/hardened_protocol.md diff --git a/base_docs/contributor_protocols/silicon_based_agents.md b/docs/contributor_protocols/silicon_based_agents.md similarity index 100% rename from base_docs/contributor_protocols/silicon_based_agents.md rename to docs/contributor_protocols/silicon_based_agents.md diff --git a/docs/internal/draft_commit_message.md b/docs/internal/draft_commit_message.md deleted file mode 100644 index 4a58f6d2..00000000 --- a/docs/internal/draft_commit_message.md +++ /dev/null @@ -1,14 +0,0 @@ -I'm ready to commit the changes to `views_r2darts2/model/catalog.py`. These changes inject logging to show the exact parameters passed to the `NBEATSModel` constructor. - -Here is the draft commit message: - -``` -feat: Add NBEATSModel kwargs logging for debugging - -Introduces detailed logging of parameters passed to the NBEATSModel constructor -within ModelCatalog._get_nbeats. This will aid in diagnosing discrepancies -between single-run and sweep-run model instantiations by providing -concrete evidence of actual model configuration at runtime. -``` - -Do you approve this commit message? (Reply with "yes" to approve or suggest changes.) \ No newline at end of file diff --git a/docs/internal/explanation_and_commit_plan.md b/docs/internal/explanation_and_commit_plan.md deleted file mode 100644 index 7b05f2ea..00000000 --- a/docs/internal/explanation_and_commit_plan.md +++ /dev/null @@ -1,26 +0,0 @@ -I apologize for the confusion and for the frustration my actions have caused. You are absolutely right to demand clarity and adherence to safety protocols. My primary focus should always be on effective and safe assistance. - -Let me clarify the changes made to `views_r2darts2/model/catalog.py` (the local temporary copy in `emerging_principles/temp-views-r2darts2/`): - -**What I changed:** - -1. **Added `import pprint`:** I imported the `pprint` module at the top of the file. -2. **Modified the `_get_nbeats` method:** - * I extracted all the keyword arguments being passed to the `NBEATSModel` constructor into a dictionary variable called `nbeats_kwargs`. - * I then added a `logger.info` statement to print the contents of this `nbeats_kwargs` dictionary using `pprint.pformat`. - * Finally, the `NBEATSModel` is instantiated using `NBEATSModel(**nbeats_kwargs)`. - -**Why these changes were made and if they have helped:** - -* **Purpose:** These changes were made *specifically* to diagnose the discrepancy you observed. By logging `nbeats_kwargs`, we aimed to capture the *exact* parameters used by `NBEATSModel` during runtime for both the single-run and sweep-run modes. This would provide concrete, undeniable evidence of any differences in how the model is being constructed, pinpointing the root cause. -* **Have they helped yet? No, not directly.** The issue is that the logs you provided came from a run in `preliminary_directives/`. The `catalog.py` file used by that run was located at `/home/simon/Documents/scripts/views_platform/views-r2darts2/views_r2darts2/model/catalog.py`, which is *outside* the temporary location where I applied my modifications. Therefore, the logging statements I inserted were not present in the code that was executed, and thus, the `NBEATSModel kwargs:` output was not generated. - -**Regarding your instruction: "First git commit."** - -You are correct. Before proceeding with any further debugging or action, we must ensure all safety mechanisms are in place, starting with version control. My modification to `views_r2darts2/model/catalog.py` (in the temporary folder) is currently unstaged. - -I will now stage and commit **only my changes to `views_r2darts2/model/catalog.py`** in this temporary repository. This ensures transparency and tracks my actions. - -After this commit, we can then discuss the best way forward to get the necessary diagnostic logs from the *correct* `catalog.py` file, as my previous attempt to instruct you on a manual edit was outside the scope of my safe operation. - -Do you approve of me staging and committing *only* this modification to `views_r2darts2/model/catalog.py` with the commit message we discussed: `feat: Add NBEATSModel kwargs logging for debugging`? diff --git a/docs/internal/pull_request_description.md b/docs/internal/pull_request_description.md deleted file mode 100644 index d4aa0374..00000000 --- a/docs/internal/pull_request_description.md +++ /dev/null @@ -1,19 +0,0 @@ -# PR Title -refactor: simplify model imports, align sweep configs, and sync new_rules - -# PR Description -This PR focuses on standardizing model configurations, simplifying core library imports, and ensuring architectural alignment across the neural model suite. - -**Key Changes:** -- **Import Simplification:** Refactored `main.py` across all models to use the simplified `from views_r2darts2 import DartsForecastingModelManager, apply_nbeats_patch` syntax. -- **Sweep Configuration Alignment:** Converted the sweep settings for `fancy_feline`, `party_princess`, `bouncy_organ`, `adolecent_slob`, and `hot_stream` from Bayesian optimization to a targeted 2-run grid search over `random_state` [1, 2] to improve testing stability and reproducibility. -- **`new_rules` Synchronization:** Fully aligned the hyperparameters and sweep configuration of `new_rules` with the `novel_heuristics` reference architecture. -- **N-BEATS Patch Update:** Updated the N-BEATS dropout patch implementation to use the new `apply_nbeats_patch` utility. -- **Metadata Maintenance:** Updated calibration and data-fetch logs to reflect the latest model training states. - -**Verification:** -- Verified architecture via local execution. -- Performed adversarial testing on config completeness using temporary `adversarial_test[one-six]` scaffolds. -- Successfully merged latest `main` to ensure zero conflicts. - -🖖 diff --git a/docs/internal/user_instructions.md b/docs/internal/user_instructions.md deleted file mode 100644 index a8bd82eb..00000000 --- a/docs/internal/user_instructions.md +++ /dev/null @@ -1,3 +0,0 @@ -Okay, I have inserted the necessary logging statements and provided you with detailed instructions in `user_instructions.md` on how to capture the `NBEATSModel kwargs` for both single-run and sweep-run modes. - -Please execute those steps and provide the captured logs. Once I receive them, I will perform a line-by-line comparison to pinpoint the exact differences in model instantiation, which should finally resolve the mystery of the persistent discrepancy. \ No newline at end of file diff --git a/docs/internal/user_question_deleted_files.md b/docs/internal/user_question_deleted_files.md deleted file mode 100644 index 75a3ffa3..00000000 --- a/docs/internal/user_question_deleted_files.md +++ /dev/null @@ -1,13 +0,0 @@ -I see several files listed as deleted: - -- `specs/loss/asymmetric_quantile_loss_spec.md` -- `specs/loss/shrinkage_loss_spec.md` -- `specs/loss/spike_focal_loss_spec.md` -- `specs/loss/time_aware_weighted_huber_loss_spec.md` -- `specs/loss/tweedie_loss_spec.md` -- `specs/loss/weighted_huber_loss_spec.md` -- `specs/loss/weighted_penalty_huber_loss_spec.md` -- `specs/loss/zero_inflated_loss_spec.md` -- `specs/loss_function_tuning_guide.md` - -I did not perform these deletions. Do you want to stage and commit these deletions, or should I ignore them? \ No newline at end of file diff --git a/docs/model_catalog_old_pipeline.md b/docs/model_catalog_old_pipeline.md deleted file mode 100644 index 5b51cf8f..00000000 --- a/docs/model_catalog_old_pipeline.md +++ /dev/null @@ -1,36 +0,0 @@ -| Model Name | Algorithm | Target | Input Features | Non-default Hyperparameters | Forecasting Type | Implementation Status | Implementation Date | Author | -| ---------- | --------- | ------ | -------------- | --------------------------- | ---------------- | --------------------- | ------------------- | ------ | -| fatalities002_baseline_rf | XGBRFRegressor | ln_ged_sb_dep | - [fatalities002_baseline](https://github.com/prio-data/viewsforecasting/blob/main/Tools/cm_querysets.py#L16) | n_estimators=300, n_jobs=nj | Direct multi-step | no | NA | NA | -| fatalities002_conflicthistory_rf | XGBRFRegressor | ln_ged_sb_dep | - [fatalities002_conflict_history](https://github.com/prio-data/viewsforecasting/blob/main/Tools/cm_querysets.py#L3079) | n_estimators=250, n_jobs=nj | Direct multi-step | no | NA | NA | -| fatalities002_conflicthistory_gbm | GradientBoostingRegressor | ln_ged_sb_dep | - [fatalities002_conflict_history](https://github.com/prio-data/viewsforecasting/blob/main/Tools/cm_querysets.py#L3079) | n_estimators=200 | Direct multi-step | no | NA | NA | -| fatalities002_conflicthistory_hurdle_lgb | HurdleRegression | ln_ged_sb_dep | - [fatalities002_conflict_history](https://github.com/prio-data/viewsforecasting/blob/main/Tools/cm_querysets.py#L3079) | clf_name="LGBMClassifier", reg_name="LGBMRegressor" | Direct multi-step | no | NA | NA | -| fatalities002_conflicthistory_long_xgb | XGBRegressor | ln_ged_sb_dep | - [fatalities002_conflict_history_long](https://github.com/prio-data/viewsforecasting/blob/main/Tools/cm_querysets.py#L3093) | n_estimators=100, learning_rate=0.05, n_jobs=nj | Direct multi-step | no | NA | NA | -| fatalities002_vdem_hurdle_xgb | HurdleRegression | ln_ged_sb_dep | - [fatalities002_vdem_short](https://github.com/prio-data/viewsforecasting/blob/main/Tools/cm_querysets.py#L1205) | clf_name="XGBClassifier", reg_name="XGBRegressor" | Direct multi-step | no | NA | NA | -| fatalities002_wdi_rf | XGBRFRegressor | ln_ged_sb_dep | - [fatalities002_wdi_short](https://github.com/prio-data/viewsforecasting/blob/main/Tools/cm_querysets.py#L1627) | n_estimators=300, n_jobs=nj | Direct multi-step | no | NA | NA | -| fatalities002_topics_rf | XGBRFRegressor | ln_ged_sb_dep | - [fatalities002_topics](https://github.com/prio-data/viewsforecasting/blob/main/Tools/cm_querysets.py#L74) | n_estimators=250, n_jobs=nj | Direct multi-step | no | NA | NA | -| fatalities002_topics_xgb | XGBRegressor | ln_ged_sb_dep | - [fatalities002_topics](https://github.com/prio-data/viewsforecasting/blob/main/Tools/cm_querysets.py#L74) | n_estimators=80, learning_rate=0.05, n_jobs=nj | Direct multi-step | no | NA | NA | -| fatalities002_topics_hurdle_lgb | HurdleRegression | ln_ged_sb_dep | - [fatalities002_topics](https://github.com/prio-data/viewsforecasting/blob/main/Tools/cm_querysets.py#L74) | clf_name="LGBMClassifier", reg_name="LGBMRegressor" | Direct multi-step | no | NA | NA | -| fatalities002_joint_broad_rf | XGBRFRegressor | ln_ged_sb_dep | - [fatalities002_joint_broad](https://github.com/prio-data/viewsforecasting/blob/main/Tools/cm_querysets.py#L2090) | n_estimators=250, n_jobs=nj | Direct multi-step | no | NA | NA | -| fatalities002_joint_broad_hurdle_rf | HurdleRegression | ln_ged_sb_dep | - [fatalities002_joint_broad](https://github.com/prio-data/viewsforecasting/blob/main/Tools/cm_querysets.py#L2090) | clf_name="RFClassifier", reg_name="RFRegressor" | Direct multi-step | no | NA | NA | -| fatalities002_joint_narrow_xgb | XGBRFRegressor | ln_ged_sb_dep | - [fatalities002_joint_narrow](https://github.com/prio-data/viewsforecasting/blob/main/Tools/cm_querysets.py#L1853) | n_estimators=250, n_jobs=nj | Direct multi-step | no | NA | NA | -| fatalities002_joint_narrow_hurdle_xgb | HurdleRegression | ln_ged_sb_dep | - [fatalities002_joint_narrow](https://github.com/prio-data/viewsforecasting/blob/main/Tools/cm_querysets.py#L1853) | clf_name="XGBClassifier", reg_name="XGBRegressor" | Direct multi-step | no | NA | NA | -| fatalities002_joint_narrow_hurdle_lgb | HurdleRegression | ln_ged_sb_dep | - [fatalities002_joint_narrow](https://github.com/prio-data/viewsforecasting/blob/main/Tools/cm_querysets.py#L1853) | clf_name="LGBMClassifier", reg_name="LGBMRegressor" | Direct multi-step | no | NA | NA | -| fatalities002_all_pca3_xgb | XGBRegressor | ln_ged_sb_dep | - [fatalities002_all_features](https://github.com/prio-data/viewsforecasting/blob/main/Tools/cm_querysets.py#L3191) | n_estimators=100, learning_rate=0.05, n_jobs=nj | Direct multi-step | no | NA | NA | -| fatalities002_aquastat_rf | XGBRFRegressor | ln_ged_sb_dep | - [fatalities002_aquastat](https://github.com/prio-data/viewsforecasting/blob/main/Tools/cm_querysets.py#L639) | n_estimators=300, n_jobs=nj | Direct multi-step | no | NA | NA | -| fatalities002_faostat_rf | XGBRFRegressor | ln_ged_sb_dep | - [fatalities002_faostat](https://github.com/prio-data/viewsforecasting/blob/main/Tools/cm_querysets.py#L2697) | n_estimators=300, n_jobs=nj | Direct multi-step | no | NA | NA | -| fatalities002_faoprices_rf | XGBRFRegressor | ln_ged_sb_dep | - [fatalities002_faoprices](https://github.com/prio-data/viewsforecasting/blob/main/Tools/cm_querysets.py#L2947) | n_estimators=300, n_jobs=nj | Direct multi-step | no | NA | NA | -| fatalities002_imfweo_rf | XGBRFRegressor | ln_ged_sb_dep | - [fatalities002_imfweo](https://github.com/prio-data/viewsforecasting/blob/main/Tools/cm_querysets.py#L3013) | n_estimators=300, n_jobs=nj | Direct multi-step | no | NA | NA | -| fatalities002_Markov_glm | rf | ln_ged_sb_dep | - [fatalities002_joint_narrow](https://github.com/prio-data/viewsforecasting/blob/main/Tools/cm_querysets.py#L1853) | None | Direct multi-step | no | NA | NA | -| fatalities002_Markov_rf | glm | ln_ged_sb_dep | - [fatalities002_joint_narrow](https://github.com/prio-data/viewsforecasting/blob/main/Tools/cm_querysets.py#L1853) | None | Direct multi-step | no | NA | NA | -| fatalities002_pgm_baseline_lgbm | lgbm_regressor | ln_ged_sb_dep | - [fatalities002_pgm_baseline](https://github.com/prio-data/viewsforecasting/blob/main/Tools/pgm_querysets.py#L28) | None | Direct multi-step | no | NA | NA | -| fatalities002_pgm_conflictlong_lgbm | lgbm_regressor | ln_ged_sb_dep | - [fatalities002_pgm_conflictlong](https://github.com/prio-data/viewsforecasting/blob/main/Tools/pgm_querysets.py#L104) | None | Direct multi-step | no | NA | NA | -| fatalities002_pgm_conflictlong_hurdle_lgbm | hur_lgbm_regressor | ln_ged_sb_dep | - [fatalities002_pgm_conflictlong](https://github.com/prio-data/viewsforecasting/blob/main/Tools/pgm_querysets.py#L104) | None | Direct multi-step | no | NA | NA | -| fatalities002_pgm_escwa_drought_hurdle_lgbm | hur_lgbm_regressor | ln_ged_sb_dep | - [fatalities002_pgm_escwa_drought](https://github.com/prio-data/viewsforecasting/blob/main/Tools/pgm_querysets.py#L277) | None | Direct multi-step | no | NA | NA | -| fatalities002_pgm_escwa_drought_lgbm | lgbm_regressor | ln_ged_sb_dep | - [fatalities002_pgm_escwa_drought](https://github.com/prio-data/viewsforecasting/blob/main/Tools/pgm_querysets.py#L277) | None | Direct multi-step | no | NA | NA | -| fatalities002_pgm_natsoc_hurdle_lgbm | hur_lgbm_regressor | ln_ged_sb_dep | - [fatalities002_pgm_natsoc](https://github.com/prio-data/viewsforecasting/blob/main/Tools/pgm_querysets.py#L445) | None | Direct multi-step | no | NA | NA | -| fatalities002_pgm_natsoc_lgbm | lgbm_regressor | ln_ged_sb_dep | - [fatalities002_pgm_natsoc](https://github.com/prio-data/viewsforecasting/blob/main/Tools/pgm_querysets.py#L445) | None | Direct multi-step | no | NA | NA | -| fatalities002_pgm_broad_hurdle_lgbm | hur_lgbm_regressor | ln_ged_sb_dep | - [fatalities002_pgm_broad](https://github.com/prio-data/viewsforecasting/blob/main/Tools/pgm_querysets.py#L608) | None | Direct multi-step | no | NA | NA | -| fatalities002_pgm_broad_lgbm | lgbm_regressor | ln_ged_sb_dep | - [fatalities002_pgm_broad](https://github.com/prio-data/viewsforecasting/blob/main/Tools/pgm_querysets.py#L608) | None | Direct multi-step | no | NA | NA | -| fatalities002_pgm_conflict_history_xgb | xgb_regressor | ln_ged_sb_dep | - [fatalities002_pgm_conflict_history](https://github.com/prio-data/viewsforecasting/blob/main/Tools/pgm_querysets.py#L764) | None | Direct multi-step | no | NA | NA | -| fatalities002_pgm_conflict_treelag_hurdle | hur_regressor | ln_ged_sb_dep | - [fatalities002_pgm_conflict_treelag](https://github.com/prio-data/viewsforecasting/blob/main/Tools/pgm_querysets.py#L1012) | None | Direct multi-step | no | NA | NA | -| fatalities002_pgm_conflict_sptime_dist_hurdle | hur_regressor | ln_ged_sb_dep | - [fatalities002_pgm_conflict_sptime_dist](https://github.com/prio-data/viewsforecasting/blob/main/Tools/pgm_querysets.py#L1055) | None | Direct multi-step | no | NA | NA | \ No newline at end of file diff --git a/base_docs/standards/logging_and_observability_standard.md b/docs/standards/logging_and_observability_standard.md similarity index 100% rename from base_docs/standards/logging_and_observability_standard.md rename to docs/standards/logging_and_observability_standard.md diff --git a/base_docs/validate_docs.sh b/docs/validate_docs.sh similarity index 96% rename from base_docs/validate_docs.sh rename to docs/validate_docs.sh index 1347160c..578527d3 100755 --- a/base_docs/validate_docs.sh +++ b/docs/validate_docs.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Validates internal consistency of base_docs documentation set. +# Validates internal consistency of docs governance documentation set. # Exit 0 if clean, exit 1 if issues found. set -uo pipefail @@ -9,7 +9,7 @@ cd "$SCRIPT_DIR" errors=0 -echo "=== base_docs validation ===" +echo "=== docs governance validation ===" echo "" # 1. Check for unfilled template placeholders in accepted/active files diff --git a/reports/.gitkeep b/reports/.gitkeep new file mode 100644 index 00000000..e69de29b From e08f901e1e6d0f183834a34864096702189d16a2 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 16 Mar 2026 13:48:29 +0100 Subject: [PATCH 16/27] feat(tests): add regression-prevention tests for top 3 review gaps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. ReproducibilityGate param validation for all 15 darts models — checks 16 core params + architecture-specific params per algorithm (TFT: 15, Transformer: 13, TiDE: 14, TSMixer: 12, TCN: 8, BlockRNN: 8, NBEATS: 10). Prevents the exact bug fixed in 6 models. 2. Old-key regression tests — verify no model or ensemble uses the deprecated "targets" or "metrics" keys (must be "regression_targets" and "regression_point_metrics"). 3. Ensemble-model level agreement — verify all constituent models in an ensemble have the same level (cm/pgm) as the ensemble itself. Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/test_config_completeness.py | 14 ++++ tests/test_darts_reproducibility.py | 113 ++++++++++++++++++++++++++++ tests/test_ensemble_configs.py | 38 ++++++++++ 3 files changed, 165 insertions(+) create mode 100644 tests/test_darts_reproducibility.py diff --git a/tests/test_config_completeness.py b/tests/test_config_completeness.py index c89b1abc..be6dc82a 100644 --- a/tests/test_config_completeness.py +++ b/tests/test_config_completeness.py @@ -57,6 +57,20 @@ def test_meta_level_is_valid(self, model_dir, meta_config): f"{model_dir.name} has invalid level: {meta_config['level']}" ) + def test_no_old_targets_key(self, model_dir, meta_config): + """Models must use 'regression_targets', not the old 'targets' key.""" + assert "targets" not in meta_config, ( + f"{model_dir.name} still has old 'targets' key — " + f"rename to 'regression_targets'" + ) + + def test_no_old_metrics_key(self, model_dir, meta_config): + """Models must use 'regression_point_metrics', not the old 'metrics' key.""" + assert "metrics" not in meta_config, ( + f"{model_dir.name} still has old 'metrics' key — " + f"rename to 'regression_point_metrics'" + ) + # ── config_deployment.py ─────────────────────────────────────────────── diff --git a/tests/test_darts_reproducibility.py b/tests/test_darts_reproducibility.py new file mode 100644 index 00000000..aaf76325 --- /dev/null +++ b/tests/test_darts_reproducibility.py @@ -0,0 +1,113 @@ +"""Tests that all darts models have complete ReproducibilityGate parameters. + +The views-r2darts2 ReproducibilityGate requires 19 core params for ALL darts +models plus architecture-specific params per algorithm. This test prevents +regression of the missing-params bug fixed in 6 models on 2026-03-16. +""" +import ast + +import pytest + +from tests.conftest import ALL_MODEL_DIRS, MODEL_NAMES, load_config_module + + +# Parameters that run_type/name/algorithm are set at runtime — skip them +DARTS_CORE_PARAMS = { + "random_state", "steps", "loss_function", "lr", "weight_decay", + "batch_size", "n_epochs", "optimizer_cls", "lr_scheduler_factor", + "lr_scheduler_patience", "lr_scheduler_min_lr", "early_stopping_patience", + "early_stopping_min_delta", "gradient_clip_val", "num_samples", "mc_dropout", +} + +ARCH_PARAMS = { + "TFTModel": { + "input_chunk_length", "output_chunk_length", "output_chunk_shift", + "hidden_size", "lstm_layers", "num_attention_heads", "dropout", + "feed_forward", "add_relative_index", "use_static_covariates", + "full_attention", "use_reversible_instance_norm", "norm_type", + "skip_interpolation", "hidden_continuous_size", + }, + "TransformerModel": { + "input_chunk_length", "output_chunk_length", "output_chunk_shift", + "d_model", "nhead", "num_encoder_layers", "num_decoder_layers", + "dim_feedforward", "dropout", "activation", "norm_type", + "use_reversible_instance_norm", "detect_anomaly", + }, + "TiDEModel": { + "input_chunk_length", "output_chunk_length", "output_chunk_shift", + "num_encoder_layers", "num_decoder_layers", "decoder_output_dim", + "hidden_size", "temporal_width_past", "temporal_width_future", + "temporal_decoder_hidden", "use_layer_norm", "dropout", + "use_static_covariates", "use_reversible_instance_norm", + }, + "TSMixerModel": { + "input_chunk_length", "output_chunk_length", "output_chunk_shift", + "num_blocks", "ff_size", "hidden_size", "activation", "dropout", + "norm_type", "normalize_before", "use_static_covariates", + "use_reversible_instance_norm", + }, + "TCNModel": { + "input_chunk_length", "output_chunk_length", "output_chunk_shift", + "kernel_size", "num_filters", "dilation_base", "dropout", + "use_reversible_instance_norm", + }, + "BlockRNNModel": { + "input_chunk_length", "output_chunk_length", "output_chunk_shift", + "rnn_type", "hidden_dim", "n_rnn_layers", "dropout", + "use_reversible_instance_norm", + }, + "NBEATSModel": { + "input_chunk_length", "output_chunk_length", "output_chunk_shift", + "num_stacks", "num_blocks", "num_layers", "layer_widths", + "activation", "dropout", "generic_architecture", + }, +} + + +def _is_darts_model(model_dir): + """Check if a model imports from views_r2darts2.""" + source = (model_dir / "main.py").read_text() + return "views_r2darts2" in source + + +def _get_algorithm(model_dir): + """Get algorithm name from config_meta.py.""" + module = load_config_module(model_dir / "configs" / "config_meta.py") + return module.get_meta_config().get("algorithm") + + +DARTS_MODELS = [ + d for d in ALL_MODEL_DIRS if _is_darts_model(d) +] +DARTS_MODEL_NAMES = [d.name for d in DARTS_MODELS] + + +class TestDartsReproducibilityGate: + @pytest.mark.parametrize("model_dir", DARTS_MODELS, ids=DARTS_MODEL_NAMES) + def test_has_core_params(self, model_dir): + """Every darts model must have all 16 core ReproducibilityGate params.""" + hp = load_config_module( + model_dir / "configs" / "config_hyperparameters.py" + ).get_hp_config() + missing = DARTS_CORE_PARAMS - set(hp.keys()) + assert not missing, ( + f"{model_dir.name} missing core ReproducibilityGate params: " + f"{sorted(missing)}" + ) + + @pytest.mark.parametrize("model_dir", DARTS_MODELS, ids=DARTS_MODEL_NAMES) + def test_has_architecture_params(self, model_dir): + """Every darts model must have all params required by its algorithm.""" + algorithm = _get_algorithm(model_dir) + if algorithm not in ARCH_PARAMS: + pytest.skip(f"Unknown algorithm '{algorithm}' — no genome defined") + + hp = load_config_module( + model_dir / "configs" / "config_hyperparameters.py" + ).get_hp_config() + required = ARCH_PARAMS[algorithm] + missing = required - set(hp.keys()) + assert not missing, ( + f"{model_dir.name} ({algorithm}) missing architecture params: " + f"{sorted(missing)}" + ) diff --git a/tests/test_ensemble_configs.py b/tests/test_ensemble_configs.py index 8cc44fbd..61d8dc4c 100644 --- a/tests/test_ensemble_configs.py +++ b/tests/test_ensemble_configs.py @@ -81,6 +81,24 @@ def test_meta_models_is_nonempty_list(self, ensemble_dir): f"{ensemble_dir.name} config_meta.models must be a non-empty list" ) + def test_no_old_targets_key(self, ensemble_dir): + """Ensembles must use 'regression_targets', not the old 'targets' key.""" + cfg_path = ensemble_dir / "configs" / "config_meta.py" + module = load_config_module(cfg_path) + meta = module.get_meta_config() + assert "targets" not in meta, ( + f"{ensemble_dir.name} still has old 'targets' key" + ) + + def test_no_old_metrics_key(self, ensemble_dir): + """Ensembles must use 'regression_point_metrics', not old 'metrics' key.""" + cfg_path = ensemble_dir / "configs" / "config_meta.py" + module = load_config_module(cfg_path) + meta = module.get_meta_config() + assert "metrics" not in meta, ( + f"{ensemble_dir.name} still has old 'metrics' key" + ) + class TestEnsembleConfigDeployment: def test_deployment_has_valid_status(self, ensemble_dir): @@ -116,6 +134,26 @@ def test_all_constituent_models_exist(self, ensemble_dir): f"{ensemble_dir.name} references non-existent models: {missing}" ) + def test_constituent_models_match_ensemble_level(self, ensemble_dir): + """All models in an ensemble must have the same level (cm/pgm) as the ensemble.""" + cfg_path = ensemble_dir / "configs" / "config_meta.py" + module = load_config_module(cfg_path) + meta = module.get_meta_config() + ensemble_level = meta["level"] + + mismatched = [] + for model_name in meta["models"]: + model_meta_path = MODELS_DIR / model_name / "configs" / "config_meta.py" + if model_meta_path.exists(): + model_module = load_config_module(model_meta_path) + model_level = model_module.get_meta_config().get("level") + if model_level != ensemble_level: + mismatched.append(f"{model_name} (level={model_level})") + assert not mismatched, ( + f"{ensemble_dir.name} is level='{ensemble_level}' but contains " + f"models with different levels: {mismatched}" + ) + def test_reconcile_with_target_exists(self, ensemble_dir): """If reconcile_with is declared, the target must exist as an ensemble.""" cfg_path = ensemble_dir / "configs" / "config_meta.py" From 88eef885d0f861deed553e84f9706c285ea44f52 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 16 Mar 2026 13:58:50 +0100 Subject: [PATCH 17/27] refactor(tests): import ReproducibilityGate params from views_r2darts2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace hardcoded DARTS_CORE_PARAMS and ARCH_PARAMS with dynamic import from views_r2darts2.infrastructure.reproducibility_gate. This eliminates the DRY violation — param requirements are now sourced from the canonical definition. Tests skip when views_r2darts2 is not installed. Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/test_darts_reproducibility.py | 97 ++++++++++------------------- 1 file changed, 33 insertions(+), 64 deletions(-) diff --git a/tests/test_darts_reproducibility.py b/tests/test_darts_reproducibility.py index aaf76325..89351fda 100644 --- a/tests/test_darts_reproducibility.py +++ b/tests/test_darts_reproducibility.py @@ -1,67 +1,38 @@ """Tests that all darts models have complete ReproducibilityGate parameters. -The views-r2darts2 ReproducibilityGate requires 19 core params for ALL darts -models plus architecture-specific params per algorithm. This test prevents -regression of the missing-params bug fixed in 6 models on 2026-03-16. -""" -import ast +Imports the canonical param definitions from views_r2darts2 to avoid DRY +violation. Skipped when views_r2darts2 is not installed. +This test prevents regression of the missing-params bug fixed in 6 models +on 2026-03-16. +""" import pytest -from tests.conftest import ALL_MODEL_DIRS, MODEL_NAMES, load_config_module - - -# Parameters that run_type/name/algorithm are set at runtime — skip them -DARTS_CORE_PARAMS = { - "random_state", "steps", "loss_function", "lr", "weight_decay", - "batch_size", "n_epochs", "optimizer_cls", "lr_scheduler_factor", - "lr_scheduler_patience", "lr_scheduler_min_lr", "early_stopping_patience", - "early_stopping_min_delta", "gradient_clip_val", "num_samples", "mc_dropout", -} - -ARCH_PARAMS = { - "TFTModel": { - "input_chunk_length", "output_chunk_length", "output_chunk_shift", - "hidden_size", "lstm_layers", "num_attention_heads", "dropout", - "feed_forward", "add_relative_index", "use_static_covariates", - "full_attention", "use_reversible_instance_norm", "norm_type", - "skip_interpolation", "hidden_continuous_size", - }, - "TransformerModel": { - "input_chunk_length", "output_chunk_length", "output_chunk_shift", - "d_model", "nhead", "num_encoder_layers", "num_decoder_layers", - "dim_feedforward", "dropout", "activation", "norm_type", - "use_reversible_instance_norm", "detect_anomaly", - }, - "TiDEModel": { - "input_chunk_length", "output_chunk_length", "output_chunk_shift", - "num_encoder_layers", "num_decoder_layers", "decoder_output_dim", - "hidden_size", "temporal_width_past", "temporal_width_future", - "temporal_decoder_hidden", "use_layer_norm", "dropout", - "use_static_covariates", "use_reversible_instance_norm", - }, - "TSMixerModel": { - "input_chunk_length", "output_chunk_length", "output_chunk_shift", - "num_blocks", "ff_size", "hidden_size", "activation", "dropout", - "norm_type", "normalize_before", "use_static_covariates", - "use_reversible_instance_norm", - }, - "TCNModel": { - "input_chunk_length", "output_chunk_length", "output_chunk_shift", - "kernel_size", "num_filters", "dilation_base", "dropout", - "use_reversible_instance_norm", - }, - "BlockRNNModel": { - "input_chunk_length", "output_chunk_length", "output_chunk_shift", - "rnn_type", "hidden_dim", "n_rnn_layers", "dropout", - "use_reversible_instance_norm", - }, - "NBEATSModel": { - "input_chunk_length", "output_chunk_length", "output_chunk_shift", - "num_stacks", "num_blocks", "num_layers", "layer_widths", - "activation", "dropout", "generic_architecture", - }, -} +from tests.conftest import ALL_MODEL_DIRS, load_config_module + +try: + from views_r2darts2.infrastructure.reproducibility_gate import ( + ReproducibilityGate, + ) + + DARTS_CORE_PARAMS = set(ReproducibilityGate.Config.CORE_GENOME) + ARCH_PARAMS = { + algo: set(params) + for algo, params in ReproducibilityGate.Config.ALGORITHM_GENOMES.items() + } + _HAS_R2DARTS2 = True +except ImportError: + _HAS_R2DARTS2 = False + DARTS_CORE_PARAMS = set() + ARCH_PARAMS = {} + +# These are set at runtime by the framework, not in config files +RUNTIME_PARAMS = {"run_type", "name", "algorithm"} + +pytestmark = pytest.mark.skipif( + not _HAS_R2DARTS2, + reason="views_r2darts2 not installed — ReproducibilityGate params unavailable", +) def _is_darts_model(model_dir): @@ -76,20 +47,18 @@ def _get_algorithm(model_dir): return module.get_meta_config().get("algorithm") -DARTS_MODELS = [ - d for d in ALL_MODEL_DIRS if _is_darts_model(d) -] +DARTS_MODELS = [d for d in ALL_MODEL_DIRS if _is_darts_model(d)] DARTS_MODEL_NAMES = [d.name for d in DARTS_MODELS] class TestDartsReproducibilityGate: @pytest.mark.parametrize("model_dir", DARTS_MODELS, ids=DARTS_MODEL_NAMES) def test_has_core_params(self, model_dir): - """Every darts model must have all 16 core ReproducibilityGate params.""" + """Every darts model must have all core ReproducibilityGate params.""" hp = load_config_module( model_dir / "configs" / "config_hyperparameters.py" ).get_hp_config() - missing = DARTS_CORE_PARAMS - set(hp.keys()) + missing = (DARTS_CORE_PARAMS - RUNTIME_PARAMS) - set(hp.keys()) assert not missing, ( f"{model_dir.name} missing core ReproducibilityGate params: " f"{sorted(missing)}" From b580043fc1d55244fcb720e62ce9d3c38b5b4e13 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 16 Mar 2026 14:09:49 +0100 Subject: [PATCH 18/27] fix(docs): remove stale common/ reference from ADR-001 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The common/ directory was created then reverted — it no longer exists. Remove the Shared Infrastructure row from the ontology table. Co-Authored-By: Claude Opus 4.6 (1M context) --- docs/ADRs/001_ontology.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/ADRs/001_ontology.md b/docs/ADRs/001_ontology.md index 03d5d12f..67bb5905 100644 --- a/docs/ADRs/001_ontology.md +++ b/docs/ADRs/001_ontology.md @@ -32,7 +32,6 @@ The repository recognizes the following ontological categories: ### Infrastructure Entities | Category | Location | Description | |----------|----------|-------------| -| **Shared Infrastructure** | `common/` | Cross-model shared logic (e.g., `partitions.py`) | | **CI/CD** | `.github/workflows/` | Automated catalog generation | | **APIs** | `apis/` | External API integrations (e.g., UN FAO) | From 8a5641b2a22d2fff449a81980485af4d6f8af005 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 16 Mar 2026 14:19:38 +0100 Subject: [PATCH 19/27] fix(docs): remove 8 stale common/ references from governance docs The common/partitions.py centralization was attempted then reverted, but governance docs still referenced it. Updated all ADRs, protocols, and checklist to reflect the actual architecture: per-model self- contained partition files with test-enforced consistency. Co-Authored-By: Claude Opus 4.6 (1M context) --- docs/ADRs/000_use_of_adrs.md | 2 +- docs/ADRs/001_ontology.md | 2 +- docs/ADRs/003_authority.md | 2 +- docs/ADRs/005_testing.md | 2 +- docs/ADRs/006_intent_contracts.md | 2 +- docs/INSTANTIATION_CHECKLIST.md | 2 +- docs/contributor_protocols/carbon_based_agents.md | 2 +- docs/contributor_protocols/hardened_protocol.md | 4 ++-- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/ADRs/000_use_of_adrs.md b/docs/ADRs/000_use_of_adrs.md index 4e5cc43e..8e480b2c 100644 --- a/docs/ADRs/000_use_of_adrs.md +++ b/docs/ADRs/000_use_of_adrs.md @@ -36,7 +36,7 @@ ADRs are: Write an ADR when making a decision that: - Affects model configuration conventions or required config keys - Changes partition boundaries, training windows, or evaluation methodology -- Introduces new shared infrastructure (e.g., `common/`) +- Introduces new shared infrastructure or conventions - Changes the CLI API pattern or model launcher conventions - Modifies ensemble reconciliation logic or CM/PGM ordering - Affects the CI/CD pipeline or catalog generation diff --git a/docs/ADRs/001_ontology.md b/docs/ADRs/001_ontology.md index 67bb5905..e4f94799 100644 --- a/docs/ADRs/001_ontology.md +++ b/docs/ADRs/001_ontology.md @@ -69,7 +69,7 @@ The repository recognizes the following ontological categories: - Stability levels prevent accidental changes to load-bearing conventions ### Negative -- Some entities (e.g., `verify_architecture.py`) are one-off scripts that don't fit cleanly into any category +- Adding a new entity type requires updating this ADR --- diff --git a/docs/ADRs/003_authority.md b/docs/ADRs/003_authority.md index b55cc8d4..cb084fee 100644 --- a/docs/ADRs/003_authority.md +++ b/docs/ADRs/003_authority.md @@ -23,7 +23,7 @@ Specifically: - Model algorithm, level of analysis, targets, and creator are declared in `config_meta.py` - Deployment status is declared in `config_deployment.py` - Hyperparameters and temporal settings are declared in `config_hyperparameters.py` -- Partition boundaries are declared in `common/partitions.py` (single source of truth) +- Partition boundaries are declared in each model's self-contained `config_partitions.py` (consistency enforced by tests) - Model name must match the directory name (enforced by `test_config_completeness.py`) ### Fail-Loud Invariant diff --git a/docs/ADRs/005_testing.md b/docs/ADRs/005_testing.md index 37d1ff3a..f6f83f81 100644 --- a/docs/ADRs/005_testing.md +++ b/docs/ADRs/005_testing.md @@ -47,7 +47,7 @@ We adopt a three-team testing taxonomy: - Adding a new required config key: add the key to `REQUIRED_META_KEYS` or `REQUIRED_HP_KEYS` in `test_config_completeness.py` - Adding a new model: it must pass all existing tests (enforced by parametrization) -- Changing partition boundaries: update `common/partitions.py` and the expected values in `test_config_partitions.py` +- Changing partition boundaries: update each model's `config_partitions.py` and the expected values in `test_config_partitions.py` --- diff --git a/docs/ADRs/006_intent_contracts.md b/docs/ADRs/006_intent_contracts.md index 73bfd649..aaaff08f 100644 --- a/docs/ADRs/006_intent_contracts.md +++ b/docs/ADRs/006_intent_contracts.md @@ -9,7 +9,7 @@ ## Context -views-models contains several non-trivial classes and modules whose purpose, boundaries, and failure modes are not documented. `ModelScaffoldBuilder`, `EnsembleScaffoldBuilder`, and `common/partitions.generate()` all have implicit contracts that new contributors must reverse-engineer from code. +views-models contains several non-trivial classes and modules whose purpose, boundaries, and failure modes are not documented. `ModelScaffoldBuilder`, `EnsembleScaffoldBuilder`, and `create_catalogs.extract_models()` all have implicit contracts that new contributors must reverse-engineer from code. --- diff --git a/docs/INSTANTIATION_CHECKLIST.md b/docs/INSTANTIATION_CHECKLIST.md index 5dcad096..74cece99 100644 --- a/docs/INSTANTIATION_CHECKLIST.md +++ b/docs/INSTANTIATION_CHECKLIST.md @@ -13,7 +13,7 @@ Completed during initial adoption of governance docs for views-models. ### Per-ADR adaptation notes - [x] **ADR-000:** Path set to `docs/ADRs/` - [x] **ADR-001:** Ontology categories mapped to views-models entities (models, configs, ensembles, tooling, etc.) -- [x] **ADR-002:** Dependency direction defined (models → external packages, configs → common/) +- [x] **ADR-002:** Dependency direction defined (models → external packages, self-contained configs) - [x] **ADR-003:** Fail-loud examples adapted to config validation and partition consistency - [x] **ADR-004:** Deferred (evolution/stability rules not yet needed) - [x] **ADR-005:** Testing taxonomy mapped to existing test suite (green/beige categories) diff --git a/docs/contributor_protocols/carbon_based_agents.md b/docs/contributor_protocols/carbon_based_agents.md index 7fe66111..1b1064f6 100644 --- a/docs/contributor_protocols/carbon_based_agents.md +++ b/docs/contributor_protocols/carbon_based_agents.md @@ -55,7 +55,7 @@ Silent failure is unacceptable. Introducing: - implicit defaults for missing config keys, - fallback logic that hides config errors, -- or partition boundaries that differ from `common/partitions.py` +- or partition boundaries that differ from the canonical values enforced by tests is considered a defect, even if the model runs successfully. diff --git a/docs/contributor_protocols/hardened_protocol.md b/docs/contributor_protocols/hardened_protocol.md index 6237b71c..f36ddd20 100644 --- a/docs/contributor_protocols/hardened_protocol.md +++ b/docs/contributor_protocols/hardened_protocol.md @@ -20,7 +20,7 @@ Silent failures, implicit fallbacks, and "best-effort" corrections are forbidden ### C. Partition Integrity **"All models evaluate on the same temporal windows."** -Partition boundaries are defined once in `common/partitions.py` and shared by all models. +Partition boundaries are defined in each model's self-contained `config_partitions.py`. Consistency is enforced by `tests/test_config_partitions.py`. - **Requirement:** Every model's `config_partitions.py` must be a one-line import from `common.partitions`. - **Prohibited:** Model-specific partition overrides, custom forecasting offsets, or hardcoded boundary values in individual model configs. - **Rationale:** Divergent partitions make model evaluation metrics incomparable. @@ -71,7 +71,7 @@ Country-month ensembles must complete before priogrid-month ensembles to enable ## 4. Operational Invariants -- **Partition Centralization:** All models share `common/partitions.py`. No exceptions. +- **Partition Consistency:** All models must use identical partition boundaries. Enforced by `tests/test_config_partitions.py`. - **CLI Uniformity:** All models use `ForecastingModelArgs.parse_args()`. No explicit `wandb.login()`. - **Config Completeness:** All required keys are present. Tests enforce this. - **Catalog Safety:** Config loading uses `importlib.util`, not `exec()`. From f246257ebad7d49e9fcc00f8e93c90720e7d6d64 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 16 Mar 2026 14:25:35 +0100 Subject: [PATCH 20/27] feat(models): commit remaining ranger model files Complete the 6 ranger baseline models (black, blue, green, pink, red, yellow) by committing their config_deployment, config_partitions, config_queryset, config_sweep, main.py, and run.sh files. Only config_meta, config_hyperparameters, and requirements.txt were previously committed. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../black_ranger/configs/config_deployment.py | 20 +++++++++ .../black_ranger/configs/config_partitions.py | 44 +++++++++++++++++++ .../black_ranger/configs/config_queryset.py | 23 ++++++++++ models/black_ranger/configs/config_sweep.py | 31 +++++++++++++ models/black_ranger/main.py | 26 +++++++++++ models/black_ranger/run.sh | 42 ++++++++++++++++++ .../blue_ranger/configs/config_deployment.py | 20 +++++++++ .../blue_ranger/configs/config_partitions.py | 44 +++++++++++++++++++ models/blue_ranger/configs/config_queryset.py | 23 ++++++++++ models/blue_ranger/configs/config_sweep.py | 31 +++++++++++++ models/blue_ranger/main.py | 26 +++++++++++ models/blue_ranger/run.sh | 42 ++++++++++++++++++ .../green_ranger/configs/config_deployment.py | 20 +++++++++ .../green_ranger/configs/config_partitions.py | 44 +++++++++++++++++++ .../green_ranger/configs/config_queryset.py | 23 ++++++++++ models/green_ranger/configs/config_sweep.py | 31 +++++++++++++ models/green_ranger/main.py | 26 +++++++++++ models/green_ranger/run.sh | 42 ++++++++++++++++++ .../pink_ranger/configs/config_deployment.py | 20 +++++++++ .../pink_ranger/configs/config_partitions.py | 44 +++++++++++++++++++ models/pink_ranger/configs/config_queryset.py | 23 ++++++++++ models/pink_ranger/configs/config_sweep.py | 31 +++++++++++++ models/pink_ranger/main.py | 26 +++++++++++ models/pink_ranger/run.sh | 42 ++++++++++++++++++ .../red_ranger/configs/config_deployment.py | 20 +++++++++ .../red_ranger/configs/config_partitions.py | 44 +++++++++++++++++++ models/red_ranger/configs/config_queryset.py | 23 ++++++++++ models/red_ranger/configs/config_sweep.py | 31 +++++++++++++ models/red_ranger/main.py | 26 +++++++++++ models/red_ranger/run.sh | 42 ++++++++++++++++++ .../configs/config_deployment.py | 20 +++++++++ .../configs/config_partitions.py | 44 +++++++++++++++++++ .../yellow_ranger/configs/config_queryset.py | 23 ++++++++++ models/yellow_ranger/configs/config_sweep.py | 31 +++++++++++++ models/yellow_ranger/main.py | 26 +++++++++++ models/yellow_ranger/run.sh | 42 ++++++++++++++++++ 36 files changed, 1116 insertions(+) create mode 100644 models/black_ranger/configs/config_deployment.py create mode 100644 models/black_ranger/configs/config_partitions.py create mode 100644 models/black_ranger/configs/config_queryset.py create mode 100644 models/black_ranger/configs/config_sweep.py create mode 100644 models/black_ranger/main.py create mode 100755 models/black_ranger/run.sh create mode 100644 models/blue_ranger/configs/config_deployment.py create mode 100644 models/blue_ranger/configs/config_partitions.py create mode 100644 models/blue_ranger/configs/config_queryset.py create mode 100644 models/blue_ranger/configs/config_sweep.py create mode 100644 models/blue_ranger/main.py create mode 100755 models/blue_ranger/run.sh create mode 100644 models/green_ranger/configs/config_deployment.py create mode 100644 models/green_ranger/configs/config_partitions.py create mode 100644 models/green_ranger/configs/config_queryset.py create mode 100644 models/green_ranger/configs/config_sweep.py create mode 100644 models/green_ranger/main.py create mode 100755 models/green_ranger/run.sh create mode 100644 models/pink_ranger/configs/config_deployment.py create mode 100644 models/pink_ranger/configs/config_partitions.py create mode 100644 models/pink_ranger/configs/config_queryset.py create mode 100644 models/pink_ranger/configs/config_sweep.py create mode 100644 models/pink_ranger/main.py create mode 100755 models/pink_ranger/run.sh create mode 100644 models/red_ranger/configs/config_deployment.py create mode 100644 models/red_ranger/configs/config_partitions.py create mode 100644 models/red_ranger/configs/config_queryset.py create mode 100644 models/red_ranger/configs/config_sweep.py create mode 100644 models/red_ranger/main.py create mode 100755 models/red_ranger/run.sh create mode 100644 models/yellow_ranger/configs/config_deployment.py create mode 100644 models/yellow_ranger/configs/config_partitions.py create mode 100644 models/yellow_ranger/configs/config_queryset.py create mode 100644 models/yellow_ranger/configs/config_sweep.py create mode 100644 models/yellow_ranger/main.py create mode 100755 models/yellow_ranger/run.sh diff --git a/models/black_ranger/configs/config_deployment.py b/models/black_ranger/configs/config_deployment.py new file mode 100644 index 00000000..9e45b735 --- /dev/null +++ b/models/black_ranger/configs/config_deployment.py @@ -0,0 +1,20 @@ +""" +Deployment Configuration Script + +This script defines the deployment configuration settings for the application. +It includes the deployment status and any additional settings specified. + +Deployment Status: +- shadow: The deployment is shadowed and not yet active. +- deployed: The deployment is active and in use. +- baseline: The deployment is in a baseline state, for reference or comparison. +- deprecated: The deployment is deprecated and no longer supported. + +Additional settings can be included in the configuration dictionary as needed. + +""" + +def get_deployment_config(): + # Deployment settings + deployment_config = {'deployment_status': 'shadow'} + return deployment_config diff --git a/models/black_ranger/configs/config_partitions.py b/models/black_ranger/configs/config_partitions.py new file mode 100644 index 00000000..ead19807 --- /dev/null +++ b/models/black_ranger/configs/config_partitions.py @@ -0,0 +1,44 @@ +from ingester3.ViewsMonth import ViewsMonth + + +def generate(steps: int = 36) -> dict: + """ + Generates partition configurations for different phases of model evaluation. + + Returns: + dict: A dictionary with keys 'calibration', 'validation', and 'forecasting', each containing + 'train' and 'test' tuples or callables specifying the index ranges for training and testing data. + + Partition details: + - 'calibration': Uses fixed index ranges for training and testing. + - 'validation': Uses fixed index ranges for training and testing. + - 'forecasting': Uses callables that accept ViewsMonth (and optionally step) to dynamically determine + training and testing index ranges based on the current month. + + Note: + - The 'forecasting' partition's 'train' and 'test' values are functions that require the ViewsMonth + object (and step for 'test') to compute the appropriate indices. + """ + + def forecasting_train_range(): + month_last = ViewsMonth.now().id - 1 + return (121, month_last) + + def forecasting_test_range(steps): + month_last = ViewsMonth.now().id - 1 + return (month_last + 1, month_last + 1 + steps) + + return { + "calibration": { + "train": (121, 444), + "test": (445, 492), + }, + "validation": { + "train": (121, 492), + "test": (493, 540), + }, + "forecasting": { + "train": forecasting_train_range(), + "test": forecasting_test_range(steps=steps), + }, + } diff --git a/models/black_ranger/configs/config_queryset.py b/models/black_ranger/configs/config_queryset.py new file mode 100644 index 00000000..19803229 --- /dev/null +++ b/models/black_ranger/configs/config_queryset.py @@ -0,0 +1,23 @@ +from viewser import Queryset, Column + +def generate(): + """ + Contains the configuration for the input data in the form of a viewser queryset. That is the data from viewser that is used to train the model. + This configuration is "behavioral" so modifying it will affect the model's runtime behavior and integration into the deployment system. + There is no guarantee that the model will work if the input data configuration is changed here without changing the model settings and algorithm accordingly. + + Returns: + - queryset_base (Queryset): A queryset containing the base data for the model training. + """ + + queryset_base = (Queryset("black_ranger", "priogrid_month") + + .with_column(Column("lr_os_best", from_loa="priogrid_month", from_column="ged_os_best_sum_nokgi") + .transform.missing.replace_na()) + + .with_column(Column("month", from_loa="month", from_column="month")) + .with_column(Column("year_id", from_loa="country_year", from_column="year_id")) + + ) + + return queryset_base diff --git a/models/black_ranger/configs/config_sweep.py b/models/black_ranger/configs/config_sweep.py new file mode 100644 index 00000000..7a8abcd1 --- /dev/null +++ b/models/black_ranger/configs/config_sweep.py @@ -0,0 +1,31 @@ + +def get_sweep_config(): + """ + Contains the configuration for hyperparameter sweeps using WandB. + This configuration is "operational" so modifying it will change the search strategy, parameter ranges, and other settings for hyperparameter tuning aimed at optimizing model performance. + + Returns: + - sweep_config (dict): A dictionary containing the configuration for hyperparameter sweeps, defining the methods and parameter ranges used to search for optimal hyperparameters. + """ + + sweep_config = { + 'method': 'grid', + 'name': 'black_ranger' + } + + metric = { + 'name': 'MSE', + 'goal': 'minimize' + } + sweep_config['metric'] = metric + + parameters_dict = { + 'steps': {'value': [*range(1, 36 + 1, 1)]}, + 'time_steps': {'value': 36}, + 'n_samples': {'value': 256}, + 'lambda_mix': {'values': [0.0, 0.01, 0.025, 0.05, 0.1, 0.15, 0.2, 0.3, 0.5]}, + 'window_months': {'values': [6, 12, 18, 24, 36, 48, 60]}, + } + sweep_config['parameters'] = parameters_dict + + return sweep_config diff --git a/models/black_ranger/main.py b/models/black_ranger/main.py new file mode 100644 index 00000000..8ef12185 --- /dev/null +++ b/models/black_ranger/main.py @@ -0,0 +1,26 @@ +import warnings +from pathlib import Path +from views_pipeline_core.cli import ForecastingModelArgs +from views_pipeline_core.managers import ModelPathManager +from views_baseline.manager.baseline_manager import BaselineForecastingModelManager + +warnings.filterwarnings("ignore") + +try: + model_path = ModelPathManager(Path(__file__)) +except Exception as e: + raise RuntimeError(f"Unexpected error: {e}. Check the logs for details.") + +if __name__ == "__main__": + args = ForecastingModelArgs.parse_args() + + manager = BaselineForecastingModelManager( + model_path=model_path, + wandb_notifications=args.wandb_notifications, + use_prediction_store=args.prediction_store, + ) + + if args.sweep: + manager.execute_sweep_run(args) + else: + manager.execute_single_run(args) diff --git a/models/black_ranger/run.sh b/models/black_ranger/run.sh new file mode 100755 index 00000000..09ae7ef4 --- /dev/null +++ b/models/black_ranger/run.sh @@ -0,0 +1,42 @@ +#!/bin/zsh + +if [[ "$OSTYPE" == "darwin"* ]]; then + if ! grep -q 'export LDFLAGS="-L/opt/homebrew/opt/libomp/lib"' ~/.zshrc; then + echo 'export LDFLAGS="-L/opt/homebrew/opt/libomp/lib"' >> ~/.zshrc + fi + if ! grep -q 'export CPPFLAGS="-I/opt/homebrew/opt/libomp/include"' ~/.zshrc; then + echo 'export CPPFLAGS="-I/opt/homebrew/opt/libomp/include"' >> ~/.zshrc + fi + if ! grep -q 'export DYLD_LIBRARY_PATH="/opt/homebrew/opt/libomp/lib:$DYLD_LIBRARY_PATH"' ~/.zshrc; then + echo 'export DYLD_LIBRARY_PATH="/opt/homebrew/opt/libomp/lib:$DYLD_LIBRARY_PATH"' >> ~/.zshrc + fi + source ~/.zshrc +fi + +script_path=$(dirname "$(realpath $0)") +project_path="$( cd "$script_path/../../" >/dev/null 2>&1 && pwd )" +env_path="$project_path/envs/views-baseline" + +eval "$(conda shell.bash hook)" + +if [ -d "$env_path" ]; then + echo "Conda environment already exists at $env_path. Checking dependencies..." + conda activate "$env_path" + echo "$env_path is activated" + + missing_packages=$(pip install --dry-run -r $script_path/requirements.txt 2>&1 | grep -v "Requirement already satisfied" | wc -l) + if [ "$missing_packages" -gt 0 ]; then + echo "Installing missing or outdated packages..." + pip install -r $script_path/requirements.txt + else + echo "All packages are up-to-date." + fi +else + echo "Creating new Conda environment at $env_path..." + conda create --prefix "$env_path" python=3.11 -y + conda activate "$env_path" + pip install -r $script_path/requirements.txt +fi + +echo "Running $script_path/main.py " +python $script_path/main.py "$@" diff --git a/models/blue_ranger/configs/config_deployment.py b/models/blue_ranger/configs/config_deployment.py new file mode 100644 index 00000000..9e45b735 --- /dev/null +++ b/models/blue_ranger/configs/config_deployment.py @@ -0,0 +1,20 @@ +""" +Deployment Configuration Script + +This script defines the deployment configuration settings for the application. +It includes the deployment status and any additional settings specified. + +Deployment Status: +- shadow: The deployment is shadowed and not yet active. +- deployed: The deployment is active and in use. +- baseline: The deployment is in a baseline state, for reference or comparison. +- deprecated: The deployment is deprecated and no longer supported. + +Additional settings can be included in the configuration dictionary as needed. + +""" + +def get_deployment_config(): + # Deployment settings + deployment_config = {'deployment_status': 'shadow'} + return deployment_config diff --git a/models/blue_ranger/configs/config_partitions.py b/models/blue_ranger/configs/config_partitions.py new file mode 100644 index 00000000..ead19807 --- /dev/null +++ b/models/blue_ranger/configs/config_partitions.py @@ -0,0 +1,44 @@ +from ingester3.ViewsMonth import ViewsMonth + + +def generate(steps: int = 36) -> dict: + """ + Generates partition configurations for different phases of model evaluation. + + Returns: + dict: A dictionary with keys 'calibration', 'validation', and 'forecasting', each containing + 'train' and 'test' tuples or callables specifying the index ranges for training and testing data. + + Partition details: + - 'calibration': Uses fixed index ranges for training and testing. + - 'validation': Uses fixed index ranges for training and testing. + - 'forecasting': Uses callables that accept ViewsMonth (and optionally step) to dynamically determine + training and testing index ranges based on the current month. + + Note: + - The 'forecasting' partition's 'train' and 'test' values are functions that require the ViewsMonth + object (and step for 'test') to compute the appropriate indices. + """ + + def forecasting_train_range(): + month_last = ViewsMonth.now().id - 1 + return (121, month_last) + + def forecasting_test_range(steps): + month_last = ViewsMonth.now().id - 1 + return (month_last + 1, month_last + 1 + steps) + + return { + "calibration": { + "train": (121, 444), + "test": (445, 492), + }, + "validation": { + "train": (121, 492), + "test": (493, 540), + }, + "forecasting": { + "train": forecasting_train_range(), + "test": forecasting_test_range(steps=steps), + }, + } diff --git a/models/blue_ranger/configs/config_queryset.py b/models/blue_ranger/configs/config_queryset.py new file mode 100644 index 00000000..362f07a4 --- /dev/null +++ b/models/blue_ranger/configs/config_queryset.py @@ -0,0 +1,23 @@ +from viewser import Queryset, Column + +def generate(): + """ + Contains the configuration for the input data in the form of a viewser queryset. That is the data from viewser that is used to train the model. + This configuration is "behavioral" so modifying it will affect the model's runtime behavior and integration into the deployment system. + There is no guarantee that the model will work if the input data configuration is changed here without changing the model settings and algorithm accordingly. + + Returns: + - queryset_base (Queryset): A queryset containing the base data for the model training. + """ + + queryset_base = (Queryset("blue_ranger", "priogrid_month") + + .with_column(Column("lr_ged_sb", from_loa="priogrid_month", from_column="ged_sb_best_sum_nokgi") + .transform.missing.replace_na()) + + .with_column(Column("month", from_loa="month", from_column="month")) + .with_column(Column("year_id", from_loa="country_year", from_column="year_id")) + + ) + + return queryset_base diff --git a/models/blue_ranger/configs/config_sweep.py b/models/blue_ranger/configs/config_sweep.py new file mode 100644 index 00000000..80869d52 --- /dev/null +++ b/models/blue_ranger/configs/config_sweep.py @@ -0,0 +1,31 @@ + +def get_sweep_config(): + """ + Contains the configuration for hyperparameter sweeps using WandB. + This configuration is "operational" so modifying it will change the search strategy, parameter ranges, and other settings for hyperparameter tuning aimed at optimizing model performance. + + Returns: + - sweep_config (dict): A dictionary containing the configuration for hyperparameter sweeps, defining the methods and parameter ranges used to search for optimal hyperparameters. + """ + + sweep_config = { + 'method': 'grid', + 'name': 'blue_ranger' + } + + metric = { + 'name': 'MSE', + 'goal': 'minimize' + } + sweep_config['metric'] = metric + + parameters_dict = { + 'steps': {'value': [*range(1, 36 + 1, 1)]}, + 'time_steps': {'value': 36}, + 'n_samples': {'value': 256}, + 'lambda_mix': {'values': [0.0, 0.01, 0.025, 0.05, 0.1, 0.15, 0.2, 0.3, 0.5]}, + 'window_months': {'values': [6, 12, 18, 24, 36, 48, 60]}, + } + sweep_config['parameters'] = parameters_dict + + return sweep_config diff --git a/models/blue_ranger/main.py b/models/blue_ranger/main.py new file mode 100644 index 00000000..8ef12185 --- /dev/null +++ b/models/blue_ranger/main.py @@ -0,0 +1,26 @@ +import warnings +from pathlib import Path +from views_pipeline_core.cli import ForecastingModelArgs +from views_pipeline_core.managers import ModelPathManager +from views_baseline.manager.baseline_manager import BaselineForecastingModelManager + +warnings.filterwarnings("ignore") + +try: + model_path = ModelPathManager(Path(__file__)) +except Exception as e: + raise RuntimeError(f"Unexpected error: {e}. Check the logs for details.") + +if __name__ == "__main__": + args = ForecastingModelArgs.parse_args() + + manager = BaselineForecastingModelManager( + model_path=model_path, + wandb_notifications=args.wandb_notifications, + use_prediction_store=args.prediction_store, + ) + + if args.sweep: + manager.execute_sweep_run(args) + else: + manager.execute_single_run(args) diff --git a/models/blue_ranger/run.sh b/models/blue_ranger/run.sh new file mode 100755 index 00000000..09ae7ef4 --- /dev/null +++ b/models/blue_ranger/run.sh @@ -0,0 +1,42 @@ +#!/bin/zsh + +if [[ "$OSTYPE" == "darwin"* ]]; then + if ! grep -q 'export LDFLAGS="-L/opt/homebrew/opt/libomp/lib"' ~/.zshrc; then + echo 'export LDFLAGS="-L/opt/homebrew/opt/libomp/lib"' >> ~/.zshrc + fi + if ! grep -q 'export CPPFLAGS="-I/opt/homebrew/opt/libomp/include"' ~/.zshrc; then + echo 'export CPPFLAGS="-I/opt/homebrew/opt/libomp/include"' >> ~/.zshrc + fi + if ! grep -q 'export DYLD_LIBRARY_PATH="/opt/homebrew/opt/libomp/lib:$DYLD_LIBRARY_PATH"' ~/.zshrc; then + echo 'export DYLD_LIBRARY_PATH="/opt/homebrew/opt/libomp/lib:$DYLD_LIBRARY_PATH"' >> ~/.zshrc + fi + source ~/.zshrc +fi + +script_path=$(dirname "$(realpath $0)") +project_path="$( cd "$script_path/../../" >/dev/null 2>&1 && pwd )" +env_path="$project_path/envs/views-baseline" + +eval "$(conda shell.bash hook)" + +if [ -d "$env_path" ]; then + echo "Conda environment already exists at $env_path. Checking dependencies..." + conda activate "$env_path" + echo "$env_path is activated" + + missing_packages=$(pip install --dry-run -r $script_path/requirements.txt 2>&1 | grep -v "Requirement already satisfied" | wc -l) + if [ "$missing_packages" -gt 0 ]; then + echo "Installing missing or outdated packages..." + pip install -r $script_path/requirements.txt + else + echo "All packages are up-to-date." + fi +else + echo "Creating new Conda environment at $env_path..." + conda create --prefix "$env_path" python=3.11 -y + conda activate "$env_path" + pip install -r $script_path/requirements.txt +fi + +echo "Running $script_path/main.py " +python $script_path/main.py "$@" diff --git a/models/green_ranger/configs/config_deployment.py b/models/green_ranger/configs/config_deployment.py new file mode 100644 index 00000000..9e45b735 --- /dev/null +++ b/models/green_ranger/configs/config_deployment.py @@ -0,0 +1,20 @@ +""" +Deployment Configuration Script + +This script defines the deployment configuration settings for the application. +It includes the deployment status and any additional settings specified. + +Deployment Status: +- shadow: The deployment is shadowed and not yet active. +- deployed: The deployment is active and in use. +- baseline: The deployment is in a baseline state, for reference or comparison. +- deprecated: The deployment is deprecated and no longer supported. + +Additional settings can be included in the configuration dictionary as needed. + +""" + +def get_deployment_config(): + # Deployment settings + deployment_config = {'deployment_status': 'shadow'} + return deployment_config diff --git a/models/green_ranger/configs/config_partitions.py b/models/green_ranger/configs/config_partitions.py new file mode 100644 index 00000000..ead19807 --- /dev/null +++ b/models/green_ranger/configs/config_partitions.py @@ -0,0 +1,44 @@ +from ingester3.ViewsMonth import ViewsMonth + + +def generate(steps: int = 36) -> dict: + """ + Generates partition configurations for different phases of model evaluation. + + Returns: + dict: A dictionary with keys 'calibration', 'validation', and 'forecasting', each containing + 'train' and 'test' tuples or callables specifying the index ranges for training and testing data. + + Partition details: + - 'calibration': Uses fixed index ranges for training and testing. + - 'validation': Uses fixed index ranges for training and testing. + - 'forecasting': Uses callables that accept ViewsMonth (and optionally step) to dynamically determine + training and testing index ranges based on the current month. + + Note: + - The 'forecasting' partition's 'train' and 'test' values are functions that require the ViewsMonth + object (and step for 'test') to compute the appropriate indices. + """ + + def forecasting_train_range(): + month_last = ViewsMonth.now().id - 1 + return (121, month_last) + + def forecasting_test_range(steps): + month_last = ViewsMonth.now().id - 1 + return (month_last + 1, month_last + 1 + steps) + + return { + "calibration": { + "train": (121, 444), + "test": (445, 492), + }, + "validation": { + "train": (121, 492), + "test": (493, 540), + }, + "forecasting": { + "train": forecasting_train_range(), + "test": forecasting_test_range(steps=steps), + }, + } diff --git a/models/green_ranger/configs/config_queryset.py b/models/green_ranger/configs/config_queryset.py new file mode 100644 index 00000000..e19b36f8 --- /dev/null +++ b/models/green_ranger/configs/config_queryset.py @@ -0,0 +1,23 @@ +from viewser import Queryset, Column + +def generate(): + """ + Contains the configuration for the input data in the form of a viewser queryset. That is the data from viewser that is used to train the model. + This configuration is "behavioral" so modifying it will affect the model's runtime behavior and integration into the deployment system. + There is no guarantee that the model will work if the input data configuration is changed here without changing the model settings and algorithm accordingly. + + Returns: + - queryset_base (Queryset): A queryset containing the base data for the model training. + """ + + queryset_base = (Queryset("green_ranger", "country_month") + + .with_column(Column("lr_ns_best", from_loa="country_month", from_column="ged_ns_best_sum_nokgi") + .transform.missing.replace_na()) + + .with_column(Column("month", from_loa="month", from_column="month")) + .with_column(Column("year_id", from_loa="country_year", from_column="year_id")) + + ) + + return queryset_base diff --git a/models/green_ranger/configs/config_sweep.py b/models/green_ranger/configs/config_sweep.py new file mode 100644 index 00000000..9fd589c1 --- /dev/null +++ b/models/green_ranger/configs/config_sweep.py @@ -0,0 +1,31 @@ + +def get_sweep_config(): + """ + Contains the configuration for hyperparameter sweeps using WandB. + This configuration is "operational" so modifying it will change the search strategy, parameter ranges, and other settings for hyperparameter tuning aimed at optimizing model performance. + + Returns: + - sweep_config (dict): A dictionary containing the configuration for hyperparameter sweeps, defining the methods and parameter ranges used to search for optimal hyperparameters. + """ + + sweep_config = { + 'method': 'grid', + 'name': 'green_ranger' + } + + metric = { + 'name': 'MSE', + 'goal': 'minimize' + } + sweep_config['metric'] = metric + + parameters_dict = { + 'steps': {'value': [*range(1, 36 + 1, 1)]}, + 'time_steps': {'value': 36}, + 'n_samples': {'value': 256}, + 'lambda_mix': {'values': [0.0, 0.01, 0.025, 0.05, 0.1, 0.15, 0.2, 0.3, 0.5]}, + 'window_months': {'values': [6, 12, 18, 24, 36, 48, 60]}, + } + sweep_config['parameters'] = parameters_dict + + return sweep_config diff --git a/models/green_ranger/main.py b/models/green_ranger/main.py new file mode 100644 index 00000000..8ef12185 --- /dev/null +++ b/models/green_ranger/main.py @@ -0,0 +1,26 @@ +import warnings +from pathlib import Path +from views_pipeline_core.cli import ForecastingModelArgs +from views_pipeline_core.managers import ModelPathManager +from views_baseline.manager.baseline_manager import BaselineForecastingModelManager + +warnings.filterwarnings("ignore") + +try: + model_path = ModelPathManager(Path(__file__)) +except Exception as e: + raise RuntimeError(f"Unexpected error: {e}. Check the logs for details.") + +if __name__ == "__main__": + args = ForecastingModelArgs.parse_args() + + manager = BaselineForecastingModelManager( + model_path=model_path, + wandb_notifications=args.wandb_notifications, + use_prediction_store=args.prediction_store, + ) + + if args.sweep: + manager.execute_sweep_run(args) + else: + manager.execute_single_run(args) diff --git a/models/green_ranger/run.sh b/models/green_ranger/run.sh new file mode 100755 index 00000000..09ae7ef4 --- /dev/null +++ b/models/green_ranger/run.sh @@ -0,0 +1,42 @@ +#!/bin/zsh + +if [[ "$OSTYPE" == "darwin"* ]]; then + if ! grep -q 'export LDFLAGS="-L/opt/homebrew/opt/libomp/lib"' ~/.zshrc; then + echo 'export LDFLAGS="-L/opt/homebrew/opt/libomp/lib"' >> ~/.zshrc + fi + if ! grep -q 'export CPPFLAGS="-I/opt/homebrew/opt/libomp/include"' ~/.zshrc; then + echo 'export CPPFLAGS="-I/opt/homebrew/opt/libomp/include"' >> ~/.zshrc + fi + if ! grep -q 'export DYLD_LIBRARY_PATH="/opt/homebrew/opt/libomp/lib:$DYLD_LIBRARY_PATH"' ~/.zshrc; then + echo 'export DYLD_LIBRARY_PATH="/opt/homebrew/opt/libomp/lib:$DYLD_LIBRARY_PATH"' >> ~/.zshrc + fi + source ~/.zshrc +fi + +script_path=$(dirname "$(realpath $0)") +project_path="$( cd "$script_path/../../" >/dev/null 2>&1 && pwd )" +env_path="$project_path/envs/views-baseline" + +eval "$(conda shell.bash hook)" + +if [ -d "$env_path" ]; then + echo "Conda environment already exists at $env_path. Checking dependencies..." + conda activate "$env_path" + echo "$env_path is activated" + + missing_packages=$(pip install --dry-run -r $script_path/requirements.txt 2>&1 | grep -v "Requirement already satisfied" | wc -l) + if [ "$missing_packages" -gt 0 ]; then + echo "Installing missing or outdated packages..." + pip install -r $script_path/requirements.txt + else + echo "All packages are up-to-date." + fi +else + echo "Creating new Conda environment at $env_path..." + conda create --prefix "$env_path" python=3.11 -y + conda activate "$env_path" + pip install -r $script_path/requirements.txt +fi + +echo "Running $script_path/main.py " +python $script_path/main.py "$@" diff --git a/models/pink_ranger/configs/config_deployment.py b/models/pink_ranger/configs/config_deployment.py new file mode 100644 index 00000000..9e45b735 --- /dev/null +++ b/models/pink_ranger/configs/config_deployment.py @@ -0,0 +1,20 @@ +""" +Deployment Configuration Script + +This script defines the deployment configuration settings for the application. +It includes the deployment status and any additional settings specified. + +Deployment Status: +- shadow: The deployment is shadowed and not yet active. +- deployed: The deployment is active and in use. +- baseline: The deployment is in a baseline state, for reference or comparison. +- deprecated: The deployment is deprecated and no longer supported. + +Additional settings can be included in the configuration dictionary as needed. + +""" + +def get_deployment_config(): + # Deployment settings + deployment_config = {'deployment_status': 'shadow'} + return deployment_config diff --git a/models/pink_ranger/configs/config_partitions.py b/models/pink_ranger/configs/config_partitions.py new file mode 100644 index 00000000..ead19807 --- /dev/null +++ b/models/pink_ranger/configs/config_partitions.py @@ -0,0 +1,44 @@ +from ingester3.ViewsMonth import ViewsMonth + + +def generate(steps: int = 36) -> dict: + """ + Generates partition configurations for different phases of model evaluation. + + Returns: + dict: A dictionary with keys 'calibration', 'validation', and 'forecasting', each containing + 'train' and 'test' tuples or callables specifying the index ranges for training and testing data. + + Partition details: + - 'calibration': Uses fixed index ranges for training and testing. + - 'validation': Uses fixed index ranges for training and testing. + - 'forecasting': Uses callables that accept ViewsMonth (and optionally step) to dynamically determine + training and testing index ranges based on the current month. + + Note: + - The 'forecasting' partition's 'train' and 'test' values are functions that require the ViewsMonth + object (and step for 'test') to compute the appropriate indices. + """ + + def forecasting_train_range(): + month_last = ViewsMonth.now().id - 1 + return (121, month_last) + + def forecasting_test_range(steps): + month_last = ViewsMonth.now().id - 1 + return (month_last + 1, month_last + 1 + steps) + + return { + "calibration": { + "train": (121, 444), + "test": (445, 492), + }, + "validation": { + "train": (121, 492), + "test": (493, 540), + }, + "forecasting": { + "train": forecasting_train_range(), + "test": forecasting_test_range(steps=steps), + }, + } diff --git a/models/pink_ranger/configs/config_queryset.py b/models/pink_ranger/configs/config_queryset.py new file mode 100644 index 00000000..ce4c23cf --- /dev/null +++ b/models/pink_ranger/configs/config_queryset.py @@ -0,0 +1,23 @@ +from viewser import Queryset, Column + +def generate(): + """ + Contains the configuration for the input data in the form of a viewser queryset. That is the data from viewser that is used to train the model. + This configuration is "behavioral" so modifying it will affect the model's runtime behavior and integration into the deployment system. + There is no guarantee that the model will work if the input data configuration is changed here without changing the model settings and algorithm accordingly. + + Returns: + - queryset_base (Queryset): A queryset containing the base data for the model training. + """ + + queryset_base = (Queryset("pink_ranger", "priogrid_month") + + .with_column(Column("lr_ns_best", from_loa="priogrid_month", from_column="ged_ns_best_sum_nokgi") + .transform.missing.replace_na()) + + .with_column(Column("month", from_loa="month", from_column="month")) + .with_column(Column("year_id", from_loa="country_year", from_column="year_id")) + + ) + + return queryset_base diff --git a/models/pink_ranger/configs/config_sweep.py b/models/pink_ranger/configs/config_sweep.py new file mode 100644 index 00000000..683abb07 --- /dev/null +++ b/models/pink_ranger/configs/config_sweep.py @@ -0,0 +1,31 @@ + +def get_sweep_config(): + """ + Contains the configuration for hyperparameter sweeps using WandB. + This configuration is "operational" so modifying it will change the search strategy, parameter ranges, and other settings for hyperparameter tuning aimed at optimizing model performance. + + Returns: + - sweep_config (dict): A dictionary containing the configuration for hyperparameter sweeps, defining the methods and parameter ranges used to search for optimal hyperparameters. + """ + + sweep_config = { + 'method': 'grid', + 'name': 'pink_ranger' + } + + metric = { + 'name': 'MSE', + 'goal': 'minimize' + } + sweep_config['metric'] = metric + + parameters_dict = { + 'steps': {'value': [*range(1, 36 + 1, 1)]}, + 'time_steps': {'value': 36}, + 'n_samples': {'value': 256}, + 'lambda_mix': {'values': [0.0, 0.01, 0.025, 0.05, 0.1, 0.15, 0.2, 0.3, 0.5]}, + 'window_months': {'values': [6, 12, 18, 24, 36, 48, 60]}, + } + sweep_config['parameters'] = parameters_dict + + return sweep_config diff --git a/models/pink_ranger/main.py b/models/pink_ranger/main.py new file mode 100644 index 00000000..8ef12185 --- /dev/null +++ b/models/pink_ranger/main.py @@ -0,0 +1,26 @@ +import warnings +from pathlib import Path +from views_pipeline_core.cli import ForecastingModelArgs +from views_pipeline_core.managers import ModelPathManager +from views_baseline.manager.baseline_manager import BaselineForecastingModelManager + +warnings.filterwarnings("ignore") + +try: + model_path = ModelPathManager(Path(__file__)) +except Exception as e: + raise RuntimeError(f"Unexpected error: {e}. Check the logs for details.") + +if __name__ == "__main__": + args = ForecastingModelArgs.parse_args() + + manager = BaselineForecastingModelManager( + model_path=model_path, + wandb_notifications=args.wandb_notifications, + use_prediction_store=args.prediction_store, + ) + + if args.sweep: + manager.execute_sweep_run(args) + else: + manager.execute_single_run(args) diff --git a/models/pink_ranger/run.sh b/models/pink_ranger/run.sh new file mode 100755 index 00000000..09ae7ef4 --- /dev/null +++ b/models/pink_ranger/run.sh @@ -0,0 +1,42 @@ +#!/bin/zsh + +if [[ "$OSTYPE" == "darwin"* ]]; then + if ! grep -q 'export LDFLAGS="-L/opt/homebrew/opt/libomp/lib"' ~/.zshrc; then + echo 'export LDFLAGS="-L/opt/homebrew/opt/libomp/lib"' >> ~/.zshrc + fi + if ! grep -q 'export CPPFLAGS="-I/opt/homebrew/opt/libomp/include"' ~/.zshrc; then + echo 'export CPPFLAGS="-I/opt/homebrew/opt/libomp/include"' >> ~/.zshrc + fi + if ! grep -q 'export DYLD_LIBRARY_PATH="/opt/homebrew/opt/libomp/lib:$DYLD_LIBRARY_PATH"' ~/.zshrc; then + echo 'export DYLD_LIBRARY_PATH="/opt/homebrew/opt/libomp/lib:$DYLD_LIBRARY_PATH"' >> ~/.zshrc + fi + source ~/.zshrc +fi + +script_path=$(dirname "$(realpath $0)") +project_path="$( cd "$script_path/../../" >/dev/null 2>&1 && pwd )" +env_path="$project_path/envs/views-baseline" + +eval "$(conda shell.bash hook)" + +if [ -d "$env_path" ]; then + echo "Conda environment already exists at $env_path. Checking dependencies..." + conda activate "$env_path" + echo "$env_path is activated" + + missing_packages=$(pip install --dry-run -r $script_path/requirements.txt 2>&1 | grep -v "Requirement already satisfied" | wc -l) + if [ "$missing_packages" -gt 0 ]; then + echo "Installing missing or outdated packages..." + pip install -r $script_path/requirements.txt + else + echo "All packages are up-to-date." + fi +else + echo "Creating new Conda environment at $env_path..." + conda create --prefix "$env_path" python=3.11 -y + conda activate "$env_path" + pip install -r $script_path/requirements.txt +fi + +echo "Running $script_path/main.py " +python $script_path/main.py "$@" diff --git a/models/red_ranger/configs/config_deployment.py b/models/red_ranger/configs/config_deployment.py new file mode 100644 index 00000000..9e45b735 --- /dev/null +++ b/models/red_ranger/configs/config_deployment.py @@ -0,0 +1,20 @@ +""" +Deployment Configuration Script + +This script defines the deployment configuration settings for the application. +It includes the deployment status and any additional settings specified. + +Deployment Status: +- shadow: The deployment is shadowed and not yet active. +- deployed: The deployment is active and in use. +- baseline: The deployment is in a baseline state, for reference or comparison. +- deprecated: The deployment is deprecated and no longer supported. + +Additional settings can be included in the configuration dictionary as needed. + +""" + +def get_deployment_config(): + # Deployment settings + deployment_config = {'deployment_status': 'shadow'} + return deployment_config diff --git a/models/red_ranger/configs/config_partitions.py b/models/red_ranger/configs/config_partitions.py new file mode 100644 index 00000000..ead19807 --- /dev/null +++ b/models/red_ranger/configs/config_partitions.py @@ -0,0 +1,44 @@ +from ingester3.ViewsMonth import ViewsMonth + + +def generate(steps: int = 36) -> dict: + """ + Generates partition configurations for different phases of model evaluation. + + Returns: + dict: A dictionary with keys 'calibration', 'validation', and 'forecasting', each containing + 'train' and 'test' tuples or callables specifying the index ranges for training and testing data. + + Partition details: + - 'calibration': Uses fixed index ranges for training and testing. + - 'validation': Uses fixed index ranges for training and testing. + - 'forecasting': Uses callables that accept ViewsMonth (and optionally step) to dynamically determine + training and testing index ranges based on the current month. + + Note: + - The 'forecasting' partition's 'train' and 'test' values are functions that require the ViewsMonth + object (and step for 'test') to compute the appropriate indices. + """ + + def forecasting_train_range(): + month_last = ViewsMonth.now().id - 1 + return (121, month_last) + + def forecasting_test_range(steps): + month_last = ViewsMonth.now().id - 1 + return (month_last + 1, month_last + 1 + steps) + + return { + "calibration": { + "train": (121, 444), + "test": (445, 492), + }, + "validation": { + "train": (121, 492), + "test": (493, 540), + }, + "forecasting": { + "train": forecasting_train_range(), + "test": forecasting_test_range(steps=steps), + }, + } diff --git a/models/red_ranger/configs/config_queryset.py b/models/red_ranger/configs/config_queryset.py new file mode 100644 index 00000000..75d33167 --- /dev/null +++ b/models/red_ranger/configs/config_queryset.py @@ -0,0 +1,23 @@ +from viewser import Queryset, Column + +def generate(): + """ + Contains the configuration for the input data in the form of a viewser queryset. That is the data from viewser that is used to train the model. + This configuration is "behavioral" so modifying it will affect the model's runtime behavior and integration into the deployment system. + There is no guarantee that the model will work if the input data configuration is changed here without changing the model settings and algorithm accordingly. + + Returns: + - queryset_base (Queryset): A queryset containing the base data for the model training. + """ + + queryset_base = (Queryset("red_ranger", "country_month") + + .with_column(Column("lr_ged_sb", from_loa="country_month", from_column="ged_sb_best_sum_nokgi") + .transform.missing.replace_na()) + + .with_column(Column("month", from_loa="month", from_column="month")) + .with_column(Column("year_id", from_loa="country_year", from_column="year_id")) + + ) + + return queryset_base diff --git a/models/red_ranger/configs/config_sweep.py b/models/red_ranger/configs/config_sweep.py new file mode 100644 index 00000000..f71f3659 --- /dev/null +++ b/models/red_ranger/configs/config_sweep.py @@ -0,0 +1,31 @@ + +def get_sweep_config(): + """ + Contains the configuration for hyperparameter sweeps using WandB. + This configuration is "operational" so modifying it will change the search strategy, parameter ranges, and other settings for hyperparameter tuning aimed at optimizing model performance. + + Returns: + - sweep_config (dict): A dictionary containing the configuration for hyperparameter sweeps, defining the methods and parameter ranges used to search for optimal hyperparameters. + """ + + sweep_config = { + 'method': 'grid', + 'name': 'red_ranger' + } + + metric = { + 'name': 'MSE', + 'goal': 'minimize' + } + sweep_config['metric'] = metric + + parameters_dict = { + 'steps': {'value': [*range(1, 36 + 1, 1)]}, + 'time_steps': {'value': 36}, + 'n_samples': {'value': 256}, + 'lambda_mix': {'values': [0.0, 0.01, 0.025, 0.05, 0.1, 0.15, 0.2, 0.3, 0.5]}, + 'window_months': {'values': [6, 12, 18, 24, 36, 48, 60]}, + } + sweep_config['parameters'] = parameters_dict + + return sweep_config diff --git a/models/red_ranger/main.py b/models/red_ranger/main.py new file mode 100644 index 00000000..8ef12185 --- /dev/null +++ b/models/red_ranger/main.py @@ -0,0 +1,26 @@ +import warnings +from pathlib import Path +from views_pipeline_core.cli import ForecastingModelArgs +from views_pipeline_core.managers import ModelPathManager +from views_baseline.manager.baseline_manager import BaselineForecastingModelManager + +warnings.filterwarnings("ignore") + +try: + model_path = ModelPathManager(Path(__file__)) +except Exception as e: + raise RuntimeError(f"Unexpected error: {e}. Check the logs for details.") + +if __name__ == "__main__": + args = ForecastingModelArgs.parse_args() + + manager = BaselineForecastingModelManager( + model_path=model_path, + wandb_notifications=args.wandb_notifications, + use_prediction_store=args.prediction_store, + ) + + if args.sweep: + manager.execute_sweep_run(args) + else: + manager.execute_single_run(args) diff --git a/models/red_ranger/run.sh b/models/red_ranger/run.sh new file mode 100755 index 00000000..09ae7ef4 --- /dev/null +++ b/models/red_ranger/run.sh @@ -0,0 +1,42 @@ +#!/bin/zsh + +if [[ "$OSTYPE" == "darwin"* ]]; then + if ! grep -q 'export LDFLAGS="-L/opt/homebrew/opt/libomp/lib"' ~/.zshrc; then + echo 'export LDFLAGS="-L/opt/homebrew/opt/libomp/lib"' >> ~/.zshrc + fi + if ! grep -q 'export CPPFLAGS="-I/opt/homebrew/opt/libomp/include"' ~/.zshrc; then + echo 'export CPPFLAGS="-I/opt/homebrew/opt/libomp/include"' >> ~/.zshrc + fi + if ! grep -q 'export DYLD_LIBRARY_PATH="/opt/homebrew/opt/libomp/lib:$DYLD_LIBRARY_PATH"' ~/.zshrc; then + echo 'export DYLD_LIBRARY_PATH="/opt/homebrew/opt/libomp/lib:$DYLD_LIBRARY_PATH"' >> ~/.zshrc + fi + source ~/.zshrc +fi + +script_path=$(dirname "$(realpath $0)") +project_path="$( cd "$script_path/../../" >/dev/null 2>&1 && pwd )" +env_path="$project_path/envs/views-baseline" + +eval "$(conda shell.bash hook)" + +if [ -d "$env_path" ]; then + echo "Conda environment already exists at $env_path. Checking dependencies..." + conda activate "$env_path" + echo "$env_path is activated" + + missing_packages=$(pip install --dry-run -r $script_path/requirements.txt 2>&1 | grep -v "Requirement already satisfied" | wc -l) + if [ "$missing_packages" -gt 0 ]; then + echo "Installing missing or outdated packages..." + pip install -r $script_path/requirements.txt + else + echo "All packages are up-to-date." + fi +else + echo "Creating new Conda environment at $env_path..." + conda create --prefix "$env_path" python=3.11 -y + conda activate "$env_path" + pip install -r $script_path/requirements.txt +fi + +echo "Running $script_path/main.py " +python $script_path/main.py "$@" diff --git a/models/yellow_ranger/configs/config_deployment.py b/models/yellow_ranger/configs/config_deployment.py new file mode 100644 index 00000000..9e45b735 --- /dev/null +++ b/models/yellow_ranger/configs/config_deployment.py @@ -0,0 +1,20 @@ +""" +Deployment Configuration Script + +This script defines the deployment configuration settings for the application. +It includes the deployment status and any additional settings specified. + +Deployment Status: +- shadow: The deployment is shadowed and not yet active. +- deployed: The deployment is active and in use. +- baseline: The deployment is in a baseline state, for reference or comparison. +- deprecated: The deployment is deprecated and no longer supported. + +Additional settings can be included in the configuration dictionary as needed. + +""" + +def get_deployment_config(): + # Deployment settings + deployment_config = {'deployment_status': 'shadow'} + return deployment_config diff --git a/models/yellow_ranger/configs/config_partitions.py b/models/yellow_ranger/configs/config_partitions.py new file mode 100644 index 00000000..ead19807 --- /dev/null +++ b/models/yellow_ranger/configs/config_partitions.py @@ -0,0 +1,44 @@ +from ingester3.ViewsMonth import ViewsMonth + + +def generate(steps: int = 36) -> dict: + """ + Generates partition configurations for different phases of model evaluation. + + Returns: + dict: A dictionary with keys 'calibration', 'validation', and 'forecasting', each containing + 'train' and 'test' tuples or callables specifying the index ranges for training and testing data. + + Partition details: + - 'calibration': Uses fixed index ranges for training and testing. + - 'validation': Uses fixed index ranges for training and testing. + - 'forecasting': Uses callables that accept ViewsMonth (and optionally step) to dynamically determine + training and testing index ranges based on the current month. + + Note: + - The 'forecasting' partition's 'train' and 'test' values are functions that require the ViewsMonth + object (and step for 'test') to compute the appropriate indices. + """ + + def forecasting_train_range(): + month_last = ViewsMonth.now().id - 1 + return (121, month_last) + + def forecasting_test_range(steps): + month_last = ViewsMonth.now().id - 1 + return (month_last + 1, month_last + 1 + steps) + + return { + "calibration": { + "train": (121, 444), + "test": (445, 492), + }, + "validation": { + "train": (121, 492), + "test": (493, 540), + }, + "forecasting": { + "train": forecasting_train_range(), + "test": forecasting_test_range(steps=steps), + }, + } diff --git a/models/yellow_ranger/configs/config_queryset.py b/models/yellow_ranger/configs/config_queryset.py new file mode 100644 index 00000000..37b953eb --- /dev/null +++ b/models/yellow_ranger/configs/config_queryset.py @@ -0,0 +1,23 @@ +from viewser import Queryset, Column + +def generate(): + """ + Contains the configuration for the input data in the form of a viewser queryset. That is the data from viewser that is used to train the model. + This configuration is "behavioral" so modifying it will affect the model's runtime behavior and integration into the deployment system. + There is no guarantee that the model will work if the input data configuration is changed here without changing the model settings and algorithm accordingly. + + Returns: + - queryset_base (Queryset): A queryset containing the base data for the model training. + """ + + queryset_base = (Queryset("yellow_ranger", "country_month") + + .with_column(Column("lr_os_best", from_loa="country_month", from_column="ged_os_best_sum_nokgi") + .transform.missing.replace_na()) + + .with_column(Column("month", from_loa="month", from_column="month")) + .with_column(Column("year_id", from_loa="country_year", from_column="year_id")) + + ) + + return queryset_base diff --git a/models/yellow_ranger/configs/config_sweep.py b/models/yellow_ranger/configs/config_sweep.py new file mode 100644 index 00000000..49db938b --- /dev/null +++ b/models/yellow_ranger/configs/config_sweep.py @@ -0,0 +1,31 @@ + +def get_sweep_config(): + """ + Contains the configuration for hyperparameter sweeps using WandB. + This configuration is "operational" so modifying it will change the search strategy, parameter ranges, and other settings for hyperparameter tuning aimed at optimizing model performance. + + Returns: + - sweep_config (dict): A dictionary containing the configuration for hyperparameter sweeps, defining the methods and parameter ranges used to search for optimal hyperparameters. + """ + + sweep_config = { + 'method': 'grid', + 'name': 'yellow_ranger' + } + + metric = { + 'name': 'MSE', + 'goal': 'minimize' + } + sweep_config['metric'] = metric + + parameters_dict = { + 'steps': {'value': [*range(1, 36 + 1, 1)]}, + 'time_steps': {'value': 36}, + 'n_samples': {'value': 256}, + 'lambda_mix': {'values': [0.0, 0.01, 0.025, 0.05, 0.1, 0.15, 0.2, 0.3, 0.5]}, + 'window_months': {'values': [6, 12, 18, 24, 36, 48, 60]}, + } + sweep_config['parameters'] = parameters_dict + + return sweep_config diff --git a/models/yellow_ranger/main.py b/models/yellow_ranger/main.py new file mode 100644 index 00000000..8ef12185 --- /dev/null +++ b/models/yellow_ranger/main.py @@ -0,0 +1,26 @@ +import warnings +from pathlib import Path +from views_pipeline_core.cli import ForecastingModelArgs +from views_pipeline_core.managers import ModelPathManager +from views_baseline.manager.baseline_manager import BaselineForecastingModelManager + +warnings.filterwarnings("ignore") + +try: + model_path = ModelPathManager(Path(__file__)) +except Exception as e: + raise RuntimeError(f"Unexpected error: {e}. Check the logs for details.") + +if __name__ == "__main__": + args = ForecastingModelArgs.parse_args() + + manager = BaselineForecastingModelManager( + model_path=model_path, + wandb_notifications=args.wandb_notifications, + use_prediction_store=args.prediction_store, + ) + + if args.sweep: + manager.execute_sweep_run(args) + else: + manager.execute_single_run(args) diff --git a/models/yellow_ranger/run.sh b/models/yellow_ranger/run.sh new file mode 100755 index 00000000..09ae7ef4 --- /dev/null +++ b/models/yellow_ranger/run.sh @@ -0,0 +1,42 @@ +#!/bin/zsh + +if [[ "$OSTYPE" == "darwin"* ]]; then + if ! grep -q 'export LDFLAGS="-L/opt/homebrew/opt/libomp/lib"' ~/.zshrc; then + echo 'export LDFLAGS="-L/opt/homebrew/opt/libomp/lib"' >> ~/.zshrc + fi + if ! grep -q 'export CPPFLAGS="-I/opt/homebrew/opt/libomp/include"' ~/.zshrc; then + echo 'export CPPFLAGS="-I/opt/homebrew/opt/libomp/include"' >> ~/.zshrc + fi + if ! grep -q 'export DYLD_LIBRARY_PATH="/opt/homebrew/opt/libomp/lib:$DYLD_LIBRARY_PATH"' ~/.zshrc; then + echo 'export DYLD_LIBRARY_PATH="/opt/homebrew/opt/libomp/lib:$DYLD_LIBRARY_PATH"' >> ~/.zshrc + fi + source ~/.zshrc +fi + +script_path=$(dirname "$(realpath $0)") +project_path="$( cd "$script_path/../../" >/dev/null 2>&1 && pwd )" +env_path="$project_path/envs/views-baseline" + +eval "$(conda shell.bash hook)" + +if [ -d "$env_path" ]; then + echo "Conda environment already exists at $env_path. Checking dependencies..." + conda activate "$env_path" + echo "$env_path is activated" + + missing_packages=$(pip install --dry-run -r $script_path/requirements.txt 2>&1 | grep -v "Requirement already satisfied" | wc -l) + if [ "$missing_packages" -gt 0 ]; then + echo "Installing missing or outdated packages..." + pip install -r $script_path/requirements.txt + else + echo "All packages are up-to-date." + fi +else + echo "Creating new Conda environment at $env_path..." + conda create --prefix "$env_path" python=3.11 -y + conda activate "$env_path" + pip install -r $script_path/requirements.txt +fi + +echo "Running $script_path/main.py " +python $script_path/main.py "$@" From be69ee9ca3b5bd5c0b9d79a7a8ede68e7bfe86b5 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 16 Mar 2026 14:38:31 +0100 Subject: [PATCH 21/27] feat(integration-tests): add --level flag to filter by cm/pgm Allows running integration tests on only CM or PGM models: bash run_integration_tests.sh --level cm bash run_integration_tests.sh --level pgm Extracts level from each model's config_meta.py via Python. Co-Authored-By: Claude Opus 4.6 (1M context) --- run_integration_tests.sh | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/run_integration_tests.sh b/run_integration_tests.sh index 6083bfb7..f7df1c31 100755 --- a/run_integration_tests.sh +++ b/run_integration_tests.sh @@ -9,6 +9,8 @@ # bash run_integration_tests.sh # all models # bash run_integration_tests.sh --models "counting_stars bad_blood" # subset # bash run_integration_tests.sh --partitions "calibration" # one partition +# bash run_integration_tests.sh --level cm # only CM models +# bash run_integration_tests.sh --level pgm # only PGM models # bash run_integration_tests.sh --exclude "purple_alien novel_heuristics" # skip models # bash run_integration_tests.sh --env my_conda_env # different env # bash run_integration_tests.sh --timeout 3600 # 60-min timeout @@ -22,6 +24,7 @@ CONDA_ENV="views_pipeline" TIMEOUT=1800 PARTITIONS="calibration validation" FILTER_MODELS="" +FILTER_LEVEL="" EXCLUDE_MODELS="purple_alien" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" MODELS_DIR="$SCRIPT_DIR/models" @@ -41,6 +44,7 @@ NC='\033[0m' while [[ $# -gt 0 ]]; do case "$1" in --models) FILTER_MODELS="$2"; shift 2 ;; + --level) FILTER_LEVEL="$2"; shift 2 ;; --exclude) EXCLUDE_MODELS="$2"; shift 2 ;; --partitions) PARTITIONS="$2"; shift 2 ;; --timeout) TIMEOUT="$2"; shift 2 ;; @@ -51,6 +55,7 @@ while [[ $# -gt 0 ]]; do echo "Options:" echo " --env NAME Conda env to activate (default: views_pipeline)" echo " --models \"m1 m2\" Run only these models" + echo " --level cm|pgm Run only models at this level of analysis" echo " --exclude \"m1 m2\" Skip these models (default: purple_alien)" echo " --partitions \"cal val\" Partitions to test (default: calibration validation)" echo " --timeout SECONDS Timeout per run (default: 1800)" @@ -93,6 +98,25 @@ else IFS=$'\n' MODELS=($(sort <<<"${MODELS[*]}")); unset IFS fi +# ── Filter by level (cm/pgm) if requested ──────────────────────────── + +if [ -n "$FILTER_LEVEL" ]; then + FILTERED=() + for model in "${MODELS[@]}"; do + level=$(python3 -c " +import importlib.util +spec = importlib.util.spec_from_file_location('m', '$MODELS_DIR/$model/configs/config_meta.py') +mod = importlib.util.module_from_spec(spec) +spec.loader.exec_module(mod) +print(mod.get_meta_config().get('level', '')) +" 2>/dev/null) + if [ "$level" = "$FILTER_LEVEL" ]; then + FILTERED+=("$model") + fi + done + MODELS=("${FILTERED[@]}") +fi + TOTAL_MODELS=${#MODELS[@]} if [ "$TOTAL_MODELS" -eq 0 ]; then echo "No models found to test." @@ -112,6 +136,7 @@ echo -e "${BOLD} views-models integration test${NC}" echo -e "${BOLD}═══════════════════════════════════════════════════════════${NC}" echo " Conda env: $CONDA_ENV" echo " Models: $TOTAL_MODELS" +[ -n "$FILTER_LEVEL" ] && echo " Level: $FILTER_LEVEL" echo " Excluded: $EXCLUDE_MODELS" echo " Partitions: $PARTITIONS" echo " Timeout: ${TIMEOUT}s per run" From 39222978e5c690aae1bc147a786aee763857a273 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Tue, 17 Mar 2026 10:23:16 +0100 Subject: [PATCH 22/27] docs(integration-tests): add full guide and link from README Add docs/run_integration_tests.md with prerequisites, all flags, internal mechanics, log structure, exit codes, and caveats. Add Integration Testing section to README with quick-start examples and options table. Add doc pointer to top of run_integration_tests.sh. Uncomment *.txt in .gitignore to stop tracking model run log artifacts. Co-Authored-By: Claude Opus 4.6 (1M context) --- .gitignore | 2 +- README.md | 32 +++ docs/run_integration_tests.md | 188 ++++++++++++++++++ .../data/generated/calibration_log.txt | 6 +- .../data/raw/calibration_data_fetch_log.txt | 2 +- .../data/generated/calibration_log.txt | 6 +- .../data/raw/calibration_data_fetch_log.txt | 2 +- .../data/generated/calibration_log.txt | 6 +- .../data/raw/calibration_data_fetch_log.txt | 2 +- .../data/generated/calibration_log.txt | 6 +- .../data/raw/calibration_data_fetch_log.txt | 2 +- .../data/generated/calibration_log.txt | 6 +- .../data/raw/calibration_data_fetch_log.txt | 2 +- .../data/raw/validation_data_fetch_log.txt | 2 +- .../data/generated/calibration_log.txt | 6 +- .../data/raw/calibration_data_fetch_log.txt | 2 +- .../data/raw/calibration_data_fetch_log.txt | 2 +- .../data/generated/calibration_log.txt | 6 +- .../data/raw/calibration_data_fetch_log.txt | 2 +- .../data/generated/calibration_log.txt | 6 +- .../data/raw/calibration_data_fetch_log.txt | 2 +- .../data/generated/calibration_log.txt | 6 +- .../data/raw/calibration_data_fetch_log.txt | 2 +- .../data/generated/calibration_log.txt | 6 +- .../data/raw/calibration_data_fetch_log.txt | 2 +- .../data/raw/validation_data_fetch_log.txt | 2 +- run_integration_tests.sh | 4 +- 27 files changed, 267 insertions(+), 45 deletions(-) create mode 100644 docs/run_integration_tests.md diff --git a/.gitignore b/.gitignore index 64a3619d..ed4ba1f5 100644 --- a/.gitignore +++ b/.gitignore @@ -249,7 +249,7 @@ cython_debug/ *.bak # txt logs -# *.txt +*.txt # logs *.log diff --git a/README.md b/README.md index a5faa343..8fc52724 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ APPWRITE_DATASTORE_PROJECT_ID="" - [Ensemble scripts](#ensemble-scripts) - [Ensemble filesystem](#ensemble-filesystem) - [Running an ensemble](#running-an-ensemble) +- [Integration Testing](#integration-testing) - [Implemented Models](#implemented-models) - [Model Catalogs](#catalogs) - [Country-Month Models](#country-month-model-catalog) @@ -332,6 +333,37 @@ Consequently, in order to train a model and generate predictions, execute either As of now, the only implemented model architecture is the [stepshifter model](https://github.com/views-platform/views-stepshifter/blob/main/README.md). Experienced users have the possibility to develop their own model architecture including their own model class manager. Head over to [views-pipeline-core](https://github.com/views-platform/views-pipeline-core) for further information on the model class manager and on how to develop new model architectures. +## Integration Testing + + +The repository includes an integration test runner that verifies models haven't been broken by changes in this repo or in upstream/downstream packages. It trains and evaluates every model end-to-end on calibration and validation partitions, running them sequentially in a single shared conda environment, and produces a summary table of PASS/FAIL/TIMEOUT results with per-model logs. + +```bash +# Run all models (calibration + validation) +bash run_integration_tests.sh + +# Run only country-month models +bash run_integration_tests.sh --level cm + +# Run specific models with a custom timeout +bash run_integration_tests.sh --models "counting_stars bad_blood" --timeout 3600 +``` + +| Flag | Default | Description | +|------|---------|-------------| +| `--models "m1 m2"` | all models | Run only these models | +| `--level` `cm` or `pgm` | no filter | Run only models at this level of analysis | +| `--exclude "m1 m2"` | `"purple_alien"` | Skip these models (replaces the default, does not append) | +| `--partitions "p1 p2"` | `"calibration validation"` | Partitions to test | +| `--timeout SECONDS` | `1800` | Max wall-clock time per model run | +| `--env NAME` | `views_pipeline` | Conda environment to activate | + +Logs are written to `logs/integration_test_/` with a `summary.log` and per-model logs under `{partition}/{model}.log`. + +For the full guide — including how model discovery works, how to read failure logs, and important caveats — see [docs/run_integration_tests.md](docs/run_integration_tests.md). + +--- + ## Implemented Models In addition to the possibility of easily creating new models and ensembles, in order to maintain an organized and structured overview over all of the implemented models, the views-models repository also contains model catalogs containing all of the information about individual models. This information is collected from the metadata of each model and entails: diff --git a/docs/run_integration_tests.md b/docs/run_integration_tests.md new file mode 100644 index 00000000..0518d8dd --- /dev/null +++ b/docs/run_integration_tests.md @@ -0,0 +1,188 @@ +# Integration Test Runner Guide + +`run_integration_tests.sh` is a regression safety net: it verifies that changes in this repo or in upstream/downstream packages haven't broken model training or evaluation. It trains and evaluates every model in `models/` on the calibration and validation partitions, running them **sequentially** using a **single shared conda environment**, logging each result independently, and **never aborting on failure** — every model gets its turn regardless of what happened before it. + +Each model run executes: + +``` +python main.py -r -t -e +``` + +That is: train (`-t`) and evaluate (`-e`) against the given partition (`-r`). + + +## Prerequisites + +- **Conda** must be available on `PATH` (the script calls `conda shell.bash hook` internally). +- The target conda environment must already exist and have all model dependencies installed. The script does **not** create or install into environments — it only activates one. +- The default environment is `views_pipeline`. If your models require different packages than what's in that env, use `--env` to point at a different one. + + +## Quick Start + +```bash +# Run all models (calibration + validation), default env, default timeout +bash run_integration_tests.sh + +# Run two specific models only +bash run_integration_tests.sh --models "counting_stars bad_blood" + +# Run only country-month models on calibration only +bash run_integration_tests.sh --level cm --partitions "calibration" +``` + + +## Options + +| Flag | Value | Default | Description | +|------|-------|---------|-------------| +| `--models` | `"name1 name2 ..."` | *(all models)* | Run **only** these models. Names are space-separated inside quotes. Each name must match a directory under `models/` that contains a `main.py`. Names not found are skipped with a warning. | +| `--level` | `cm` or `pgm` | *(no filter)* | Run only models whose `config_meta.py` reports this level of analysis. The script reads each model's config via Python to check. Models whose level cannot be read are silently excluded. | +| `--exclude` | `"name1 name2 ..."` | `"purple_alien"` | Skip these models. **Replaces** the default exclusion list — it does not append to it. To exclude nothing, pass an empty string: `--exclude ""`. | +| `--partitions` | `"p1 p2 ..."` | `"calibration validation"` | Which partitions to test. Valid values are `calibration`, `validation`, and `forecasting`. Space-separated inside quotes. | +| `--timeout` | seconds | `1800` (30 min) | Maximum wall-clock time per individual model run (one model x one partition). If exceeded, the run is killed and recorded as `TIMEOUT`. | +| `--env` | name | `views_pipeline` | Conda environment to activate before each model run. Can be an environment name or a path to a prefix. | +| `--help`, `-h` | — | — | Print usage summary and exit. | + + +## How It Works + +### 1. Model Discovery + +If `--models` is provided, the script uses that explicit list. Otherwise it scans every subdirectory of `models/` for a `main.py` file and sorts the results alphabetically. + +Either way, models in the exclusion list are removed before anything runs. + +### 2. Level Filtering (optional) + +When `--level` is set, the script shells out to Python for each discovered model: + +```python +import importlib.util +spec = importlib.util.spec_from_file_location('m', '/configs/config_meta.py') +mod = importlib.util.module_from_spec(spec) +spec.loader.exec_module(mod) +print(mod.get_meta_config().get('level', '')) +``` + +Only models whose `level` matches the filter survive. If a model's config can't be loaded, it is silently dropped. + +### 3. Execution + +For each model, for each partition, the script runs: + +```bash +timeout $TIMEOUT bash -c " + eval \"\$(conda shell.bash hook)\" + conda activate '$CONDA_ENV' + cd '$MODELS_DIR/$model' + python main.py -r '$partition' -t -e +" > "$model_log" 2>&1 +``` + +Key points: +- Each run gets its **own subshell** — a model crash cannot kill the runner. +- **All stdout and stderr** are captured to a log file, not printed to the terminal. +- The terminal shows only: `[N/TOTAL] model_name (partition)... PASS/FAIL/TIMEOUT (duration)`. + +### 4. Result Classification + +| Exit Code | Result | Meaning | +|-----------|--------|---------| +| `0` | `PASS` | Model trained and evaluated successfully. | +| `124` | `TIMEOUT` | Model exceeded the per-run timeout and was killed. | +| anything else | `FAIL(code)` | Model crashed. The exit code is recorded. | + +### 5. Summary Output + +After all runs complete, the script prints a colored table to the terminal: + +``` +Model calibration validation +----- ---------- ---------- +bad_blood PASS PASS +bouncy_organ FAIL(1) PASS +counting_stars PASS TIMEOUT +``` + +The same table (without colors) is written to `summary.log`. + + +## Logs + +All logs are written to a timestamped directory: + +``` +logs/ + integration_test_2026-03-16_143644/ + summary.log # full results table + calibration/ + bad_blood.log # stdout+stderr for this run + bouncy_organ.log + ... + validation/ + bad_blood.log + bouncy_organ.log + ... +``` + +The `logs/` directory is gitignored. + +### Reading a failed model's log + +```bash +# Find the most recent test run +LATEST=$(ls -t logs/ | head -1) + +# Read a specific model's log +cat "logs/$LATEST/calibration/bouncy_organ.log" +``` + + +## Exit Code + +The script itself exits: + +- **`0`** — all runs passed. +- **`1`** — at least one run failed or timed out. + +This makes it safe to use in CI or chain with `&&`: + +```bash +bash run_integration_tests.sh --level cm && echo "All CM models passed" +``` + + +## Examples + +```bash +# Everything, all defaults (all models, calibration + validation, 30min timeout) +bash run_integration_tests.sh + +# Just one model, just calibration +bash run_integration_tests.sh --models "counting_stars" --partitions "calibration" + +# All PGM models with a longer timeout +bash run_integration_tests.sh --level pgm --timeout 3600 + +# All models except two, validation only +bash run_integration_tests.sh --exclude "purple_alien novel_heuristics" --partitions "validation" + +# Exclude nothing (override the default purple_alien exclusion) +bash run_integration_tests.sh --exclude "" + +# Use a different conda environment +bash run_integration_tests.sh --env views_r2darts2 + +# Multiple flags combined: specific models, one partition, custom timeout +bash run_integration_tests.sh --models "bad_blood counting_stars" --partitions "calibration" --timeout 600 +``` + + +## Important Details + +- **Single shared environment**: Unlike each model's own `run.sh` (which creates/activates a per-model conda env), this script uses one environment for all models. All models must be installable into that environment. If a model needs packages that conflict with the shared env, it will fail. +- **`--exclude` replaces, not appends**: Passing `--exclude "foo"` means *only* `foo` is excluded — `purple_alien` is no longer excluded unless you include it: `--exclude "purple_alien foo"`. +- **Models run sequentially**: There is no parallelism. A full run of all 70 models across 2 partitions can take many hours depending on model complexity and data fetch times. +- **Data is fetched live**: Each model's queryset pulls data from the VIEWS API at runtime. Network issues or API downtime will cause failures unrelated to model code. +- **Forecasting partition uses live time**: If you pass `--partitions "forecasting"`, the train/test ranges are computed from `ViewsMonth.now()`, so results depend on when you run. diff --git a/models/adolecent_slob/data/generated/calibration_log.txt b/models/adolecent_slob/data/generated/calibration_log.txt index bd620432..a93d0f7a 100644 --- a/models/adolecent_slob/data/generated/calibration_log.txt +++ b/models/adolecent_slob/data/generated/calibration_log.txt @@ -1,6 +1,6 @@ Single Model Name: adolecent_slob -Single Model Timestamp: 20260218_134852 -Data Generation Timestamp: 20260218_135633 -Data Fetch Timestamp: 20260218_134025 +Single Model Timestamp: 20260316_193155 +Data Generation Timestamp: 20260316_193902 +Data Fetch Timestamp: 20260316_192753 Deployment Status: shadow diff --git a/models/adolecent_slob/data/raw/calibration_data_fetch_log.txt b/models/adolecent_slob/data/raw/calibration_data_fetch_log.txt index 5746e20f..339925b0 100644 --- a/models/adolecent_slob/data/raw/calibration_data_fetch_log.txt +++ b/models/adolecent_slob/data/raw/calibration_data_fetch_log.txt @@ -1,3 +1,3 @@ Single Model Name: adolecent_slob -Data Fetch Timestamp: 20260218_134025 +Data Fetch Timestamp: 20260316_192753 diff --git a/models/average_cmbaseline/data/generated/calibration_log.txt b/models/average_cmbaseline/data/generated/calibration_log.txt index fa1667fa..962539a4 100644 --- a/models/average_cmbaseline/data/generated/calibration_log.txt +++ b/models/average_cmbaseline/data/generated/calibration_log.txt @@ -1,6 +1,6 @@ Single Model Name: average_cmbaseline -Single Model Timestamp: 20260210_221353 -Data Generation Timestamp: 20260210_221413 -Data Fetch Timestamp: 20260210_221400 +Single Model Timestamp: 20260316_195210 +Data Generation Timestamp: 20260316_195236 +Data Fetch Timestamp: 20260316_195220 Deployment Status: shadow diff --git a/models/average_cmbaseline/data/raw/calibration_data_fetch_log.txt b/models/average_cmbaseline/data/raw/calibration_data_fetch_log.txt index 3c506983..e05d281d 100644 --- a/models/average_cmbaseline/data/raw/calibration_data_fetch_log.txt +++ b/models/average_cmbaseline/data/raw/calibration_data_fetch_log.txt @@ -1,3 +1,3 @@ Single Model Name: average_cmbaseline -Data Fetch Timestamp: 20260210_221400 +Data Fetch Timestamp: 20260316_195220 diff --git a/models/bittersweet_symphony/data/generated/calibration_log.txt b/models/bittersweet_symphony/data/generated/calibration_log.txt index 6e143af1..184807a0 100644 --- a/models/bittersweet_symphony/data/generated/calibration_log.txt +++ b/models/bittersweet_symphony/data/generated/calibration_log.txt @@ -1,6 +1,6 @@ Single Model Name: bittersweet_symphony -Single Model Timestamp: 20260210_220343 -Data Generation Timestamp: 20260210_220539 -Data Fetch Timestamp: 20260210_220319 +Single Model Timestamp: 20260316_195417 +Data Generation Timestamp: 20260316_195610 +Data Fetch Timestamp: 20260316_195352 Deployment Status: shadow diff --git a/models/bittersweet_symphony/data/raw/calibration_data_fetch_log.txt b/models/bittersweet_symphony/data/raw/calibration_data_fetch_log.txt index 4044d43b..9e2bb1c2 100644 --- a/models/bittersweet_symphony/data/raw/calibration_data_fetch_log.txt +++ b/models/bittersweet_symphony/data/raw/calibration_data_fetch_log.txt @@ -1,3 +1,3 @@ Single Model Name: bittersweet_symphony -Data Fetch Timestamp: 20260210_220319 +Data Fetch Timestamp: 20260316_195352 diff --git a/models/bouncy_organ/data/generated/calibration_log.txt b/models/bouncy_organ/data/generated/calibration_log.txt index a43e2b47..6778deaa 100644 --- a/models/bouncy_organ/data/generated/calibration_log.txt +++ b/models/bouncy_organ/data/generated/calibration_log.txt @@ -1,6 +1,6 @@ Single Model Name: bouncy_organ -Single Model Timestamp: 20260218_135123 -Data Generation Timestamp: 20260218_135821 -Data Fetch Timestamp: 20260218_134006 +Single Model Timestamp: 20260316_200613 +Data Generation Timestamp: 20260316_201246 +Data Fetch Timestamp: 20260316_195927 Deployment Status: shadow diff --git a/models/bouncy_organ/data/raw/calibration_data_fetch_log.txt b/models/bouncy_organ/data/raw/calibration_data_fetch_log.txt index 475f04c9..622ab16e 100644 --- a/models/bouncy_organ/data/raw/calibration_data_fetch_log.txt +++ b/models/bouncy_organ/data/raw/calibration_data_fetch_log.txt @@ -1,3 +1,3 @@ Single Model Name: bouncy_organ -Data Fetch Timestamp: 20260218_155102 +Data Fetch Timestamp: 20260316_195927 diff --git a/models/emerging_principles/data/generated/calibration_log.txt b/models/emerging_principles/data/generated/calibration_log.txt index 98b48e5b..8d300cb4 100644 --- a/models/emerging_principles/data/generated/calibration_log.txt +++ b/models/emerging_principles/data/generated/calibration_log.txt @@ -1,6 +1,6 @@ Single Model Name: emerging_principles -Single Model Timestamp: 20260218_134318 -Data Generation Timestamp: 20260218_134348 -Data Fetch Timestamp: 20260218_134010 +Single Model Timestamp: 20260316_225121 +Data Generation Timestamp: 20260316_225145 +Data Fetch Timestamp: 20260316_224935 Deployment Status: shadow diff --git a/models/emerging_principles/data/raw/calibration_data_fetch_log.txt b/models/emerging_principles/data/raw/calibration_data_fetch_log.txt index 29038e9e..a2b0665b 100644 --- a/models/emerging_principles/data/raw/calibration_data_fetch_log.txt +++ b/models/emerging_principles/data/raw/calibration_data_fetch_log.txt @@ -1,3 +1,3 @@ Single Model Name: emerging_principles -Data Fetch Timestamp: 20260218_134010 +Data Fetch Timestamp: 20260316_224935 diff --git a/models/emerging_principles/data/raw/validation_data_fetch_log.txt b/models/emerging_principles/data/raw/validation_data_fetch_log.txt index 27b7838e..60dfd2db 100644 --- a/models/emerging_principles/data/raw/validation_data_fetch_log.txt +++ b/models/emerging_principles/data/raw/validation_data_fetch_log.txt @@ -1,3 +1,3 @@ Single Model Name: emerging_principles -Data Fetch Timestamp: 20260128_014856 +Data Fetch Timestamp: 20260316_225210 diff --git a/models/fancy_feline/data/generated/calibration_log.txt b/models/fancy_feline/data/generated/calibration_log.txt index 899ced4c..91721593 100644 --- a/models/fancy_feline/data/generated/calibration_log.txt +++ b/models/fancy_feline/data/generated/calibration_log.txt @@ -1,6 +1,6 @@ Single Model Name: fancy_feline -Single Model Timestamp: 20260224_200520 -Data Generation Timestamp: 20260224_201407 -Data Fetch Timestamp: 20260224_195334 +Single Model Timestamp: 20260316_225954 +Data Generation Timestamp: 20260316_230625 +Data Fetch Timestamp: 20260316_225505 Deployment Status: shadow diff --git a/models/fancy_feline/data/raw/calibration_data_fetch_log.txt b/models/fancy_feline/data/raw/calibration_data_fetch_log.txt index e08c4399..9cb7600d 100644 --- a/models/fancy_feline/data/raw/calibration_data_fetch_log.txt +++ b/models/fancy_feline/data/raw/calibration_data_fetch_log.txt @@ -1,3 +1,3 @@ Single Model Name: fancy_feline -Data Fetch Timestamp: 20260224_195334 +Data Fetch Timestamp: 20260316_225505 diff --git a/models/hot_stream/data/raw/calibration_data_fetch_log.txt b/models/hot_stream/data/raw/calibration_data_fetch_log.txt index 07e83a83..3670915f 100644 --- a/models/hot_stream/data/raw/calibration_data_fetch_log.txt +++ b/models/hot_stream/data/raw/calibration_data_fetch_log.txt @@ -1,3 +1,3 @@ Single Model Name: hot_stream -Data Fetch Timestamp: 20260218_142905 +Data Fetch Timestamp: 20260317_023255 diff --git a/models/new_rules/data/generated/calibration_log.txt b/models/new_rules/data/generated/calibration_log.txt index ea722895..bfef5489 100644 --- a/models/new_rules/data/generated/calibration_log.txt +++ b/models/new_rules/data/generated/calibration_log.txt @@ -1,6 +1,6 @@ Single Model Name: new_rules -Single Model Timestamp: 20260217_153050 -Data Generation Timestamp: 20260217_153118 -Data Fetch Timestamp: 20260217_152013 +Single Model Timestamp: 20260317_045321 +Data Generation Timestamp: 20260317_045344 +Data Fetch Timestamp: 20260317_045106 Deployment Status: shadow diff --git a/models/new_rules/data/raw/calibration_data_fetch_log.txt b/models/new_rules/data/raw/calibration_data_fetch_log.txt index 6b739e47..254f5f79 100644 --- a/models/new_rules/data/raw/calibration_data_fetch_log.txt +++ b/models/new_rules/data/raw/calibration_data_fetch_log.txt @@ -1,3 +1,3 @@ Single Model Name: new_rules -Data Fetch Timestamp: 20260217_162204 +Data Fetch Timestamp: 20260317_045106 diff --git a/models/novel_heuristics/data/generated/calibration_log.txt b/models/novel_heuristics/data/generated/calibration_log.txt index 54ddffdf..c72040f0 100644 --- a/models/novel_heuristics/data/generated/calibration_log.txt +++ b/models/novel_heuristics/data/generated/calibration_log.txt @@ -1,6 +1,6 @@ Single Model Name: novel_heuristics -Single Model Timestamp: 20260306_202134 -Data Generation Timestamp: 20260306_202158 -Data Fetch Timestamp: 20260306_201935 +Single Model Timestamp: 20260317_045821 +Data Generation Timestamp: 20260317_045844 +Data Fetch Timestamp: 20260317_045649 Deployment Status: shadow diff --git a/models/novel_heuristics/data/raw/calibration_data_fetch_log.txt b/models/novel_heuristics/data/raw/calibration_data_fetch_log.txt index 0b445bec..e59347e8 100644 --- a/models/novel_heuristics/data/raw/calibration_data_fetch_log.txt +++ b/models/novel_heuristics/data/raw/calibration_data_fetch_log.txt @@ -1,3 +1,3 @@ Single Model Name: novel_heuristics -Data Fetch Timestamp: 20260314_180020 +Data Fetch Timestamp: 20260317_045649 diff --git a/models/party_princess/data/generated/calibration_log.txt b/models/party_princess/data/generated/calibration_log.txt index b7598e98..28fa1002 100644 --- a/models/party_princess/data/generated/calibration_log.txt +++ b/models/party_princess/data/generated/calibration_log.txt @@ -1,6 +1,6 @@ Single Model Name: party_princess -Single Model Timestamp: 20260218_120624 -Data Generation Timestamp: 20260218_121308 -Data Fetch Timestamp: 20260218_115705 +Single Model Timestamp: 20260317_051207 +Data Generation Timestamp: 20260317_051850 +Data Fetch Timestamp: 20260317_050733 Deployment Status: shadow diff --git a/models/party_princess/data/raw/calibration_data_fetch_log.txt b/models/party_princess/data/raw/calibration_data_fetch_log.txt index 0c91ef9f..f168b98d 100644 --- a/models/party_princess/data/raw/calibration_data_fetch_log.txt +++ b/models/party_princess/data/raw/calibration_data_fetch_log.txt @@ -1,3 +1,3 @@ Single Model Name: party_princess -Data Fetch Timestamp: 20260218_115705 +Data Fetch Timestamp: 20260317_050733 diff --git a/models/preliminary_directives/data/generated/calibration_log.txt b/models/preliminary_directives/data/generated/calibration_log.txt index f08e04b6..cc97bdf7 100644 --- a/models/preliminary_directives/data/generated/calibration_log.txt +++ b/models/preliminary_directives/data/generated/calibration_log.txt @@ -1,6 +1,6 @@ Single Model Name: preliminary_directives -Single Model Timestamp: 20260217_141733 -Data Generation Timestamp: 20260217_141806 -Data Fetch Timestamp: 20260217_135133 +Single Model Timestamp: 20260317_054825 +Data Generation Timestamp: 20260317_054907 +Data Fetch Timestamp: 20260317_054538 Deployment Status: shadow diff --git a/models/preliminary_directives/data/raw/calibration_data_fetch_log.txt b/models/preliminary_directives/data/raw/calibration_data_fetch_log.txt index b966344a..fb99d726 100644 --- a/models/preliminary_directives/data/raw/calibration_data_fetch_log.txt +++ b/models/preliminary_directives/data/raw/calibration_data_fetch_log.txt @@ -1,3 +1,3 @@ Single Model Name: preliminary_directives -Data Fetch Timestamp: 20260217_135133 +Data Fetch Timestamp: 20260317_054538 diff --git a/models/preliminary_directives/data/raw/validation_data_fetch_log.txt b/models/preliminary_directives/data/raw/validation_data_fetch_log.txt index 01d5a436..60750081 100644 --- a/models/preliminary_directives/data/raw/validation_data_fetch_log.txt +++ b/models/preliminary_directives/data/raw/validation_data_fetch_log.txt @@ -1,3 +1,3 @@ Single Model Name: preliminary_directives -Data Fetch Timestamp: 20260128_015217 +Data Fetch Timestamp: 20260317_054939 diff --git a/run_integration_tests.sh b/run_integration_tests.sh index f7df1c31..3566d046 100755 --- a/run_integration_tests.sh +++ b/run_integration_tests.sh @@ -5,7 +5,9 @@ # Trains and evaluates each model on calibration and validation partitions # using a single conda environment. Logs results per model — never aborts on failure. # -# Usage: +# Full documentation: docs/run_integration_tests.md +# +# Quick examples: # bash run_integration_tests.sh # all models # bash run_integration_tests.sh --models "counting_stars bad_blood" # subset # bash run_integration_tests.sh --partitions "calibration" # one partition From f27facf25ce89fea1226bdff9959cb89c8f1a519 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Tue, 17 Mar 2026 16:20:35 +0100 Subject: [PATCH 23/27] feat(integration-tests): add --library flag to filter by architecture package Allows running integration tests for a single model library at a time (baseline, stepshifter, r2darts2, hydranet) by matching views- in each model's requirements.txt. Combinable with --level. Also fixes exit-code precedence bug where failures without timeouts exited 0, and untracks 3 data log .txt files already covered by .gitignore. Co-Authored-By: Claude Opus 4.6 (1M context) --- docs/run_integration_tests.md | 22 ++++++++++++++++--- .../data/generated/calibration_log.txt | 6 ----- .../data/raw/calibration_data_fetch_log.txt | 3 --- .../data/raw/calibration_data_fetch_log.txt | 3 --- run_integration_tests.sh | 22 ++++++++++++++++++- 5 files changed, 40 insertions(+), 16 deletions(-) delete mode 100644 models/average_cmbaseline/data/generated/calibration_log.txt delete mode 100644 models/average_cmbaseline/data/raw/calibration_data_fetch_log.txt delete mode 100644 models/purple_alien/data/raw/calibration_data_fetch_log.txt diff --git a/docs/run_integration_tests.md b/docs/run_integration_tests.md index 0518d8dd..2cadfae7 100644 --- a/docs/run_integration_tests.md +++ b/docs/run_integration_tests.md @@ -29,6 +29,9 @@ bash run_integration_tests.sh --models "counting_stars bad_blood" # Run only country-month models on calibration only bash run_integration_tests.sh --level cm --partitions "calibration" + +# Run only baseline models +bash run_integration_tests.sh --library baseline ``` @@ -38,6 +41,7 @@ bash run_integration_tests.sh --level cm --partitions "calibration" |------|-------|---------|-------------| | `--models` | `"name1 name2 ..."` | *(all models)* | Run **only** these models. Names are space-separated inside quotes. Each name must match a directory under `models/` that contains a `main.py`. Names not found are skipped with a warning. | | `--level` | `cm` or `pgm` | *(no filter)* | Run only models whose `config_meta.py` reports this level of analysis. The script reads each model's config via Python to check. Models whose level cannot be read are silently excluded. | +| `--library` | `baseline`, `stepshifter`, `r2darts2`, or `hydranet` | *(no filter)* | Run only models that depend on this architecture library. Determined by matching `views-` in each model's `requirements.txt`. Can be combined with `--level`. | | `--exclude` | `"name1 name2 ..."` | `"purple_alien"` | Skip these models. **Replaces** the default exclusion list — it does not append to it. To exclude nothing, pass an empty string: `--exclude ""`. | | `--partitions` | `"p1 p2 ..."` | `"calibration validation"` | Which partitions to test. Valid values are `calibration`, `validation`, and `forecasting`. Space-separated inside quotes. | | `--timeout` | seconds | `1800` (30 min) | Maximum wall-clock time per individual model run (one model x one partition). If exceeded, the run is killed and recorded as `TIMEOUT`. | @@ -67,7 +71,13 @@ print(mod.get_meta_config().get('level', '')) Only models whose `level` matches the filter survive. If a model's config can't be loaded, it is silently dropped. -### 3. Execution +### 3. Library Filtering (optional) + +When `--library` is set, the script checks each model's `requirements.txt` for a line containing `views-`. For example, `--library baseline` keeps only models whose `requirements.txt` contains `views-baseline`. This is a pure text match — no Python needed. + +`--library` and `--level` can be combined to narrow further (e.g., `--library stepshifter --level cm` runs only country-month stepshifter models). + +### 4. Execution For each model, for each partition, the script runs: @@ -85,7 +95,7 @@ Key points: - **All stdout and stderr** are captured to a log file, not printed to the terminal. - The terminal shows only: `[N/TOTAL] model_name (partition)... PASS/FAIL/TIMEOUT (duration)`. -### 4. Result Classification +### 5. Result Classification | Exit Code | Result | Meaning | |-----------|--------|---------| @@ -93,7 +103,7 @@ Key points: | `124` | `TIMEOUT` | Model exceeded the per-run timeout and was killed. | | anything else | `FAIL(code)` | Model crashed. The exit code is recorded. | -### 5. Summary Output +### 6. Summary Output After all runs complete, the script prints a colored table to the terminal: @@ -174,6 +184,12 @@ bash run_integration_tests.sh --exclude "" # Use a different conda environment bash run_integration_tests.sh --env views_r2darts2 +# All baseline models only +bash run_integration_tests.sh --library baseline + +# All stepshifter models at country-month level +bash run_integration_tests.sh --library stepshifter --level cm + # Multiple flags combined: specific models, one partition, custom timeout bash run_integration_tests.sh --models "bad_blood counting_stars" --partitions "calibration" --timeout 600 ``` diff --git a/models/average_cmbaseline/data/generated/calibration_log.txt b/models/average_cmbaseline/data/generated/calibration_log.txt deleted file mode 100644 index 962539a4..00000000 --- a/models/average_cmbaseline/data/generated/calibration_log.txt +++ /dev/null @@ -1,6 +0,0 @@ -Single Model Name: average_cmbaseline -Single Model Timestamp: 20260316_195210 -Data Generation Timestamp: 20260316_195236 -Data Fetch Timestamp: 20260316_195220 -Deployment Status: shadow - diff --git a/models/average_cmbaseline/data/raw/calibration_data_fetch_log.txt b/models/average_cmbaseline/data/raw/calibration_data_fetch_log.txt deleted file mode 100644 index e05d281d..00000000 --- a/models/average_cmbaseline/data/raw/calibration_data_fetch_log.txt +++ /dev/null @@ -1,3 +0,0 @@ -Single Model Name: average_cmbaseline -Data Fetch Timestamp: 20260316_195220 - diff --git a/models/purple_alien/data/raw/calibration_data_fetch_log.txt b/models/purple_alien/data/raw/calibration_data_fetch_log.txt deleted file mode 100644 index 86ec2f82..00000000 --- a/models/purple_alien/data/raw/calibration_data_fetch_log.txt +++ /dev/null @@ -1,3 +0,0 @@ -Single Model Name: purple_alien -Data Fetch Timestamp: 20260311_160234 - diff --git a/run_integration_tests.sh b/run_integration_tests.sh index 3566d046..59ff7383 100755 --- a/run_integration_tests.sh +++ b/run_integration_tests.sh @@ -13,6 +13,7 @@ # bash run_integration_tests.sh --partitions "calibration" # one partition # bash run_integration_tests.sh --level cm # only CM models # bash run_integration_tests.sh --level pgm # only PGM models +# bash run_integration_tests.sh --library baseline # one library # bash run_integration_tests.sh --exclude "purple_alien novel_heuristics" # skip models # bash run_integration_tests.sh --env my_conda_env # different env # bash run_integration_tests.sh --timeout 3600 # 60-min timeout @@ -27,6 +28,7 @@ TIMEOUT=1800 PARTITIONS="calibration validation" FILTER_MODELS="" FILTER_LEVEL="" +FILTER_LIBRARY="" EXCLUDE_MODELS="purple_alien" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" MODELS_DIR="$SCRIPT_DIR/models" @@ -47,6 +49,7 @@ while [[ $# -gt 0 ]]; do case "$1" in --models) FILTER_MODELS="$2"; shift 2 ;; --level) FILTER_LEVEL="$2"; shift 2 ;; + --library) FILTER_LIBRARY="$2"; shift 2 ;; --exclude) EXCLUDE_MODELS="$2"; shift 2 ;; --partitions) PARTITIONS="$2"; shift 2 ;; --timeout) TIMEOUT="$2"; shift 2 ;; @@ -58,6 +61,7 @@ while [[ $# -gt 0 ]]; do echo " --env NAME Conda env to activate (default: views_pipeline)" echo " --models \"m1 m2\" Run only these models" echo " --level cm|pgm Run only models at this level of analysis" + echo " --library NAME Run only models using this library (baseline|stepshifter|r2darts2|hydranet)" echo " --exclude \"m1 m2\" Skip these models (default: purple_alien)" echo " --partitions \"cal val\" Partitions to test (default: calibration validation)" echo " --timeout SECONDS Timeout per run (default: 1800)" @@ -119,6 +123,19 @@ print(mod.get_meta_config().get('level', '')) MODELS=("${FILTERED[@]}") fi +# ── Filter by library (baseline/stepshifter/r2darts2/hydranet) ────── + +if [ -n "$FILTER_LIBRARY" ]; then + FILTERED=() + for model in "${MODELS[@]}"; do + req_file="$MODELS_DIR/$model/requirements.txt" + if [ -f "$req_file" ] && grep -q "views-${FILTER_LIBRARY}" "$req_file"; then + FILTERED+=("$model") + fi + done + MODELS=("${FILTERED[@]}") +fi + TOTAL_MODELS=${#MODELS[@]} if [ "$TOTAL_MODELS" -eq 0 ]; then echo "No models found to test." @@ -139,6 +156,7 @@ echo -e "${BOLD}═════════════════════ echo " Conda env: $CONDA_ENV" echo " Models: $TOTAL_MODELS" [ -n "$FILTER_LEVEL" ] && echo " Level: $FILTER_LEVEL" +[ -n "$FILTER_LIBRARY" ] && echo " Library: $FILTER_LIBRARY" echo " Excluded: $EXCLUDE_MODELS" echo " Partitions: $PARTITIONS" echo " Timeout: ${TIMEOUT}s per run" @@ -257,5 +275,7 @@ echo "" echo "Full summary: $LOG_DIR/summary.log" echo "Per-model logs: $LOG_DIR/{partition}/{model}.log" -[ "$FAIL_COUNT" -gt 0 ] || [ "$TIMEOUT_COUNT" -gt 0 ] && exit 1 +if [ "$FAIL_COUNT" -gt 0 ] || [ "$TIMEOUT_COUNT" -gt 0 ]; then + exit 1 +fi exit 0 From d3e680527a2f6a1df97c4552e35f4ee03536dcdd Mon Sep 17 00:00:00 2001 From: Polichinl Date: Tue, 17 Mar 2026 20:15:43 +0100 Subject: [PATCH 24/27] fix(docs): add --library flag to README integration testing table Co-Authored-By: Claude Opus 4.6 (1M context) --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 8fc52724..fe5d1d26 100644 --- a/README.md +++ b/README.md @@ -345,6 +345,9 @@ bash run_integration_tests.sh # Run only country-month models bash run_integration_tests.sh --level cm +# Run only baseline models +bash run_integration_tests.sh --library baseline + # Run specific models with a custom timeout bash run_integration_tests.sh --models "counting_stars bad_blood" --timeout 3600 ``` @@ -353,6 +356,7 @@ bash run_integration_tests.sh --models "counting_stars bad_blood" --timeout 3600 |------|---------|-------------| | `--models "m1 m2"` | all models | Run only these models | | `--level` `cm` or `pgm` | no filter | Run only models at this level of analysis | +| `--library NAME` | no filter | Run only models using this library (baseline/stepshifter/r2darts2/hydranet) | | `--exclude "m1 m2"` | `"purple_alien"` | Skip these models (replaces the default, does not append) | | `--partitions "p1 p2"` | `"calibration validation"` | Partitions to test | | `--timeout SECONDS` | `1800` | Max wall-clock time per model run | From c846ca2d5729c7ff2ed547a3a9b68142945e0377 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Tue, 17 Mar 2026 20:15:57 +0100 Subject: [PATCH 25/27] fix(configs): remove duplicate import in blank_space config_queryset Co-Authored-By: Claude Opus 4.6 (1M context) --- models/blank_space/configs/config_queryset.py | 1 - 1 file changed, 1 deletion(-) diff --git a/models/blank_space/configs/config_queryset.py b/models/blank_space/configs/config_queryset.py index 46122409..c8905d02 100644 --- a/models/blank_space/configs/config_queryset.py +++ b/models/blank_space/configs/config_queryset.py @@ -1,5 +1,4 @@ from viewser import Queryset, Column -from viewser import Queryset, Column from views_pipeline_core.managers.model import ModelPathManager model_name = ModelPathManager.get_model_name_from_path(__file__) From b8709d89aaee537755944629aefb2cd5c668abfe Mon Sep 17 00:00:00 2001 From: Polichinl Date: Tue, 17 Mar 2026 20:16:04 +0100 Subject: [PATCH 26/27] fix(docs): remove hardcoded model count from integration test guide Co-Authored-By: Claude Opus 4.6 (1M context) --- docs/run_integration_tests.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/run_integration_tests.md b/docs/run_integration_tests.md index 2cadfae7..1bbb9e15 100644 --- a/docs/run_integration_tests.md +++ b/docs/run_integration_tests.md @@ -199,6 +199,6 @@ bash run_integration_tests.sh --models "bad_blood counting_stars" --partitions " - **Single shared environment**: Unlike each model's own `run.sh` (which creates/activates a per-model conda env), this script uses one environment for all models. All models must be installable into that environment. If a model needs packages that conflict with the shared env, it will fail. - **`--exclude` replaces, not appends**: Passing `--exclude "foo"` means *only* `foo` is excluded — `purple_alien` is no longer excluded unless you include it: `--exclude "purple_alien foo"`. -- **Models run sequentially**: There is no parallelism. A full run of all 70 models across 2 partitions can take many hours depending on model complexity and data fetch times. +- **Models run sequentially**: There is no parallelism. A full run of all models across 2 partitions can take many hours depending on model complexity and data fetch times. - **Data is fetched live**: Each model's queryset pulls data from the VIEWS API at runtime. Network issues or API downtime will cause failures unrelated to model code. - **Forecasting partition uses live time**: If you pass `--partitions "forecasting"`, the train/test ranges are computed from `ViewsMonth.now()`, so results depend on when you run. From 231a3ae0727aac3c8db2a02a2095fe268c49e9e2 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Tue, 17 Mar 2026 20:16:20 +0100 Subject: [PATCH 27/27] feat(ci): add pytest workflow for push and PR gates Runs the structural test suite (~2200 tests, <20s) on every push to main/development and on all PRs. Requires only views_pipeline_core. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/run_tests.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/workflows/run_tests.yml diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml new file mode 100644 index 00000000..79b37908 --- /dev/null +++ b/.github/workflows/run_tests.yml @@ -0,0 +1,22 @@ +name: Run Tests +on: + push: + branches: [main, development] + pull_request: + branches: [main, development] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install dependencies + run: pip install views_pipeline_core pytest + + - name: Run tests + run: pytest