From f72cc861f8b740fe4fa5e529bfa7c9530af5ecfa Mon Sep 17 00:00:00 2001 From: paddy-r Date: Sun, 16 Mar 2025 13:33:33 +0000 Subject: [PATCH 01/12] Bundling wiki instructions + argparse(r) --- setup.py | 11 +- .../synthesizer/correct_and_train.py | 63 ++++++ .../imputation/adults_imputation.R | 189 ++++++++++++------ src/synthwave/synthesizer/uk/generator.py | 132 ++++++------ src/synthwave/utils/uk/pre_process.py | 24 +++ .../utils/uk/understanding_society.py | 8 +- 6 files changed, 297 insertions(+), 130 deletions(-) create mode 100644 src/synthwave/synthesizer/correct_and_train.py create mode 100644 src/synthwave/utils/uk/pre_process.py diff --git a/setup.py b/setup.py index f786363..8f08331 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,16 @@ "scikit-learn>=1.6.1", "sdv>=1.17.3", "torch>=2.5.1", - "matplotlib>=3.10.0" + "matplotlib>=3.10.0", + "jupyter", + "r-arrow", + "r-mice", + "r-dplyr", + "r-lattice", + "r-argparser", + "r-parallelly", + "r-future", + "r-furrr", ], extras_require={ "dev": ["check-manifest"], diff --git a/src/synthwave/synthesizer/correct_and_train.py b/src/synthwave/synthesizer/correct_and_train.py new file mode 100644 index 0000000..c1b3b2e --- /dev/null +++ b/src/synthwave/synthesizer/correct_and_train.py @@ -0,0 +1,63 @@ +# HR 15/03/25 Correct and train imputed data + +import os +import pandas as pd +from synthwave.synthesizer.postimputation.correction import correct_imputed_data +from synthwave.synthesizer.uk.generator import Syntets +import argparse + + +def main(data_path, save_path=None): + + if not save_path: + save_path = os.path.join(data_path, "synthwave", "trained") + + # 1. Correct imputed data + print("Correcting imputed data...") + adults = pd.read_csv(os.path.join(data_path, "synthwave", "imputed", "imputed_data.csv"), dtype_backend="pyarrow") + adults = correct_imputed_data(adults) + print("Done!") + + # 2. Train model + print("Creating generator and restructure data...") + generator = Syntets(adults) + generator.split_data() + generator.restructure_data() + print("Done!") + + # load dataset + print("Tidying up child data...") + children = pd.read_parquet(os.path.join(data_path, "children_non_imputed_middle_fidelity.parquet")).drop(columns=["id_person"]) + + # convert data types + children[["ordinal_person_age", "category_person_ethnic_group"]] = children[["ordinal_person_age", "category_person_ethnic_group"]].astype("uint8[pyarrow]") + + # drop households with incomplete records + crooked_records = pd.unique(children[children["category_person_ethnic_group"].isna()]["id_household"]) + children = children[~children["id_household"].isin(crooked_records)] # NOTE do not drop duplicates ever, this destroys twins + print("Done!") + + print("Training child data...") + # children = children.sample(frac=2.0, replace=True) + generator.train_children(children, verbose=True) + print("Done!") + + generator.drop_id_columns() # we need ids to learn how children are formed + generator.locate_degenerate_distributions() + generator.convert_types() + generator.init_models(_epochs=5) + generator.attach_constraints() + + print("Running main training...") + generator.train(save_path=save_path) + print("Done!") + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + parser.add_argument("p", type=str, help="Data source path") + + args = parser.parse_args() + data_path = args.p + main(data_path) diff --git a/src/synthwave/synthesizer/imputation/adults_imputation.R b/src/synthwave/synthesizer/imputation/adults_imputation.R index 726dcb1..0182129 100644 --- a/src/synthwave/synthesizer/imputation/adults_imputation.R +++ b/src/synthwave/synthesizer/imputation/adults_imputation.R @@ -2,93 +2,152 @@ require(mice) require(lattice) require(dplyr) require(arrow) +require(argparser) -set.seed(123) +N_CORES_DEFAULT <- 128 +MAXIT_DEFAULT <- 20 +PROP_DEFAULT <- 1e-2 -ind <- read_parquet("./adults_non_imputed_middle_fidelity.parquet") -ind[grepl("^(indicator_)", colnames(ind))] <- lapply(ind[grepl("^(indicator_)", colnames(ind))], as.logical) -ind[grepl("^(mlb_)", colnames(ind))] <- lapply(ind[grepl("^(mlb_)", colnames(ind))], as.logical) +do_adults_imputation <- function(path.to.data, subset, n_cores, maxit) { -ind[grepl("^(category_)", colnames(ind))] <- lapply(ind[grepl("^(category_)", colnames(ind))], as.factor) + set.seed(123) -ghq = grepl("^(ordinal_person_ghq)", colnames(ind)) -ind[ghq] <- lapply(ind[ghq], factor, order=TRUE, levels=seq(min(ind[colnames(ind)[ghq]], na.rm = TRUE), - max(ind[colnames(ind)[ghq]], na.rm = TRUE))) -# This way we can disregard any potential shifts in the variable; they all also have the same levels + print("Getting parquet file...") + adults.file <- "adults_non_imputed_middle_fidelity.parquet" + path.to.file <- file.path(path.to.data, adults.file) + ind <- read_parquet(path.to.file) + print("Done!") -ind["ordinal_person_sf_1"] <- lapply(ind["ordinal_person_sf_1"], factor, order=TRUE, levels=c(5, 4, 3, 2, 1)) -ind["ordinal_person_sf_2a"] <- lapply(ind["ordinal_person_sf_2a"], factor, order=TRUE, levels=c(1, 2, 3)) -ind["ordinal_person_sf_2b"] <- lapply(ind["ordinal_person_sf_2b"], factor, order=TRUE, levels=c(1, 2, 3)) -ind["ordinal_person_sf_3a"] <- lapply(ind["ordinal_person_sf_3a"], factor, order=TRUE, levels=c(1, 2, 3, 4, 5)) -ind["ordinal_person_sf_3b"] <- lapply(ind["ordinal_person_sf_3b"], factor, order=TRUE, levels=c(1, 2, 3, 4, 5)) -ind["ordinal_person_sf_4a"] <- lapply(ind["ordinal_person_sf_4a"], factor, order=TRUE, levels=c(1, 2, 3, 4, 5)) -ind["ordinal_person_sf_4b"] <- lapply(ind["ordinal_person_sf_4b"], factor, order=TRUE, levels=c(1, 2, 3, 4, 5)) -ind["ordinal_person_sf_5"] <- lapply(ind["ordinal_person_sf_5"], factor, order=TRUE, levels=c(5, 4, 3, 2, 1)) -ind["ordinal_person_sf_6a"] <- lapply(ind["ordinal_person_sf_6a"], factor, order=TRUE, levels=c(5, 4, 3, 2, 1)) -ind["ordinal_person_sf_6b"] <- lapply(ind["ordinal_person_sf_6b"], factor, order=TRUE, levels=c(5, 4, 3, 2, 1)) -ind["ordinal_person_sf_6c"] <- lapply(ind["ordinal_person_sf_6c"], factor, order=TRUE, levels=c(1, 2, 3, 4, 5)) -ind["ordinal_person_sf_7"] <- lapply(ind["ordinal_person_sf_7"], factor, order=TRUE, levels=c(1, 2, 3, 4, 5)) + print("Tidying data...") + ind[grepl("^(indicator_)", colnames(ind))] <- lapply(ind[grepl("^(indicator_)", colnames(ind))], as.logical) + ind[grepl("^(mlb_)", colnames(ind))] <- lapply(ind[grepl("^(mlb_)", colnames(ind))], as.logical) -ind["ordinal_person_financial_situation"] <- lapply(ind["ordinal_person_financial_situation"], factor, order=TRUE, levels=c(5, 4, 3, 2, 1)) -ind["ordinal_person_life_satisfaction"] <- lapply(ind["ordinal_person_life_satisfaction"], factor, order=TRUE, levels=c(1, 2, 3, 4, 5, 6, 7)) + ind[grepl("^(category_)", colnames(ind))] <- lapply(ind[grepl("^(category_)", colnames(ind))], as.factor) -# NOTE for some methods values must be shifted to start from 0, converted to ordinals, processed, converted to int (!), and shifted back -min_age <- min(ind["ordinal_person_age"], na.rm = TRUE) -max_age <- max(ind["ordinal_person_age"], na.rm = TRUE) + ghq <- grepl("^(ordinal_person_ghq)", colnames(ind)) + ind[ghq] <- lapply(ind[ghq], factor, order=TRUE, levels=seq(min(ind[colnames(ind)[ghq]], na.rm = TRUE), + max(ind[colnames(ind)[ghq]], na.rm = TRUE))) + # This way we can disregard any potential shifts in the variable; they all also have the same levels -ind["ordinal_person_age"] <- lapply(ind["ordinal_person_age"], - factor, - order=TRUE, - levels=seq(min_age, max_age)) + ind["ordinal_person_sf_1"] <- lapply(ind["ordinal_person_sf_1"], factor, order=TRUE, levels=c(5, 4, 3, 2, 1)) -min_year <- min(ind["ordinal_household_year"], na.rm = TRUE) -max_year <- max(ind["ordinal_household_year"], na.rm = TRUE) + ind["ordinal_person_sf_2a"] <- lapply(ind["ordinal_person_sf_2a"], factor, order=TRUE, levels=c(1, 2, 3)) + ind["ordinal_person_sf_2b"] <- lapply(ind["ordinal_person_sf_2b"], factor, order=TRUE, levels=c(1, 2, 3)) + ind["ordinal_person_sf_3a"] <- lapply(ind["ordinal_person_sf_3a"], factor, order=TRUE, levels=c(1, 2, 3, 4, 5)) + ind["ordinal_person_sf_3b"] <- lapply(ind["ordinal_person_sf_3b"], factor, order=TRUE, levels=c(1, 2, 3, 4, 5)) + ind["ordinal_person_sf_4a"] <- lapply(ind["ordinal_person_sf_4a"], factor, order=TRUE, levels=c(1, 2, 3, 4, 5)) + ind["ordinal_person_sf_4b"] <- lapply(ind["ordinal_person_sf_4b"], factor, order=TRUE, levels=c(1, 2, 3, 4, 5)) + ind["ordinal_person_sf_5"] <- lapply(ind["ordinal_person_sf_5"], factor, order=TRUE, levels=c(5, 4, 3, 2, 1)) + ind["ordinal_person_sf_6a"] <- lapply(ind["ordinal_person_sf_6a"], factor, order=TRUE, levels=c(5, 4, 3, 2, 1)) + ind["ordinal_person_sf_6b"] <- lapply(ind["ordinal_person_sf_6b"], factor, order=TRUE, levels=c(5, 4, 3, 2, 1)) + ind["ordinal_person_sf_6c"] <- lapply(ind["ordinal_person_sf_6c"], factor, order=TRUE, levels=c(1, 2, 3, 4, 5)) + ind["ordinal_person_sf_7"] <- lapply(ind["ordinal_person_sf_7"], factor, order=TRUE, levels=c(1, 2, 3, 4, 5)) -ind["ordinal_household_year"] <- lapply(ind["ordinal_household_year"], + ind["ordinal_person_financial_situation"] <- lapply(ind["ordinal_person_financial_situation"], factor, order=TRUE, levels=c(5, 4, 3, 2, 1)) + ind["ordinal_person_life_satisfaction"] <- lapply(ind["ordinal_person_life_satisfaction"], factor, order=TRUE, levels=c(1, 2, 3, 4, 5, 6, 7)) + + # NOTE for some methods values must be shifted to start from 0, converted to ordinals, processed, converted to int (!), and shifted back + min_age <- min(ind["ordinal_person_age"], na.rm = TRUE) + max_age <- max(ind["ordinal_person_age"], na.rm = TRUE) + + ind["ordinal_person_age"] <- lapply(ind["ordinal_person_age"], factor, - order = TRUE, - levels=seq(min_year, max_year)) + order=TRUE, + levels=seq(min_age, max_age)) + + min_year <- min(ind["ordinal_household_year"], na.rm = TRUE) + max_year <- max(ind["ordinal_household_year"], na.rm = TRUE) + + ind["ordinal_household_year"] <- lapply(ind["ordinal_household_year"], + factor, + order = TRUE, + levels=seq(min_year, max_year)) + + + ind["total_individuals"] <- lapply(ind["total_individuals"], factor, order=TRUE, levels=seq(min(ind["total_individuals"]), max(ind["total_individuals"]))) + ind["total_children"] <- lapply(ind["total_children"], factor, order=TRUE, levels=seq(min(ind["total_children"]), max(ind["total_children"]))) + + ind["has_partner"] <- lapply(ind["has_partner"], as.logical) + print("Done!") + + + if (subset == TRUE) + { + print("Subsetting data for testing...") + print(paste0("Current size: ", nrow(ind), " by ", ncol(ind))) + + prop <- PROP_DEFAULT + ind <- ind %>% slice_sample(prop=prop, replace=FALSE) + print(paste0("Length after subsetting (", as.character(prop*100), "%): ", nrow(ind), " by ", ncol(ind))) + } + + + id_names <- grepl("^(id_)", colnames(ind)) # Boolean mask, not actual values + ids <- ind[id_names] + ind <- ind[ , !id_names] + + + print("Doing quickpred...") + pred <- quickpred(ind, mincor = 0.01) # about 10 minutes + # current version of quickpred does remove complete columns from the list of vars to be predicted + # automatically + + #to_predict <- names(which(colSums(is.na(ind)) > 0)) # contain NA values + #do_not_predict <- colnames(ind)[!colnames(ind) %in% to_predict] + #pred <- quickpred(ind, mincor = 0) + #pred[do_not_predict, ] <- 0 + print("Done!") -ind["total_individuals"] <- lapply(ind["total_individuals"], factor, order=TRUE, levels=seq(min(ind["total_individuals"]), max(ind["total_individuals"]))) -ind["total_children"] <- lapply(ind["total_children"], factor, order=TRUE, levels=seq(min(ind["total_children"]), max(ind["total_children"]))) -ind["has_partner"] <- lapply(ind["has_partner"], as.logical) + print("Doing imputation via MICE...") + options(future.globals.maxSize=10485760000) -id_names <- grepl("^(id_)", colnames(ind)) # Boolean mask, not actual values -ids <- ind[id_names] -ind <- ind[ , !id_names] + start_time <- Sys.time() + imp <- futuremice(ind, + parallelseed = 123, + n.core = n_cores, + visitSequence = "monotone", + m = 1, + maxit = maxit, + method = "pmm", + pred = pred) + end_time <- Sys.time() + end_time - start_time + print("Done!") -pred <- quickpred(ind, mincor = 0.01) # about 10 minutes -# current version of quickpred does remove complete columns from the list of vars to be predicted -# automatically + imputed.data <- complete(imp, "long") -#to_predict <- names(which(colSums(is.na(ind)) > 0)) # contain NA values -#do_not_predict <- colnames(ind)[!colnames(ind) %in% to_predict] + # TODO converting from ordinal to integer/double increases the value by one + # TODO education is an ordinal variable + imputed.data <- cbind(ids, imputed.data) -#pred <- quickpred(ind, mincor = 0) -#pred[do_not_predict, ] <- 0 -options(future.globals.maxSize=10485760000) + print("Saving imputed data...") + out.path <- file.path(path.to.data, "synthwave", "imputed") + if (!dir.exists(out.path)) { + dir.create(out.path, recursive=TRUE) + } + out.full <- file.path(out.path, "imputed_data.csv") + write.csv(imputed.data, out.full) + print(paste0("Saved to: ", out.full)) +} -start_time <- Sys.time() -imp <- futuremice(ind, - parallelseed = 123, - n.core = 128, - visitSequence = "monotone", - m = 1, - maxit = 20, - method = "pmm", - pred = pred) -end_time <- Sys.time() -end_time - start_time -imputed_data <- complete(imp, "long") +ap <- arg_parser("imputation_stage") +ap <- add_argument(ap, "path_to_data", default=NULL, help="Data source path") +ap <- add_argument(ap, "--subset", default=FALSE, help="Take tiny subset for testing") +ap <- add_argument(ap, "--n_cores", default=N_CORES_DEFAULT, help="Number of cores to use for imputation") +ap <- add_argument(ap, "--maxit", default=MAXIT_DEFAULT, help="Maximum number of iterations in imputation" ) +args <- parse_args(ap) -# TODO converting from ordinal to integer/double increases the value by one -# TODO education is an ordinal variable -imputed_data <- cbind(ids, imputed_data) +path.to.data <- args$path_to_data +subset.data <- args$subset +n_cores <- args$n_cores +maxit <- args$maxit -write.csv(imputed_data, "out20.csv") +paste0('Imputing parquet data in folder ', path.to.data) +do_adults_imputation(path.to.data, subset.data, n_cores, maxit) +print('Done!') diff --git a/src/synthwave/synthesizer/uk/generator.py b/src/synthwave/synthesizer/uk/generator.py index a31d6f2..4f76201 100644 --- a/src/synthwave/synthesizer/uk/generator.py +++ b/src/synthwave/synthesizer/uk/generator.py @@ -1,3 +1,4 @@ +import os from sklearn.dummy import DummyClassifier import logging @@ -24,7 +25,7 @@ from sklearn.model_selection import train_test_split from importlib.resources import files -HOUSEHOLD_ID_MAP = {_v: _i for _i, _v in enumerate(["a0", "a1+", "c0", "c1", "c2", "c3+", "m2", "m3", "mc3", "m4", "mc4"])} +HOUSEHOLD_ID_MAP = {_v: _i for _i, _v in enumerate(["a0", "a1+", "c0", "c1", "c2", "c3+", "m2", "m3", "mc3", "m4", "mc4"])} # their relative order is irrelevant at the moment as long as it is the same across all data MAX_CHILDREN = yaml.safe_load(files("synthwave.data.understanding_society").joinpath('syntet.yaml').read_text())["MAX_CHILDREN"] @@ -83,8 +84,8 @@ def split_data(self): if not _g.endswith("+"): d = d.drop(columns=["total_children"]) if len(d) > 0: - self.subsets.append((_g, _r)) - self.groups[(_g, _r)] = {"data": d, "model": None, "dropouts": None} + self.subsets.append((_g, _r)) + self.groups[(_g, _r)] = {"data": d, "model": None, "dropouts": None} @staticmethod def _splitter(_df: pd.DataFrame, _type: list, _location: int) -> pd.DataFrame: @@ -296,44 +297,49 @@ def get_inequality(_columns: list, _table_name: str) -> dict: # the number of columns can vary from group to group # FIXME the benefits have been corrupted by imputation and therefore do not pass the constraint check ]) - if "income_person_second_job" + _p in self.groups[(_g, _r)]["data"].columns: + if "income_person_second_job" + _p in self.groups[(_g, _r)]["data"].columns: self.groups[(_g, _r)]["model"].add_constraints([ - MetaEmployment.get_schema( - ["indicator_person_is_self_employed" + _p, - "indicator_person_is_employed" + _p, + MetaEmployment.get_schema( + ["indicator_person_is_self_employed" + _p, + "indicator_person_is_employed" + _p, - "minutes_person_employment" + _p, - "income_person_pay" + _p, - "hours_person_overtime" + _p, + "minutes_person_employment" + _p, + "income_person_pay" + _p, + "hours_person_overtime" + _p, - "hours_person_self_employment" + _p, - "income_person_self_employment" + _p, + "hours_person_self_employment" + _p, + "income_person_self_employment" + _p, - "category_person_job_nssec" + _p, - "category_person_job_sic" + _p, + "category_person_job_nssec" + _p, + "category_person_job_sic" + _p, - "category_person_job_status" + _p, + "category_person_job_status" + _p, - "income_person_second_job" + _p])]) + "income_person_second_job" + _p])]) else: self.groups[(_g, _r)]["model"].add_constraints([ - MetaEmploymentNoSecondJob.get_schema( - ["indicator_person_is_self_employed" + _p, - "indicator_person_is_employed" + _p, + MetaEmploymentNoSecondJob.get_schema( + ["indicator_person_is_self_employed" + _p, + "indicator_person_is_employed" + _p, - "minutes_person_employment" + _p, - "income_person_pay" + _p, - "hours_person_overtime" + _p, + "minutes_person_employment" + _p, + "income_person_pay" + _p, + "hours_person_overtime" + _p, - "hours_person_self_employment" + _p, - "income_person_self_employment" + _p, + "hours_person_self_employment" + _p, + "income_person_self_employment" + _p, - "category_person_job_nssec" + _p, - "category_person_job_sic" + _p, + "category_person_job_nssec" + _p, + "category_person_job_sic" + _p, - "category_person_job_status" + _p])]) + "category_person_job_status" + _p])]) def train(self, save_path, verbose=False): + + if not os.path.exists(save_path): + print("Folder for trained data not found; creating at {}".format(save_path)) + os.makedirs(save_path) + for _g, _r in self.subsets: if verbose: print(_g, _r) @@ -341,9 +347,11 @@ def train(self, save_path, verbose=False): print(len(self.groups[(_g, _r)]["data"])) self.groups[(_g, _r)]["model"].fit(self.groups[(_g, _r)]["data"]) - self.groups[(_g, _r)]["model"].save(filepath=save_path + f'model_{_g}_{_r}.pkl') + fullpath = os.path.join(save_path, f'model_{_g}_{_r}.pkl') + self.groups[(_g, _r)]["model"].save(fullpath) - with open(save_path + f'dropouts_{_g}_{_r}.yaml', 'w') as yml: + yaml_path = os.path.join(save_path, f'dropouts_{_g}_{_r}.yaml') + with open(yaml_path, 'w') as yml: yaml.dump(self.groups[(_g, _r)]["dropouts"], yml, allow_unicode=True) @staticmethod @@ -402,12 +410,12 @@ def _generate_model(_target_code): # TODO rf is better than knn in that it can work fine without one hot encoding. still, current implementation does see *all* columns as numeric. works well though. see https://github.com/scikit-learn/scikit-learn/pull/12866 _rf = RandomForestClassifier(n_estimators=1024, - criterion='entropy', - max_depth=None, - max_features=None, - random_state=1, - n_jobs=4, - ) + criterion='entropy', + max_depth=None, + max_features=None, + random_state=1, + n_jobs=4, + ) X_train, X_test, y_train, y_test = train_test_split(_data_household[_full_base_predictors + _extra_children_predictors], _data_household[_target_map[_target_code]], @@ -560,14 +568,14 @@ def pad_children(_df: pd.DataFrame) -> pd.DataFrame: return _df def add_children(self, - _df: pd.DataFrame, - _max_household_children: int, - _household_type: str, - _household_location: int, - _model_location: str, - _mini_batch_id_: int = None, - _micro_batch_id_: int = None, - ) -> pd.DataFrame: + _df: pd.DataFrame, + _max_household_children: int, + _household_type: str, + _household_location: int, + _model_location: str, + _mini_batch_id_: int = None, + _micro_batch_id_: int = None, + ) -> pd.DataFrame: # this is the highest level function """Adds children to provided households @@ -672,12 +680,12 @@ def generator(self, # loop over children in a1+ _split.append( self.add_children(_df=synthetic_data[synthetic_data["total_children"] == _children], - _max_household_children = _children, - _household_type = _household_type, - _household_location = _household_location, - _model_location = "/tmp", # FIXME no hardcoded values - _mini_batch_id_ = _mini_batch_id, - _micro_batch_id_ = _micro_batch_id) + _max_household_children = _children, + _household_type = _household_type, + _household_location = _household_location, + _model_location = "/tmp", # FIXME no hardcoded values + _mini_batch_id_ = _mini_batch_id, + _micro_batch_id_ = _micro_batch_id) ) synthetic_data = pd.concat(_split) @@ -690,12 +698,12 @@ def generator(self, # couple, several children if _household_type in ["c1", "c2"]: synthetic_data = self.add_children(_df=synthetic_data, - _max_household_children = int(''.join(filter(str.isdigit, _household_type))), - _household_type = _household_type, - _household_location = _household_location, - _model_location = "/tmp", # FIXME no hardcoded values - _mini_batch_id_ = _mini_batch_id, - _micro_batch_id_ = _micro_batch_id) + _max_household_children = int(''.join(filter(str.isdigit, _household_type))), + _household_type = _household_type, + _household_location = _household_location, + _model_location = "/tmp", # FIXME no hardcoded values + _mini_batch_id_ = _mini_batch_id, + _micro_batch_id_ = _micro_batch_id) else: # loop over children c3+ _split = [] @@ -703,19 +711,19 @@ def generator(self, # loop over children in a1+ _split.append( self.add_children(_df=synthetic_data[synthetic_data["total_children"] == _children], - _max_household_children = _children, - _household_type = _household_type, - _household_location = _household_location, - _model_location = "/tmp", # FIXME no hardcoded values - _mini_batch_id_ = _mini_batch_id, - _micro_batch_id_ = _micro_batch_id) + _max_household_children = _children, + _household_type = _household_type, + _household_location = _household_location, + _model_location = "/tmp", # FIXME no hardcoded values + _mini_batch_id_ = _mini_batch_id, + _micro_batch_id_ = _micro_batch_id) ) synthetic_data = pd.concat(_split) elif _household_type.startswith("mc"): # couple + some other people synthetic_data = generate_personal_ids(synthetic_data, contains_couples=True) - else: # TODO this only works for m3, m4 not mf + else: # TODO this only works for m3, m4 not mf synthetic_data = generate_personal_ids(synthetic_data, contains_couples=False) return synthetic_data diff --git a/src/synthwave/utils/uk/pre_process.py b/src/synthwave/utils/uk/pre_process.py new file mode 100644 index 0000000..010c682 --- /dev/null +++ b/src/synthwave/utils/uk/pre_process.py @@ -0,0 +1,24 @@ +# HR 13/03/25 Getting pre-processing running from command line + +from synthwave.utils.uk.understanding_society import preprocess_usoc_data +import argparse + +def do_it(_path): + try: + print('Trying to get pre-processed ind and hh pickle files...') + preprocess_usoc_data(_path, skip_conversion=True) + print('Done!') + except: + print("Exception occurred, probably because the pickle files weren't found; running pre-processing step and caching pickles... ") + preprocess_usoc_data(_path, skip_conversion=False) + print('Done!') + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + parser.add_argument("p", type=str, help="Data source path") + + args = parser.parse_args() + us_path = args.p + do_it(us_path) diff --git a/src/synthwave/utils/uk/understanding_society.py b/src/synthwave/utils/uk/understanding_society.py index babe693..4763a8d 100644 --- a/src/synthwave/utils/uk/understanding_society.py +++ b/src/synthwave/utils/uk/understanding_society.py @@ -353,8 +353,12 @@ def preprocess_usoc_data(_path="~/Work/data/", skip_conversion=True): individuals, households = merge_usoc_data(_path + "synthwave") - individuals.to_pickle(_path + "synthwave/md/individuals.pkl") - households.to_pickle(_path + "synthwave/md/households.pkl") + _md_path = "synthwave/md/" + if not os.path.exists(_path + _md_path): + print("Preprocessing folder not found; creating at {}".format(_path + _md_path)) + os.makedirs(_path + _md_path) + individuals.to_pickle(_path + _md_path + "individuals.pkl") + households.to_pickle(_path + _md_path + "households.pkl") households = process_households(households) # NOTE this procedure might decrease the number of households From 3cdfa422471f6f4d3aa80be3210a338ec15a96d2 Mon Sep 17 00:00:00 2001 From: paddy-r Date: Sun, 23 Mar 2025 17:50:48 +0000 Subject: [PATCH 02/12] Avoiding horrible df size-dependent error in uk/generator --- src/synthwave/synthesizer/uk/generator.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/synthwave/synthesizer/uk/generator.py b/src/synthwave/synthesizer/uk/generator.py index fb2791e..fcb1822 100644 --- a/src/synthwave/synthesizer/uk/generator.py +++ b/src/synthwave/synthesizer/uk/generator.py @@ -481,7 +481,8 @@ def train_children(self, _df_children: pd.DataFrame, save_path: str = "/tmp/", v how="inner", # inner merge to avoid incomplete records on="id_household").drop(columns=["id_household"], errors="ignore") - comb = _df.apply(pd.unique) + # comb = _df.apply(pd.unique) + comb = _df.T.apply(lambda x: x.unique(), axis=1) # HR 88/89 Avoids horrible Pandas error exclude = comb[comb.map(len) == 1].map(lambda x: x[0]).to_dict().keys() exclude = [_e for _e in exclude if not _e.endswith(tuple(self._get_postfixes_children(_children)))] # do not drop out degenerate attributes of children @@ -528,7 +529,8 @@ def train_children(self, _df_children: pd.DataFrame, save_path: str = "/tmp/", v how="inner", # inner merge to avoid incomplete records on="id_household").drop(columns=["id_household"], errors="ignore") - comb = _df.apply(pd.unique) + # comb = _df.apply(pd.unique) + comb = _df.T.apply(lambda x: x.unique(), axis=1) # HR 88/89 Avoids horrible Pandas error exclude = comb[comb.map(len) == 1].map(lambda x: x[0]).to_dict().keys() exclude = [_e for _e in exclude if not _e.endswith(tuple(self._get_postfixes_children(_children)))] # do not drop out degenerate attributes of children From c5eff34f5291033ad28b63d3633a9778d8b067ee Mon Sep 17 00:00:00 2001 From: paddy-r Date: Mon, 24 Mar 2025 14:35:59 +0000 Subject: [PATCH 03/12] From 33e193b - removing degenerate groups (0/1 unique hhs) --- src/synthwave/synthesizer/uk/generator.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/synthwave/synthesizer/uk/generator.py b/src/synthwave/synthesizer/uk/generator.py index fcb1822..51663e4 100644 --- a/src/synthwave/synthesizer/uk/generator.py +++ b/src/synthwave/synthesizer/uk/generator.py @@ -482,7 +482,13 @@ def train_children(self, _df_children: pd.DataFrame, save_path: str = "/tmp/", v on="id_household").drop(columns=["id_household"], errors="ignore") # comb = _df.apply(pd.unique) - comb = _df.T.apply(lambda x: x.unique(), axis=1) # HR 88/89 Avoids horrible Pandas error + # comb = _df.T.apply(lambda x: x.unique(), axis=1) # HR 88/89 Avoids horrible Pandas error + if len(_df.drop_duplicates()) <= 1: + print(f"Extreme degeneracy detected in {_g}/{_r} with {_total_children} children, can't deal with 0 or 1 unique household in a group; dropping them out") + self.groups[(_g, _r)]["data"] = self.groups[(_g, _r)]["data"][self.groups[(_g, _r)]["data"]["total_children"].ne(_total_children)] + continue + + comb = _df.apply(pd.unique) exclude = comb[comb.map(len) == 1].map(lambda x: x[0]).to_dict().keys() exclude = [_e for _e in exclude if not _e.endswith(tuple(self._get_postfixes_children(_children)))] # do not drop out degenerate attributes of children @@ -530,7 +536,13 @@ def train_children(self, _df_children: pd.DataFrame, save_path: str = "/tmp/", v on="id_household").drop(columns=["id_household"], errors="ignore") # comb = _df.apply(pd.unique) - comb = _df.T.apply(lambda x: x.unique(), axis=1) # HR 88/89 Avoids horrible Pandas error + # comb = _df.T.apply(lambda x: x.unique(), axis=1) # HR 88/89 Avoids horrible Pandas error + if len(_df.drop_duplicates()) <= 1: + print(f"Extreme degeneracy detected in {_g}/{_r} with {_total_children} children, can't deal with 0 or 1 unique household in a group; dropping them out") + self.groups[(_g, _r)]["data"] = self.groups[(_g, _r)]["data"][self.groups[(_g, _r)]["data"]["total_children"].ne(_total_children)] + continue + + comb = _df.apply(pd.unique) exclude = comb[comb.map(len) == 1].map(lambda x: x[0]).to_dict().keys() exclude = [_e for _e in exclude if not _e.endswith(tuple(self._get_postfixes_children(_children)))] # do not drop out degenerate attributes of children From 0cdaf24b2e933903b44449a57b63ed2df65bb673 Mon Sep 17 00:00:00 2001 From: paddy-r Date: Tue, 25 Mar 2025 09:56:57 +0000 Subject: [PATCH 04/12] Simplifying adults_imputation subsetting - testing --- .../synthesizer/imputation/adults_imputation.R | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/synthwave/synthesizer/imputation/adults_imputation.R b/src/synthwave/synthesizer/imputation/adults_imputation.R index 0182129..db109e4 100644 --- a/src/synthwave/synthesizer/imputation/adults_imputation.R +++ b/src/synthwave/synthesizer/imputation/adults_imputation.R @@ -6,10 +6,10 @@ require(argparser) N_CORES_DEFAULT <- 128 MAXIT_DEFAULT <- 20 -PROP_DEFAULT <- 1e-2 +FRAC_DEFAULT <- 1e-2 -do_adults_imputation <- function(path.to.data, subset, n_cores, maxit) { +do_adults_imputation <- function(path.to.data, fraction, n_cores, maxit) { set.seed(123) @@ -73,14 +73,13 @@ do_adults_imputation <- function(path.to.data, subset, n_cores, maxit) { print("Done!") - if (subset == TRUE) + if (fraction != 1.0) { print("Subsetting data for testing...") print(paste0("Current size: ", nrow(ind), " by ", ncol(ind))) - prop <- PROP_DEFAULT - ind <- ind %>% slice_sample(prop=prop, replace=FALSE) - print(paste0("Length after subsetting (", as.character(prop*100), "%): ", nrow(ind), " by ", ncol(ind))) + ind <- ind %>% slice_sample(prop=fraction, replace=FALSE) + print(paste0("Length after subsetting (", as.character(fraction*100), "%): ", nrow(ind), " by ", ncol(ind))) } @@ -138,16 +137,16 @@ do_adults_imputation <- function(path.to.data, subset, n_cores, maxit) { ap <- arg_parser("imputation_stage") ap <- add_argument(ap, "path_to_data", default=NULL, help="Data source path") -ap <- add_argument(ap, "--subset", default=FALSE, help="Take tiny subset for testing") +ap <- add_argument(ap, "--fraction", default=FRAC_DEFAULT, help="Fraction to subset for testing") ap <- add_argument(ap, "--n_cores", default=N_CORES_DEFAULT, help="Number of cores to use for imputation") ap <- add_argument(ap, "--maxit", default=MAXIT_DEFAULT, help="Maximum number of iterations in imputation" ) args <- parse_args(ap) path.to.data <- args$path_to_data -subset.data <- args$subset +fraction <- args$fraction n_cores <- args$n_cores maxit <- args$maxit paste0('Imputing parquet data in folder ', path.to.data) -do_adults_imputation(path.to.data, subset.data, n_cores, maxit) +do_adults_imputation(path.to.data, fraction, n_cores, maxit) print('Done!') From 809de33ba07d8eff98ea509317e14e8f0b9f6f43 Mon Sep 17 00:00:00 2001 From: paddy-r Date: Tue, 25 Mar 2025 11:24:02 +0000 Subject: [PATCH 05/12] Draft Aire sumbission scripts - testing --- scripts/aire_run.sh | 34 ++++++++++++++++++++++++++++++++++ scripts/aire_submit.sh | 16 ++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 scripts/aire_run.sh create mode 100644 scripts/aire_submit.sh diff --git a/scripts/aire_run.sh b/scripts/aire_run.sh new file mode 100644 index 0000000..a859331 --- /dev/null +++ b/scripts/aire_run.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +################ +# Slurm settings +################ +#SBATCH --job-name=synthwave_preprocessing # Job name +#SBATCH --mail-type=FAIL # Mail events (NONE, BEGIN, END, FAIL, ALL) +#SBATCH --mail-type=END +#SBATCH --mail-user="$6" # Where to send mail +#SBATCH --array=1 # Number of runs, --array=1-X will run X jobs (X >= 1) +###SBATCH --ntasks=1 # Number of tasks to run, change as desired - disabled 21/03/24 as distinction with array not clear +#SBATCH --cpus-per-task=2 # Number of CPU cores per task +#SBATCH --mem=256gb # Job memory request +#SBATCH --time=01:00:00 # Time limit hrs:min:sec +#SBATCH --output=logs/errors/batch-%A-%a.out +#SBATCH --error=logs/logs/batch-%A-%a.err + + +echo -e "\nRunning Synthwave pre-processing steps... \n Source data path: $2\n Subset fraction: $4\n Emails to: $6\n Time: $8" +echo -e "Task $SLURM_JOB_ID" +echo -e "Running with $SLURM_CPUS_PER_TASK CPU cores, $SLURM_CPUS_ON_NODE CPU cores per node" +echo -e "Running task $SLURM_ARRAY_TASK_ID of $SLURM_ARRAY_TASK_MAX\n" + +export MAXIT=1 # Testing +export NCORES=2 # Testing + +python src/synthwave/utils/uk/pre_process.py "$2" +Rscript src/synthwave/synthesizer/imputation/adults_imputation.R "$2" -f "$4" -n $NCORES -m $MAXIT # Testing +#Rscript src/synthwave/synthesizer/imputation/adults_imputation.R "$2" -f "$4" -n $SLURM_CPUS_PER_TASK -m $MAXIT +#Rscript src/synthwave/synthesizer/imputation/adults_imputation.R "$2" -f "$4" -n $SLURM_CPUS_ON_NODE -m $MAXIT +# python src/synthwave/synthesizer/correct_and_train.py "$2" + +# If no errors... +exit 0 diff --git a/scripts/aire_submit.sh b/scripts/aire_submit.sh new file mode 100644 index 0000000..9cf7d57 --- /dev/null +++ b/scripts/aire_submit.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Set current time for directory naming +TIME=`date +%Y_%m_%d_%H_%M_%S` + +# Create these if they don't exist +mkdir -p logs +mkdir -p logs/log +mkdir -p logs/errors + + +#sbatch scripts/aire_run.sh -d "$2" -f "$4" -e "$6" -t $TIME +bash scripts/aire_run.sh -d "$2" -f "$4" -e "$6" -t $TIME # Testing + +# If no errors... +exit 0 From d882d8a273159960444b07d61f928be28fb79cc0 Mon Sep 17 00:00:00 2001 From: paddy-r Date: Tue, 25 Mar 2025 11:51:06 +0000 Subject: [PATCH 06/12] Removing R dependencies from setup.py --- setup.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index 8f08331..beb7d3c 100644 --- a/setup.py +++ b/setup.py @@ -39,14 +39,14 @@ "torch>=2.5.1", "matplotlib>=3.10.0", "jupyter", - "r-arrow", - "r-mice", - "r-dplyr", - "r-lattice", - "r-argparser", - "r-parallelly", - "r-future", - "r-furrr", + # "r-arrow", + # "r-mice", + # "r-dplyr", + # "r-lattice", + # "r-argparser", + # "r-parallelly", + # "r-future", + # "r-furrr", ], extras_require={ "dev": ["check-manifest"], From 5ad31bb16a88e913bfa773b7c38a248c416f1598 Mon Sep 17 00:00:00 2001 From: paddy-r Date: Wed, 26 Mar 2025 10:00:44 +0000 Subject: [PATCH 07/12] Moving logs/errors output + testing ncores --- scripts/aire_run.sh | 16 ++++++++-------- scripts/aire_submit.sh | 10 +++++----- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/scripts/aire_run.sh b/scripts/aire_run.sh index a859331..5e2b3ea 100644 --- a/scripts/aire_run.sh +++ b/scripts/aire_run.sh @@ -7,13 +7,13 @@ #SBATCH --mail-type=FAIL # Mail events (NONE, BEGIN, END, FAIL, ALL) #SBATCH --mail-type=END #SBATCH --mail-user="$6" # Where to send mail -#SBATCH --array=1 # Number of runs, --array=1-X will run X jobs (X >= 1) -###SBATCH --ntasks=1 # Number of tasks to run, change as desired - disabled 21/03/24 as distinction with array not clear +###SBATCH --array=1 # Number of runs, --array=1-X will run X jobs (X >= 1) +#SBATCH --ntasks=1 # Number of tasks to run, change as desired #SBATCH --cpus-per-task=2 # Number of CPU cores per task -#SBATCH --mem=256gb # Job memory request +#SBATCH --mem=64gb # Job memory request #SBATCH --time=01:00:00 # Time limit hrs:min:sec -#SBATCH --output=logs/errors/batch-%A-%a.out -#SBATCH --error=logs/logs/batch-%A-%a.err +#SBATCH --output="$2"/logs/logs/batch-%A-%a.out +#SBATCH --error="$2"/logs/errors/batch-%A-%a.err echo -e "\nRunning Synthwave pre-processing steps... \n Source data path: $2\n Subset fraction: $4\n Emails to: $6\n Time: $8" @@ -24,9 +24,9 @@ echo -e "Running task $SLURM_ARRAY_TASK_ID of $SLURM_ARRAY_TASK_MAX\n" export MAXIT=1 # Testing export NCORES=2 # Testing -python src/synthwave/utils/uk/pre_process.py "$2" -Rscript src/synthwave/synthesizer/imputation/adults_imputation.R "$2" -f "$4" -n $NCORES -m $MAXIT # Testing -#Rscript src/synthwave/synthesizer/imputation/adults_imputation.R "$2" -f "$4" -n $SLURM_CPUS_PER_TASK -m $MAXIT +#python src/synthwave/utils/uk/pre_process.py "$2" +#Rscript src/synthwave/synthesizer/imputation/adults_imputation.R "$2" -f "$4" -n $NCORES -m $MAXIT # Testing +Rscript src/synthwave/synthesizer/imputation/adults_imputation.R "$2" -f "$4" -n $SLURM_CPUS_PER_TASK -m $MAXIT #Rscript src/synthwave/synthesizer/imputation/adults_imputation.R "$2" -f "$4" -n $SLURM_CPUS_ON_NODE -m $MAXIT # python src/synthwave/synthesizer/correct_and_train.py "$2" diff --git a/scripts/aire_submit.sh b/scripts/aire_submit.sh index 9cf7d57..54bd2e6 100644 --- a/scripts/aire_submit.sh +++ b/scripts/aire_submit.sh @@ -4,13 +4,13 @@ TIME=`date +%Y_%m_%d_%H_%M_%S` # Create these if they don't exist -mkdir -p logs -mkdir -p logs/log -mkdir -p logs/errors +mkdir -p "$2"/logs +mkdir -p "$2"/logs/logs +mkdir -p "$2"/logs/errors -#sbatch scripts/aire_run.sh -d "$2" -f "$4" -e "$6" -t $TIME -bash scripts/aire_run.sh -d "$2" -f "$4" -e "$6" -t $TIME # Testing +sbatch scripts/aire_run.sh -d "$2" -f "$4" -e "$6" -t $TIME +#bash scripts/aire_run.sh -d "$2" -f "$4" -e "$6" -t $TIME # Testing # If no errors... exit 0 From 4c53896d842b1ddab21dff8f33905b1f6e022107 Mon Sep 17 00:00:00 2001 From: prehr prehr Date: Wed, 26 Mar 2025 11:16:00 +0000 Subject: [PATCH 08/12] Unbreaking log paths + gitignore - can't pass arguments to sbatch - testing --- .gitignore | 7 ++++++- scripts/aire_run.sh | 8 ++++---- scripts/aire_submit.sh | 6 +++--- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 84c4117..ea1ef21 100644 --- a/.gitignore +++ b/.gitignore @@ -129,4 +129,9 @@ dmypy.json .pyre/ # IDE -.idea/ \ No newline at end of file +.idea/ + +# HPC output +logs/ +logs/* +logs/*/* diff --git a/scripts/aire_run.sh b/scripts/aire_run.sh index 5e2b3ea..f3b0d84 100644 --- a/scripts/aire_run.sh +++ b/scripts/aire_run.sh @@ -6,14 +6,14 @@ #SBATCH --job-name=synthwave_preprocessing # Job name #SBATCH --mail-type=FAIL # Mail events (NONE, BEGIN, END, FAIL, ALL) #SBATCH --mail-type=END -#SBATCH --mail-user="$6" # Where to send mail -###SBATCH --array=1 # Number of runs, --array=1-X will run X jobs (X >= 1) +#SBATCH --mail-user=h.p.rice@leeds.ac.uk # Where to send mail +#SBATCH --array=1 # Number of runs, --array=1-X will run X jobs (X >= 1) #SBATCH --ntasks=1 # Number of tasks to run, change as desired #SBATCH --cpus-per-task=2 # Number of CPU cores per task #SBATCH --mem=64gb # Job memory request #SBATCH --time=01:00:00 # Time limit hrs:min:sec -#SBATCH --output="$2"/logs/logs/batch-%A-%a.out -#SBATCH --error="$2"/logs/errors/batch-%A-%a.err +#SBATCH --output=logs/logs/batch-%A-%a.out +#SBATCH --error=logs/errors/batch-%A-%a.err echo -e "\nRunning Synthwave pre-processing steps... \n Source data path: $2\n Subset fraction: $4\n Emails to: $6\n Time: $8" diff --git a/scripts/aire_submit.sh b/scripts/aire_submit.sh index 54bd2e6..e1f99a8 100644 --- a/scripts/aire_submit.sh +++ b/scripts/aire_submit.sh @@ -4,9 +4,9 @@ TIME=`date +%Y_%m_%d_%H_%M_%S` # Create these if they don't exist -mkdir -p "$2"/logs -mkdir -p "$2"/logs/logs -mkdir -p "$2"/logs/errors +mkdir -p logs +mkdir -p logs/logs +mkdir -p logs/errors sbatch scripts/aire_run.sh -d "$2" -f "$4" -e "$6" -t $TIME From 17f62474387beee245c8414082cae02bdcd73850 Mon Sep 17 00:00:00 2001 From: prehr prehr Date: Wed, 26 Mar 2025 15:39:19 +0000 Subject: [PATCH 09/12] Tinkering with ncores - testing --- scripts/aire_run.sh | 3 +-- .../synthesizer/imputation/adults_imputation.R | 12 ++++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/scripts/aire_run.sh b/scripts/aire_run.sh index f3b0d84..8d3c903 100644 --- a/scripts/aire_run.sh +++ b/scripts/aire_run.sh @@ -9,7 +9,7 @@ #SBATCH --mail-user=h.p.rice@leeds.ac.uk # Where to send mail #SBATCH --array=1 # Number of runs, --array=1-X will run X jobs (X >= 1) #SBATCH --ntasks=1 # Number of tasks to run, change as desired -#SBATCH --cpus-per-task=2 # Number of CPU cores per task +#SBATCH --cpus-per-task=8 # Number of CPU cores per task #SBATCH --mem=64gb # Job memory request #SBATCH --time=01:00:00 # Time limit hrs:min:sec #SBATCH --output=logs/logs/batch-%A-%a.out @@ -22,7 +22,6 @@ echo -e "Running with $SLURM_CPUS_PER_TASK CPU cores, $SLURM_CPUS_ON_NODE CPU co echo -e "Running task $SLURM_ARRAY_TASK_ID of $SLURM_ARRAY_TASK_MAX\n" export MAXIT=1 # Testing -export NCORES=2 # Testing #python src/synthwave/utils/uk/pre_process.py "$2" #Rscript src/synthwave/synthesizer/imputation/adults_imputation.R "$2" -f "$4" -n $NCORES -m $MAXIT # Testing diff --git a/src/synthwave/synthesizer/imputation/adults_imputation.R b/src/synthwave/synthesizer/imputation/adults_imputation.R index db109e4..3248e12 100644 --- a/src/synthwave/synthesizer/imputation/adults_imputation.R +++ b/src/synthwave/synthesizer/imputation/adults_imputation.R @@ -3,6 +3,8 @@ require(lattice) require(dplyr) require(arrow) require(argparser) +require(future) +require(parallel) N_CORES_DEFAULT <- 128 MAXIT_DEFAULT <- 20 @@ -11,6 +13,11 @@ FRAC_DEFAULT <- 1e-2 do_adults_imputation <- function(path.to.data, fraction, n_cores, maxit) { + print("availableCores:") + print(availableCores()) + print("detectCores:") + print(detectCores()) + set.seed(123) print("Getting parquet file...") @@ -101,7 +108,8 @@ do_adults_imputation <- function(path.to.data, fraction, n_cores, maxit) { print("Done!") - print("Doing imputation via MICE...") + print("Doing imputation via MICE with n cores...") + print(n_cores) options(future.globals.maxSize=10485760000) start_time <- Sys.time() @@ -114,7 +122,7 @@ do_adults_imputation <- function(path.to.data, fraction, n_cores, maxit) { method = "pmm", pred = pred) end_time <- Sys.time() - end_time - start_time + print(end_time - start_time) print("Done!") imputed.data <- complete(imp, "long") From 8db626e9b223be0841c5fd13ca6467377f876a54 Mon Sep 17 00:00:00 2001 From: paddy-r Date: Fri, 4 Apr 2025 17:05:12 +0100 Subject: [PATCH 10/12] Updating setup with seaborn --- setup.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/setup.py b/setup.py index beb7d3c..6764383 100644 --- a/setup.py +++ b/setup.py @@ -39,14 +39,7 @@ "torch>=2.5.1", "matplotlib>=3.10.0", "jupyter", - # "r-arrow", - # "r-mice", - # "r-dplyr", - # "r-lattice", - # "r-argparser", - # "r-parallelly", - # "r-future", - # "r-furrr", + "seaborn", ], extras_require={ "dev": ["check-manifest"], From b0a3dc54aa77a2da8cae276aa10f771a01c5d6d9 Mon Sep 17 00:00:00 2001 From: paddy-r Date: Mon, 28 Apr 2025 08:28:32 +0100 Subject: [PATCH 11/12] Avoiding type warnings with infer_objects --- src/synthwave/synthesizer/uk/generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/synthwave/synthesizer/uk/generator.py b/src/synthwave/synthesizer/uk/generator.py index e075dbe..1f89b1c 100644 --- a/src/synthwave/synthesizer/uk/generator.py +++ b/src/synthwave/synthesizer/uk/generator.py @@ -71,7 +71,7 @@ def convert_types(self): if _c.split("_")[0] in ["category", "hours", "ordinal", "income", "ordinal", "total", "minutes"] and _c != "category_household_type": _int_target.append(_c) - self.groups[_g]["data"][_int_target] = self.groups[_g]["data"][_int_target].astype(int) + self.groups[_g]["data"][_int_target] = self.groups[_g]["data"][_int_target].infer_objects(copy=False).fillna(0).astype(int) _bools = [_c for _c in _columns if _c.startswith(("indicator_", "mlb_"))] self.groups[_g]["data"][_bools] = self.groups[_g]["data"][_bools].astype(bool) From 387914b6850f11912613bb17619124be19c48022 Mon Sep 17 00:00:00 2001 From: paddy-r Date: Fri, 30 May 2025 16:40:38 +0100 Subject: [PATCH 12/12] Chopping correct_and_train for testing to avoid rerunning main method --- .../synthesizer/correct_and_train.py | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/synthwave/synthesizer/correct_and_train.py b/src/synthwave/synthesizer/correct_and_train.py index 770761c..c36a3fd 100644 --- a/src/synthwave/synthesizer/correct_and_train.py +++ b/src/synthwave/synthesizer/correct_and_train.py @@ -47,17 +47,20 @@ def main(data_path, save_path=None): generator.convert_types() generator.init_models(_epochs=1) generator.attach_constraints() - - print("Running main training...") - generator.train(save_path=save_path) - print("Done!") + return generator if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("p", type=str, help="Data source path") + # parser = argparse.ArgumentParser() + # parser.add_argument("p", type=str, help="Data source path") + # + # args = parser.parse_args() + # data_path = args.p - args = parser.parse_args() - data_path = args.p - main(data_path) + # Run main training - popping this out for testing + data_path = '/home/hpr/data/' + g = main(data_path) + print("Running main training...") + g.train(save_path=data_path) + print("Done!")