From 0923a6197d4736a7211d56c781323397ca82d7d7 Mon Sep 17 00:00:00 2001 From: ens-lgil Date: Mon, 18 May 2026 13:51:48 +0100 Subject: [PATCH] Update the way to filter the list of datasets data to export, by creating a new common class. --- exports/config_template.py | 9 +++++++-- exports/datasets.py | 19 +++++++++++++++++++ exports/scripts/generate_metadata_exports.py | 10 +++++----- exports/scripts/generate_phewas_exports.py | 19 +++++-------------- exports/scripts/generate_sqlite_files.py | 4 +++- 5 files changed, 39 insertions(+), 22 deletions(-) create mode 100644 exports/datasets.py diff --git a/exports/config_template.py b/exports/config_template.py index 50e9040..f7f17d4 100644 --- a/exports/config_template.py +++ b/exports/config_template.py @@ -1,3 +1,10 @@ +## Dataset selection +dataset_selection = { '': '' } +# e.g.: +# 'publication_id': 1 +# 'num': 208 +# 'id__in': ['OPD000056','OPD000105','OPD000154','OPD000203'] +# 'num__in': ['56','105','154','203'] # Metadata exports metadata_exports_publication_id = '' @@ -6,7 +13,6 @@ # PheWAS exports phewas_exports_dir = '' - # SQLite exports sqlite_default_values = { 'opp_id': '', # e.g. OPP000003 @@ -17,7 +23,6 @@ 'use_different_id_as_gene': '' # Use the OmicsPred ID, the score name in the 'gene' column or not. None if the key is missing } - # Scoring files exports - only for PredictDB studies scoring_file_config = { 'pmid': '', # e.g. 32913098 diff --git a/exports/datasets.py b/exports/datasets.py new file mode 100644 index 0000000..102b46f --- /dev/null +++ b/exports/datasets.py @@ -0,0 +1,19 @@ +from django.db.models import Q +from omicspred.models import Dataset +from exports.config import dataset_selection + +class DatasetsSelection(): + + def __init__(self): + print(f'dataset_selection: {dataset_selection}') + ds_keys = list(dataset_selection.keys()) + print(f'dataset_selection keys: {ds_keys}') + self.dataset_col = ds_keys[0] + self.dataset_value = dataset_selection[self.dataset_col] + + def get_datasets(self) -> list: + param = self.dataset_col + param = Q(**{f'{self.dataset_col}':self.dataset_value}) + + datasets = Dataset.objects.filter(param).order_by('num') + return datasets \ No newline at end of file diff --git a/exports/scripts/generate_metadata_exports.py b/exports/scripts/generate_metadata_exports.py index f7de17d..3076a96 100644 --- a/exports/scripts/generate_metadata_exports.py +++ b/exports/scripts/generate_metadata_exports.py @@ -1,16 +1,16 @@ from datetime import date from omicspred.models import Dataset from exports.metadata_build_export import MetadataExport +from exports.datasets import DatasetsSelection from exports.config import metadata_exports_dir, metadata_exports_publication_id, sqlite_exports_dir from django.db.models import Q def run(): - # datasets = Dataset.objects.filter(Q(publication_id=metadata_exports_publication_id) & Q(name__icontains='sQTL - Enet')).order_by('num') - # datasets = Dataset.objects.filter(Q(publication_id=metadata_exports_publication_id) & Q(num__lte=56)).order_by('num') - datasets = Dataset.objects.filter(publication_id=metadata_exports_publication_id).order_by('num') - # datasets = Dataset.objects.all().order_by('num') - # datasets = Dataset.objects.filter(num=6).order_by('num') + # Fetch dataset(s) + ds_selection = DatasetsSelection() + datasets = ds_selection.get_datasets() + print("## Start metadata exports, dataset by dataset") datasets_total = len(datasets) count_dataset = 0 diff --git a/exports/scripts/generate_phewas_exports.py b/exports/scripts/generate_phewas_exports.py index 6b8f879..4838d25 100644 --- a/exports/scripts/generate_phewas_exports.py +++ b/exports/scripts/generate_phewas_exports.py @@ -1,20 +1,16 @@ from omicspred.models import Dataset from exports.phewas_export import PheWASExport +from exports.datasets import DatasetsSelection from exports.config import phewas_exports_dir -# from django.db.models import Q # Need to run first the script import/phewas/add_mappings (and store the SQLite file in /metadata) def run(): - # datasets = Dataset.objects.filter(Q(publication_id=metadata_exports_publication_id) & Q(name__icontains='sQTL - Enet')).order_by('num') - # datasets = Dataset.objects.filter(Q(publication_id=metadata_exports_publication_id) & Q(num__lte=56)).order_by('num') - # datasets = Dataset.objects.filter(publication_id=metadata_exports_publication_id).order_by('num') - # datasets = Dataset.objects.all().order_by('num') - datasets = Dataset.objects.filter(publication_id=6).order_by('num') - # datasets = Dataset.objects.filter(id__in=['OPD000056','OPD000105','OPD000154','OPD000203']) - # datasets = Dataset.objects.filter(id__in=['OPD000204','OPD000205','OPD000208','OPD000209','OPD000210','OPD000211','OPD000212','OPD000213']).order_by('num') - # datasets = Dataset.objects.filter(num__in=[2]).order_by('num') + # Fetch dataset(s) + ds_selection = DatasetsSelection() + datasets = ds_selection.get_datasets() + print("## Start PheWAS exports, dataset by dataset") datasets_total = len(datasets) count_dataset = 0 @@ -26,10 +22,5 @@ def run(): # Prepare data for exports - only DB phewas_export = PheWASExport(phewas_export_file, phewas_exports_dir, dataset) - - # Prepare data for exports - DB + file - # raw_file_dir = '/Users/lg10/Workspace/datafiles/OmicsPred/phenotypes' - # phewas_exports_dir_all = f'{phewas_exports_dir}/all' - # phewas_export = PheWASExport(phewas_export_file, phewas_exports_dir_all, dataset, raw_file_dir) phewas_export.generate_export() \ No newline at end of file diff --git a/exports/scripts/generate_sqlite_files.py b/exports/scripts/generate_sqlite_files.py index 57d9aa4..5aa3bfa 100644 --- a/exports/scripts/generate_sqlite_files.py +++ b/exports/scripts/generate_sqlite_files.py @@ -1,5 +1,6 @@ from omicspred.models import * from exports.sqlite_export import SqliteExport +from exports.datasets import DatasetsSelection from exports.config import sqlite_default_values @@ -98,7 +99,8 @@ def run(*args): skip_zip = sqlite_default_values['skip_zip'] # Fetch dataset(s) - datasets = Dataset.objects.filter(publication__id=opp_id).order_by('id') + ds_selection = DatasetsSelection() + datasets = ds_selection.get_datasets() # Create SqliteExport object sqlite_export = SqliteExport(opp_id,output_sqlite_dir,scoring_files_dir,datasets,use_different_id_as_gene) # Generate SQLite export file(s)