diff --git a/.github/workflows/django_ci.yml b/.github/workflows/django_ci.yml index eea3801..d7cf620 100644 --- a/.github/workflows/django_ci.yml +++ b/.github/workflows/django_ci.yml @@ -39,7 +39,7 @@ jobs: services: postgres: - image: postgres:15 + image: postgres:18 # Provide the user/password/db for postgres env: POSTGRES_USER: postgres @@ -58,7 +58,7 @@ jobs: strategy: max-parallel: 2 matrix: - python-version: [3.13] + python-version: [3.14] steps: - uses: actions/checkout@v4 diff --git a/config/settings.py b/config/settings.py index 516f4ec..9fb25e5 100644 --- a/config/settings.py +++ b/config/settings.py @@ -267,7 +267,7 @@ # REST API Settings # #---------------------# -REST_API_VERSION = '1.2.0' +REST_API_VERSION = '1.2.1' #REST_SAFELIST_IPS = [ # '127.0.0.1' diff --git a/exports/config_template.py b/exports/config_template.py index f7f17d4..4a93a8c 100644 --- a/exports/config_template.py +++ b/exports/config_template.py @@ -7,8 +7,8 @@ # 'num__in': ['56','105','154','203'] # Metadata exports -metadata_exports_publication_id = '' metadata_exports_dir = '' +sqlite_exports_dir = '' # Uncompressed databases # PheWAS exports phewas_exports_dir = '' diff --git a/exports/exports.py b/exports/exports.py index 1214cdb..e061610 100644 --- a/exports/exports.py +++ b/exports/exports.py @@ -17,6 +17,7 @@ {'name': 'name', 'label': 'Dataset Name'}, {'name': 'omics_type', 'label': 'Omics Type' }, {'name': 'method_name', 'label': 'Development Method'}, + {'name': 'training_window', 'label': 'Training Window'}, {'name': 'scores_count', 'label': 'Scores Count'}, {'name': 'phewas_count', 'label': 'PheWAS Count'}, {'name': 'platform__name', 'label': 'Platform Name'}, @@ -29,7 +30,6 @@ {'name': 'file_url_scoring_files_hm_38', 'label': 'Harmonised scoring files (mapped to GRCh38)', 'skip_auto_import': True}, {'name': 'file_url_scoring_files', 'label': 'Scoring files', 'skip_auto_import': True}, {'name': 'file_url_predictdb', 'label': 'PredictDB', 'skip_auto_import': True}, - {'name': 'file_url_covariance', 'label': 'Covariance', 'skip_auto_import': True}, {'name': 'file_url_phewas', 'label': 'PheWAS', 'skip_auto_import': True}, {'name': 'file_url_phewas_full', 'label': 'PheWAS Full', 'skip_auto_import': True}, {'name': 'file_url_validation_results', 'label': 'Validation data file', 'skip_auto_import': True}, @@ -71,7 +71,9 @@ {'name': 'sample__sample_age', 'label': 'Mean Sample Age'}, {'name': 'sample__sample_age_sd', 'label': 'Standard Deviation of Age'}, {'name': 'sample__ancestry_broad', 'label': 'Broad Ancestry Category'}, + {'name': 'sample__ancestry_assignment', 'label': 'Ancestry Assignment Method'}, {'name': 'cohorts', 'label': 'Cohort(s)', 'skip_auto_import': True}, + {'name': 'sample__cohorts_additional', 'label': 'Cohort(s) additional information'}, {'name': 'metrics_r2', 'label': 'R2', 'skip_auto_import': True}, {'name': 'metrics_r2_pval', 'label': 'R2 - p-value', 'skip_auto_import': True}, {'name': 'metrics_rho', 'label': 'Rho', 'skip_auto_import': True}, diff --git a/exports/metadata_build_export.py b/exports/metadata_build_export.py index 302ba81..f20a43d 100644 --- a/exports/metadata_build_export.py +++ b/exports/metadata_build_export.py @@ -13,7 +13,7 @@ class MetadataExport: def __init__(self, exports_dir:str, sqlite_dir:str, dataset:Dataset): self.dataset = dataset self.dataset_id = dataset.id - self.sqlite_dir = sqlite_dir + self.sqlite_dir = f'{sqlite_dir}/{self.dataset_id}' self.data = { 'dataset': self.get_data_attr(self.dataset,'Dataset'), 'publication': self.get_data_attr(self.dataset.publication,'Publication'), @@ -26,15 +26,16 @@ def __init__(self, exports_dir:str, sqlite_dir:str, dataset:Dataset): # Find corresponding SQLite file self.sqlite_file = None - for s_file in os.listdir(sqlite_dir): + for s_file in os.listdir(self.sqlite_dir): if s_file.startswith(dataset.id) and s_file.endswith('.db'): self.sqlite_file = (s_file) break if not self.sqlite_file: - print(f"ERROR: Can't find a SQLite file for the dataset {dataset.id} in {sqlite_dir}") + print(f"ERROR: Can't find a SQLite file for the dataset {self.dataset_id} in {self.sqlite_dir}") exit() - self.filename = f'{exports_dir}/{self.dataset_id}_metadata.xlsx' + filename = self.sqlite_file.replace('.db','_metadata.xlsx') + self.filepath = f'{exports_dir}/{filename}' def get_data_attr(self, model:object, model_type:str)-> dict: @@ -99,6 +100,9 @@ def build_performance_metadata(self, score:Score): # Update eval_type using the long name sample_perf_data['eval_type'] = perf.get_eval_type_display() + # Update the Sample ancestry_assignment using the long name + sample_perf_data['sample__ancestry_assignment'] = perf.sample.get_ancestry_assignment_display() + # Cohorts cohorts_list = [] for cohort in perf.sample.cohorts.all(): @@ -206,6 +210,6 @@ def generate_metadata(self): self.build_performance_metadata(score) # Create export - op_export = OPExport(self.filename, self.data) + op_export = OPExport(self.filepath, self.data) op_export.generate_sheets() op_export.save() \ No newline at end of file diff --git a/exports/scripts/generate_metadata_exports.py b/exports/scripts/generate_metadata_exports.py index 3076a96..43b651c 100644 --- a/exports/scripts/generate_metadata_exports.py +++ b/exports/scripts/generate_metadata_exports.py @@ -2,8 +2,7 @@ from omicspred.models import Dataset from exports.metadata_build_export import MetadataExport from exports.datasets import DatasetsSelection -from exports.config import metadata_exports_dir, metadata_exports_publication_id, sqlite_exports_dir -from django.db.models import Q +from exports.config import metadata_exports_dir, sqlite_exports_dir def run(): diff --git a/exports/tests/config.py b/exports/tests/config.py index 3da21cc..34c81e4 100644 --- a/exports/tests/config.py +++ b/exports/tests/config.py @@ -1,7 +1,9 @@ import os +dataset_selection = { 'publication_id': 1 } + current_dir = os.getcwd() -metadata_exports_publication_id = 1 +sqlite_exports_dir = current_dir+'/exports/tests/datas' metadata_exports_dir = current_dir+'/exports/tests/output' phewas_exports_dir = current_dir+'/exports/tests/output' sqlite_exports_dir = current_dir+'/exports/tests/data/' diff --git a/exports/tests/data/OPD000001_DS1.db b/exports/tests/data/OPD000001_DS1.db deleted file mode 100644 index 7d2eef0..0000000 Binary files a/exports/tests/data/OPD000001_DS1.db and /dev/null differ diff --git a/exports/tests/data/OPD000001_metadata.xlsx b/exports/tests/data/OPD000001_metadata.xlsx deleted file mode 100644 index c3db4af..0000000 Binary files a/exports/tests/data/OPD000001_metadata.xlsx and /dev/null differ diff --git a/exports/tests/test_sqlite_export.py b/exports/tests/test_sqlite_export.py index 1c119ea..21e2fb8 100644 --- a/exports/tests/test_sqlite_export.py +++ b/exports/tests/test_sqlite_export.py @@ -23,8 +23,8 @@ class ExportSQLiteTest(TestCase): sqlite_filename = f'{opd_id}_{dataset_label}.db' # SQLite ref - ref_sqlite_dir = current_dir+'/exports/tests/data/' - ref_sqlite_filepath = f'{ref_sqlite_dir}{sqlite_filename}' + ref_sqlite_dir = current_dir+'/exports/tests/data/'+opd_id + ref_sqlite_filepath = f'{ref_sqlite_dir}/{sqlite_filename}' # SQLite Output output_sqlite_dir = sqlite_default_values['sqlite_dir'] output_sqlite_filepath = f'{output_sqlite_dir}{sqlite_filename}' diff --git a/exports/tests/tests_metadata_export.py b/exports/tests/tests_metadata_export.py index 21e312d..acfe1ba 100644 --- a/exports/tests/tests_metadata_export.py +++ b/exports/tests/tests_metadata_export.py @@ -1,9 +1,10 @@ import os import pandas as pd from django.test import TestCase +from django.db.models import Q from omicspred.models import * from exports.metadata_build_export import MetadataExport -from exports.tests.config import metadata_exports_dir,metadata_exports_publication_id, sqlite_exports_dir +from exports.tests.config import metadata_exports_dir, sqlite_exports_dir, dataset_selection class ExportMetadataTest(TestCase): @@ -12,13 +13,13 @@ class ExportMetadataTest(TestCase): # Load data in DB - from the rest_api/fixtures/ directory fixtures = ['db_test'] databases = {'default'} - filename = 'OPD000001_metadata.xlsx' + filename = 'OPD000001_DS1_metadata.xlsx' spreadsheets = ['Publication','Dataset','Scores','Performances','Cohorts'] current_dir = os.getcwd() output_export_dir = metadata_exports_dir - data_export_dir = current_dir+'/exports/tests/data' + data_export_dir = current_dir+'/exports/tests/data/OPD000001' output_filepath = f'{output_export_dir}/{filename}' # Remove output metadata file before creating it again if os.path.isdir(output_export_dir): @@ -36,7 +37,8 @@ class ExportMetadataTest(TestCase): except OSError: print (f'Creation of the directory {output_export_dir} failed') exit() - + + def check_file(self): ''' Check file exists and is not empty ''' print(f' - Check file {self.filename}') @@ -58,14 +60,20 @@ def check_spreadsheets(self): df_test = pd.read_excel(self.output_filepath, sheet_name=sp_name, index_col=0) df_ref = pd.read_excel(ref_filepath, sheet_name=sp_name, index_col=0) # Columns - self.assertEqual(df_test.columns.all(),df_ref.columns.all()) + self.assertEqual(len(df_test.columns),len(df_ref.columns)) # Rows self.assertEqual(len(df_test.index), len(df_ref.index)) def test_metadata_export(self): print("# Test Metadata export") - datasets = Dataset.objects.filter(publication_id=metadata_exports_publication_id).order_by('num') + + # Mimic DatasetsSelection + col = list(dataset_selection.keys())[0] + value = dataset_selection[col] + param = Q(**{f'{col}':value}) + + datasets = Dataset.objects.filter(param).order_by('num') dataset = datasets[0] metadata2export = MetadataExport(self.output_export_dir,sqlite_exports_dir, dataset) metadata2export.generate_metadata() diff --git a/imports/omicspred/models/platform.py b/imports/omicspred/models/platform.py index f084fae..7e907b0 100644 --- a/imports/omicspred/models/platform.py +++ b/imports/omicspred/models/platform.py @@ -8,7 +8,7 @@ class PlatformMasterData(GenericData): - def __init__(self,name,type,full_name=None,technic=None): + def __init__(self,name,type,full_name=None,technique=None): GenericData.__init__(self) self.name = name self.data = { @@ -17,8 +17,8 @@ def __init__(self,name,type,full_name=None,technic=None): } if full_name: self.data['full_name'] = full_name - if technic: - self.data['technic'] = technic + if technique: + self.data['technique'] = technique def check_model_exist(self): diff --git a/imports/omicspred/parsers/data_content.py b/imports/omicspred/parsers/data_content.py index 752db81..dc9e1d5 100644 --- a/imports/omicspred/parsers/data_content.py +++ b/imports/omicspred/parsers/data_content.py @@ -16,7 +16,7 @@ 'method_name': method_name, 'full_name': 'Olink', 'version': 'Explore', - 'technic': 'antibody-based proximity extension assay for proteins', + 'technique': 'antibody-based proximity extension assay for proteins', 'dataset_name': 'UKB European', 'internal_cohort': 'UKB', 'internal_label': internal_label, @@ -43,7 +43,7 @@ 'method_name': method_name, 'full_name': 'Olink', 'version': 'Explore', - 'technic': 'antibody-based proximity extension assay for proteins', + 'technique': 'antibody-based proximity extension assay for proteins', 'dataset_name': 'UKB Multi-ancestry', 'internal_cohort': 'UKB', 'internal_label': internal_label, diff --git a/omicspred/migrations/0001_initial.py b/omicspred/migrations/0001_initial.py index f5ffa4a..09fda48 100644 --- a/omicspred/migrations/0001_initial.py +++ b/omicspred/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 6.0.5 on 2026-05-12 10:40 +# Generated by Django 6.0.6 on 2026-07-03 13:41 import django.contrib.postgres.fields.ranges import django.core.validators @@ -34,6 +34,7 @@ class Migration(migrations.Migration): ('omics_count', models.IntegerField(verbose_name='Omics Entities count')), ('omics_type', models.CharField(max_length=50, verbose_name='Omics type')), ('method_name', models.TextField(verbose_name='Score Development Method')), + ('training_window', models.CharField(choices=[('', ''), ('genome-wide', 'Genome-wide'), ('cis-only', 'Cis-only')], db_index=True, default='', max_length=25)), ('scores_count', models.IntegerField(verbose_name='Associated Scores count')), ('phewas_count', models.IntegerField(default=0, verbose_name='Associated PheWAS data count')), ('files_ids', models.JSONField(default=dict, verbose_name='Files IDs on Box')), @@ -43,6 +44,15 @@ class Migration(migrations.Migration): 'get_latest_by': 'num', }, ), + migrations.CreateModel( + name='ExternalSource', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=50, verbose_name='External Source Name')), + ('version', models.CharField(max_length=50, null=True, verbose_name='External Source Version')), + ('url', models.CharField(max_length=100, verbose_name='External Source URL')), + ], + ), migrations.CreateModel( name='Pathway', fields=[ @@ -89,7 +99,7 @@ class Migration(migrations.Migration): ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('name', models.CharField(max_length=100, verbose_name='Platform name')), ('full_name', models.CharField(max_length=100, verbose_name='Platform full name')), - ('technic', models.CharField(max_length=100, verbose_name='Platform technic')), + ('technique', models.CharField(max_length=100, verbose_name='Platform technique')), ('type', models.CharField(max_length=100, verbose_name='Platform type')), ], ), @@ -280,6 +290,7 @@ class Migration(migrations.Migration): ('ancestry_free', models.TextField(null=True, verbose_name='Ancestry (e.g. French, Chinese)')), ('ancestry_country', models.TextField(null=True, verbose_name='Country of Recruitment')), ('ancestry_additional', models.TextField(null=True, verbose_name='Additional Ancestry Description')), + ('ancestry_assignment', models.CharField(choices=[('SR', 'Self reported'), ('GS', 'Genetic similarity'), ('GSR', 'Genetic similarity to reference panel'), ('AS', 'Author statement - no method reported'), ('INF', 'Curator inferred ancestry label from cohort description (e.g. country of recruitment)'), ('NR', 'No population descriptor or inferrable CoR provided')], default='SR', max_length=4)), ('source_gwas_catalog', models.CharField(max_length=20, null=True, verbose_name='GWAS Catalog Study ID (GCST...)')), ('source_pmid', models.IntegerField(null=True, verbose_name='Source PubMed ID (PMID)')), ('source_doi', models.CharField(max_length=100, null=True, verbose_name='Source DOI')), @@ -352,7 +363,7 @@ class Migration(migrations.Migration): ('var_gene_exp', models.FloatField(null=True, verbose_name='Variance of the gene expression')), ('variants_number_used', models.IntegerField(null=True, verbose_name='Number of variants used')), ('variants_fraction_found', models.FloatField(null=True, verbose_name='Fraction of variants found')), - ('dataset', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='dataset_phenotype', to='omicspred.dataset', verbose_name='Dataset')), + ('dataset', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='dataset_phewas', to='omicspred.dataset', verbose_name='Dataset')), ('phenotypes', models.ManyToManyField(related_name='phenotype_scores', to='omicspred.phenotype', verbose_name='Phenotype(s)')), ('publication', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='score_phewas_publication', to='omicspred.publication', verbose_name='PheWAS Publication')), ('samples', models.ManyToManyField(related_name='sample_scores', to='omicspred.sample', verbose_name='Sample(s)')), diff --git a/omicspred/models.py b/omicspred/models.py index e14624f..9c8e3ed 100644 --- a/omicspred/models.py +++ b/omicspred/models.py @@ -107,7 +107,7 @@ class PlatformMaster(models.Model): """ Class to describe the platform used to get the omics data """ name = models.CharField('Platform name', max_length=100) full_name = models.CharField('Platform full name', max_length=100) - technic = models.CharField('Platform technic', max_length=100) + technique = models.CharField('Platform technique', max_length=100) type = models.CharField('Platform type', max_length=100) @property @@ -132,10 +132,7 @@ def scores_count(self): class Platform(models.Model): """ Class to describe the versioned platform used to get the omics data """ name = models.CharField('Platform name', max_length=100, db_index=True) - # full_name = models.CharField('Platform full name', max_length=100) version = models.CharField('Platform version', max_length=50) - # technic = models.CharField('Platform technic', max_length=100) - # type = models.CharField('Platform type', max_length=100) platform_master = models.ForeignKey(PlatformMaster, on_delete=models.CASCADE, related_name='platform_version', verbose_name='Platform') @property @@ -185,6 +182,19 @@ class Sample(models.Model): ancestry_free = models.TextField('Ancestry (e.g. French, Chinese)', null=True) ancestry_country = models.TextField('Country of Recruitment', null=True) ancestry_additional = models.TextField('Additional Ancestry Description', null=True) + ANCESTRY_ASSIGNMENT_CHOICES = [ + ('SR', 'Self reported'), + ('GS', 'Genetic similarity'), + ('GSR', 'Genetic similarity to reference panel'), + ('AS', 'Author statement - no method reported'), + ('INF', 'Curator inferred ancestry label from cohort description (e.g. country of recruitment)'), + ('NR', 'No population descriptor or inferrable CoR provided') + ] + ancestry_assignment = models.CharField(max_length=4, + choices=ANCESTRY_ASSIGNMENT_CHOICES, + default='SR', + db_index=False + ) ## Cohorts/Sources source_gwas_catalog = models.CharField('GWAS Catalog Study ID (GCST...)', max_length=20, null=True) @@ -324,6 +334,17 @@ class Dataset(models.Model): omics_count = models.IntegerField('Omics Entities count', null=False) omics_type = models.CharField('Omics type', max_length=50) method_name = models.TextField('Score Development Method') + # Cis-variant, Trans-variants + TRAINING_WINDOW_CHOICES = [ + ('',''), + ('genome-wide', 'Genome-wide'), + ('cis-only', 'Cis-only') + ] + training_window = models.CharField(max_length=25, + choices=TRAINING_WINDOW_CHOICES, + default='', + db_index=True + ) tissue = models.ForeignKey(Tissue, on_delete=models.PROTECT, related_name='tissue_dataset', verbose_name='Tissue', null=True) # Tissue trait defining the sampled tissue scores_count = models.IntegerField('Associated Scores count', null=False) phewas_count = models.IntegerField('Associated PheWAS data count', default=0) @@ -782,4 +803,11 @@ def values_dict(self): 'bonferroni': self.bonferroni, 'effect_size': self.effect_size, 'var_gene_exp': self.var_gene_exp - } \ No newline at end of file + } + + +class ExternalSource(models.Model): + """ Class to hold ExternalSource values """ + name = models.CharField('External Source Name', max_length=50) + version = models.CharField('External Source Version', max_length=50, null=True) + url = models.CharField('External Source URL', max_length=100) \ No newline at end of file diff --git a/rest_api/fixtures/db_test.default.json b/rest_api/fixtures/db_test.default.json index ae8a242..2bbf967 100644 --- a/rest_api/fixtures/db_test.default.json +++ b/rest_api/fixtures/db_test.default.json @@ -1,4 +1,22 @@ [ + { + "model": "omicspred.ExternalSource", + "pk": 1, + "fields": { + "name": "Ensembl", + "version": "113", + "url": "https://www.ensembl.org" + } + }, + { + "model": "omicspred.ExternalSource", + "pk": 2, + "fields": { + "name": "UniProt", + "version": "2026_01", + "url": "https://www.uniprot.org" + } + }, { "model": "omicspred.Publication", "pk": 1, @@ -39,7 +57,7 @@ "fields": { "name": "Somalogic", "full_name": "Somalogic", - "technic": "aptamer-based multiplex protein assay", + "technique": "aptamer-based multiplex protein assay", "type": "Proteomics" } }, @@ -49,7 +67,7 @@ "fields": { "name": "Olink", "full_name": "Olink", - "technic": "antibody-based proximity extension assay for proteins", + "technique": "antibody-based proximity extension assay for proteins", "type": "Proteomics" } }, @@ -59,7 +77,7 @@ "fields": { "name": "Metabolon", "full_name": "Metabolon Discovery HD4", - "technic": "untargeted mass spectrometry metabolomics platform", + "technique": "untargeted mass spectrometry metabolomics platform", "type": "Metabolomics" } }, @@ -69,7 +87,7 @@ "fields": { "name": "Illumina RNAseq", "full_name": "Illumina NovaSeq 6000", - "technic": "whole-transcriptome sequencing platform", + "technique": "whole-transcriptome sequencing platform", "type": "Transcriptomics" } }, @@ -147,6 +165,7 @@ "sample_age": 46.3, "sample_age_sd": 14.2, "ancestry_broad": "European", + "ancestry_assignment": "SR", "cohorts": [ 1 ] } }, @@ -159,6 +178,7 @@ "sample_age": 56.5, "sample_age_sd": 8.1, "ancestry_broad": "European", + "ancestry_assignment": "SR", "cohorts": [ 2 ] } }, @@ -172,6 +192,7 @@ "source_gwas_catalog": "GCST90475715", "source_pmid": 39024449, "ancestry_broad": "European", + "ancestry_assignment": "SR", "cohorts": [ 3 ] } }, @@ -185,6 +206,7 @@ "source_gwas_catalog": "GCST90475799", "source_pmid": 39024449, "ancestry_broad": "European", + "ancestry_assignment": "SR", "cohorts": [ 3 ] } }, @@ -198,6 +220,7 @@ "source_gwas_catalog": "GCST90475670", "source_pmid": 39024449, "ancestry_broad": "European", + "ancestry_assignment": "NR", "cohorts": [ 3 ] } }, @@ -211,7 +234,9 @@ "source_gwas_catalog": "GCST90475569", "source_pmid": 39024449, "ancestry_broad": "European", - "cohorts": [ 3 ] + "ancestry_assignment": "NR", + "cohorts": [ 3 ], + "cohorts_additional": "withheld" } }, { @@ -263,6 +288,7 @@ "platform": 1, "publication": 1, "method_name": "Bayesian Ridge regression", + "training_window": "genome-wide", "species_id": 9606, "tissue_id": "UBERON_0001969", "license": "Creative Commons Attribution 4.0 International (CC BY 4.0)", @@ -289,6 +315,7 @@ }, "platform": 2, "publication": 1, + "training_window": "genome-wide", "species_id": 9606, "tissue_id": "BTO_0000133", "license": "Creative Commons Attribution 4.0 International (CC BY 4.0)", @@ -316,6 +343,7 @@ "platform": 4, "publication": 1, "method_name": "Bayesian Ridge regression", + "training_window": "genome-wide", "species_id": 9606, "tissue_id": "UBERON_0001969", "license": "Creative Commons Attribution 4.0 International (CC BY 4.0)", @@ -343,6 +371,7 @@ "platform": 5, "publication": 1, "method_name": "Bayesian Ridge regression", + "training_window": "genome-wide", "species_id": 9606, "tissue_id": "UBERON_0001969", "license": "Creative Commons Attribution 4.0 International (CC BY 4.0)", diff --git a/rest_api/serializers.py b/rest_api/serializers.py index 5228209..c6bfd30 100644 --- a/rest_api/serializers.py +++ b/rest_api/serializers.py @@ -13,6 +13,16 @@ #### OmicsPred #### ################### +#### External Source #### +class ExternalSourceSerializer(serializers.ModelSerializer): + + class Meta: + model = ExternalSource + meta_fields = ('name', 'version', 'url') + fields = meta_fields + read_only_fields = meta_fields + + #### Cohort #### class CohortSerializer(serializers.ModelSerializer): @@ -39,15 +49,19 @@ def get_ancestries(self, obj): #### Sample #### class SampleSerializer(serializers.ModelSerializer): cohorts = CohortSerializer(many=True, read_only=True) + ancestry_assignment = serializers.SerializerMethodField('get_ancestry_assignment_label') class Meta: model = Sample meta_fields = ('sample_number', 'sample_cases', 'sample_controls', 'sample_age', 'sample_age_sd', 'sample_percent_male', - 'ancestry_broad', 'ancestry_free', 'ancestry_country', 'ancestry_additional', + 'ancestry_broad', 'ancestry_free', 'ancestry_country', 'ancestry_additional', 'ancestry_assignment', 'source_gwas_catalog', 'source_pmid','source_doi','cohorts','cohorts_additional')#,'tissue_name') fields = meta_fields read_only_fields = meta_fields + + def get_ancestry_assignment_label(self, obj): + return obj.get_ancestry_assignment_display() #### Tissue #### @@ -79,19 +93,19 @@ class Meta: class PlatformMasterSerializer(serializers.ModelSerializer): class Meta: model = PlatformMaster - meta_fields = ('name', 'full_name', 'versions', 'technic', 'type', 'scores_count') + meta_fields = ('name', 'full_name', 'versions', 'technique', 'type', 'scores_count') fields = meta_fields read_only_fields = meta_fields class PlatformSerializer(serializers.ModelSerializer): full_name = serializers.SerializerMethodField('get_full_name') - technic = serializers.SerializerMethodField('get_technic') + technique = serializers.SerializerMethodField('get_technique') type = serializers.SerializerMethodField('get_type') class Meta: model = Platform - meta_fields = ('name', 'full_name', 'version', 'technic', 'type') + meta_fields = ('name', 'full_name', 'version', 'technique', 'type') fields = meta_fields read_only_fields = meta_fields @@ -102,8 +116,8 @@ def get_full_name(self, obj): else: return obj.name - def get_technic(self, obj): - return obj.platform_master.technic + def get_technique(self, obj): + return obj.platform_master.technique def get_type(self, obj): return obj.platform_master.type @@ -123,14 +137,18 @@ class DatasetLightSerializer(serializers.ModelSerializer): tissue = TissueSerializer(many=False, read_only=True) samples_training = SampleSerializer(many=True, read_only=True) samples_validation = SampleSerializer(many=True, read_only=True) + training_window = serializers.SerializerMethodField('get_training_window_label') class Meta: model = Dataset meta_fields = ('id', 'name', 'platform', 'scores_count', 'omics_count', 'phewas_count', 'omics_type', 'method_name', - 'tissue', 'samples_training', 'samples_validation','scoring_files_urls') + 'training_window', 'tissue', 'samples_training', 'samples_validation','scoring_files_urls') fields = meta_fields read_only_fields = meta_fields + def get_training_window_label(self, obj): + return obj.get_training_window_display() + class DatasetSerializer(DatasetLightSerializer): publication = PublicationSerializer(many=False, read_only=True) @@ -456,44 +474,6 @@ def get_metabolites_count(self, obj): return obj.pathway_metabolites.count() -#### Score #### -# class ScoreWebsiteSerializer(serializers.ModelSerializer): -# dataset_id = serializers.SerializerMethodField() -# dataset_name = serializers.SerializerMethodField() -# publication = serializers.SerializerMethodField() -# platform_name = serializers.SerializerMethodField() - -# genes = GeneSerializerMinimal(many=True, read_only=True) -# transcripts = TranscriptSerializer(many=True, read_only=True) -# proteins = ProteinBaseSerializer(many=True, read_only=True) -# metabolites = MetaboliteBaseSerializer(many=True, read_only=True) - -# class Meta: -# model = Score -# meta_fields = ('id', 'name', 'variants_number', 'ancestry', -# 'dataset_id', 'dataset_name', 'publication', 'platform_name', -# 'genes', 'transcripts', 'proteins', 'metabolites') -# fields = meta_fields -# read_only_fields = meta_fields - -# def get_dataset_id(self,obj): -# ''' Get Dataset ID ''' -# return obj.dataset.id - -# def get_dataset_name(self,obj): -# ''' Get Dataset name ''' -# return obj.dataset.name - -# def get_publication(self, obj): -# ''' Get Publication model ''' -# publication = obj.dataset.publication -# return PublicationSerializer(publication, many=False, read_only=True).data - -# def get_platform_name(self, obj): -# ''' Get Platform model ''' -# return obj.dataset.platform.name - - class ScoreLightSerializer(serializers.ModelSerializer): dataset_id = serializers.SerializerMethodField() dataset_name = serializers.SerializerMethodField() diff --git a/rest_api/static/rest_api/openapi/openapi-schema.yml b/rest_api/static/rest_api/openapi/openapi-schema.yml index 8b66857..d556920 100644 --- a/rest_api/static/rest_api/openapi/openapi-schema.yml +++ b/rest_api/static/rest_api/openapi/openapi-schema.yml @@ -3,7 +3,7 @@ servers: - url: https://rest.omicspred.org info: title: 'OmicsPred REST API' - version: '1.2.0' + version: '1.2.1' description: | Programmatic access to the OmicsPred metadata. @@ -69,6 +69,13 @@ info: REST API version changelog
+ * 1.2.1 - July 2026: + * New endpoint `/api/external_source/all` (endpoint group 'Other') to retrieve information from the external sources used in OmicsPred. + * New field **training_window** in the Dataset schemas (`/api/dataset/` endpoints), containing the type of training window for the variants (Genome-wide, Cis-only, ...). + * New field **ancestry_assignment** in the Sample schemas, containing the description about how the ancestry category was assigned. + * Rename field **technic** by **technique** in the Platform schemas. + * New Sample endpoint `/api/sample/search` allowing to search the sample by cohort name. + * 1.2.0 - May 2026: * Refactor the Phenotype endpoints (`/api/phenotype/`), due to a change in the phenotype data structure. * New PheWAS endpoints (`/api/score/phewas/`), replacing the Score Applications endpoints (`/api/applications_score/`), due to a change in the phewas data structure. @@ -145,7 +152,7 @@ tags: - name: "Dataset endpoints" description: "Dataset metadata (name, publication, platform, samples, ...)" - name: "Platform endpoints" - description: "Platform metadata (name, type, version, technic, ...)" + description: "Platform metadata (name, type, version, technique, ...)" - name: "Tissue endpoints" description: "Tissue ontology information (name, label, description, ...)" - name: "Sample/Cohort endpoints" @@ -157,7 +164,7 @@ tags: - name: "Pathway endpoints" description: "Pathway metadata (Reactome)" - name: "Other endpoints" - description: "Extra endpoints: **/api/info** (REST API version, data counts, ...)" + description: "Extra endpoints: **/api/info** (REST API version, data counts, ...), **/api/external_source/all** (External sources information)" components: @@ -291,6 +298,10 @@ components: type: string description: "The name or description of the method or computational algorithm used to develop the PGS." example: "Bayesian Ridge regression" + training_window: + type: string + description: "The type of training window used for the variants" + example: "Genome-wide" tissue: $ref: '#/components/schemas/Tissue' samples_training: @@ -330,12 +341,8 @@ components: example: "https://app.box.com/shared/static/u4gq3qypwley3m0wfduuisrou3ij0iso" predictdb: type: string - description: "PredictDB prediction weigths - SQLite database" - example: "https://app.box.com/shared/static/unjn1bmt2xhwkd33pp0wczldfzrkevu2" - covariance: - type: string - description: "Covariance file, to be used alongside the PredictDB database in MetaXcan tools" - example: "https://app.box.com/shared/static/24avhdwb2gfo083oz3r6j8w6qqburbp2" + description: "Compressed archive contining the PredictDB prediction weigths - SQLite database file and the Covariance file. They are used together in MetaXcan tools" + example: "https://app.box.com/shared/static/m8yd9yrd6afbiq7jxm5ob66pbj3yj2qt" score_variant_info: type: string description: "Variant information file" @@ -371,6 +378,23 @@ components: type: string example: "This REST endpoint does not exist" + # Cohort # + ExternalSource: + type: object + description: "Structure containing information about an external source used in OmicsPred." + properties: + name: + type: string + description: "Name of the external source" + example: "Ensembl" + version: + type: string + description: "Version used in OmicsPred" + example: "113" + url: + type: string + description: "URL of the external source" + example: "https://www.ensembl.org" # Gene # Gene_basic: @@ -741,9 +765,9 @@ components: type: string description: "Platform's version" example: "3.0" - technic: + technique: type: string - description: "Technic used by the platform" + description: "Technique used by the platform" example: "aptamer-based multiplex protein assay" type: type: string @@ -920,6 +944,10 @@ components: type: string description: "Author reported countries of recruitment (if available)." example: "Canada" + ancestry_assignment: + type: string + description: "Description about how the ancestry category was assigned." + example: "Self-reported" ancestry_additional: type: string description: "Any additional description not captured in the structured data (e.g. founder or genetically isolated populations, or further description of admixed samples)." @@ -2915,7 +2943,7 @@ paths: Retrieve all the Platforms, including for each of them: * Name * Versions - * Technic + * Technique * Type (Transcriptomics, Proteomics, Metabolomics) * Number ofLinked genetic Scores @@ -2955,7 +2983,7 @@ paths: Retrieve a Platform, including: * Name * Versions - * Technic + * Technique * Type (Transcriptomics, Proteomics, Metabolomics) * Number ofLinked genetic Scores @@ -2995,9 +3023,11 @@ paths: * Publication information * Platform information * Number of Genetic Scores + * Method used to develop the scores + * Training window used to select the variants * Tissue information * Training Samples and Cohorts - * Validation Samples and Cohorts + * Validation Samples and Cohorts (if available) * URLs to the data files Example of request: @@ -3039,9 +3069,11 @@ paths: * Publication information * Platform information * Number of Genetic Scores + * Method used to develop the scores + * Training window used to select the variants * Tissue information * Training Samples and Cohorts - * Validation Samples and Cohorts + * Validation Samples and Cohorts (if available) * URLs to the data files Examples of request: @@ -3080,9 +3112,11 @@ paths: * Publication information * Platform information * Number of Genetic Scores + * Method used to develop the scores + * Training window used to select the variants * Tissue information * Training Samples and Cohorts - * Validation Samples and Cohorts + * Validation Samples and Cohorts (if available) * URLs to the data files Examples of request: @@ -3463,3 +3497,43 @@ paths: description: 'Diverse information about the project, such as REST API version, data counts, ...' '4XX': $ref: '#/components/responses/4XX' + + + # External Sources # + /api/external_source/all: + get: + tags: + - "Other endpoints" + operationId: getAllExternalSources + description: | + Retrieve all External Sources, including for each of them: + * External source name + * External source version used + * External source URL + + Example of request: + ``` + https://rest.omicspred.org/api/external_source/all + ``` + responses: + '200': + content: + application/json: + schema: + type: object + allOf: + - $ref: '#/components/schemas/Pagination' + properties: + results: + description: "List of External Sources used in OmicsPred" + type: array + items: + $ref: '#/components/schemas/ExternalSource' + description: | + All External Sources used in OmicsPred + + --- + + __Notes:__ This endpoint uses pagination. + '4XX': + $ref: '#/components/responses/4XX' diff --git a/rest_api/tests.py b/rest_api/tests.py index f67284d..978a6b7 100644 --- a/rest_api/tests.py +++ b/rest_api/tests.py @@ -5,6 +5,10 @@ sleep_time = 60 +status_ok = 200 +status_not_found = 404 +status_too_many = 429 + class BrowseEndpointTest(APITestCase): """ Test the REST endpoints """ @@ -63,6 +67,7 @@ class BrowseEndpointTest(APITestCase): seach_mt = f'molecular_trait={proteins_list[0]}' search_combined = f'{search_opgs_id}&{search_pmid}' search_combined_2 = f'{search_opgs_ids}&{search_pmid}' + search_cohort = f'cohort=INTERVAL' omics_common_columns = ['id', 'trait_reported_id', 'trait_reported', 'variants_number', 'dataset__publication', 'dataset__platform__version', 'dataset__name'] @@ -96,6 +101,7 @@ class BrowseEndpointTest(APITestCase): ('Publication Search', 'publication/search', 1, {'query': [search_opgs_id,search_pmid,search_author]}), # Sample endpoint ('Samples', 'sample/all', 1), + ('Samples Search', 'sample/search', 1, {'query': [search_cohort]}), # Score endpoints ('Scores', 'score/all', 1), ('Score/ID', 'score', 0, {'path': scores_list}), @@ -123,7 +129,8 @@ class BrowseEndpointTest(APITestCase): ('Score PheWAS', 'score/phewas', 1, {'path': scores_list}), ('Score PheWAS Search', 'score/phewas/search', 1, {'query': [search_phenotype,search_opgs_ids,search_opgs_ids_2,search_opp_id,search_combined_2]}), # Other endpoints - ('Info', 'info', 0) + ('Info', 'info', 0), + ('External Sources - all', 'external_source/all', 1) ] @@ -246,7 +253,7 @@ class BrowseEndpointTest(APITestCase): def client_response(self,url:str): resp = self.client.get(url) # In case the number of REST API calls is too high (i.e. greater than the modified 'rate4test' variable) - if resp.status_code == 429: + if resp.status_code == status_too_many: print(f"\nNeed to put the test on sleep for {sleep_time}s as it reaches the maximum number of calls/min.\nMight be worth increasing the value of the 'rate4test' variable (currently {self.rate4test}).") sleep(sleep_time) resp = self.client.get(url) @@ -286,9 +293,11 @@ def check_filter_ids(self,url:str,ids_list:list): def send_request(self, url:str): """ Send REST API request and check the reponse status code """ resp = self.client_response(url) - self.assertEqual(resp.status_code, 200) + self.assertEqual(resp.status_code, status_ok) content = resp.content.decode("utf-8") for empty_content in self.empty_resp: + if content == empty_content: + print(f"\t>> Equal data on endpoint: {url}") self.assertNotEqual(content, empty_content) return content @@ -296,20 +305,20 @@ def send_request(self, url:str): def get_empty_response(self, url:str, index:int): """ Send REST API request and check the reponse status code """ resp = self.client_response(url) - self.assertEqual(resp.status_code, 200) + self.assertEqual(resp.status_code, status_ok) self.assertEqual(resp.content.decode("utf-8"), self.empty_resp[index]) def get_not_found_response(self, url:str): """ Send REST API request on non existing endpoint and check reponse status code """ resp = self.client_response(url) - self.assertEqual(resp.status_code, 404) + self.assertEqual(resp.status_code, status_not_found) def get_paginated_response(self, url:str): """ Send REST API request with limit and offset parameters, and check the reponse status code """ resp = self.client_response(url+'?limit=20&offset=20') - self.assertEqual(resp.status_code, 200) + self.assertEqual(resp.status_code, status_ok) def test_endpoints(self): diff --git a/rest_api/urls.py b/rest_api/urls.py index 8fad9c9..2082847 100644 --- a/rest_api/urls.py +++ b/rest_api/urls.py @@ -10,28 +10,31 @@ slash = '/?' +url_prefix = 'api' + rest_urls = { - 'applications_score': 'api/applications_score/', - 'applications_sample': 'api/applications_sample/', - 'cohort': 'api/cohort/', - 'dataset': 'api/dataset/', - 'gene': 'api/gene/', - 'info': 'api/info', - 'metabolite': 'api/metabolite/', - 'metabolomics': 'api/metabolomics/', - 'pathway': 'api/pathway/', - 'phenotype': 'api/phenotype/', - 'phenotype_old': 'api/phenotype_old/', - 'platform': 'api/platform/', - 'plot': 'api/plot/', - 'performance': 'api/performance/', - 'protein': 'api/protein/', - 'proteomics': 'api/proteomics/', - 'publication': 'api/publication/', - 'sample': 'api/sample/', - 'score': 'api/score/', - 'tissue': 'api/tissue/', - 'transcriptomics': 'api/transcriptomics/' + 'applications_score': f'{url_prefix}/applications_score/', + 'applications_sample': f'{url_prefix}/applications_sample/', + 'cohort': f'{url_prefix}/cohort/', + 'dataset': f'{url_prefix}/dataset/', + 'external_source': f'{url_prefix}/external_source/', + 'gene': f'{url_prefix}/gene/', + 'info': f'{url_prefix}/info', + 'metabolite': f'{url_prefix}/metabolite/', + 'metabolomics': f'{url_prefix}/metabolomics/', + 'pathway': f'{url_prefix}/pathway/', + 'phenotype': f'{url_prefix}/phenotype/', + 'phenotype_old': f'{url_prefix}/phenotype_old/', + 'platform': f'{url_prefix}/platform/', + 'plot': f'{url_prefix}/plot/', + 'performance': f'{url_prefix}/performance/', + 'protein': f'{url_prefix}/protein/', + 'proteomics': f'{url_prefix}/proteomics/', + 'publication': f'{url_prefix}/publication/', + 'sample': f'{url_prefix}/sample/', + 'score': f'{url_prefix}/score/', + 'tissue': f'{url_prefix}/tissue/', + 'transcriptomics': f'{url_prefix}/transcriptomics/' } urlpatterns = [ @@ -64,6 +67,7 @@ re_path(r'^'+rest_urls['publication']+'(?P[^/]+)'+slash, RestPublication.as_view(), name="getPublication"), # Samples re_path(r'^'+rest_urls['sample']+'all'+slash, cache_page(cache_time)(RestListSamples.as_view()), name="getAllSamples"), + re_path(r'^'+rest_urls['sample']+'search'+slash, cache_page(cache_time)(RestSampleSearch.as_view()), name="searchSamples"), # Score PheWAS re_path(r'^'+rest_urls['score']+'phewas/all'+slash, cache_page(cache_time)(RestListScorePheWAS.as_view()), name="getAllScorePheWAS"), re_path(r'^'+rest_urls['score']+'phewas/search'+slash, RestScorePheWASSearch.as_view(), name="searchScorePheWAS"), @@ -93,14 +97,17 @@ re_path(r'^'+rest_urls['plot']+'file/search'+slash, cache_page(cache_time)(RestPlotFileSearch.as_view()), name="searchFilePlots"), re_path(r'^'+rest_urls['plot']+'score/search'+slash, cache_page(cache_time)(RestPlotScoreSearch.as_view()), name="searchScorePlots"), - # Applications + # Applications [DEPRECATED] re_path(r'^'+rest_urls['phenotype_old']+'(?P[^/]+)'+slash, RestPhenotypeOld.as_view(), name="getPhenotypeOld"), re_path(r'^'+rest_urls['applications_score']+'all'+slash, cache_page(cache_time)(RestListPhenotypeScore.as_view()), name="getAllPhenotypeScores"), re_path(r'^'+rest_urls['applications_score']+'search'+slash, RestPhenotypeScoreSearch.as_view(), name="searchPhenotypeScores"), re_path(r'^'+rest_urls['applications_score']+'(?P[^/]+)'+slash, RestPhenotypeScore.as_view(), name="getPhenotypeScore"), re_path(r'^'+rest_urls['applications_sample']+'all'+slash, cache_page(cache_time)(RestListPhenotypeSample.as_view()), name="getAllPhenotypeSamples"), + # Other re_path(r'^'+rest_urls['info']+slash, RestInfo.as_view(), name="getInfo"), + # -> External sources + re_path(r'^'+rest_urls['external_source']+'all'+slash, RestListExternalSources.as_view(), name="getAllExternalSources"), # Setup URL used to warmup the Django app in the Google App Engine path('_ah/warmup', warmup, name="Warmup"), diff --git a/rest_api/views.py b/rest_api/views.py index 8254d09..a6e7095 100644 --- a/rest_api/views.py +++ b/rest_api/views.py @@ -1167,6 +1167,29 @@ class RestListSamples(generics.ListAPIView): queryset = Sample.objects.all().prefetch_related('cohorts').order_by('id') +class RestSampleSearch(generics.ListAPIView): + """ + Search the Sample(s) using parameters as query + Endpoint: /api/sample/search + """ + serializer_class = SampleSerializer + + def get_queryset(self): + queryset = Sample.objects.all().prefetch_related('cohorts').order_by('id') + + params = 0 + # Cohort + cohort = self.request.query_params.get('cohort') + if cohort and cohort is not None: + queryset = queryset.filter(Q(cohorts__name_short__iexact=cohort) | Q(cohorts__name_full__iexact=cohort)) + params += 1 + + if params == 0: + queryset = [] + return queryset + + + ## Scores ## class RestListScores(generics.ListAPIView): @@ -1975,4 +1998,15 @@ def get(self, request): } } - return Response(data) \ No newline at end of file + return Response(data) + + +class RestListExternalSources(generics.ListAPIView): + """ + Retrieve all the External Sources + Endpoint: /api/external_source/all + """ + serializer_class = ExternalSourceSerializer + + def get_queryset(self): + return ExternalSource.objects.all()