From e75544d8d5a312b42853d2ee10a0e16194ee2688 Mon Sep 17 00:00:00 2001 From: ens-lgil Date: Tue, 16 Jun 2026 13:21:39 +0100 Subject: [PATCH 1/3] Update export scripts --- exports/covariance_validation.py | 111 +++++++++++++++++++++++++ exports/migrations/__init__.py | 0 exports/scripts/validate_covariance.py | 23 +++++ 3 files changed, 134 insertions(+) create mode 100644 exports/covariance_validation.py delete mode 100644 exports/migrations/__init__.py create mode 100644 exports/scripts/validate_covariance.py diff --git a/exports/covariance_validation.py b/exports/covariance_validation.py new file mode 100644 index 0000000..c2738a7 --- /dev/null +++ b/exports/covariance_validation.py @@ -0,0 +1,111 @@ +import os +import sqlite3 +import gzip +import csv + +class CovarianceValidation: + + # Expected covariance columns + covariance_columns = ('GENE','RSID1','RSID2','VALUE') + + def __init__(self, covariance_dir:str, sqlite_dir:str, datasets:list,var_type:str): + self.covariance_dir = covariance_dir + self.sqlite_dir = sqlite_dir + self.datasets = datasets # List of Dataset models + self.dataset_format_checked = [] + self.var_type = var_type # rsid or VarID + self.variants = set() + + # Get the list of SQLite files in the directory + self.sqlite_files = {} + for file in os.listdir(sqlite_dir): + if file.endswith('.db'): + files_parts = file.split('_') + self.sqlite_files[files_parts[0]] = file + + # Get the list of covariance files in the directory + self.covariance_files = {} + for file in os.listdir(covariance_dir): + if file.endswith('.txt.gz'): + files_parts = file.split('_') + self.covariance_files[files_parts[0]] = file + + + def fetch_varids(self, sqlite_file:str) -> list: + ''' Fetch the variants information from the SQLite DB. ''' + sql_con = sqlite3.connect(f'{self.sqlite_dir}/{sqlite_file}') + sql_cur = sql_con.cursor() + sql_statement = "SELECT distinct gene,rsid FROM weights;" + if self.var_type == 'VarID': + sql_statement = "SELECT distinct gene,VarID FROM weights;" + sql_cur.execute(sql_statement) + var_table = sql_cur.fetchall() + sql_cur.close() + sql_con.close() + data = {} + for var_entry in var_table: + gene = var_entry[0] + variant_id = var_entry[1] + if gene not in data.keys(): + data[gene] = set() + data[gene].add(variant_id) + self.variants.add(variant_id) + return data + + + def check_format(self, dataset_id:str, row:dict) -> None: + ''' Check that the file columns match the expected covariance columns. ''' + file_columns = sorted(row.keys()) + if file_columns != list(self.covariance_columns): + print(f' >> Covariance file having different columns for {dataset_id}:') + print(f' >> -> File: {file_columns}') + print(f' >> -> Expected: {list(self.covariance_columns)}') + self.dataset_format_checked.append(dataset_id) + + + def validate_covariance_files(self) -> None: + ''' Run the validation for each file/dataset. ''' + for dataset in self.datasets: + dataset_id = dataset.id + missing_genes = 0 + different_rsid1 = 0 + unknown_rsid2 = 0 + if dataset_id in self.sqlite_files.keys() and dataset_id in self.covariance_files.keys(): + print(f'# Dataset {dataset_id} ({dataset.name})') + varid_list = self.fetch_varids(self.sqlite_files[dataset_id]) + varid_list_keys = varid_list.keys() + with gzip.open(f'{self.covariance_dir}/{self.covariance_files[dataset_id]}', 'rt') as f: + reader = csv.DictReader(f, delimiter='\t') + for row in reader: + if dataset_id not in self.dataset_format_checked: + self.check_format(dataset_id, row) + gene = row['GENE'] + rsid1 = row['RSID1'] + rsid2 = row['RSID2'] + # Checks + if gene in varid_list_keys: + vars = varid_list[gene] + # Check the variant is associated with the gene + if rsid1 not in vars: + different_rsid1 += 1 + # Check that rsid2 is in the dataset + if rsid2 not in self.variants: + unknown_rsid2 += 1 + + else: + missing_genes += 1 + if missing_genes != 0: + print(f' - missing_genes: {missing_genes}') + if different_rsid1 != 0: + print(f' - different_rsid1: {different_rsid1}') + if unknown_rsid2 != 0: + print(f' - unknown_rsid2: {unknown_rsid2}') + if not missing_genes and not different_rsid1 and not unknown_rsid2: + print(f' > No missing data') + + elif dataset_id in self.sqlite_files.keys(): + print(f' >> Covariance file missing for {dataset_id}') + elif dataset_id in self.covariance_files.keys(): + print(f' >> SQLite file missing for {dataset_id}') + else: + print(f' >> SQLite and Covariance files missing for {dataset_id}') \ No newline at end of file diff --git a/exports/migrations/__init__.py b/exports/migrations/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/exports/scripts/validate_covariance.py b/exports/scripts/validate_covariance.py new file mode 100644 index 0000000..bd1e09e --- /dev/null +++ b/exports/scripts/validate_covariance.py @@ -0,0 +1,23 @@ +from exports.datasets import DatasetsSelection +from exports.covariance_validation import CovarianceValidation + + +# covariance_dir = '/Users/lg10/Workspace/datafiles/OmicsPred/PCAIR/covariances' +# sqlite_dir = '/Users/lg10/Workspace/datafiles/OmicsPred/PCAIR/sqlite_exports' +# var_type = 'rsid' + +# covariance_dir = '/Users/lg10/Workspace/datafiles/OmicsPred/GTEx_V8/covariances/MASHR_eQTL' +# sqlite_dir = '/Users/lg10/Workspace/datafiles/OmicsPred/GTEx_V8/sqlite_exports_tmp_eQTL_MASHR' +# var_type = 'VarID' + +covariance_dir = '/Users/lg10/Workspace/datafiles/OmicsPred/GTEx_V8/covariances/MASHR_sQTL' +sqlite_dir = '/Users/lg10/Workspace/datafiles/OmicsPred/GTEx_V8/sqlite_exports_tmp_sQTL_MASHR' +var_type = 'VarID' + +def run(): + ds_selection = DatasetsSelection() + datasets = ds_selection.get_datasets() + + print("## Start Covariance Validation, dataset by dataset") + covariance_validation = CovarianceValidation(covariance_dir,sqlite_dir,datasets,var_type) + covariance_validation.validate_covariance_files() \ No newline at end of file From 7e32df04c9928fba36cfe4d9555b4e78de942d29 Mon Sep 17 00:00:00 2001 From: ens-lgil Date: Tue, 16 Jun 2026 13:24:03 +0100 Subject: [PATCH 2/3] Update REST API (update on internal use only) --- omicspred/models.py | 7 ++++++- rest_api/serializers.py | 14 ++++++++++++++ rest_api/views.py | 3 ++- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/omicspred/models.py b/omicspred/models.py index 0fe66c6..e14624f 100644 --- a/omicspred/models.py +++ b/omicspred/models.py @@ -369,6 +369,11 @@ def cohorts_validation(self): cohorts.add(cohort.name_short) return list(cohorts) + @property + def phewas_publications(self): + score_phewas_list = self.dataset_phewas.only('dataset_id','publication_id').select_related('publication').all().distinct('publication_id') + return [x.publication for x in score_phewas_list] + class MolecularTrait(models.Model): """ Generic Class to describe a molecular trait """ @@ -732,7 +737,7 @@ def traits_reported_list(self): class ScorePheWAS(models.Model): """ Class to hold Score PheWAS values """ score = models.ForeignKey(Score, on_delete=models.CASCADE, verbose_name='Score', related_name='score_phewas') # Score that the PheWAS data are associated with - dataset = models.ForeignKey(Dataset, on_delete=models.PROTECT, verbose_name='Dataset', related_name='dataset_phenotype') + dataset = models.ForeignKey(Dataset, on_delete=models.PROTECT, verbose_name='Dataset', related_name='dataset_phewas') publication = models.ForeignKey(Publication, on_delete=models.PROTECT, verbose_name='PheWAS Publication', related_name='score_phewas_publication') # Method diff --git a/rest_api/serializers.py b/rest_api/serializers.py index dcd20ce..5228209 100644 --- a/rest_api/serializers.py +++ b/rest_api/serializers.py @@ -141,6 +141,20 @@ class Meta(DatasetLightSerializer.Meta): read_only_fields = DatasetLightSerializer.Meta.fields + meta_fields +class DatasetPheWASPublicationSerializer(DatasetSerializer): + phewas_publications = PublicationSerializer(many=True, read_only=True) + + class Meta(DatasetSerializer.Meta): + meta_fields = ('phewas_publications',) + fields = DatasetSerializer.Meta.fields + meta_fields + read_only_fields = DatasetSerializer.Meta.fields + meta_fields + + def get_phewas_publications(self, obj): + if (obj.phewas_publications): + return obj.phewas_publications + return [] + + #### Publication - Extended (with Platform) #### class PublicationExtendedSerializer(PublicationSerializer): date_release = serializers.SerializerMethodField('get_date_released') diff --git a/rest_api/views.py b/rest_api/views.py index d66cd84..8254d09 100644 --- a/rest_api/views.py +++ b/rest_api/views.py @@ -993,10 +993,11 @@ class RestListDatasets(generics.ListAPIView): def get_queryset(self): queryset = Dataset.objects.select_related(*related_dict['dataset_select']).all().prefetch_related(*related_dict['dataset_prefetch']).order_by('num') - # Filter by list of Publications IDs + # Filter PheWAS study only - FOR PRIVATE USE CASE only_phewas = self.request.query_params.get('only_phewas') if only_phewas and only_phewas is not None: queryset = queryset.filter(phewas_count__gt=0) + self.serializer_class = DatasetPheWASPublicationSerializer return queryset From a2ce30e31ab61ebe3b8ba0b5ae6f36bc49578747 Mon Sep 17 00:00:00 2001 From: ens-lgil Date: Tue, 16 Jun 2026 13:27:52 +0100 Subject: [PATCH 3/3] Move to ElasticSearch v8 --- requirements.txt | 6 +++--- search_es/analyzers.py | 7 +++---- search_es/documents/gene.py | 2 +- search_es/documents/metabolite.py | 2 +- search_es/views.py | 2 +- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/requirements.txt b/requirements.txt index 4195748..6516e69 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ #### Django library #### -Django==6.0.5 +Django==6.0.6 #### Python PostgreSQL library #### # psycopg2-binary==2.9.11 psycopg[binary]==3.3.4 @@ -8,5 +8,5 @@ psycopg[binary]==3.3.4 djangorestframework==3.17.1 django-cors-headers==4.9.0 #### Elasticsearch libraries #### -elasticsearch-dsl==7.4.1 -django-elasticsearch-dsl==7.4 +elasticsearch==8.19.3 +django-elasticsearch-dsl==8.0 \ No newline at end of file diff --git a/search_es/analyzers.py b/search_es/analyzers.py index 4cefa2c..e20882d 100644 --- a/search_es/analyzers.py +++ b/search_es/analyzers.py @@ -1,10 +1,9 @@ -from elasticsearch_dsl import analysis, analyzer, tokenizer +from elasticsearch.dsl import analysis, analyzer, tokenizer # Documentation about analysers: -# v7.17: -# https://www.elastic.co/guide/en/elasticsearch/reference/7.17/analysis-keyword-tokenizer.html -# https://www.elastic.co/guide/en/elasticsearch/reference/7.17/analysis-tokenizers.html +# https://www.elastic.co/docs/reference/text-analysis/analyzer-reference +# https://www.elastic.co/docs/reference/text-analysis/tokenizer-reference def id_analyzer(): ''' Analyser for the different IDs in the OmicsPred indexes ''' diff --git a/search_es/documents/gene.py b/search_es/documents/gene.py index 9ddcaca..7f846a7 100644 --- a/search_es/documents/gene.py +++ b/search_es/documents/gene.py @@ -2,7 +2,7 @@ from django_elasticsearch_dsl import Document, fields from django_elasticsearch_dsl.registries import registry from search_es.analyzers import id_analyzer, name_delimiter_analyzer, word_delimiter_analyzer -from omicspred.models import Gene, Dataset +from omicspred.models import Gene # PGS index analyzer diff --git a/search_es/documents/metabolite.py b/search_es/documents/metabolite.py index f9c4a22..ba8619f 100644 --- a/search_es/documents/metabolite.py +++ b/search_es/documents/metabolite.py @@ -2,7 +2,7 @@ from django_elasticsearch_dsl import Document, fields from django_elasticsearch_dsl.registries import registry from search_es.analyzers import id_analyzer, name_delimiter_analyzer, word_delimiter_analyzer -from omicspred.models import Metabolite, Dataset +from omicspred.models import Metabolite # PGS index analyzer diff --git a/search_es/views.py b/search_es/views.py index ba9a00b..1e8e977 100644 --- a/search_es/views.py +++ b/search_es/views.py @@ -1,6 +1,6 @@ from rest_framework import generics from rest_framework.response import Response -from elasticsearch_dsl import Search +from elasticsearch.dsl import Search # No need to redeclare "custom_exception_handler" as it is already used in rest_api.views