Move to ElasticSearch v8

exports/covariance_validation.py

@@ -0,0 +1,111 @@
+import os
+import sqlite3
+import gzip
+import csv
+
+class CovarianceValidation:
+
+    # Expected covariance columns
+    covariance_columns = ('GENE','RSID1','RSID2','VALUE')
+
+    def __init__(self, covariance_dir:str, sqlite_dir:str, datasets:list,var_type:str):
+        self.covariance_dir = covariance_dir
+        self.sqlite_dir = sqlite_dir
+        self.datasets = datasets # List of Dataset models
+        self.dataset_format_checked = []
+        self.var_type = var_type # rsid or VarID
+        self.variants = set()
+
+        # Get the list of SQLite files in the directory
+        self.sqlite_files = {}
+        for file in os.listdir(sqlite_dir):
+            if file.endswith('.db'):
+                files_parts = file.split('_')
+                self.sqlite_files[files_parts[0]] = file
+
+        # Get the list of covariance files in the directory
+        self.covariance_files = {}
+        for file in os.listdir(covariance_dir):
+            if file.endswith('.txt.gz'):
+                files_parts = file.split('_')
+                self.covariance_files[files_parts[0]] = file
+
+
+    def fetch_varids(self, sqlite_file:str) -> list:
+        ''' Fetch the variants information from the SQLite DB. '''
+        sql_con = sqlite3.connect(f'{self.sqlite_dir}/{sqlite_file}')
+        sql_cur = sql_con.cursor()
+        sql_statement = "SELECT distinct gene,rsid FROM weights;"
+        if self.var_type == 'VarID':
+            sql_statement = "SELECT distinct gene,VarID FROM weights;"
+        sql_cur.execute(sql_statement)
+        var_table = sql_cur.fetchall()
+        sql_cur.close()
+        sql_con.close()
+        data = {}
+        for var_entry in var_table:
+            gene = var_entry[0]
+            variant_id = var_entry[1]
+            if gene not in data.keys():
+                data[gene] = set()
+            data[gene].add(variant_id)
+            self.variants.add(variant_id)
+        return data
+
+
+    def check_format(self, dataset_id:str, row:dict) -> None:
+        ''' Check that the file columns match the expected covariance columns. '''
+        file_columns = sorted(row.keys())
+        if file_columns != list(self.covariance_columns):
+            print(f'  >> Covariance file having different columns for {dataset_id}:')
+            print(f'  >> -> File: {file_columns}')
+            print(f'  >> -> Expected: {list(self.covariance_columns)}')
+        self.dataset_format_checked.append(dataset_id)
+
+
+    def validate_covariance_files(self) -> None:
+        ''' Run the validation for each file/dataset. '''
+        for dataset in self.datasets:
+            dataset_id = dataset.id
+            missing_genes = 0
+            different_rsid1 = 0
+            unknown_rsid2 = 0
+            if dataset_id in self.sqlite_files.keys() and dataset_id in self.covariance_files.keys():
+                print(f'# Dataset {dataset_id} ({dataset.name})')
+                varid_list = self.fetch_varids(self.sqlite_files[dataset_id])
+                varid_list_keys = varid_list.keys()
+                with gzip.open(f'{self.covariance_dir}/{self.covariance_files[dataset_id]}', 'rt') as f:
+                    reader = csv.DictReader(f, delimiter='\t')
+                    for row in reader:
+                        if dataset_id not in self.dataset_format_checked:
+                            self.check_format(dataset_id, row)
+                        gene = row['GENE']
+                        rsid1 = row['RSID1']
+                        rsid2 = row['RSID2']
+                        # Checks
+                        if gene in varid_list_keys:
+                            vars = varid_list[gene]
+                            # Check the variant is associated with the gene
+                            if rsid1 not in vars:
+                                different_rsid1 += 1
+                            # Check that rsid2 is in the dataset
+                            if rsid2 not in self.variants:
+                                unknown_rsid2 += 1
+
+                        else:
+                            missing_genes += 1
+                if missing_genes != 0:
+                    print(f'  - missing_genes: {missing_genes}')
+                if different_rsid1 != 0:
+                    print(f'  - different_rsid1: {different_rsid1}')
+                if unknown_rsid2 != 0:
+                    print(f'  - unknown_rsid2: {unknown_rsid2}')
+                if not missing_genes and not different_rsid1 and not unknown_rsid2:
+                    print(f'  > No missing data')
+
+            elif dataset_id in self.sqlite_files.keys():
+                print(f'  >> Covariance file missing for {dataset_id}')
+            elif dataset_id in self.covariance_files.keys():
+                print(f'  >> SQLite file missing for {dataset_id}')
+            else:
+                print(f'  >> SQLite and Covariance files missing for {dataset_id}')

exports/scripts/validate_covariance.py

@@ -0,0 +1,23 @@
+from exports.datasets import DatasetsSelection
+from exports.covariance_validation import CovarianceValidation
+
+
+# covariance_dir = '/Users/lg10/Workspace/datafiles/OmicsPred/PCAIR/covariances'
+# sqlite_dir = '/Users/lg10/Workspace/datafiles/OmicsPred/PCAIR/sqlite_exports'
+# var_type = 'rsid'
+
+# covariance_dir = '/Users/lg10/Workspace/datafiles/OmicsPred/GTEx_V8/covariances/MASHR_eQTL'
+# sqlite_dir = '/Users/lg10/Workspace/datafiles/OmicsPred/GTEx_V8/sqlite_exports_tmp_eQTL_MASHR'
+# var_type = 'VarID'
+
+covariance_dir = '/Users/lg10/Workspace/datafiles/OmicsPred/GTEx_V8/covariances/MASHR_sQTL'
+sqlite_dir = '/Users/lg10/Workspace/datafiles/OmicsPred/GTEx_V8/sqlite_exports_tmp_sQTL_MASHR'
+var_type = 'VarID'
+
+def run():
+    ds_selection = DatasetsSelection()
+    datasets = ds_selection.get_datasets()
+
+    print("## Start Covariance Validation, dataset by dataset")
+    covariance_validation = CovarianceValidation(covariance_dir,sqlite_dir,datasets,var_type)
+    covariance_validation.validate_covariance_files()

omicspred/models.py

@@ -369,6 +369,11 @@ def cohorts_validation(self):
                 cohorts.add(cohort.name_short)
         return list(cohorts)
 
+    @property
+    def phewas_publications(self):
+        score_phewas_list = self.dataset_phewas.only('dataset_id','publication_id').select_related('publication').all().distinct('publication_id')
+        return [x.publication for x in score_phewas_list]
+
 
 class MolecularTrait(models.Model):
     """ Generic Class to describe a molecular trait """
@@ -732,7 +737,7 @@ def traits_reported_list(self):
 class ScorePheWAS(models.Model):
     """ Class to hold Score PheWAS values """
     score = models.ForeignKey(Score, on_delete=models.CASCADE, verbose_name='Score', related_name='score_phewas') # Score that the PheWAS data are associated with
-    dataset = models.ForeignKey(Dataset, on_delete=models.PROTECT, verbose_name='Dataset', related_name='dataset_phenotype')
+    dataset = models.ForeignKey(Dataset, on_delete=models.PROTECT, verbose_name='Dataset', related_name='dataset_phewas')
     publication = models.ForeignKey(Publication, on_delete=models.PROTECT, verbose_name='PheWAS Publication', related_name='score_phewas_publication')
 
     # Method

requirements.txt

@@ -1,5 +1,5 @@
 #### Django library ####
-Django==6.0.5
+Django==6.0.6
 #### Python PostgreSQL library ####
 # psycopg2-binary==2.9.11
 psycopg[binary]==3.3.4
@@ -8,5 +8,5 @@ psycopg[binary]==3.3.4
 djangorestframework==3.17.1
 django-cors-headers==4.9.0
 #### Elasticsearch libraries ####
-elasticsearch-dsl==7.4.1
-django-elasticsearch-dsl==7.4
+elasticsearch==8.19.3
+django-elasticsearch-dsl==8.0

rest_api/serializers.py

@@ -141,6 +141,20 @@ class Meta(DatasetLightSerializer.Meta):
         read_only_fields = DatasetLightSerializer.Meta.fields + meta_fields
 
 
+class DatasetPheWASPublicationSerializer(DatasetSerializer):
+    phewas_publications = PublicationSerializer(many=True, read_only=True)
+
+    class Meta(DatasetSerializer.Meta):
+        meta_fields = ('phewas_publications',)
+        fields = DatasetSerializer.Meta.fields + meta_fields
+        read_only_fields = DatasetSerializer.Meta.fields + meta_fields
+
+    def get_phewas_publications(self, obj):
+        if (obj.phewas_publications):
+            return obj.phewas_publications
+        return []
+
+
 #### Publication - Extended (with Platform) ####
 class PublicationExtendedSerializer(PublicationSerializer):
     date_release = serializers.SerializerMethodField('get_date_released')

rest_api/views.py

@@ -993,10 +993,11 @@ class RestListDatasets(generics.ListAPIView):
 
     def get_queryset(self):
         queryset = Dataset.objects.select_related(*related_dict['dataset_select']).all().prefetch_related(*related_dict['dataset_prefetch']).order_by('num')
-        # Filter by list of Publications IDs
+        # Filter PheWAS study only - FOR PRIVATE USE CASE
         only_phewas = self.request.query_params.get('only_phewas')
         if only_phewas and only_phewas is not None:
             queryset = queryset.filter(phewas_count__gt=0)
+            self.serializer_class = DatasetPheWASPublicationSerializer
         return queryset
 
 

search_es/analyzers.py

@@ -1,10 +1,9 @@
-from elasticsearch_dsl import analysis, analyzer, tokenizer
+from elasticsearch.dsl import analysis, analyzer, tokenizer
 
 
 # Documentation about analysers:
-# v7.17:
-# https://www.elastic.co/guide/en/elasticsearch/reference/7.17/analysis-keyword-tokenizer.html
-# https://www.elastic.co/guide/en/elasticsearch/reference/7.17/analysis-tokenizers.html
+# https://www.elastic.co/docs/reference/text-analysis/analyzer-reference
+# https://www.elastic.co/docs/reference/text-analysis/tokenizer-reference
 
 def id_analyzer():
     ''' Analyser for the different IDs in the OmicsPred indexes '''

search_es/documents/gene.py

@@ -2,7 +2,7 @@
 from django_elasticsearch_dsl import Document, fields
 from django_elasticsearch_dsl.registries import registry
 from search_es.analyzers import id_analyzer, name_delimiter_analyzer, word_delimiter_analyzer
-from omicspred.models import Gene, Dataset
+from omicspred.models import Gene
 
 
 # PGS index analyzer

search_es/documents/metabolite.py

@@ -2,7 +2,7 @@
 from django_elasticsearch_dsl import Document, fields
 from django_elasticsearch_dsl.registries import registry
 from search_es.analyzers import id_analyzer, name_delimiter_analyzer, word_delimiter_analyzer
-from omicspred.models import Metabolite, Dataset
+from omicspred.models import Metabolite
 
 
 # PGS index analyzer

search_es/views.py

@@ -1,6 +1,6 @@
 from rest_framework import generics
 from rest_framework.response import Response
-from elasticsearch_dsl import Search
+from elasticsearch.dsl import Search
 
 
 # No need to redeclare "custom_exception_handler" as it is already used in rest_api.views