From 116809f0c40f1c1609b513f292f38751373e057a Mon Sep 17 00:00:00 2001 From: ens-lgil Date: Fri, 3 Jul 2026 14:36:46 +0100 Subject: [PATCH 1/4] Update export scripts and tests --- .github/workflows/django_ci.yml | 4 ++-- exports/config_template.py | 2 +- exports/exports.py | 4 +++- exports/metadata_build_export.py | 14 ++++++++----- exports/scripts/generate_metadata_exports.py | 3 +-- exports/tests/config.py | 4 +++- exports/tests/data/OPD000001_DS1.db | Bin 20480 -> 0 bytes exports/tests/data/OPD000001_metadata.xlsx | Bin 14076 -> 0 bytes exports/tests/test_sqlite_export.py | 4 ++-- exports/tests/tests_metadata_export.py | 20 +++++++++++++------ 10 files changed, 35 insertions(+), 20 deletions(-) delete mode 100644 exports/tests/data/OPD000001_DS1.db delete mode 100644 exports/tests/data/OPD000001_metadata.xlsx diff --git a/.github/workflows/django_ci.yml b/.github/workflows/django_ci.yml index eea3801..d7cf620 100644 --- a/.github/workflows/django_ci.yml +++ b/.github/workflows/django_ci.yml @@ -39,7 +39,7 @@ jobs: services: postgres: - image: postgres:15 + image: postgres:18 # Provide the user/password/db for postgres env: POSTGRES_USER: postgres @@ -58,7 +58,7 @@ jobs: strategy: max-parallel: 2 matrix: - python-version: [3.13] + python-version: [3.14] steps: - uses: actions/checkout@v4 diff --git a/exports/config_template.py b/exports/config_template.py index f7f17d4..4a93a8c 100644 --- a/exports/config_template.py +++ b/exports/config_template.py @@ -7,8 +7,8 @@ # 'num__in': ['56','105','154','203'] # Metadata exports -metadata_exports_publication_id = '' metadata_exports_dir = '' +sqlite_exports_dir = '' # Uncompressed databases # PheWAS exports phewas_exports_dir = '' diff --git a/exports/exports.py b/exports/exports.py index 1214cdb..e061610 100644 --- a/exports/exports.py +++ b/exports/exports.py @@ -17,6 +17,7 @@ {'name': 'name', 'label': 'Dataset Name'}, {'name': 'omics_type', 'label': 'Omics Type' }, {'name': 'method_name', 'label': 'Development Method'}, + {'name': 'training_window', 'label': 'Training Window'}, {'name': 'scores_count', 'label': 'Scores Count'}, {'name': 'phewas_count', 'label': 'PheWAS Count'}, {'name': 'platform__name', 'label': 'Platform Name'}, @@ -29,7 +30,6 @@ {'name': 'file_url_scoring_files_hm_38', 'label': 'Harmonised scoring files (mapped to GRCh38)', 'skip_auto_import': True}, {'name': 'file_url_scoring_files', 'label': 'Scoring files', 'skip_auto_import': True}, {'name': 'file_url_predictdb', 'label': 'PredictDB', 'skip_auto_import': True}, - {'name': 'file_url_covariance', 'label': 'Covariance', 'skip_auto_import': True}, {'name': 'file_url_phewas', 'label': 'PheWAS', 'skip_auto_import': True}, {'name': 'file_url_phewas_full', 'label': 'PheWAS Full', 'skip_auto_import': True}, {'name': 'file_url_validation_results', 'label': 'Validation data file', 'skip_auto_import': True}, @@ -71,7 +71,9 @@ {'name': 'sample__sample_age', 'label': 'Mean Sample Age'}, {'name': 'sample__sample_age_sd', 'label': 'Standard Deviation of Age'}, {'name': 'sample__ancestry_broad', 'label': 'Broad Ancestry Category'}, + {'name': 'sample__ancestry_assignment', 'label': 'Ancestry Assignment Method'}, {'name': 'cohorts', 'label': 'Cohort(s)', 'skip_auto_import': True}, + {'name': 'sample__cohorts_additional', 'label': 'Cohort(s) additional information'}, {'name': 'metrics_r2', 'label': 'R2', 'skip_auto_import': True}, {'name': 'metrics_r2_pval', 'label': 'R2 - p-value', 'skip_auto_import': True}, {'name': 'metrics_rho', 'label': 'Rho', 'skip_auto_import': True}, diff --git a/exports/metadata_build_export.py b/exports/metadata_build_export.py index 302ba81..f20a43d 100644 --- a/exports/metadata_build_export.py +++ b/exports/metadata_build_export.py @@ -13,7 +13,7 @@ class MetadataExport: def __init__(self, exports_dir:str, sqlite_dir:str, dataset:Dataset): self.dataset = dataset self.dataset_id = dataset.id - self.sqlite_dir = sqlite_dir + self.sqlite_dir = f'{sqlite_dir}/{self.dataset_id}' self.data = { 'dataset': self.get_data_attr(self.dataset,'Dataset'), 'publication': self.get_data_attr(self.dataset.publication,'Publication'), @@ -26,15 +26,16 @@ def __init__(self, exports_dir:str, sqlite_dir:str, dataset:Dataset): # Find corresponding SQLite file self.sqlite_file = None - for s_file in os.listdir(sqlite_dir): + for s_file in os.listdir(self.sqlite_dir): if s_file.startswith(dataset.id) and s_file.endswith('.db'): self.sqlite_file = (s_file) break if not self.sqlite_file: - print(f"ERROR: Can't find a SQLite file for the dataset {dataset.id} in {sqlite_dir}") + print(f"ERROR: Can't find a SQLite file for the dataset {self.dataset_id} in {self.sqlite_dir}") exit() - self.filename = f'{exports_dir}/{self.dataset_id}_metadata.xlsx' + filename = self.sqlite_file.replace('.db','_metadata.xlsx') + self.filepath = f'{exports_dir}/{filename}' def get_data_attr(self, model:object, model_type:str)-> dict: @@ -99,6 +100,9 @@ def build_performance_metadata(self, score:Score): # Update eval_type using the long name sample_perf_data['eval_type'] = perf.get_eval_type_display() + # Update the Sample ancestry_assignment using the long name + sample_perf_data['sample__ancestry_assignment'] = perf.sample.get_ancestry_assignment_display() + # Cohorts cohorts_list = [] for cohort in perf.sample.cohorts.all(): @@ -206,6 +210,6 @@ def generate_metadata(self): self.build_performance_metadata(score) # Create export - op_export = OPExport(self.filename, self.data) + op_export = OPExport(self.filepath, self.data) op_export.generate_sheets() op_export.save() \ No newline at end of file diff --git a/exports/scripts/generate_metadata_exports.py b/exports/scripts/generate_metadata_exports.py index 3076a96..43b651c 100644 --- a/exports/scripts/generate_metadata_exports.py +++ b/exports/scripts/generate_metadata_exports.py @@ -2,8 +2,7 @@ from omicspred.models import Dataset from exports.metadata_build_export import MetadataExport from exports.datasets import DatasetsSelection -from exports.config import metadata_exports_dir, metadata_exports_publication_id, sqlite_exports_dir -from django.db.models import Q +from exports.config import metadata_exports_dir, sqlite_exports_dir def run(): diff --git a/exports/tests/config.py b/exports/tests/config.py index 3da21cc..34c81e4 100644 --- a/exports/tests/config.py +++ b/exports/tests/config.py @@ -1,7 +1,9 @@ import os +dataset_selection = { 'publication_id': 1 } + current_dir = os.getcwd() -metadata_exports_publication_id = 1 +sqlite_exports_dir = current_dir+'/exports/tests/datas' metadata_exports_dir = current_dir+'/exports/tests/output' phewas_exports_dir = current_dir+'/exports/tests/output' sqlite_exports_dir = current_dir+'/exports/tests/data/' diff --git a/exports/tests/data/OPD000001_DS1.db b/exports/tests/data/OPD000001_DS1.db deleted file mode 100644 index 7d2eef06e22edabb004d0eea47027f05a955b8cb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 20480 zcmeI%&yLzg7yxjaWfu+$?x|?Is%qG#T2w+ko*9qllmOYFNGl~FNIi~B7{C%3VpEbF z(@RcygC6<9a7*&YzF(`^{X;-n)Aq&b2+5 z#*<)fQ{yYcG>z-FZ5YNO-1~5Uyzz-|jXxn<@`rIPvUgL%*!YyD^*2g|oit8>=qp4r_- z^Zd+ScAXA&q-%Ei9tD$iOU)n8pgD`Da~(!CyAsutXqME&s6LtY^|(^A?=MuxRl#Bi zqqA7|>oXk>>Kzw`BAv{2U%`w)n2dC+W{Y54v*Xe9U-Q!uSg{e|!K!CA3T?>sym~Wx zP`Yx(yt}LfvYbw|>irUq`sN(Y5mJ3Nu>%$01BW03ZMWApa2S>!2eL7 zX%@;y)gv=!81OP~-)Y}-(u?10wL5Ne{ezLZKkbLn@D3$Xvi!9_UcdSJ>5rLfe;s`P z_NQ-tPoMwug+C4JrS-^S#lMTsi(&Bx=)emFPyhu`00mG01yBG5Pyhu`V5bCbW)$I4 z?)VPz6@y9AqZ}f%(in{vgrJ`EeV6(kla7>r^B`9)e;R@ik8{BsB9sR$%7j$iRg7*L z!;yp&=1M6Y$|Y~@uVQQ!Lm7OFpzbOLgTfd@=%#tJx@j#gfjnKp1!Te01BW03ZMWApuiUiWX(+GaJYK@e`i?lzGx9nLID&& h0Te(16hHwKKmim$0Te(16xbPoyqP&_cUmJVJ^L=GP#R{n4MV9*HnN7EbTT55LVd|6)XJ1MO)i@3-A*J;4nd8zu z!*!)iHZpyO= z4(B#NQW;vu)MH#Zm^ga0syuPeopd1w;=&PYi%8>mWZx%UPs>^qE?CE4NoXV%1I)tI zq|pZ^Tv5?zedG?`l&jIjFW8=*MfDDBPVq5`U3$si2}G`>*lK3M^^d05EczfW2IN?l;`-??Py}_E9An)=;n*@84>{S@&W;n`!}Z6DKnB@fx=G;L_`D-Q+4c( zEFBo=|9Jf$L;s6u`7eKZX{@wtHzQ)usn}!i!0qgEG^(KF$9G~aKqW69@goUCsiUbB%mD{|lZA4R0ZZWZJ)W|nDVJHLex^M)iK~vG zWyhs(+E?@^4?3Cj0R<9Pv^(bU_x- z9GC25_fzAvP-c~qajjfz;a&p1vv2^C(E=w=AwKi7l(p3&i0txTP)B0746 zE3nD7A9wdD3XV&x^MDnGSO~sqhoLKDJuxAYe~@NHzn6z`!y7lkrxve_EV@-TM?}LK z>_BHsxRZEp>e^KKsUV-H2;3El?S=w9(S({iIA8r@*&o%lxg5HVszc(@M;P|D!9LTJ zVUtB2`-UiBsmtF075d!~?>n5iz-@Jf;a~~%iFJzd3iI+#q8)gQlc#lWi)=0otUUz% zz?kX)nbCmnOC?S*B#*kUT{K7DiOI@E6y$_&HdW#jTl->)^(ScF@a9`~8U@`KD}2YZ z1mZ)R%qX-{>bt2ky#Nh{G_4uQOb9!EJafvU2Fs`y$&}u#Jv(2aM;=DDcdYvgS4&&1 zHc#)rl`KHNc>ESt@np)2s3Y#R+%?G>#b^z;5=tOTos|6CX)o%m zMGNv1AaS_w2N`o(A=PU>?}sB8>`MYF^cP-7s9cEmbRsbEq81Vcjoj9pp95|=*+#J% z*=i`$z*AHQ?IC9Dij7FT%0DUCMs2TJ|@%}M;(&@+oX4fZQL0Nz74T;y!H!T}lvRBp z1`*(|6ya0#aYF@>=L8Y}zyrX7fhh84D*0EU{4=e9feJs+v;W;!Yy7f#03&MPDS{`= zpl3ugKb_I$N_^K!{UQu3H=dd_8Ozi8F%N7RX5y7-E1XnN`sFs_;QHNc-6V{QWx6s+ z1o|sqI?aW7^-m3#m&^Vk(Y16=hi|Bu0HFq6?hY=#D(J+5(^oiEp(5mn@-I~6SkT-N zRfnDSh4$^!J^2+z6=7lJ?~KK(lb$Pr224^Xi{v+qACjMAelyY|&^n$H|`pTajzj&~~Qp5%2)10$vT#@=Y~xol6X>JwA)84mD0IRaC^P1iWIK(EP)A_Xd=jjzo&qKDqbf) z(m7)pLKmScUMIdy2H(vjFNM`;95TvAz%5OVEjijCfl*GuL}G$`(2(-x_>=Ux#U2Jr z#o|Sfg)(@r`p#6Favlyf(N{}&t9kBV2jabGJ4eQ|$Tb$u{GxB}De+sP8X3)iWN|Hr zmpesf@^B8Rr%N3Xq2Qj7j4N!Hg3z}rXoPHkcg5J>&ZXPjsiyC56HhrKZplsb%LE|<^pfM7@LRtxj zIv;yqkjr$n>pd;_7NzZW-W(?0;N7=nlJpot)6S?622V&UQlcLrUm{d*Uiq_cR7K&` zzbbjFKO@R}{c4kaqbaJ<3Keq3f(+7KhYqNVe@;PoPH!+BX_>_IUEXs6-Wk%{5{gH? zLcKF`M7B)#z1|+N_ly6zqXaeI2Mq*8j`znku~t1FjM=Lzsj$tqw=}|~tgab-ObR7W zPcL(V9DIG-!mxM!P2}Vdino0T5)Zx-_Sq+0Lt+HF_(5vntpT9sa_Mb~DDvP*|h94&SEUkyZlF7xpF8W<$8X zB|wlLlyR5qz6KTodOGPha&r0(H1t$b!_1AXk zK`w$_%8>}#adPQEZnSWwxfnVs#c z8AR|AAL042pI59?4qPqrd6B5~$ab`J>&>$k7l~CN5CV}D-CrmtIr+jioM^xs3+=z| z#5#PlZE|Nm4Kv(DN{J5pQNjv8;(fC_>03f72mtNbY2GYy?=*noklr8bjhH&sw$fjwufOQ3BwDqEAfc*c|DurD>TS z|2pNCIs{h|&;V&mFi46ux*f4;QUIFS@Ow~WJz~?W00FLjp>HFzVS%p;^OiQm{$Ml= zlCb=(JcRs==ac}3Qx+Sl$TIdHKdM7I$8cI@jc#kkGSrO=l7p6!6#{U;?NL`XzGX#@ zrwY$c3AH1^O;}rMRBmq3)?t#$SCeS9CeKAH;u-wXV@iRWu(u>BN=}aw+?RTQUsN1I;PPneK_(O+&K{q%dj6gN7HJM z;)Pb=ZMo8b&>BH1VN+N=4?JKkww*_ac?Bx`AYPY&Rq-~^qlJJJ1DqqEO_A>^57Hjgsd(1L(KFml7FKw+8)O;4{M|Tm_16BzT{lLDcYYDOfRg2+1cBF(+7kq#I(NtpK@NOA_wh>x-TFnPdL2vvKaw zJ%aK+KcCx>g>2HWXW^Ug6jYpuuM#P2@cM!`FO($aji7HGys&^k0ehT`&6dsUXBU(v zOP}?PlbCXo_KJ2r^b+15)%rz|t(xKN2p5qq0_dz!RcJ z-__K|Q97|7IwT>N0ST4%y$=3s6~`>R*D8^jhtYec0#6kWw4vlaBVz6 z+9X9GQzJWL*s?9nHq836+9jOS#549O?B39oCAlhl7aQ2JBfeIXolSRSLE^W4We(vQ zDIwJviEY^ek{2O&u{&Mj;yDk5SJ2w*Jsti{hJgouCKFazQi3`>r7CCT0-x0&i5}g_ z!Uwwr>y8eN4K6^9=wFE|C9t*4Ii z-HVE~GYJ_#)%g@1YpN)}v>Awr*{V{j$J?fDvap!+k&1q3;bmbDKo|BT{bheEc4|;`qnK?UJTv=h2xA3B7Jo#2_}5TO;1nQ8Fnly2-pFu~L_3B6AuJ&e zAUWg`NzQwbPbVQ`t3WrzvgnlVn_&RTL_$3b>$+wV2FwQM+?0W(;+%t(0e8Kji5ne> z{Yews$zRKdwMRgi2#YoMqKt$sf;aLc32M_N=G_X<*5>@Y-Cfh!EwXj-tx2Xmo&$v{ z4V`ZNVet2>55%3x7SC;ftgPeH?cv982q_^gH!^h9N5%Pj&u8|x?o4Dv;R4}HN)ctIs$3&-OvY;|bBn#btJQKt0X>;p$&Hd?Sr z9l*52oo9j*Kj)6?Ps+x0E8he*tm1sFIVQ{=P6!}Bt3ODia!q^nN{CdbBe6t3L=75w z-EvFGN^DQ4wWNHpjo;7}>0>Ka>~^qNF@d=sO&o5b3-gAhg@8p-sbncTO=$Fs3}8)7 zuL84C9o!dA&YtBQhSVLRy2pKBl*WR;?m+MDB<9?X0 zN?4kr!53T;>sd)6cvkWDnV@z9GwKlwfBwo^g{5!MTN-Y-8en@+mbi6Oe5cfD=5-i; zubi6%0bV-UAj9kZTB8|-G}i==?ul!sC-bmX{^<(#E11Okje7QgM63=iT7Xu2qP&5} z7hFPGTmSX_?1Zax9|V5`uezCQKzYB)r=%^oOLDq^n3&qsXkW84w*Wuv=;Q=*YiLFp zY^ko-yu4*=EdCt?4sT#NjiiW(W}|nZeUbRU zdSiVgJpzbcj``hhk*oQ2z)-$V7bvn$VC^uAg>Oh(6m+1IZtM@Z6z9^e>tK)YrwKO0?Z`tg z{^c^5N^HIw-^|0o1eSSXbfeGx_V-w2)T%#+`^wm})(9FMh9u;Jo73_HOuzEwwj zZs`T65TW_iXHLW)s5&j9Hfw!qtF5!6YXEXRh#HPr7$v;1$Y}L}&6p*`|E|v3w~A2t zBx*`m>`>wJksD%_o85FMU}86ty5pJl6=+!hH%EL|@ZyChC@kmj|Jbel?2H{vjjW6q ze!l)}ijUMJZSXlz+p+h&U*BZz>512e5BQQx+NE(aS(1xH&V+t9>d~SX8;Z0Rv411? z*;M^X;d28~XgwI(=cKE-M(N1{foH+sEeUmXM@vfRAJxe9Jx`pcxMQ{zTHeWwjsgLB zyjD;NCEI`))-9n_(Xut`uOE%*1QgaCLvUrqX_L`#HCJ6;+ALhOddYyWH-+|u4SuRc zmTgM0z&8f22_|HLCZzOerE7&D5<E4UG+Vsfj|0fi-d z@WqL=+qwpQUxs}&iu1H#Q|9L^Ls}m0+SeO%Chci>q&`E8sDqsmUv$59{Q}-osdV-k zGk%nk%1V=V{n`Uvk@AS{=rBdLone)c-~*EEE2JGM+BS6uCkvTXK(czDQ_HeU19PDX zfD`eZ1+8R^aan_k*|;f|z|A4^u}qV8dD9*EQv!KP+Oc!X%7{^wFw%fm8QuEaP{eoR z*-rfE-npc+x*OxdY1=bs!}ADo51YD$FHJ5Gp}5lhlBUNK+6;wngTpx&Z@T6H?J8sI z2pGcMyi=@UuViP{Ht0K18u!3dZS++qF^bGYML7e+PS*@yQ+*pb>E0(tuJ-N%C5`--$6LPkL@8OpVzeE(xk$H04f}2&DrBSRU70H^|))}N_ zxPKW! zOf~FA0z469zJAJfc3p1oF8CCN5gO^qYNIz6HbD3Gq=$2`m3f+#UY|eO09HQSx#^#!~9U_UpZfjv*wRy}BdD z2T-^6{lr-`t4XA$T@HH?2SxmZwCuJzil$GzMl~fcrvJTL0dMbnC~Ily8^XqnUU3oU zI`sL`)?0e*RHP{chJAv!iIH#UC@3W-^=M7Mm0SY?IW3?_anjZ;VtwcGt=}+(t$EQX zbp<^Y4O(_Nt&bM)zoE6I)jw{}t5k$x2F_w7Rv{r>QlWq)rUF+7bjtkI7$m^(_icc^ zLwJzgs}xZ4lTLKF5n^9L$LLFwc~Bt+B&oBgNNzUQfq0dl=DXLt2#Y_DM@uw5F=D=Z z@v-2svbr3g8*`n)WEe&qD>q2&i#=?qa0H(O?)rw5nQ7%OC3ZGRJqqW6C~!6S=x&q z!QKn|?i-}IZ;Z;te0acj>%0n8uMM$>Vn4MjhA3au496OB0nz25?2L?iiJ~F8el$a~ zUo8-Q#t~drifcv0SvtRS@ZQB`4RMUSls)#aWX@{fn^^X>sycdRxgn_&oMa$?l0IE9 zaVvJ^a_R2$8q58rQc50Ca?L_qhpR~ge1aL{rx6sD!a0Pf8^8TrPvTOcw@3>q@7c>v zB!(@fK?F0wxgdfsl^SoQdU0~owMf-?KE|`mr86+quok9qK5WW z3`!a?hg@@-b?N-t*U*VS_BiSa@+>W**Ss(wX?0L3?6UM|Ly5&nLcA<3D&%q~f(UU; zc8=`z$U={UbCpsFI0cTo$f@P9%N+x_`K&ie8E#&cP87>0H2Jo+7t^RrB-wI2_fhu7 z!!duQQ^a|AcU5{L^Ob3OYVPWj+P!U}l0+&rW^n>VF9N`vdj74%b;HOpt(?o*Q0xwZ z8wtT1;7v`QoTizf0JrX#8riGb+eY$qLe?5<%va%A?hb}XGRK{=jB6`+o?*G~wHIhB_q<9gv!Op)e8%d#4_p|Cg$;V+i^ZxmOn_zT;>9^9kR32RrMUiB?n2DQ z6)ew-aJC%OkvT=4J|?f&;$Sm`CPNFcZR*Us&?3*Ki86$&UGmDsN}NA*8p%k= zPKIYd89uu@%YD$wr+$Q%)+Nq;5Ip9r&iGa#YTS1nj>}U?fh8kR!+>h;O69`^@9+f0 z66wnUyFluJ*)8d`6z<3hQB_0X@y!n#Gn$Z_`kBz!e@E(Vq+^G1aH3szUcU1&#rgc0 z&Maf=Rl=V_p=O~eyU^QPT9=uwra(Ap(lP3-~~w`c#vQp~ilIg^{b zaLJgB{rlVup7kNYf{gq5j!GXMb2iCy{*NuI><=xx7vDOqkLj>kZ`jYRPkVEj;{C&KTlNN)zO*Bw?k;zwN>oLZ$_&hl5z-|a0DAb`&HyO_^L{Qy^ zG1c4;XGv}@IKyvBwomBCHgRw6B8X2qP`aP7JN%*=g5coGn9SK|SR4GrDj#KDJ%0({ zf66p^0d+cmoh6)4Bn57O)_JHP-eUZNx~6*eMuv)x_GZ>5KN%eUVbOY#6SVr)ebHMh z#0Vi@80%FdqxKM2INi~omudE-F*SQrlP{KlO}S99nq9k)mjgb7!qf#c2}*Yu3zy}8 ziwW-h1?j^@>2z2u@@FaKJ|nMhO*&f6!Gw&7gW+)G=cSk71()ZKx2rM{RRSjYT!)rr zbT1(Wan@D36bj@L)t2$}$@$(+L}?17!y{OeSR^NugYue;^gPitlytq)RN--t!QJ9* z_V)8!{A`bpGnuQHylf5;xFgHzXYp!7m^UXomne*BnSD2ptT#M|%SoDw#-t_<5*${v z)==!&FG@0!6l;l?zQvethv&zYcXBm!kcCXh8mMg>3E9Ua!)S_;U?=6Sp;Nl8 zf?7&|MvhMDC%%xa%5}fjRUdXJo>G}((8$ALsp_$C0B7jafw4=dVtnx{1PrQbQ#8Os zbEXC4pw7vxF1Cv24#;tH^=a%9dvni^^Gu_-vq8fV=F864uC=&44U6&I1qhU zv+5m+W`;WnyI~u3)q4M=rfbh2q!Y*M<>)*`<=i^AP+0V;Pf^}mdL(5$FStZje({T*UOXo+-(?K<&*q$)J4IU=2=s=( zP%RHFKqcmBLmnky_ZN(3mC=oOmQX0xh9jUe$gPbDxs0(o5zvSUtnx^Eb~ohF#up1r z$xB;h+D#|H-Mqz}r?gLW@eyh^(Su)$7hIsl_{T_1sw!;3A;~vQjgJn?VnkTfLq&!O5Mp`m2`(%% zt@{P5TPh1EG;g&$*d$9gRGOw7@vf7zHGZy@O$dP3UzQv> zLPITi=^mp(20WwE%2$YNSLf>HLh`NL3x@>=hUmPnai$qkFaN%r>6hP61_u>7+n|z$ z^v^T#4}rr!*3y3}9)7<0x$gU^cj$?emV{tJ4LbFE0gfz>*3Z_9)wF$?@W(FV*OfBn zw``YESom=mfzhe|?q-81IED9AC|Rw{J$Hd8aGr}+S0TP*K%$U)5!PkpttOpsZ-7d+ zb0Q=GRabL!t14eiUp#Sd{bh_%XaxB8_(!^3`FcY}tg*P!jPFqX)JfUvMBB*t8`IV4 zmuGQcvv6@%@jjLEEV_&SEY z#7tba!sdFf+;=962)tJ*;j!4xE;DsDUrz6<(&xH)DK{T?HT=7IZA8zP#?ARgF5NI> z1yfKi8@s>W#$^$hT!U)9zor#E**mm-kn3(hZu~!8_;bMa(}k@uo#sCkX#Q>flb&|p zyK!>G88otkwMB0M#bsm^kpz~7W-DHJ29<&f);#dyt1m#S`lQYhmfkJ!50x|$ay=BP z{Z>$?{mb3xU*covesm~x3c}T2KP~S>@EWU0WZ=i4!56|er}vwb#qFDRCBP0OE~K9}re6j^!`u?><1spnoa;3NQ}ZBzR* zj>eGAJlWVKFSeq${d&L>wrSqPs$#IN$l7%*HM849gY<_N(*sq_u|37iqYKyhRj2FO zhJ!yuWqP2YKJ*}XZNpR73$YVjEGUpB}wuNp!4>)*qb1J=Pr9uJg+QhZ%#dRC2Zzm^x z1@&!EB`JiAI#l3NMXZf((xNSVm&!iVtTR`PXLe zo2Vzp+=ZB&Jzzqc$T%HQIm(nrf~Tyj^d94^z)Z}zyWx4!M_8613><(t=n7{DQ%6=T z0LcCcAMj&B3b;;wZ4{Lzetc-74W%h!>EwNkk3(1<7z`)=ao7fPPU1-@OoCp1t8*nu zdNZmAULOeLlwoC>pmsDgFkRFGdf7(qo#xL){TlgIXzLsFqW8qvq1qE6%*Ebb2nYia z2;;I68XniHv~y@AYt+(>*JlDIEM;mQbG@HFcxPF)$`vBMpj~oGTEtErMaJF-xIhQI zs(2Q<{#$7>46)61TL1!K6zlNWlt?mD!4-|I)xYFaKWtSL12YkXEO3Y~~1l<>P zF40-R-s}*umWdiW3jDC1NZy)BYj3_dVf^YqZMU>2jyRLIrPepn8IGuvP+6bCL}<{? z*fGXzSk7zjU7xkc#QLxckcpV>02mxUj>3ivOo6cwQjG!qQcA+d3KF%LtkBr$H|f+u zmWWmO4b1WRHMaOiQ;DJPcY=a)ra;C3oeeWAY?8y0iBp*rF{V35r!ezX$fFnDM@}3oyt6ai zVz>UGpaL2Uzp$&l1GejvVoyXa-dWX%be?QT#ZC`I_W;kvR!h>R;chhgNg2>O>FsEl zRfz4>;*}b*T_b_b3s5zXhVdl06R~Co>&4t0GxggY1SQ2`(q+Znj5RoEmt9ohySj3z z0KX6X$FA^S+_m*eA4p1u?&vVykM;|Ddyj9DPGir7U0Dplu!T|IKc`=T2Il`bWx>Gd zKrPMRe_Q3B|JHxr|K__Za+3cJ@b4n}e;WRAF9vbvUnKRv8~!dV{$)B2n(+T7GydK9 z-zAs7OaXukP~!d10?glWewW()LJ~y$|G&gvgm=HA{4Rm|g>r`cw Date: Fri, 3 Jul 2026 14:37:44 +0100 Subject: [PATCH 2/4] Update import scripts --- imports/omicspred/models/platform.py | 6 +++--- imports/omicspred/parsers/data_content.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/imports/omicspred/models/platform.py b/imports/omicspred/models/platform.py index f084fae..7e907b0 100644 --- a/imports/omicspred/models/platform.py +++ b/imports/omicspred/models/platform.py @@ -8,7 +8,7 @@ class PlatformMasterData(GenericData): - def __init__(self,name,type,full_name=None,technic=None): + def __init__(self,name,type,full_name=None,technique=None): GenericData.__init__(self) self.name = name self.data = { @@ -17,8 +17,8 @@ def __init__(self,name,type,full_name=None,technic=None): } if full_name: self.data['full_name'] = full_name - if technic: - self.data['technic'] = technic + if technique: + self.data['technique'] = technique def check_model_exist(self): diff --git a/imports/omicspred/parsers/data_content.py b/imports/omicspred/parsers/data_content.py index 752db81..dc9e1d5 100644 --- a/imports/omicspred/parsers/data_content.py +++ b/imports/omicspred/parsers/data_content.py @@ -16,7 +16,7 @@ 'method_name': method_name, 'full_name': 'Olink', 'version': 'Explore', - 'technic': 'antibody-based proximity extension assay for proteins', + 'technique': 'antibody-based proximity extension assay for proteins', 'dataset_name': 'UKB European', 'internal_cohort': 'UKB', 'internal_label': internal_label, @@ -43,7 +43,7 @@ 'method_name': method_name, 'full_name': 'Olink', 'version': 'Explore', - 'technic': 'antibody-based proximity extension assay for proteins', + 'technique': 'antibody-based proximity extension assay for proteins', 'dataset_name': 'UKB Multi-ancestry', 'internal_cohort': 'UKB', 'internal_label': internal_label, From 6c6a18784d3ae3c37ff7c64cd801f9a02b39b4bd Mon Sep 17 00:00:00 2001 From: ens-lgil Date: Fri, 3 Jul 2026 14:42:16 +0100 Subject: [PATCH 3/4] Add 'ExternalSource' model and add extra parameters in other models: 'training_window' in Dataset and 'ancestry_assignment' in Sample. --- omicspred/migrations/0001_initial.py | 17 ++++++++++--- omicspred/models.py | 38 ++++++++++++++++++++++++---- 2 files changed, 47 insertions(+), 8 deletions(-) diff --git a/omicspred/migrations/0001_initial.py b/omicspred/migrations/0001_initial.py index f5ffa4a..09fda48 100644 --- a/omicspred/migrations/0001_initial.py +++ b/omicspred/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 6.0.5 on 2026-05-12 10:40 +# Generated by Django 6.0.6 on 2026-07-03 13:41 import django.contrib.postgres.fields.ranges import django.core.validators @@ -34,6 +34,7 @@ class Migration(migrations.Migration): ('omics_count', models.IntegerField(verbose_name='Omics Entities count')), ('omics_type', models.CharField(max_length=50, verbose_name='Omics type')), ('method_name', models.TextField(verbose_name='Score Development Method')), + ('training_window', models.CharField(choices=[('', ''), ('genome-wide', 'Genome-wide'), ('cis-only', 'Cis-only')], db_index=True, default='', max_length=25)), ('scores_count', models.IntegerField(verbose_name='Associated Scores count')), ('phewas_count', models.IntegerField(default=0, verbose_name='Associated PheWAS data count')), ('files_ids', models.JSONField(default=dict, verbose_name='Files IDs on Box')), @@ -43,6 +44,15 @@ class Migration(migrations.Migration): 'get_latest_by': 'num', }, ), + migrations.CreateModel( + name='ExternalSource', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=50, verbose_name='External Source Name')), + ('version', models.CharField(max_length=50, null=True, verbose_name='External Source Version')), + ('url', models.CharField(max_length=100, verbose_name='External Source URL')), + ], + ), migrations.CreateModel( name='Pathway', fields=[ @@ -89,7 +99,7 @@ class Migration(migrations.Migration): ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('name', models.CharField(max_length=100, verbose_name='Platform name')), ('full_name', models.CharField(max_length=100, verbose_name='Platform full name')), - ('technic', models.CharField(max_length=100, verbose_name='Platform technic')), + ('technique', models.CharField(max_length=100, verbose_name='Platform technique')), ('type', models.CharField(max_length=100, verbose_name='Platform type')), ], ), @@ -280,6 +290,7 @@ class Migration(migrations.Migration): ('ancestry_free', models.TextField(null=True, verbose_name='Ancestry (e.g. French, Chinese)')), ('ancestry_country', models.TextField(null=True, verbose_name='Country of Recruitment')), ('ancestry_additional', models.TextField(null=True, verbose_name='Additional Ancestry Description')), + ('ancestry_assignment', models.CharField(choices=[('SR', 'Self reported'), ('GS', 'Genetic similarity'), ('GSR', 'Genetic similarity to reference panel'), ('AS', 'Author statement - no method reported'), ('INF', 'Curator inferred ancestry label from cohort description (e.g. country of recruitment)'), ('NR', 'No population descriptor or inferrable CoR provided')], default='SR', max_length=4)), ('source_gwas_catalog', models.CharField(max_length=20, null=True, verbose_name='GWAS Catalog Study ID (GCST...)')), ('source_pmid', models.IntegerField(null=True, verbose_name='Source PubMed ID (PMID)')), ('source_doi', models.CharField(max_length=100, null=True, verbose_name='Source DOI')), @@ -352,7 +363,7 @@ class Migration(migrations.Migration): ('var_gene_exp', models.FloatField(null=True, verbose_name='Variance of the gene expression')), ('variants_number_used', models.IntegerField(null=True, verbose_name='Number of variants used')), ('variants_fraction_found', models.FloatField(null=True, verbose_name='Fraction of variants found')), - ('dataset', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='dataset_phenotype', to='omicspred.dataset', verbose_name='Dataset')), + ('dataset', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='dataset_phewas', to='omicspred.dataset', verbose_name='Dataset')), ('phenotypes', models.ManyToManyField(related_name='phenotype_scores', to='omicspred.phenotype', verbose_name='Phenotype(s)')), ('publication', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='score_phewas_publication', to='omicspred.publication', verbose_name='PheWAS Publication')), ('samples', models.ManyToManyField(related_name='sample_scores', to='omicspred.sample', verbose_name='Sample(s)')), diff --git a/omicspred/models.py b/omicspred/models.py index e14624f..9c8e3ed 100644 --- a/omicspred/models.py +++ b/omicspred/models.py @@ -107,7 +107,7 @@ class PlatformMaster(models.Model): """ Class to describe the platform used to get the omics data """ name = models.CharField('Platform name', max_length=100) full_name = models.CharField('Platform full name', max_length=100) - technic = models.CharField('Platform technic', max_length=100) + technique = models.CharField('Platform technique', max_length=100) type = models.CharField('Platform type', max_length=100) @property @@ -132,10 +132,7 @@ def scores_count(self): class Platform(models.Model): """ Class to describe the versioned platform used to get the omics data """ name = models.CharField('Platform name', max_length=100, db_index=True) - # full_name = models.CharField('Platform full name', max_length=100) version = models.CharField('Platform version', max_length=50) - # technic = models.CharField('Platform technic', max_length=100) - # type = models.CharField('Platform type', max_length=100) platform_master = models.ForeignKey(PlatformMaster, on_delete=models.CASCADE, related_name='platform_version', verbose_name='Platform') @property @@ -185,6 +182,19 @@ class Sample(models.Model): ancestry_free = models.TextField('Ancestry (e.g. French, Chinese)', null=True) ancestry_country = models.TextField('Country of Recruitment', null=True) ancestry_additional = models.TextField('Additional Ancestry Description', null=True) + ANCESTRY_ASSIGNMENT_CHOICES = [ + ('SR', 'Self reported'), + ('GS', 'Genetic similarity'), + ('GSR', 'Genetic similarity to reference panel'), + ('AS', 'Author statement - no method reported'), + ('INF', 'Curator inferred ancestry label from cohort description (e.g. country of recruitment)'), + ('NR', 'No population descriptor or inferrable CoR provided') + ] + ancestry_assignment = models.CharField(max_length=4, + choices=ANCESTRY_ASSIGNMENT_CHOICES, + default='SR', + db_index=False + ) ## Cohorts/Sources source_gwas_catalog = models.CharField('GWAS Catalog Study ID (GCST...)', max_length=20, null=True) @@ -324,6 +334,17 @@ class Dataset(models.Model): omics_count = models.IntegerField('Omics Entities count', null=False) omics_type = models.CharField('Omics type', max_length=50) method_name = models.TextField('Score Development Method') + # Cis-variant, Trans-variants + TRAINING_WINDOW_CHOICES = [ + ('',''), + ('genome-wide', 'Genome-wide'), + ('cis-only', 'Cis-only') + ] + training_window = models.CharField(max_length=25, + choices=TRAINING_WINDOW_CHOICES, + default='', + db_index=True + ) tissue = models.ForeignKey(Tissue, on_delete=models.PROTECT, related_name='tissue_dataset', verbose_name='Tissue', null=True) # Tissue trait defining the sampled tissue scores_count = models.IntegerField('Associated Scores count', null=False) phewas_count = models.IntegerField('Associated PheWAS data count', default=0) @@ -782,4 +803,11 @@ def values_dict(self): 'bonferroni': self.bonferroni, 'effect_size': self.effect_size, 'var_gene_exp': self.var_gene_exp - } \ No newline at end of file + } + + +class ExternalSource(models.Model): + """ Class to hold ExternalSource values """ + name = models.CharField('External Source Name', max_length=50) + version = models.CharField('External Source Version', max_length=50, null=True) + url = models.CharField('External Source URL', max_length=100) \ No newline at end of file From 4db2bff2ce2b758c5bf714cad859b95abced9922 Mon Sep 17 00:00:00 2001 From: ens-lgil Date: Fri, 3 Jul 2026 14:48:06 +0100 Subject: [PATCH 4/4] REST API v.1.2.1: Add a couple of new REST API endpoints and update few others --- config/settings.py | 2 +- rest_api/fixtures/db_test.default.json | 39 ++++++- rest_api/serializers.py | 70 +++++------- .../rest_api/openapi/openapi-schema.yml | 106 +++++++++++++++--- rest_api/tests.py | 21 +++- rest_api/urls.py | 51 +++++---- rest_api/views.py | 36 +++++- 7 files changed, 229 insertions(+), 96 deletions(-) diff --git a/config/settings.py b/config/settings.py index 516f4ec..9fb25e5 100644 --- a/config/settings.py +++ b/config/settings.py @@ -267,7 +267,7 @@ # REST API Settings # #---------------------# -REST_API_VERSION = '1.2.0' +REST_API_VERSION = '1.2.1' #REST_SAFELIST_IPS = [ # '127.0.0.1' diff --git a/rest_api/fixtures/db_test.default.json b/rest_api/fixtures/db_test.default.json index ae8a242..2bbf967 100644 --- a/rest_api/fixtures/db_test.default.json +++ b/rest_api/fixtures/db_test.default.json @@ -1,4 +1,22 @@ [ + { + "model": "omicspred.ExternalSource", + "pk": 1, + "fields": { + "name": "Ensembl", + "version": "113", + "url": "https://www.ensembl.org" + } + }, + { + "model": "omicspred.ExternalSource", + "pk": 2, + "fields": { + "name": "UniProt", + "version": "2026_01", + "url": "https://www.uniprot.org" + } + }, { "model": "omicspred.Publication", "pk": 1, @@ -39,7 +57,7 @@ "fields": { "name": "Somalogic", "full_name": "Somalogic", - "technic": "aptamer-based multiplex protein assay", + "technique": "aptamer-based multiplex protein assay", "type": "Proteomics" } }, @@ -49,7 +67,7 @@ "fields": { "name": "Olink", "full_name": "Olink", - "technic": "antibody-based proximity extension assay for proteins", + "technique": "antibody-based proximity extension assay for proteins", "type": "Proteomics" } }, @@ -59,7 +77,7 @@ "fields": { "name": "Metabolon", "full_name": "Metabolon Discovery HD4", - "technic": "untargeted mass spectrometry metabolomics platform", + "technique": "untargeted mass spectrometry metabolomics platform", "type": "Metabolomics" } }, @@ -69,7 +87,7 @@ "fields": { "name": "Illumina RNAseq", "full_name": "Illumina NovaSeq 6000", - "technic": "whole-transcriptome sequencing platform", + "technique": "whole-transcriptome sequencing platform", "type": "Transcriptomics" } }, @@ -147,6 +165,7 @@ "sample_age": 46.3, "sample_age_sd": 14.2, "ancestry_broad": "European", + "ancestry_assignment": "SR", "cohorts": [ 1 ] } }, @@ -159,6 +178,7 @@ "sample_age": 56.5, "sample_age_sd": 8.1, "ancestry_broad": "European", + "ancestry_assignment": "SR", "cohorts": [ 2 ] } }, @@ -172,6 +192,7 @@ "source_gwas_catalog": "GCST90475715", "source_pmid": 39024449, "ancestry_broad": "European", + "ancestry_assignment": "SR", "cohorts": [ 3 ] } }, @@ -185,6 +206,7 @@ "source_gwas_catalog": "GCST90475799", "source_pmid": 39024449, "ancestry_broad": "European", + "ancestry_assignment": "SR", "cohorts": [ 3 ] } }, @@ -198,6 +220,7 @@ "source_gwas_catalog": "GCST90475670", "source_pmid": 39024449, "ancestry_broad": "European", + "ancestry_assignment": "NR", "cohorts": [ 3 ] } }, @@ -211,7 +234,9 @@ "source_gwas_catalog": "GCST90475569", "source_pmid": 39024449, "ancestry_broad": "European", - "cohorts": [ 3 ] + "ancestry_assignment": "NR", + "cohorts": [ 3 ], + "cohorts_additional": "withheld" } }, { @@ -263,6 +288,7 @@ "platform": 1, "publication": 1, "method_name": "Bayesian Ridge regression", + "training_window": "genome-wide", "species_id": 9606, "tissue_id": "UBERON_0001969", "license": "Creative Commons Attribution 4.0 International (CC BY 4.0)", @@ -289,6 +315,7 @@ }, "platform": 2, "publication": 1, + "training_window": "genome-wide", "species_id": 9606, "tissue_id": "BTO_0000133", "license": "Creative Commons Attribution 4.0 International (CC BY 4.0)", @@ -316,6 +343,7 @@ "platform": 4, "publication": 1, "method_name": "Bayesian Ridge regression", + "training_window": "genome-wide", "species_id": 9606, "tissue_id": "UBERON_0001969", "license": "Creative Commons Attribution 4.0 International (CC BY 4.0)", @@ -343,6 +371,7 @@ "platform": 5, "publication": 1, "method_name": "Bayesian Ridge regression", + "training_window": "genome-wide", "species_id": 9606, "tissue_id": "UBERON_0001969", "license": "Creative Commons Attribution 4.0 International (CC BY 4.0)", diff --git a/rest_api/serializers.py b/rest_api/serializers.py index 5228209..c6bfd30 100644 --- a/rest_api/serializers.py +++ b/rest_api/serializers.py @@ -13,6 +13,16 @@ #### OmicsPred #### ################### +#### External Source #### +class ExternalSourceSerializer(serializers.ModelSerializer): + + class Meta: + model = ExternalSource + meta_fields = ('name', 'version', 'url') + fields = meta_fields + read_only_fields = meta_fields + + #### Cohort #### class CohortSerializer(serializers.ModelSerializer): @@ -39,15 +49,19 @@ def get_ancestries(self, obj): #### Sample #### class SampleSerializer(serializers.ModelSerializer): cohorts = CohortSerializer(many=True, read_only=True) + ancestry_assignment = serializers.SerializerMethodField('get_ancestry_assignment_label') class Meta: model = Sample meta_fields = ('sample_number', 'sample_cases', 'sample_controls', 'sample_age', 'sample_age_sd', 'sample_percent_male', - 'ancestry_broad', 'ancestry_free', 'ancestry_country', 'ancestry_additional', + 'ancestry_broad', 'ancestry_free', 'ancestry_country', 'ancestry_additional', 'ancestry_assignment', 'source_gwas_catalog', 'source_pmid','source_doi','cohorts','cohorts_additional')#,'tissue_name') fields = meta_fields read_only_fields = meta_fields + + def get_ancestry_assignment_label(self, obj): + return obj.get_ancestry_assignment_display() #### Tissue #### @@ -79,19 +93,19 @@ class Meta: class PlatformMasterSerializer(serializers.ModelSerializer): class Meta: model = PlatformMaster - meta_fields = ('name', 'full_name', 'versions', 'technic', 'type', 'scores_count') + meta_fields = ('name', 'full_name', 'versions', 'technique', 'type', 'scores_count') fields = meta_fields read_only_fields = meta_fields class PlatformSerializer(serializers.ModelSerializer): full_name = serializers.SerializerMethodField('get_full_name') - technic = serializers.SerializerMethodField('get_technic') + technique = serializers.SerializerMethodField('get_technique') type = serializers.SerializerMethodField('get_type') class Meta: model = Platform - meta_fields = ('name', 'full_name', 'version', 'technic', 'type') + meta_fields = ('name', 'full_name', 'version', 'technique', 'type') fields = meta_fields read_only_fields = meta_fields @@ -102,8 +116,8 @@ def get_full_name(self, obj): else: return obj.name - def get_technic(self, obj): - return obj.platform_master.technic + def get_technique(self, obj): + return obj.platform_master.technique def get_type(self, obj): return obj.platform_master.type @@ -123,14 +137,18 @@ class DatasetLightSerializer(serializers.ModelSerializer): tissue = TissueSerializer(many=False, read_only=True) samples_training = SampleSerializer(many=True, read_only=True) samples_validation = SampleSerializer(many=True, read_only=True) + training_window = serializers.SerializerMethodField('get_training_window_label') class Meta: model = Dataset meta_fields = ('id', 'name', 'platform', 'scores_count', 'omics_count', 'phewas_count', 'omics_type', 'method_name', - 'tissue', 'samples_training', 'samples_validation','scoring_files_urls') + 'training_window', 'tissue', 'samples_training', 'samples_validation','scoring_files_urls') fields = meta_fields read_only_fields = meta_fields + def get_training_window_label(self, obj): + return obj.get_training_window_display() + class DatasetSerializer(DatasetLightSerializer): publication = PublicationSerializer(many=False, read_only=True) @@ -456,44 +474,6 @@ def get_metabolites_count(self, obj): return obj.pathway_metabolites.count() -#### Score #### -# class ScoreWebsiteSerializer(serializers.ModelSerializer): -# dataset_id = serializers.SerializerMethodField() -# dataset_name = serializers.SerializerMethodField() -# publication = serializers.SerializerMethodField() -# platform_name = serializers.SerializerMethodField() - -# genes = GeneSerializerMinimal(many=True, read_only=True) -# transcripts = TranscriptSerializer(many=True, read_only=True) -# proteins = ProteinBaseSerializer(many=True, read_only=True) -# metabolites = MetaboliteBaseSerializer(many=True, read_only=True) - -# class Meta: -# model = Score -# meta_fields = ('id', 'name', 'variants_number', 'ancestry', -# 'dataset_id', 'dataset_name', 'publication', 'platform_name', -# 'genes', 'transcripts', 'proteins', 'metabolites') -# fields = meta_fields -# read_only_fields = meta_fields - -# def get_dataset_id(self,obj): -# ''' Get Dataset ID ''' -# return obj.dataset.id - -# def get_dataset_name(self,obj): -# ''' Get Dataset name ''' -# return obj.dataset.name - -# def get_publication(self, obj): -# ''' Get Publication model ''' -# publication = obj.dataset.publication -# return PublicationSerializer(publication, many=False, read_only=True).data - -# def get_platform_name(self, obj): -# ''' Get Platform model ''' -# return obj.dataset.platform.name - - class ScoreLightSerializer(serializers.ModelSerializer): dataset_id = serializers.SerializerMethodField() dataset_name = serializers.SerializerMethodField() diff --git a/rest_api/static/rest_api/openapi/openapi-schema.yml b/rest_api/static/rest_api/openapi/openapi-schema.yml index 8b66857..d556920 100644 --- a/rest_api/static/rest_api/openapi/openapi-schema.yml +++ b/rest_api/static/rest_api/openapi/openapi-schema.yml @@ -3,7 +3,7 @@ servers: - url: https://rest.omicspred.org info: title: 'OmicsPred REST API' - version: '1.2.0' + version: '1.2.1' description: | Programmatic access to the OmicsPred metadata. @@ -69,6 +69,13 @@ info: REST API version changelog
+ * 1.2.1 - July 2026: + * New endpoint `/api/external_source/all` (endpoint group 'Other') to retrieve information from the external sources used in OmicsPred. + * New field **training_window** in the Dataset schemas (`/api/dataset/` endpoints), containing the type of training window for the variants (Genome-wide, Cis-only, ...). + * New field **ancestry_assignment** in the Sample schemas, containing the description about how the ancestry category was assigned. + * Rename field **technic** by **technique** in the Platform schemas. + * New Sample endpoint `/api/sample/search` allowing to search the sample by cohort name. + * 1.2.0 - May 2026: * Refactor the Phenotype endpoints (`/api/phenotype/`), due to a change in the phenotype data structure. * New PheWAS endpoints (`/api/score/phewas/`), replacing the Score Applications endpoints (`/api/applications_score/`), due to a change in the phewas data structure. @@ -145,7 +152,7 @@ tags: - name: "Dataset endpoints" description: "Dataset metadata (name, publication, platform, samples, ...)" - name: "Platform endpoints" - description: "Platform metadata (name, type, version, technic, ...)" + description: "Platform metadata (name, type, version, technique, ...)" - name: "Tissue endpoints" description: "Tissue ontology information (name, label, description, ...)" - name: "Sample/Cohort endpoints" @@ -157,7 +164,7 @@ tags: - name: "Pathway endpoints" description: "Pathway metadata (Reactome)" - name: "Other endpoints" - description: "Extra endpoints: **/api/info** (REST API version, data counts, ...)" + description: "Extra endpoints: **/api/info** (REST API version, data counts, ...), **/api/external_source/all** (External sources information)" components: @@ -291,6 +298,10 @@ components: type: string description: "The name or description of the method or computational algorithm used to develop the PGS." example: "Bayesian Ridge regression" + training_window: + type: string + description: "The type of training window used for the variants" + example: "Genome-wide" tissue: $ref: '#/components/schemas/Tissue' samples_training: @@ -330,12 +341,8 @@ components: example: "https://app.box.com/shared/static/u4gq3qypwley3m0wfduuisrou3ij0iso" predictdb: type: string - description: "PredictDB prediction weigths - SQLite database" - example: "https://app.box.com/shared/static/unjn1bmt2xhwkd33pp0wczldfzrkevu2" - covariance: - type: string - description: "Covariance file, to be used alongside the PredictDB database in MetaXcan tools" - example: "https://app.box.com/shared/static/24avhdwb2gfo083oz3r6j8w6qqburbp2" + description: "Compressed archive contining the PredictDB prediction weigths - SQLite database file and the Covariance file. They are used together in MetaXcan tools" + example: "https://app.box.com/shared/static/m8yd9yrd6afbiq7jxm5ob66pbj3yj2qt" score_variant_info: type: string description: "Variant information file" @@ -371,6 +378,23 @@ components: type: string example: "This REST endpoint does not exist" + # Cohort # + ExternalSource: + type: object + description: "Structure containing information about an external source used in OmicsPred." + properties: + name: + type: string + description: "Name of the external source" + example: "Ensembl" + version: + type: string + description: "Version used in OmicsPred" + example: "113" + url: + type: string + description: "URL of the external source" + example: "https://www.ensembl.org" # Gene # Gene_basic: @@ -741,9 +765,9 @@ components: type: string description: "Platform's version" example: "3.0" - technic: + technique: type: string - description: "Technic used by the platform" + description: "Technique used by the platform" example: "aptamer-based multiplex protein assay" type: type: string @@ -920,6 +944,10 @@ components: type: string description: "Author reported countries of recruitment (if available)." example: "Canada" + ancestry_assignment: + type: string + description: "Description about how the ancestry category was assigned." + example: "Self-reported" ancestry_additional: type: string description: "Any additional description not captured in the structured data (e.g. founder or genetically isolated populations, or further description of admixed samples)." @@ -2915,7 +2943,7 @@ paths: Retrieve all the Platforms, including for each of them: * Name * Versions - * Technic + * Technique * Type (Transcriptomics, Proteomics, Metabolomics) * Number ofLinked genetic Scores @@ -2955,7 +2983,7 @@ paths: Retrieve a Platform, including: * Name * Versions - * Technic + * Technique * Type (Transcriptomics, Proteomics, Metabolomics) * Number ofLinked genetic Scores @@ -2995,9 +3023,11 @@ paths: * Publication information * Platform information * Number of Genetic Scores + * Method used to develop the scores + * Training window used to select the variants * Tissue information * Training Samples and Cohorts - * Validation Samples and Cohorts + * Validation Samples and Cohorts (if available) * URLs to the data files Example of request: @@ -3039,9 +3069,11 @@ paths: * Publication information * Platform information * Number of Genetic Scores + * Method used to develop the scores + * Training window used to select the variants * Tissue information * Training Samples and Cohorts - * Validation Samples and Cohorts + * Validation Samples and Cohorts (if available) * URLs to the data files Examples of request: @@ -3080,9 +3112,11 @@ paths: * Publication information * Platform information * Number of Genetic Scores + * Method used to develop the scores + * Training window used to select the variants * Tissue information * Training Samples and Cohorts - * Validation Samples and Cohorts + * Validation Samples and Cohorts (if available) * URLs to the data files Examples of request: @@ -3463,3 +3497,43 @@ paths: description: 'Diverse information about the project, such as REST API version, data counts, ...' '4XX': $ref: '#/components/responses/4XX' + + + # External Sources # + /api/external_source/all: + get: + tags: + - "Other endpoints" + operationId: getAllExternalSources + description: | + Retrieve all External Sources, including for each of them: + * External source name + * External source version used + * External source URL + + Example of request: + ``` + https://rest.omicspred.org/api/external_source/all + ``` + responses: + '200': + content: + application/json: + schema: + type: object + allOf: + - $ref: '#/components/schemas/Pagination' + properties: + results: + description: "List of External Sources used in OmicsPred" + type: array + items: + $ref: '#/components/schemas/ExternalSource' + description: | + All External Sources used in OmicsPred + + --- + + __Notes:__ This endpoint uses pagination. + '4XX': + $ref: '#/components/responses/4XX' diff --git a/rest_api/tests.py b/rest_api/tests.py index f67284d..978a6b7 100644 --- a/rest_api/tests.py +++ b/rest_api/tests.py @@ -5,6 +5,10 @@ sleep_time = 60 +status_ok = 200 +status_not_found = 404 +status_too_many = 429 + class BrowseEndpointTest(APITestCase): """ Test the REST endpoints """ @@ -63,6 +67,7 @@ class BrowseEndpointTest(APITestCase): seach_mt = f'molecular_trait={proteins_list[0]}' search_combined = f'{search_opgs_id}&{search_pmid}' search_combined_2 = f'{search_opgs_ids}&{search_pmid}' + search_cohort = f'cohort=INTERVAL' omics_common_columns = ['id', 'trait_reported_id', 'trait_reported', 'variants_number', 'dataset__publication', 'dataset__platform__version', 'dataset__name'] @@ -96,6 +101,7 @@ class BrowseEndpointTest(APITestCase): ('Publication Search', 'publication/search', 1, {'query': [search_opgs_id,search_pmid,search_author]}), # Sample endpoint ('Samples', 'sample/all', 1), + ('Samples Search', 'sample/search', 1, {'query': [search_cohort]}), # Score endpoints ('Scores', 'score/all', 1), ('Score/ID', 'score', 0, {'path': scores_list}), @@ -123,7 +129,8 @@ class BrowseEndpointTest(APITestCase): ('Score PheWAS', 'score/phewas', 1, {'path': scores_list}), ('Score PheWAS Search', 'score/phewas/search', 1, {'query': [search_phenotype,search_opgs_ids,search_opgs_ids_2,search_opp_id,search_combined_2]}), # Other endpoints - ('Info', 'info', 0) + ('Info', 'info', 0), + ('External Sources - all', 'external_source/all', 1) ] @@ -246,7 +253,7 @@ class BrowseEndpointTest(APITestCase): def client_response(self,url:str): resp = self.client.get(url) # In case the number of REST API calls is too high (i.e. greater than the modified 'rate4test' variable) - if resp.status_code == 429: + if resp.status_code == status_too_many: print(f"\nNeed to put the test on sleep for {sleep_time}s as it reaches the maximum number of calls/min.\nMight be worth increasing the value of the 'rate4test' variable (currently {self.rate4test}).") sleep(sleep_time) resp = self.client.get(url) @@ -286,9 +293,11 @@ def check_filter_ids(self,url:str,ids_list:list): def send_request(self, url:str): """ Send REST API request and check the reponse status code """ resp = self.client_response(url) - self.assertEqual(resp.status_code, 200) + self.assertEqual(resp.status_code, status_ok) content = resp.content.decode("utf-8") for empty_content in self.empty_resp: + if content == empty_content: + print(f"\t>> Equal data on endpoint: {url}") self.assertNotEqual(content, empty_content) return content @@ -296,20 +305,20 @@ def send_request(self, url:str): def get_empty_response(self, url:str, index:int): """ Send REST API request and check the reponse status code """ resp = self.client_response(url) - self.assertEqual(resp.status_code, 200) + self.assertEqual(resp.status_code, status_ok) self.assertEqual(resp.content.decode("utf-8"), self.empty_resp[index]) def get_not_found_response(self, url:str): """ Send REST API request on non existing endpoint and check reponse status code """ resp = self.client_response(url) - self.assertEqual(resp.status_code, 404) + self.assertEqual(resp.status_code, status_not_found) def get_paginated_response(self, url:str): """ Send REST API request with limit and offset parameters, and check the reponse status code """ resp = self.client_response(url+'?limit=20&offset=20') - self.assertEqual(resp.status_code, 200) + self.assertEqual(resp.status_code, status_ok) def test_endpoints(self): diff --git a/rest_api/urls.py b/rest_api/urls.py index 8fad9c9..2082847 100644 --- a/rest_api/urls.py +++ b/rest_api/urls.py @@ -10,28 +10,31 @@ slash = '/?' +url_prefix = 'api' + rest_urls = { - 'applications_score': 'api/applications_score/', - 'applications_sample': 'api/applications_sample/', - 'cohort': 'api/cohort/', - 'dataset': 'api/dataset/', - 'gene': 'api/gene/', - 'info': 'api/info', - 'metabolite': 'api/metabolite/', - 'metabolomics': 'api/metabolomics/', - 'pathway': 'api/pathway/', - 'phenotype': 'api/phenotype/', - 'phenotype_old': 'api/phenotype_old/', - 'platform': 'api/platform/', - 'plot': 'api/plot/', - 'performance': 'api/performance/', - 'protein': 'api/protein/', - 'proteomics': 'api/proteomics/', - 'publication': 'api/publication/', - 'sample': 'api/sample/', - 'score': 'api/score/', - 'tissue': 'api/tissue/', - 'transcriptomics': 'api/transcriptomics/' + 'applications_score': f'{url_prefix}/applications_score/', + 'applications_sample': f'{url_prefix}/applications_sample/', + 'cohort': f'{url_prefix}/cohort/', + 'dataset': f'{url_prefix}/dataset/', + 'external_source': f'{url_prefix}/external_source/', + 'gene': f'{url_prefix}/gene/', + 'info': f'{url_prefix}/info', + 'metabolite': f'{url_prefix}/metabolite/', + 'metabolomics': f'{url_prefix}/metabolomics/', + 'pathway': f'{url_prefix}/pathway/', + 'phenotype': f'{url_prefix}/phenotype/', + 'phenotype_old': f'{url_prefix}/phenotype_old/', + 'platform': f'{url_prefix}/platform/', + 'plot': f'{url_prefix}/plot/', + 'performance': f'{url_prefix}/performance/', + 'protein': f'{url_prefix}/protein/', + 'proteomics': f'{url_prefix}/proteomics/', + 'publication': f'{url_prefix}/publication/', + 'sample': f'{url_prefix}/sample/', + 'score': f'{url_prefix}/score/', + 'tissue': f'{url_prefix}/tissue/', + 'transcriptomics': f'{url_prefix}/transcriptomics/' } urlpatterns = [ @@ -64,6 +67,7 @@ re_path(r'^'+rest_urls['publication']+'(?P[^/]+)'+slash, RestPublication.as_view(), name="getPublication"), # Samples re_path(r'^'+rest_urls['sample']+'all'+slash, cache_page(cache_time)(RestListSamples.as_view()), name="getAllSamples"), + re_path(r'^'+rest_urls['sample']+'search'+slash, cache_page(cache_time)(RestSampleSearch.as_view()), name="searchSamples"), # Score PheWAS re_path(r'^'+rest_urls['score']+'phewas/all'+slash, cache_page(cache_time)(RestListScorePheWAS.as_view()), name="getAllScorePheWAS"), re_path(r'^'+rest_urls['score']+'phewas/search'+slash, RestScorePheWASSearch.as_view(), name="searchScorePheWAS"), @@ -93,14 +97,17 @@ re_path(r'^'+rest_urls['plot']+'file/search'+slash, cache_page(cache_time)(RestPlotFileSearch.as_view()), name="searchFilePlots"), re_path(r'^'+rest_urls['plot']+'score/search'+slash, cache_page(cache_time)(RestPlotScoreSearch.as_view()), name="searchScorePlots"), - # Applications + # Applications [DEPRECATED] re_path(r'^'+rest_urls['phenotype_old']+'(?P[^/]+)'+slash, RestPhenotypeOld.as_view(), name="getPhenotypeOld"), re_path(r'^'+rest_urls['applications_score']+'all'+slash, cache_page(cache_time)(RestListPhenotypeScore.as_view()), name="getAllPhenotypeScores"), re_path(r'^'+rest_urls['applications_score']+'search'+slash, RestPhenotypeScoreSearch.as_view(), name="searchPhenotypeScores"), re_path(r'^'+rest_urls['applications_score']+'(?P[^/]+)'+slash, RestPhenotypeScore.as_view(), name="getPhenotypeScore"), re_path(r'^'+rest_urls['applications_sample']+'all'+slash, cache_page(cache_time)(RestListPhenotypeSample.as_view()), name="getAllPhenotypeSamples"), + # Other re_path(r'^'+rest_urls['info']+slash, RestInfo.as_view(), name="getInfo"), + # -> External sources + re_path(r'^'+rest_urls['external_source']+'all'+slash, RestListExternalSources.as_view(), name="getAllExternalSources"), # Setup URL used to warmup the Django app in the Google App Engine path('_ah/warmup', warmup, name="Warmup"), diff --git a/rest_api/views.py b/rest_api/views.py index 8254d09..a6e7095 100644 --- a/rest_api/views.py +++ b/rest_api/views.py @@ -1167,6 +1167,29 @@ class RestListSamples(generics.ListAPIView): queryset = Sample.objects.all().prefetch_related('cohorts').order_by('id') +class RestSampleSearch(generics.ListAPIView): + """ + Search the Sample(s) using parameters as query + Endpoint: /api/sample/search + """ + serializer_class = SampleSerializer + + def get_queryset(self): + queryset = Sample.objects.all().prefetch_related('cohorts').order_by('id') + + params = 0 + # Cohort + cohort = self.request.query_params.get('cohort') + if cohort and cohort is not None: + queryset = queryset.filter(Q(cohorts__name_short__iexact=cohort) | Q(cohorts__name_full__iexact=cohort)) + params += 1 + + if params == 0: + queryset = [] + return queryset + + + ## Scores ## class RestListScores(generics.ListAPIView): @@ -1975,4 +1998,15 @@ def get(self, request): } } - return Response(data) \ No newline at end of file + return Response(data) + + +class RestListExternalSources(generics.ListAPIView): + """ + Retrieve all the External Sources + Endpoint: /api/external_source/all + """ + serializer_class = ExternalSourceSerializer + + def get_queryset(self): + return ExternalSource.objects.all()