🚧 Kids First: Chromosome 18 - Proof of Concept

dbt_project/dbt_project.yml

@@ -45,6 +45,15 @@ models:
           +schema: kf_sd_1nns3k8v_src
         int:
           +schema: kf_sd_1nns3k8v_int
+      chr_18:
+        +tags:
+        - chr_18
+        src:
+          +schema: kf_chr_18_src
+        int:
+          +schema: kf_chr_18_int
+        stb:
+          +schema: kf_chr_18_stb
     access:
       +tags:
       - access_stage

dbt_project/models/kids_first/chr_18/int/kf_chr_18_int_clinical.sql

@@ -0,0 +1,31 @@
+{{ config(materialized='table') }}
+select
+  family_id,
+  participant_id,
+  family_relationship_target_participant_id,
+  aliquot_id,
+  'GRU' as consent_group,
+  age_at_sample,
+  case
+    when sample_composition = 'Blood' then 'Peripheral Whole Blood'
+    when sample_composition = 'Lymphoblastoid Cells' then 'Lymphoblastoid Cell Lines'
+  end as sample_composition,
+  case
+    when race = 'Native American' then 'American Indian or Alaska Native'
+    when race = 'Caucasian' then 'White'
+    else race
+  end as race,
+  case
+    when sex = 'M' then 'Male'
+    when sex = 'F' then 'Female'
+  end as sex,
+  case
+    when ethnicity = 'Not Hispanic' then 'Not Hispanic or Latino'
+    when ethnicity = 'Hispanic' then 'Hispanic or Latino'
+  end as ethnicity,
+  analyte_type,
+  -- age_at_phenotype,
+  affected_status,
+  vital_status,
+  sample_tissue_type
+from {{ ref('kf_chr_18_src_chr_18_phenotype_data') }} 

dbt_project/models/kids_first/chr_18/int/kf_chr_18_int_phenotypes.sql

@@ -0,0 +1,85 @@
+{{ config(materialized='table') }}
+
+with base_data as (
+    select * from {{ ref('kf_chr_18_src_chr_18_phenotype_data') }}
+),
+unpivoted_phenotypes as (
+    select
+        participant_id,
+        age_at_phenotype,
+        phenotype
+    from base_data
+    unpivot (
+        value for phenotype in (
+            chiari_2,
+            pituitary_gland_abn,
+            cleft_palate,
+            hearing_loss,
+            tetralogy_of_fallot,
+            conductive_hearing_loss,
+            holoprosencephaly,
+            choanal_atresia,
+            aural_atresia,
+            midface_hypoplasia,
+            dysmyelination,
+            polymicrogyria,
+            cryptorchidism,
+            sensorineural_hearing_loss,
+            cleft_lip,
+            imperforate_anus,
+            microcephaly,
+            vertical_talus,
+            hypospadias,
+            rocker_bottom_foot,
+            chiari_1,
+            ptosis,
+            strabismus,
+            bilateral_coxa_valga,
+            cardiac_nos,
+            corpus_callosum_abn,
+            pulmonary_valve_stenosis,
+            metatarsus_varus,
+            decreased_white_matter_volume,
+            macrocephaly,
+            suprasellar_cyst,
+            hydrocephalus,
+            micropenis,
+            anal_stenosis,
+            congenital_heart_malformation,
+            bilateral_choanal_atresia,
+            dandy_walker_malformation,
+            dandy_walker,
+            congenital_heart_defect,
+            aural_stenosis,
+            kidney_malform,
+            pituitary_fossa_enlargement,
+            submucous_cleft,
+            virchow_robins_spaces,
+            retinal_coloboma,
+            corneal_staphyloma,
+            club_foot,
+            macroglossia,
+            dandy_walker_variant,
+            coanal_atresia,
+            occipatal_lobe_abn,
+            asd,
+            vsd,
+            pectus_excavatum,
+            peripherial_pulmonary_stenosis,
+            sagittal_stenosis,
+            hydronephrosis,
+            patent_ductus_arteriosus,
+            duanes_syndrome,
+            laryngomalacia,
+            metatarsus_adductus,
+            ptosisbilateral,
+            microphallus,
+            sacral_agenesis,
+            situs_inversus
+        )
+    )
+    where value = 'present'
+)
+
+select * from unpivoted_phenotypes
+

dbt_project/models/kids_first/chr_18/int/kf_chr_18_int_s3.sql

@@ -0,0 +1,21 @@
+{{ config(materialized='table') }}
+
+select
+  concat('s3://', bucket, '/', key) as s3_path,
+  split_part(key, '/', -1) as file_name,
+  case
+    when key like '%.cram' then 'cram'
+    when key like '%.crai' then 'crai'
+    when key like '%.md5' then 'md5'
+    else null
+  end as file_format,
+  case
+    when key like '%.cram' then 'Aligned Reads'
+    when key like '%.crai' then 'Aligned Reads Index'
+    when key like '%.md5' then 'Other'
+    else null
+  end as data_type,
+  concat('{etag', replace(etag, '\"', ''), '}') as hash_dict,
+  'False' as harmonized
+from {{ ref('kf_chr_18_src_s3_scrape_cody') }}
+

dbt_project/models/kids_first/chr_18/int/kf_chr_18_int_sequencing.sql

@@ -0,0 +1,73 @@
+{{ config(materialized='table') }}
+
+with base as (
+  select
+    -- commenting these out for now, but linking sample_id differs by study so will be used in some cases
+    -- sample_id,
+    -- collaborator_sample_id,
+    collaborator_participant_id as participant_id,
+    root_sample_id as sample_id,
+    data_type as sequencing_strategy,
+
+    -- these fields are typically included in ingest package but not sure if they will be needed for access model
+    total_reads,
+    mean_read_length,
+    reference_sequence_name
+  from {{ ref('kf_chr_18_src_terra_manifest_v2') }}
+),
+
+crams as (
+  select
+    root_sample_id,
+    replace(
+      cram_path,
+      'gs://fc-2092c58b-ce12-42e7-9a8e-64b78440537a/',
+      's3://kf-strides-study-us-east-1-prd-sd-1nns3k8v/source/'
+    ) as cram_path
+  from {{ ref('kf_chr_18_src_terra_manifest_v2') }} 
+),
+
+crais as (
+  select
+    root_sample_id,
+    replace(
+      crai_path,
+      'gs://fc-2092c58b-ce12-42e7-9a8e-64b78440537a/',
+      's3://kf-strides-study-us-east-1-prd-sd-1nns3k8v/source/'
+    ) as crai_path
+  from {{ ref('kf_chr_18_src_terra_manifest_v2') }}
+),
+
+md5s as (
+  select
+    root_sample_id,
+    replace(
+      md5_path,
+      'gs://fc-2092c58b-ce12-42e7-9a8e-64b78440537a/',
+      's3://kf-strides-study-us-east-1-prd-sd-1nns3k8v/source/'
+    ) as md5_path
+  from {{ ref('kf_chr_18_src_terra_manifest_v2') }}
+)
+
+select 
+  -- base.sample_id,
+  -- base.collaborator_sample_id,
+  base.participant_id,
+  base.sample_id,
+  base.sequencing_strategy,
+  base.total_reads,
+  base.mean_read_length,
+  base.reference_sequence_name,
+  crams.cram_path,
+  crais.crai_path,
+  md5s.md5_path,
+  'False' as harmonized,
+  'False' as paired_end,
+  'Broad Institute' as sequencing_center_name
+from base
+left join crams
+  on base.sample_id = crams.root_sample_id
+left join crais
+  on base.sample_id = crais.root_sample_id
+left join md5s
+  on base.sample_id = md5s.root_sample_id

dbt_project/models/kids_first/chr_18/src/Clinical_Data/kf_chr_18_src_chr_18_phenotype_data.sql

@@ -0,0 +1,86 @@
+{{ config(materialized='table') }}
+
+select
+  ROW_NUMBER() over () as kf_index,
+  "Family ID"::text as family_id,
+  "Participant ID"::text as participant_id,
+  "Family Relationship Target Participant ID"::text as family_relationship_target_participant_id,
+  alliquot_id::text as aliquot_id,
+  "Consent Group"::text as consent_group,
+  "Age at Sample"::text as age_at_sample,
+  "Sample Composition"::text as sample_composition,
+  "Race"::text as race,
+  "Sex"::text as sex,
+  "Ethnicity"::text as ethnicity,
+  "Analyte Type"::text as analyte_type,
+  "Age at Phenotype"::text as age_at_phenotype,
+  "Affected Status"::text as affected_status,
+  "Vital Status"::text as vital_status,
+  "chiari 2"::text as chiari_2,
+  "pituitary gland abn"::text as pituitary_gland_abn,
+  "cleft palate"::text as cleft_palate,
+  "hearing loss"::text as hearing_loss,
+  "tetralogy of fallot"::text as tetralogy_of_fallot,
+  "conductive hearing loss"::text as conductive_hearing_loss,
+  holoprosencephaly::text as holoprosencephaly,
+  "choanal atresia"::text as choanal_atresia,
+  "aural atresia"::text as aural_atresia,
+  "midface hypoplasia"::text as midface_hypoplasia,
+  dysmyelination::text as dysmyelination,
+  polymicrogyria::text as polymicrogyria,
+  cryptorchidism::text as cryptorchidism,
+  "sensorineural hearing loss"::text as sensorineural_hearing_loss,
+  "cleft lip"::text as cleft_lip,
+  "imperforate anus"::text as imperforate_anus,
+  microcephaly::text as microcephaly,
+  "vertical talus"::text as vertical_talus,
+  hypospadias::text as hypospadias,
+  "rocker bottom foot"::text as rocker_bottom_foot,
+  "chiari 1"::text as chiari_1,
+  ptosis::text as ptosis,
+  strabismus::text as strabismus,
+  "bilateral coxa valga"::text as bilateral_coxa_valga,
+  cardiac_nos::text as cardiac_nos,
+  "corpus callosum abn"::text as corpus_callosum_abn,
+  "pulmonary valve stenosis"::text as pulmonary_valve_stenosis,
+  "metatarsus varus"::text as metatarsus_varus,
+  "decreased white matter volume"::text as decreased_white_matter_volume,
+  macrocephaly::text as macrocephaly,
+  "suprasellar cyst"::text as suprasellar_cyst,
+  hydrocephalus::text as hydrocephalus,
+  micropenis::text as micropenis,
+  "anal stenosis"::text as anal_stenosis,
+  "congenital heart malformation"::text as congenital_heart_malformation,
+  "bilateral choanal atresia"::text as bilateral_choanal_atresia,
+  "dandy walker malformation"::text as dandy_walker_malformation,
+  "dandy walker"::text as dandy_walker,
+  "congenital heart defect"::text as congenital_heart_defect,
+  "aural stenosis"::text as aural_stenosis,
+  "kidney malform"::text as kidney_malform,
+  "pituitary fossa enlargement"::text as pituitary_fossa_enlargement,
+  "submucous cleft"::text as submucous_cleft,
+  "virchow robins spaces"::text as virchow_robins_spaces,
+  "retinal coloboma"::text as retinal_coloboma,
+  "corneal staphyloma"::text as corneal_staphyloma,
+  "club foot"::text as club_foot,
+  macroglossia::text as macroglossia,
+  "dandy walker variant"::text as dandy_walker_variant,
+  "coanal atresia"::text as coanal_atresia,
+  "occipatal lobe abn"::text as occipatal_lobe_abn,
+  asd::text as asd,
+  vsd::text as vsd,
+  "pectus excavatum"::text as pectus_excavatum,
+  "peripherial pulmonary stenosis"::text as peripherial_pulmonary_stenosis,
+  "sagittal stenosis"::text as sagittal_stenosis,
+  hydronephrosis::text as hydronephrosis,
+  "patent ductus arteriosus"::text as patent_ductus_arteriosus,
+  "duanes syndrome"::text as duanes_syndrome,
+  laryngomalacia::text as laryngomalacia,
+  "metatarsus adductus"::text as metatarsus_adductus,
+  ptosisbilateral::text as ptosisbilateral,
+  microphallus::text as microphallus,
+  "sacral agenesis"::text as sacral_agenesis,
+  "situs inversus"::text as situs_inversus,
+  "Sample Tissue Type"::text as sample_tissue_type
+from {{ source('chr_18', 'chr_18_phenotype_data') }}
+

dbt_project/models/kids_first/chr_18/src/S3_Bucket_Scrape/kf_chr_18_src_s3_scrape_cody.sql

@@ -0,0 +1,12 @@
+{{ config(materialized='table') }}
+
+select
+  ROW_NUMBER() over () as kf_index,
+  "Size"::bigint as size,
+  "Key"::text as key,
+  "ETag"::text as etag,
+  "Bucket"::text as bucket,
+  "LastModified"::text as lastmodified,
+  "StorageClass"::text as storageclass
+from {{ source('chr_18', 's3_scrape_cody') }}
+