diff --git a/404.html b/404.html
index 0024f5bfe..672564ac9 100644
--- a/404.html
+++ b/404.html
@@ -2182,6 +2182,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="/python_api/methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="/python_api/methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="/python_api/methods/susie_inf/" class="md-nav__link">
         
@@ -2213,7 +2255,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2229,14 +2271,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
diff --git a/assets/_mkdocstrings.css b/assets/_mkdocstrings.css
index 4b7d98b83..85449ec79 100644
--- a/assets/_mkdocstrings.css
+++ b/assets/_mkdocstrings.css
@@ -26,6 +26,11 @@
   float: right;
 }
 
+/* Backward-compatibility: docstring section titles in bold. */
+.doc-section-title {
+  font-weight: bold;
+}
+
 /* Symbols in Navigation and ToC. */
 :root,
 [data-md-color-scheme="default"] {
@@ -106,4 +111,9 @@ code.doc-symbol-module {
 
 code.doc-symbol-module::after {
   content: "mod";
-}
\ No newline at end of file
+}
+
+.doc-signature .autorefs {
+  color: inherit;
+  border-bottom: 1px dotted currentcolor;
+}
diff --git a/development/_development/index.html b/development/_development/index.html
index d3f33034d..dd364a076 100644
--- a/development/_development/index.html
+++ b/development/_development/index.html
@@ -2190,6 +2190,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../python_api/methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../python_api/methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../python_api/methods/susie_inf/" class="md-nav__link">
         
@@ -2221,7 +2263,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2237,14 +2279,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
diff --git a/development/airflow/index.html b/development/airflow/index.html
index 45581830e..41430880e 100644
--- a/development/airflow/index.html
+++ b/development/airflow/index.html
@@ -2322,6 +2322,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../python_api/methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../python_api/methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../python_api/methods/susie_inf/" class="md-nav__link">
         
@@ -2353,7 +2395,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2369,14 +2411,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
diff --git a/development/contributing/index.html b/development/contributing/index.html
index d49011571..58d7069e9 100644
--- a/development/contributing/index.html
+++ b/development/contributing/index.html
@@ -2313,6 +2313,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../python_api/methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../python_api/methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../python_api/methods/susie_inf/" class="md-nav__link">
         
@@ -2344,7 +2386,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2360,14 +2402,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
diff --git a/development/troubleshooting/index.html b/development/troubleshooting/index.html
index 485f51df8..b7bb1522a 100644
--- a/development/troubleshooting/index.html
+++ b/development/troubleshooting/index.html
@@ -2271,6 +2271,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../python_api/methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../python_api/methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../python_api/methods/susie_inf/" class="md-nav__link">
         
@@ -2302,7 +2344,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2318,14 +2360,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
diff --git a/development/workflows/index.html b/development/workflows/index.html
index d0103cf6e..c689473fa 100644
--- a/development/workflows/index.html
+++ b/development/workflows/index.html
@@ -2262,6 +2262,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../python_api/methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../python_api/methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../python_api/methods/susie_inf/" class="md-nav__link">
         
@@ -2293,7 +2335,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2309,14 +2351,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
diff --git a/howto/_howto/index.html b/howto/_howto/index.html
index 76ae066cb..6fd265561 100644
--- a/howto/_howto/index.html
+++ b/howto/_howto/index.html
@@ -2195,6 +2195,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../python_api/methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../python_api/methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../python_api/methods/susie_inf/" class="md-nav__link">
         
@@ -2226,7 +2268,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2242,14 +2284,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
diff --git a/howto/command_line/_command_line/index.html b/howto/command_line/_command_line/index.html
index d0858a329..67944f9ae 100644
--- a/howto/command_line/_command_line/index.html
+++ b/howto/command_line/_command_line/index.html
@@ -2197,6 +2197,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../python_api/methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../python_api/methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../python_api/methods/susie_inf/" class="md-nav__link">
         
@@ -2228,7 +2270,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2244,14 +2286,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
diff --git a/howto/command_line/run_step_in_cli/index.html b/howto/command_line/run_step_in_cli/index.html
index df8f1a68d..0c1d10c45 100644
--- a/howto/command_line/run_step_in_cli/index.html
+++ b/howto/command_line/run_step_in_cli/index.html
@@ -2207,6 +2207,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../python_api/methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../python_api/methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../python_api/methods/susie_inf/" class="md-nav__link">
         
@@ -2238,7 +2280,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2254,14 +2296,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
diff --git a/howto/command_line/run_step_using_config/index.html b/howto/command_line/run_step_using_config/index.html
index c4d1d7642..b2f1bf373 100644
--- a/howto/command_line/run_step_using_config/index.html
+++ b/howto/command_line/run_step_using_config/index.html
@@ -2207,6 +2207,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../python_api/methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../python_api/methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../python_api/methods/susie_inf/" class="md-nav__link">
         
@@ -2238,7 +2280,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2254,14 +2296,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
diff --git a/howto/python_api/_python_api/index.html b/howto/python_api/_python_api/index.html
index 22483a53d..daac82c54 100644
--- a/howto/python_api/_python_api/index.html
+++ b/howto/python_api/_python_api/index.html
@@ -2192,6 +2192,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../python_api/methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../python_api/methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../python_api/methods/susie_inf/" class="md-nav__link">
         
@@ -2223,7 +2265,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2239,14 +2281,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
diff --git a/howto/python_api/a_creating_spark_session/index.html b/howto/python_api/a_creating_spark_session/index.html
index ccd18a0bc..277a5d767 100644
--- a/howto/python_api/a_creating_spark_session/index.html
+++ b/howto/python_api/a_creating_spark_session/index.html
@@ -2275,6 +2275,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../python_api/methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../python_api/methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../python_api/methods/susie_inf/" class="md-nav__link">
         
@@ -2306,7 +2348,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2322,14 +2364,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
diff --git a/howto/python_api/b_create_dataset/index.html b/howto/python_api/b_create_dataset/index.html
index be2a40ae0..a0baea335 100644
--- a/howto/python_api/b_create_dataset/index.html
+++ b/howto/python_api/b_create_dataset/index.html
@@ -2269,6 +2269,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../python_api/methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../python_api/methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../python_api/methods/susie_inf/" class="md-nav__link">
         
@@ -2300,7 +2342,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2316,14 +2358,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
diff --git a/howto/python_api/c_applying_methods/index.html b/howto/python_api/c_applying_methods/index.html
index 83ffefd0d..74a0005d8 100644
--- a/howto/python_api/c_applying_methods/index.html
+++ b/howto/python_api/c_applying_methods/index.html
@@ -2260,6 +2260,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../python_api/methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../python_api/methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../python_api/methods/susie_inf/" class="md-nav__link">
         
@@ -2291,7 +2333,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2307,14 +2349,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -2931,7 +2973,7 @@ <h2 id="apply-a-dataset-instance-method">Apply a <code>Dataset</code> instance m
 <span class="kn">from</span> <span class="nn">gentropy.method.window_based_clumping</span> <span class="kn">import</span> <span class="n">WindowBasedClumping</span>
 
 <span class="n">clumped_summary_statistics</span> <span class="o">=</span> <span class="n">WindowBasedClumping</span><span class="o">.</span><span class="n">clump</span><span class="p">(</span>
-    <span class="n">summary_stats</span><span class="p">,</span> <span class="n">window_length</span><span class="o">=</span><span class="mi">500_000</span>
+    <span class="n">summary_stats</span><span class="p">,</span> <span class="n">distance</span><span class="o">=</span><span class="mi">250_000</span>
 <span class="p">)</span>
 </code></pre></div>
 </div>
diff --git a/howto/python_api/d_inspect_dataset/index.html b/howto/python_api/d_inspect_dataset/index.html
index 86c8a94f5..b7c5fc2be 100644
--- a/howto/python_api/d_inspect_dataset/index.html
+++ b/howto/python_api/d_inspect_dataset/index.html
@@ -2284,6 +2284,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../python_api/methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../python_api/methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../python_api/methods/susie_inf/" class="md-nav__link">
         
@@ -2315,7 +2357,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2331,14 +2373,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
diff --git a/index.html b/index.html
index 6c2d83d8b..790b7f508 100644
--- a/index.html
+++ b/index.html
@@ -2247,6 +2247,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="python_api/methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="python_api/methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="python_api/methods/susie_inf/" class="md-nav__link">
         
@@ -2278,7 +2320,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2294,14 +2336,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
diff --git a/installation/index.html b/installation/index.html
index 8944631a3..ad9f0b139 100644
--- a/installation/index.html
+++ b/installation/index.html
@@ -2262,6 +2262,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../python_api/methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../python_api/methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../python_api/methods/susie_inf/" class="md-nav__link">
         
@@ -2293,7 +2335,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2309,14 +2351,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
diff --git a/objects.inv b/objects.inv
index 18979e8c8..913742f08 100644
Binary files a/objects.inv and b/objects.inv differ
diff --git a/python_api/_python_api/index.html b/python_api/_python_api/index.html
index eeb5aaa34..7b0332ffc 100644
--- a/python_api/_python_api/index.html
+++ b/python_api/_python_api/index.html
@@ -2190,6 +2190,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../methods/susie_inf/" class="md-nav__link">
         
@@ -2221,7 +2263,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2237,14 +2279,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
diff --git a/python_api/datasets/_datasets/index.html b/python_api/datasets/_datasets/index.html
index ad6580b30..237f12917 100644
--- a/python_api/datasets/_datasets/index.html
+++ b/python_api/datasets/_datasets/index.html
@@ -2197,6 +2197,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../methods/susie_inf/" class="md-nav__link">
         
@@ -2228,7 +2270,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2244,14 +2286,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3374,7 +3416,7 @@ <h3 id="gentropy.dataset.dataset.Dataset.df" class="doc doc-heading">
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3383,7 +3425,7 @@ <h3 id="gentropy.dataset.dataset.Dataset.df" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>DataFrame</code></td>          <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
           </td>
@@ -3419,7 +3461,7 @@ <h3 id="gentropy.dataset.dataset.Dataset.schema" class="doc doc-heading">
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3428,7 +3470,7 @@ <h3 id="gentropy.dataset.dataset.Dataset.schema" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StructType</code></td>          <td>
                 <code><span title="pyspark.sql.types.StructType">StructType</span></code>
           </td>
@@ -3464,7 +3506,7 @@ <h3 id="gentropy.dataset.dataset.Dataset.coalesce" class="doc doc-heading">
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3475,7 +3517,7 @@ <h3 id="gentropy.dataset.dataset.Dataset.coalesce" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>num_partitions</code></td>
           <td>
                 <code>int</code>
@@ -3489,7 +3531,7 @@ <h3 id="gentropy.dataset.dataset.Dataset.coalesce" class="doc doc-heading">
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>**kwargs</code></td>
           <td>
                 <code><span title="typing.Any">Any</span></code>
@@ -3508,7 +3550,7 @@ <h3 id="gentropy.dataset.dataset.Dataset.coalesce" class="doc doc-heading">
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3517,7 +3559,7 @@ <h3 id="gentropy.dataset.dataset.Dataset.coalesce" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>Self</code></td>          <td>
                 <code><span title="typing_extensions.Self">Self</span></code>
           </td>
@@ -3582,7 +3624,7 @@ <h3 id="gentropy.dataset.dataset.Dataset.filter" class="doc doc-heading">
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3593,7 +3635,7 @@ <h3 id="gentropy.dataset.dataset.Dataset.filter" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>condition</code></td>
           <td>
                 <code><span title="pyspark.sql.Column">Column</span></code>
@@ -3612,7 +3654,7 @@ <h3 id="gentropy.dataset.dataset.Dataset.filter" class="doc doc-heading">
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3621,7 +3663,7 @@ <h3 id="gentropy.dataset.dataset.Dataset.filter" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>Self</code></td>          <td>
                 <code><span title="typing_extensions.Self">Self</span></code>
           </td>
@@ -3686,7 +3728,7 @@ <h3 id="gentropy.dataset.dataset.Dataset.from_parquet" class="doc doc-heading">
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3697,7 +3739,7 @@ <h3 id="gentropy.dataset.dataset.Dataset.from_parquet" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>session</code></td>
           <td>
                 <code><span title="gentropy.common.session.Session">Session</span></code>
@@ -3711,7 +3753,7 @@ <h3 id="gentropy.dataset.dataset.Dataset.from_parquet" class="doc doc-heading">
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>path</code></td>
           <td>
                 <code>str | list[str]</code>
@@ -3725,7 +3767,7 @@ <h3 id="gentropy.dataset.dataset.Dataset.from_parquet" class="doc doc-heading">
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>**kwargs</code></td>
           <td>
                 <code>bool | float | int | str | None</code>
@@ -3744,7 +3786,7 @@ <h3 id="gentropy.dataset.dataset.Dataset.from_parquet" class="doc doc-heading">
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3753,7 +3795,7 @@ <h3 id="gentropy.dataset.dataset.Dataset.from_parquet" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>Self</code></td>          <td>
                 <code><span title="typing_extensions.Self">Self</span></code>
           </td>
@@ -3768,7 +3810,7 @@ <h3 id="gentropy.dataset.dataset.Dataset.from_parquet" class="doc doc-heading">
 
 
 
-  <p><strong>Raises:</strong></p>
+  <p><span class="doc-section-title">Raises:</span></p>
   <table>
     <thead>
       <tr>
@@ -3777,7 +3819,7 @@ <h3 id="gentropy.dataset.dataset.Dataset.from_parquet" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td>
                 <code>ValueError</code>
           </td>
@@ -3869,7 +3911,7 @@ <h3 id="gentropy.dataset.dataset.Dataset.get_schema" class="doc doc-heading">
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3878,7 +3920,7 @@ <h3 id="gentropy.dataset.dataset.Dataset.get_schema" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StructType</code></td>          <td>
                 <code><span title="pyspark.sql.types.StructType">StructType</span></code>
           </td>
@@ -3933,7 +3975,7 @@ <h3 id="gentropy.dataset.dataset.Dataset.persist" class="doc doc-heading">
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3942,7 +3984,7 @@ <h3 id="gentropy.dataset.dataset.Dataset.persist" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>Self</code></td>          <td>
                 <code><span title="typing_extensions.Self">Self</span></code>
           </td>
@@ -3996,7 +4038,7 @@ <h3 id="gentropy.dataset.dataset.Dataset.repartition" class="doc doc-heading">
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -4007,7 +4049,7 @@ <h3 id="gentropy.dataset.dataset.Dataset.repartition" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>num_partitions</code></td>
           <td>
                 <code>int</code>
@@ -4021,7 +4063,7 @@ <h3 id="gentropy.dataset.dataset.Dataset.repartition" class="doc doc-heading">
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>**kwargs</code></td>
           <td>
                 <code><span title="typing.Any">Any</span></code>
@@ -4040,7 +4082,7 @@ <h3 id="gentropy.dataset.dataset.Dataset.repartition" class="doc doc-heading">
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -4049,7 +4091,7 @@ <h3 id="gentropy.dataset.dataset.Dataset.repartition" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>Self</code></td>          <td>
                 <code><span title="typing_extensions.Self">Self</span></code>
           </td>
@@ -4114,7 +4156,7 @@ <h3 id="gentropy.dataset.dataset.Dataset.unpersist" class="doc doc-heading">
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -4123,7 +4165,7 @@ <h3 id="gentropy.dataset.dataset.Dataset.unpersist" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>Self</code></td>          <td>
                 <code><span title="typing_extensions.Self">Self</span></code>
           </td>
@@ -4176,7 +4218,7 @@ <h3 id="gentropy.dataset.dataset.Dataset.validate_schema" class="doc doc-heading
 
 
 
-  <p><strong>Raises:</strong></p>
+  <p><span class="doc-section-title">Raises:</span></p>
   <table>
     <thead>
       <tr>
@@ -4185,7 +4227,7 @@ <h3 id="gentropy.dataset.dataset.Dataset.validate_schema" class="doc doc-heading
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td>
                 <code>ValueError</code>
           </td>
diff --git a/python_api/datasets/colocalisation/index.html b/python_api/datasets/colocalisation/index.html
index 6ef6ef461..5d89b33a1 100644
--- a/python_api/datasets/colocalisation/index.html
+++ b/python_api/datasets/colocalisation/index.html
@@ -2266,6 +2266,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../methods/susie_inf/" class="md-nav__link">
         
@@ -2297,7 +2339,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2313,14 +2355,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3003,7 +3045,7 @@ <h3 id="gentropy.dataset.colocalisation.Colocalisation.get_schema" class="doc do
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3012,7 +3054,7 @@ <h3 id="gentropy.dataset.colocalisation.Colocalisation.get_schema" class="doc do
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StructType</code></td>          <td>
                 <code><span title="pyspark.sql.types.StructType">StructType</span></code>
           </td>
diff --git a/python_api/datasets/gene_index/index.html b/python_api/datasets/gene_index/index.html
index f2c1cf181..69d10ee16 100644
--- a/python_api/datasets/gene_index/index.html
+++ b/python_api/datasets/gene_index/index.html
@@ -2293,6 +2293,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../methods/susie_inf/" class="md-nav__link">
         
@@ -2324,7 +2366,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2340,14 +2382,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3146,7 +3188,7 @@ <h3 id="gentropy.dataset.gene_index.GeneIndex.filter_by_biotypes" class="doc doc
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3157,7 +3199,7 @@ <h3 id="gentropy.dataset.gene_index.GeneIndex.filter_by_biotypes" class="doc doc
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>biotypes</code></td>
           <td>
                 <code>list[str]</code>
@@ -3176,7 +3218,7 @@ <h3 id="gentropy.dataset.gene_index.GeneIndex.filter_by_biotypes" class="doc doc
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3185,7 +3227,7 @@ <h3 id="gentropy.dataset.gene_index.GeneIndex.filter_by_biotypes" class="doc doc
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>GeneIndex</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.gene_index.GeneIndex" href="#gentropy.dataset.gene_index.GeneIndex">GeneIndex</a></code>
           </td>
@@ -3248,7 +3290,7 @@ <h3 id="gentropy.dataset.gene_index.GeneIndex.get_schema" class="doc doc-heading
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3257,7 +3299,7 @@ <h3 id="gentropy.dataset.gene_index.GeneIndex.get_schema" class="doc doc-heading
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StructType</code></td>          <td>
                 <code><span title="pyspark.sql.types.StructType">StructType</span></code>
           </td>
@@ -3310,7 +3352,7 @@ <h3 id="gentropy.dataset.gene_index.GeneIndex.locations_lut" class="doc doc-head
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3319,7 +3361,7 @@ <h3 id="gentropy.dataset.gene_index.GeneIndex.locations_lut" class="doc doc-head
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>DataFrame</code></td>          <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
           </td>
@@ -3386,7 +3428,7 @@ <h3 id="gentropy.dataset.gene_index.GeneIndex.symbols_lut" class="doc doc-headin
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3395,7 +3437,7 @@ <h3 id="gentropy.dataset.gene_index.GeneIndex.symbols_lut" class="doc doc-headin
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>DataFrame</code></td>          <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
           </td>
diff --git a/python_api/datasets/intervals/index.html b/python_api/datasets/intervals/index.html
index 737d77a00..2269daefa 100644
--- a/python_api/datasets/intervals/index.html
+++ b/python_api/datasets/intervals/index.html
@@ -2284,6 +2284,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../methods/susie_inf/" class="md-nav__link">
         
@@ -2315,7 +2357,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2331,14 +2373,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3183,7 +3225,7 @@ <h3 id="gentropy.dataset.intervals.Intervals.from_source" class="doc doc-heading
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3194,7 +3236,7 @@ <h3 id="gentropy.dataset.intervals.Intervals.from_source" class="doc doc-heading
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>spark</code></td>
           <td>
                 <code><span title="pyspark.sql.SparkSession">SparkSession</span></code>
@@ -3208,7 +3250,7 @@ <h3 id="gentropy.dataset.intervals.Intervals.from_source" class="doc doc-heading
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>source_name</code></td>
           <td>
                 <code>str</code>
@@ -3222,7 +3264,7 @@ <h3 id="gentropy.dataset.intervals.Intervals.from_source" class="doc doc-heading
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>source_path</code></td>
           <td>
                 <code>str</code>
@@ -3236,7 +3278,7 @@ <h3 id="gentropy.dataset.intervals.Intervals.from_source" class="doc doc-heading
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>gene_index</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.gene_index.GeneIndex" href="../gene_index/#gentropy.dataset.gene_index.GeneIndex">GeneIndex</a></code>
@@ -3250,7 +3292,7 @@ <h3 id="gentropy.dataset.intervals.Intervals.from_source" class="doc doc-heading
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>lift</code></td>
           <td>
                 <code><span title="gentropy.common.Liftover.LiftOverSpark">LiftOverSpark</span></code>
@@ -3269,7 +3311,7 @@ <h3 id="gentropy.dataset.intervals.Intervals.from_source" class="doc doc-heading
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3278,7 +3320,7 @@ <h3 id="gentropy.dataset.intervals.Intervals.from_source" class="doc doc-heading
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>Intervals</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.intervals.Intervals" href="#gentropy.dataset.intervals.Intervals">Intervals</a></code>
           </td>
@@ -3293,7 +3335,7 @@ <h3 id="gentropy.dataset.intervals.Intervals.from_source" class="doc doc-heading
 
 
 
-  <p><strong>Raises:</strong></p>
+  <p><span class="doc-section-title">Raises:</span></p>
   <table>
     <thead>
       <tr>
@@ -3302,7 +3344,7 @@ <h3 id="gentropy.dataset.intervals.Intervals.from_source" class="doc doc-heading
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td>
                 <code>ValueError</code>
           </td>
@@ -3427,7 +3469,7 @@ <h3 id="gentropy.dataset.intervals.Intervals.get_schema" class="doc doc-heading"
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3436,7 +3478,7 @@ <h3 id="gentropy.dataset.intervals.Intervals.get_schema" class="doc doc-heading"
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StructType</code></td>          <td>
                 <code><span title="pyspark.sql.types.StructType">StructType</span></code>
           </td>
@@ -3489,7 +3531,7 @@ <h3 id="gentropy.dataset.intervals.Intervals.v2g" class="doc doc-heading">
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3500,7 +3542,7 @@ <h3 id="gentropy.dataset.intervals.Intervals.v2g" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>variant_index</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.variant_index.VariantIndex" href="../variant_index/#gentropy.dataset.variant_index.VariantIndex">VariantIndex</a></code>
@@ -3519,7 +3561,7 @@ <h3 id="gentropy.dataset.intervals.Intervals.v2g" class="doc doc-heading">
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3528,7 +3570,7 @@ <h3 id="gentropy.dataset.intervals.Intervals.v2g" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>V2G</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.v2g.V2G" href="../variant_to_gene/#gentropy.dataset.v2g.V2G">V2G</a></code>
           </td>
diff --git a/python_api/datasets/l2g_feature/index.html b/python_api/datasets/l2g_feature/index.html
index 25881d9a4..e97840961 100644
--- a/python_api/datasets/l2g_feature/index.html
+++ b/python_api/datasets/l2g_feature/index.html
@@ -2266,6 +2266,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../methods/susie_inf/" class="md-nav__link">
         
@@ -2297,7 +2339,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2313,14 +2355,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3003,7 +3045,7 @@ <h3 id="gentropy.method.l2g.feature_factory.L2GFeature.get_schema" class="doc do
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3012,7 +3054,7 @@ <h3 id="gentropy.method.l2g.feature_factory.L2GFeature.get_schema" class="doc do
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StructType</code></td>          <td>
                 <code><span title="pyspark.sql.types.StructType">StructType</span></code>
           </td>
diff --git a/python_api/datasets/l2g_feature_matrix/index.html b/python_api/datasets/l2g_feature_matrix/index.html
index 793279af1..05f1bc877 100644
--- a/python_api/datasets/l2g_feature_matrix/index.html
+++ b/python_api/datasets/l2g_feature_matrix/index.html
@@ -2311,6 +2311,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../methods/susie_inf/" class="md-nav__link">
         
@@ -2342,7 +2384,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2358,14 +2400,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3034,7 +3076,7 @@ <h2 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix" class="doc doc-hea
 
 
 
-  <p><strong>Attributes:</strong></p>
+  <p><span class="doc-section-title">Attributes:</span></p>
   <table>
     <thead>
       <tr>
@@ -3044,7 +3086,7 @@ <h2 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix" class="doc doc-hea
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code><span title="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.features_list">features_list</span></code></td>
           <td>
                 <code>list[str] | None</code>
@@ -3228,7 +3270,7 @@ <h2 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix" class="doc doc-hea
     <span class="k">def</span> <span class="nf">generate_features</span><span class="p">(</span>
         <span class="bp">cls</span><span class="p">:</span> <span class="n">Type</span><span class="p">[</span><span class="n">L2GFeatureMatrix</span><span class="p">],</span>
         <span class="n">features_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span>
-        <span class="n">study_locus</span><span class="p">:</span> <span class="n">StudyLocus</span><span class="p">,</span>
+        <span class="n">credible_set</span><span class="p">:</span> <span class="n">StudyLocus</span><span class="p">,</span>
         <span class="n">study_index</span><span class="p">:</span> <span class="n">StudyIndex</span><span class="p">,</span>
         <span class="n">variant_gene</span><span class="p">:</span> <span class="n">V2G</span><span class="p">,</span>
         <span class="n">colocalisation</span><span class="p">:</span> <span class="n">Colocalisation</span><span class="p">,</span>
@@ -3237,7 +3279,7 @@ <h2 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix" class="doc doc-hea
 
 <span class="sd">        Args:</span>
 <span class="sd">            features_list (list[str]): List of features to generate</span>
-<span class="sd">            study_locus (StudyLocus): Study locus dataset</span>
+<span class="sd">            credible_set (StudyLocus): Credible set dataset</span>
 <span class="sd">            study_index (StudyIndex): Study index dataset</span>
 <span class="sd">            variant_gene (V2G): Variant to gene dataset</span>
 <span class="sd">            colocalisation (Colocalisation): Colocalisation dataset</span>
@@ -3250,11 +3292,13 @@ <h2 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix" class="doc doc-hea
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="k">if</span> <span class="n">features_dfs</span> <span class="o">:=</span> <span class="p">[</span>
             <span class="c1"># Extract features</span>
-            <span class="n">ColocalisationFactory</span><span class="o">.</span><span class="n">_get_coloc_features</span><span class="p">(</span>
-                <span class="n">study_locus</span><span class="p">,</span> <span class="n">study_index</span><span class="p">,</span> <span class="n">colocalisation</span>
+            <span class="n">ColocalisationFactory</span><span class="o">.</span><span class="n">_get_max_coloc_per_credible_set</span><span class="p">(</span>
+                <span class="n">colocalisation</span><span class="p">,</span>
+                <span class="n">credible_set</span><span class="p">,</span>
+                <span class="n">study_index</span><span class="p">,</span>
             <span class="p">)</span><span class="o">.</span><span class="n">df</span><span class="p">,</span>
-            <span class="n">StudyLocusFactory</span><span class="o">.</span><span class="n">_get_tss_distance_features</span><span class="p">(</span><span class="n">study_locus</span><span class="p">,</span> <span class="n">variant_gene</span><span class="p">)</span><span class="o">.</span><span class="n">df</span><span class="p">,</span>
-            <span class="n">StudyLocusFactory</span><span class="o">.</span><span class="n">_get_vep_features</span><span class="p">(</span><span class="n">study_locus</span><span class="p">,</span> <span class="n">variant_gene</span><span class="p">)</span><span class="o">.</span><span class="n">df</span><span class="p">,</span>
+            <span class="n">StudyLocusFactory</span><span class="o">.</span><span class="n">_get_tss_distance_features</span><span class="p">(</span><span class="n">credible_set</span><span class="p">,</span> <span class="n">variant_gene</span><span class="p">)</span><span class="o">.</span><span class="n">df</span><span class="p">,</span>
+            <span class="n">StudyLocusFactory</span><span class="o">.</span><span class="n">_get_vep_features</span><span class="p">(</span><span class="n">credible_set</span><span class="p">,</span> <span class="n">variant_gene</span><span class="p">)</span><span class="o">.</span><span class="n">df</span><span class="p">,</span>
         <span class="p">]:</span>
             <span class="n">fm</span> <span class="o">=</span> <span class="n">reduce</span><span class="p">(</span>
                 <span class="k">lambda</span> <span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">:</span> <span class="n">x</span><span class="o">.</span><span class="n">unionByName</span><span class="p">(</span><span class="n">y</span><span class="p">),</span>
@@ -3350,10 +3394,8 @@ <h2 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix" class="doc doc-hea
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="n">train</span><span class="p">,</span> <span class="n">test</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_df</span><span class="o">.</span><span class="n">randomSplit</span><span class="p">([</span><span class="n">fraction</span><span class="p">,</span> <span class="mi">1</span> <span class="o">-</span> <span class="n">fraction</span><span class="p">],</span> <span class="n">seed</span><span class="o">=</span><span class="mi">42</span><span class="p">)</span>
         <span class="k">return</span> <span class="p">(</span>
-            <span class="n">L2GFeatureMatrix</span><span class="p">(</span>
-                <span class="n">_df</span><span class="o">=</span><span class="n">train</span><span class="p">,</span> <span class="n">_schema</span><span class="o">=</span><span class="n">L2GFeatureMatrix</span><span class="o">.</span><span class="n">get_schema</span><span class="p">()</span>
-            <span class="p">)</span><span class="o">.</span><span class="n">persist</span><span class="p">(),</span>
-            <span class="n">L2GFeatureMatrix</span><span class="p">(</span><span class="n">_df</span><span class="o">=</span><span class="n">test</span><span class="p">,</span> <span class="n">_schema</span><span class="o">=</span><span class="n">L2GFeatureMatrix</span><span class="o">.</span><span class="n">get_schema</span><span class="p">())</span><span class="o">.</span><span class="n">persist</span><span class="p">(),</span>
+            <span class="n">L2GFeatureMatrix</span><span class="p">(</span><span class="n">_df</span><span class="o">=</span><span class="n">train</span><span class="p">,</span> <span class="n">_schema</span><span class="o">=</span><span class="n">L2GFeatureMatrix</span><span class="o">.</span><span class="n">get_schema</span><span class="p">()),</span>
+            <span class="n">L2GFeatureMatrix</span><span class="p">(</span><span class="n">_df</span><span class="o">=</span><span class="n">test</span><span class="p">,</span> <span class="n">_schema</span><span class="o">=</span><span class="n">L2GFeatureMatrix</span><span class="o">.</span><span class="n">get_schema</span><span class="p">()),</span>
         <span class="p">)</span>
 </code></pre></div></td></tr></table></div>
             </details>
@@ -3387,7 +3429,7 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.calculate_feature_m
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3396,7 +3438,7 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.calculate_feature_m
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td>
                 <code>dict[str, float]</code>
           </td>
@@ -3411,7 +3453,7 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.calculate_feature_m
 
 
 
-  <p><strong>Raises:</strong></p>
+  <p><span class="doc-section-title">Raises:</span></p>
   <table>
     <thead>
       <tr>
@@ -3420,7 +3462,7 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.calculate_feature_m
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td>
                 <code>ValueError</code>
           </td>
@@ -3435,9 +3477,7 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.calculate_feature_m
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/dataset/l2g_feature_matrix.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 96</span>
-<span class="normal"> 97</span>
-<span class="normal"> 98</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 98</span>
 <span class="normal"> 99</span>
 <span class="normal">100</span>
 <span class="normal">101</span>
@@ -3458,7 +3498,9 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.calculate_feature_m
 <span class="normal">116</span>
 <span class="normal">117</span>
 <span class="normal">118</span>
-<span class="normal">119</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">calculate_feature_missingness_rate</span><span class="p">(</span>
+<span class="normal">119</span>
+<span class="normal">120</span>
+<span class="normal">121</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">calculate_feature_missingness_rate</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">:</span> <span class="n">L2GFeatureMatrix</span><span class="p">,</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">float</span><span class="p">]:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Calculate the proportion of missing values in each feature.</span>
@@ -3505,7 +3547,7 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.fill_na" class="doc
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3516,7 +3558,7 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.fill_na" class="doc
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>value</code></td>
           <td>
                 <code>float</code>
@@ -3530,7 +3572,7 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.fill_na" class="doc
                 <code>0.0</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>subset</code></td>
           <td>
                 <code>list[str] | None</code>
@@ -3549,7 +3591,7 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.fill_na" class="doc
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3558,7 +3600,7 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.fill_na" class="doc
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>L2GFeatureMatrix</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix" href="#gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix">L2GFeatureMatrix</a></code>
           </td>
@@ -3573,9 +3615,7 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.fill_na" class="doc
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/dataset/l2g_feature_matrix.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">121</span>
-<span class="normal">122</span>
-<span class="normal">123</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">123</span>
 <span class="normal">124</span>
 <span class="normal">125</span>
 <span class="normal">126</span>
@@ -3586,7 +3626,9 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.fill_na" class="doc
 <span class="normal">131</span>
 <span class="normal">132</span>
 <span class="normal">133</span>
-<span class="normal">134</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fill_na</span><span class="p">(</span>
+<span class="normal">134</span>
+<span class="normal">135</span>
+<span class="normal">136</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fill_na</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">:</span> <span class="n">L2GFeatureMatrix</span><span class="p">,</span> <span class="n">value</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.0</span><span class="p">,</span> <span class="n">subset</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">L2GFeatureMatrix</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Fill missing values in a column with a given value.</span>
@@ -3612,7 +3654,7 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.fill_na" class="doc
 
 
 <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.generate_features" class="doc doc-heading">
-          <code class="highlight language-python"><span class="n">generate_features</span><span class="p">(</span><span class="n">features_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="n">study_locus</span><span class="p">:</span> <span class="n">StudyLocus</span><span class="p">,</span> <span class="n">study_index</span><span class="p">:</span> <span class="n">StudyIndex</span><span class="p">,</span> <span class="n">variant_gene</span><span class="p">:</span> <span class="n">V2G</span><span class="p">,</span> <span class="n">colocalisation</span><span class="p">:</span> <span class="n">Colocalisation</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">L2GFeatureMatrix</span></code>
+          <code class="highlight language-python"><span class="n">generate_features</span><span class="p">(</span><span class="n">features_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="n">credible_set</span><span class="p">:</span> <span class="n">StudyLocus</span><span class="p">,</span> <span class="n">study_index</span><span class="p">:</span> <span class="n">StudyIndex</span><span class="p">,</span> <span class="n">variant_gene</span><span class="p">:</span> <span class="n">V2G</span><span class="p">,</span> <span class="n">colocalisation</span><span class="p">:</span> <span class="n">Colocalisation</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">L2GFeatureMatrix</span></code>
   
   <span class="doc doc-labels">
       <small class="doc doc-label doc-label-classmethod"><code>classmethod</code></small>
@@ -3627,7 +3669,7 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.generate_features"
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3638,7 +3680,7 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.generate_features"
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>features_list</code></td>
           <td>
                 <code>list[str]</code>
@@ -3652,21 +3694,21 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.generate_features"
               <em>required</em>
           </td>
         </tr>
-        <tr>
-          <td><code>study_locus</code></td>
+        <tr class="doc-section-item">
+          <td><code>credible_set</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_locus.StudyLocus" href="../study_locus/#gentropy.dataset.study_locus.StudyLocus">StudyLocus</a></code>
           </td>
           <td>
             <div class="doc-md-description">
-              <p>Study locus dataset</p>
+              <p>Credible set dataset</p>
             </div>
           </td>
           <td>
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>study_index</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_index.StudyIndex" href="../study_index/#gentropy.dataset.study_index.StudyIndex">StudyIndex</a></code>
@@ -3680,7 +3722,7 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.generate_features"
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>variant_gene</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.v2g.V2G" href="../variant_to_gene/#gentropy.dataset.v2g.V2G">V2G</a></code>
@@ -3694,7 +3736,7 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.generate_features"
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>colocalisation</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.colocalisation.Colocalisation" href="../colocalisation/#gentropy.dataset.colocalisation.Colocalisation">Colocalisation</a></code>
@@ -3713,7 +3755,7 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.generate_features"
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3722,7 +3764,7 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.generate_features"
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>L2GFeatureMatrix</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix" href="#gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix">L2GFeatureMatrix</a></code>
           </td>
@@ -3737,7 +3779,7 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.generate_features"
 
 
 
-  <p><strong>Raises:</strong></p>
+  <p><span class="doc-section-title">Raises:</span></p>
   <table>
     <thead>
       <tr>
@@ -3746,7 +3788,7 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.generate_features"
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td>
                 <code>ValueError</code>
           </td>
@@ -3807,11 +3849,13 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.generate_features"
 <span class="normal">82</span>
 <span class="normal">83</span>
 <span class="normal">84</span>
-<span class="normal">85</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@classmethod</span>
+<span class="normal">85</span>
+<span class="normal">86</span>
+<span class="normal">87</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@classmethod</span>
 <span class="k">def</span> <span class="nf">generate_features</span><span class="p">(</span>
     <span class="bp">cls</span><span class="p">:</span> <span class="n">Type</span><span class="p">[</span><span class="n">L2GFeatureMatrix</span><span class="p">],</span>
     <span class="n">features_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span>
-    <span class="n">study_locus</span><span class="p">:</span> <span class="n">StudyLocus</span><span class="p">,</span>
+    <span class="n">credible_set</span><span class="p">:</span> <span class="n">StudyLocus</span><span class="p">,</span>
     <span class="n">study_index</span><span class="p">:</span> <span class="n">StudyIndex</span><span class="p">,</span>
     <span class="n">variant_gene</span><span class="p">:</span> <span class="n">V2G</span><span class="p">,</span>
     <span class="n">colocalisation</span><span class="p">:</span> <span class="n">Colocalisation</span><span class="p">,</span>
@@ -3820,7 +3864,7 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.generate_features"
 
 <span class="sd">    Args:</span>
 <span class="sd">        features_list (list[str]): List of features to generate</span>
-<span class="sd">        study_locus (StudyLocus): Study locus dataset</span>
+<span class="sd">        credible_set (StudyLocus): Credible set dataset</span>
 <span class="sd">        study_index (StudyIndex): Study index dataset</span>
 <span class="sd">        variant_gene (V2G): Variant to gene dataset</span>
 <span class="sd">        colocalisation (Colocalisation): Colocalisation dataset</span>
@@ -3833,11 +3877,13 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.generate_features"
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="k">if</span> <span class="n">features_dfs</span> <span class="o">:=</span> <span class="p">[</span>
         <span class="c1"># Extract features</span>
-        <span class="n">ColocalisationFactory</span><span class="o">.</span><span class="n">_get_coloc_features</span><span class="p">(</span>
-            <span class="n">study_locus</span><span class="p">,</span> <span class="n">study_index</span><span class="p">,</span> <span class="n">colocalisation</span>
+        <span class="n">ColocalisationFactory</span><span class="o">.</span><span class="n">_get_max_coloc_per_credible_set</span><span class="p">(</span>
+            <span class="n">colocalisation</span><span class="p">,</span>
+            <span class="n">credible_set</span><span class="p">,</span>
+            <span class="n">study_index</span><span class="p">,</span>
         <span class="p">)</span><span class="o">.</span><span class="n">df</span><span class="p">,</span>
-        <span class="n">StudyLocusFactory</span><span class="o">.</span><span class="n">_get_tss_distance_features</span><span class="p">(</span><span class="n">study_locus</span><span class="p">,</span> <span class="n">variant_gene</span><span class="p">)</span><span class="o">.</span><span class="n">df</span><span class="p">,</span>
-        <span class="n">StudyLocusFactory</span><span class="o">.</span><span class="n">_get_vep_features</span><span class="p">(</span><span class="n">study_locus</span><span class="p">,</span> <span class="n">variant_gene</span><span class="p">)</span><span class="o">.</span><span class="n">df</span><span class="p">,</span>
+        <span class="n">StudyLocusFactory</span><span class="o">.</span><span class="n">_get_tss_distance_features</span><span class="p">(</span><span class="n">credible_set</span><span class="p">,</span> <span class="n">variant_gene</span><span class="p">)</span><span class="o">.</span><span class="n">df</span><span class="p">,</span>
+        <span class="n">StudyLocusFactory</span><span class="o">.</span><span class="n">_get_vep_features</span><span class="p">(</span><span class="n">credible_set</span><span class="p">,</span> <span class="n">variant_gene</span><span class="p">)</span><span class="o">.</span><span class="n">df</span><span class="p">,</span>
     <span class="p">]:</span>
         <span class="n">fm</span> <span class="o">=</span> <span class="n">reduce</span><span class="p">(</span>
             <span class="k">lambda</span> <span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">:</span> <span class="n">x</span><span class="o">.</span><span class="n">unionByName</span><span class="p">(</span><span class="n">y</span><span class="p">),</span>
@@ -3881,7 +3927,7 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.get_schema" class="
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3890,7 +3936,7 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.get_schema" class="
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StructType</code></td>          <td>
                 <code><span title="pyspark.sql.types.StructType">StructType</span></code>
           </td>
@@ -3905,14 +3951,14 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.get_schema" class="
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/dataset/l2g_feature_matrix.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">87</span>
-<span class="normal">88</span>
-<span class="normal">89</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">89</span>
 <span class="normal">90</span>
 <span class="normal">91</span>
 <span class="normal">92</span>
 <span class="normal">93</span>
-<span class="normal">94</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@classmethod</span>
+<span class="normal">94</span>
+<span class="normal">95</span>
+<span class="normal">96</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@classmethod</span>
 <span class="k">def</span> <span class="nf">get_schema</span><span class="p">(</span><span class="bp">cls</span><span class="p">:</span> <span class="nb">type</span><span class="p">[</span><span class="n">L2GFeatureMatrix</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">StructType</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Provides the schema for the L2gFeatureMatrix dataset.</span>
 
@@ -3943,7 +3989,7 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.select_features" cl
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3954,7 +4000,7 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.select_features" cl
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>features_list</code></td>
           <td>
                 <code>list[str] | None</code>
@@ -3973,7 +4019,7 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.select_features" cl
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3982,7 +4028,7 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.select_features" cl
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>L2GFeatureMatrix</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix" href="#gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix">L2GFeatureMatrix</a></code>
           </td>
@@ -3997,9 +4043,7 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.select_features" cl
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/dataset/l2g_feature_matrix.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">136</span>
-<span class="normal">137</span>
-<span class="normal">138</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">138</span>
 <span class="normal">139</span>
 <span class="normal">140</span>
 <span class="normal">141</span>
@@ -4011,7 +4055,9 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.select_features" cl
 <span class="normal">147</span>
 <span class="normal">148</span>
 <span class="normal">149</span>
-<span class="normal">150</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">select_features</span><span class="p">(</span>
+<span class="normal">150</span>
+<span class="normal">151</span>
+<span class="normal">152</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">select_features</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">:</span> <span class="n">L2GFeatureMatrix</span><span class="p">,</span> <span class="n">features_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">L2GFeatureMatrix</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Select a subset of features from the feature matrix.</span>
@@ -4049,7 +4095,7 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.train_test_split" c
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -4060,7 +4106,7 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.train_test_split" c
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>fraction</code></td>
           <td>
                 <code>float</code>
@@ -4079,7 +4125,7 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.train_test_split" c
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -4088,7 +4134,7 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.train_test_split" c
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td>
                 <code>tuple[<a class="autorefs autorefs-internal" title="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix" href="#gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix">L2GFeatureMatrix</a>, <a class="autorefs autorefs-internal" title="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix" href="#gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix">L2GFeatureMatrix</a>]</code>
           </td>
@@ -4103,9 +4149,7 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.train_test_split" c
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/dataset/l2g_feature_matrix.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">152</span>
-<span class="normal">153</span>
-<span class="normal">154</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">154</span>
 <span class="normal">155</span>
 <span class="normal">156</span>
 <span class="normal">157</span>
@@ -4133,10 +4177,8 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.train_test_split" c
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="n">train</span><span class="p">,</span> <span class="n">test</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_df</span><span class="o">.</span><span class="n">randomSplit</span><span class="p">([</span><span class="n">fraction</span><span class="p">,</span> <span class="mi">1</span> <span class="o">-</span> <span class="n">fraction</span><span class="p">],</span> <span class="n">seed</span><span class="o">=</span><span class="mi">42</span><span class="p">)</span>
     <span class="k">return</span> <span class="p">(</span>
-        <span class="n">L2GFeatureMatrix</span><span class="p">(</span>
-            <span class="n">_df</span><span class="o">=</span><span class="n">train</span><span class="p">,</span> <span class="n">_schema</span><span class="o">=</span><span class="n">L2GFeatureMatrix</span><span class="o">.</span><span class="n">get_schema</span><span class="p">()</span>
-        <span class="p">)</span><span class="o">.</span><span class="n">persist</span><span class="p">(),</span>
-        <span class="n">L2GFeatureMatrix</span><span class="p">(</span><span class="n">_df</span><span class="o">=</span><span class="n">test</span><span class="p">,</span> <span class="n">_schema</span><span class="o">=</span><span class="n">L2GFeatureMatrix</span><span class="o">.</span><span class="n">get_schema</span><span class="p">())</span><span class="o">.</span><span class="n">persist</span><span class="p">(),</span>
+        <span class="n">L2GFeatureMatrix</span><span class="p">(</span><span class="n">_df</span><span class="o">=</span><span class="n">train</span><span class="p">,</span> <span class="n">_schema</span><span class="o">=</span><span class="n">L2GFeatureMatrix</span><span class="o">.</span><span class="n">get_schema</span><span class="p">()),</span>
+        <span class="n">L2GFeatureMatrix</span><span class="p">(</span><span class="n">_df</span><span class="o">=</span><span class="n">test</span><span class="p">,</span> <span class="n">_schema</span><span class="o">=</span><span class="n">L2GFeatureMatrix</span><span class="o">.</span><span class="n">get_schema</span><span class="p">()),</span>
     <span class="p">)</span>
 </code></pre></div></td></tr></table></div>
           </details>
@@ -4174,6 +4216,10 @@ <h3 id="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.train_test_split" c
  |-- sqtlColocClppMaximumNeighborhood: float (nullable = true)
  |-- sqtlColocLlrMaximum: float (nullable = true)
  |-- sqtlColocLlrMaximumNeighborhood: float (nullable = true)
+ |-- tuqtlColocClppMaximum: float (nullable = true)
+ |-- tuqtlColocClppMaximumNeighborhood: float (nullable = true)
+ |-- tuqtlColocLlrMaximum: float (nullable = true)
+ |-- tuqtlColocLlrMaximumNeighborhood: float (nullable = true)
 </code></pre></div>
 <hr />
 <div class="md-source-file">
diff --git a/python_api/datasets/l2g_gold_standard/index.html b/python_api/datasets/l2g_gold_standard/index.html
index 10dbb5349..c15ea62b1 100644
--- a/python_api/datasets/l2g_gold_standard/index.html
+++ b/python_api/datasets/l2g_gold_standard/index.html
@@ -2302,6 +2302,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../methods/susie_inf/" class="md-nav__link">
         
@@ -2333,7 +2375,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2349,14 +2391,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3407,7 +3449,7 @@ <h3 id="gentropy.dataset.l2g_gold_standard.L2GGoldStandard.filter_unique_associa
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3418,7 +3460,7 @@ <h3 id="gentropy.dataset.l2g_gold_standard.L2GGoldStandard.filter_unique_associa
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>study_locus_overlap</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_locus_overlap.StudyLocusOverlap" href="../study_locus_overlap/#gentropy.dataset.study_locus_overlap.StudyLocusOverlap">StudyLocusOverlap</a></code>
@@ -3437,7 +3479,7 @@ <h3 id="gentropy.dataset.l2g_gold_standard.L2GGoldStandard.filter_unique_associa
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3446,7 +3488,7 @@ <h3 id="gentropy.dataset.l2g_gold_standard.L2GGoldStandard.filter_unique_associa
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>L2GGoldStandard</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.l2g_gold_standard.L2GGoldStandard" href="#gentropy.dataset.l2g_gold_standard.L2GGoldStandard">L2GGoldStandard</a></code>
           </td>
@@ -3573,7 +3615,7 @@ <h3 id="gentropy.dataset.l2g_gold_standard.L2GGoldStandard.from_otg_curation" cl
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3584,7 +3626,7 @@ <h3 id="gentropy.dataset.l2g_gold_standard.L2GGoldStandard.from_otg_curation" cl
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>gold_standard_curation</code></td>
           <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
@@ -3598,7 +3640,7 @@ <h3 id="gentropy.dataset.l2g_gold_standard.L2GGoldStandard.from_otg_curation" cl
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>v2g</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.v2g.V2G" href="../variant_to_gene/#gentropy.dataset.v2g.V2G">V2G</a></code>
@@ -3612,7 +3654,7 @@ <h3 id="gentropy.dataset.l2g_gold_standard.L2GGoldStandard.from_otg_curation" cl
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>study_locus_overlap</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_locus_overlap.StudyLocusOverlap" href="../study_locus_overlap/#gentropy.dataset.study_locus_overlap.StudyLocusOverlap">StudyLocusOverlap</a></code>
@@ -3626,7 +3668,7 @@ <h3 id="gentropy.dataset.l2g_gold_standard.L2GGoldStandard.from_otg_curation" cl
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>interactions</code></td>
           <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
@@ -3645,7 +3687,7 @@ <h3 id="gentropy.dataset.l2g_gold_standard.L2GGoldStandard.from_otg_curation" cl
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3654,7 +3696,7 @@ <h3 id="gentropy.dataset.l2g_gold_standard.L2GGoldStandard.from_otg_curation" cl
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>L2GGoldStandard</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.l2g_gold_standard.L2GGoldStandard" href="#gentropy.dataset.l2g_gold_standard.L2GGoldStandard">L2GGoldStandard</a></code>
           </td>
@@ -3755,7 +3797,7 @@ <h3 id="gentropy.dataset.l2g_gold_standard.L2GGoldStandard.get_schema" class="do
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3764,7 +3806,7 @@ <h3 id="gentropy.dataset.l2g_gold_standard.L2GGoldStandard.get_schema" class="do
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StructType</code></td>          <td>
                 <code><span title="pyspark.sql.types.StructType">StructType</span></code>
           </td>
@@ -3821,7 +3863,7 @@ <h3 id="gentropy.dataset.l2g_gold_standard.L2GGoldStandard.process_gene_interact
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3832,7 +3874,7 @@ <h3 id="gentropy.dataset.l2g_gold_standard.L2GGoldStandard.process_gene_interact
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>interactions</code></td>
           <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
@@ -3851,7 +3893,7 @@ <h3 id="gentropy.dataset.l2g_gold_standard.L2GGoldStandard.process_gene_interact
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3860,7 +3902,7 @@ <h3 id="gentropy.dataset.l2g_gold_standard.L2GGoldStandard.process_gene_interact
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>DataFrame</code></td>          <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
           </td>
@@ -3875,7 +3917,7 @@ <h3 id="gentropy.dataset.l2g_gold_standard.L2GGoldStandard.process_gene_interact
 
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="n">interactions</span> <span class="o">=</span> <span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">([(</span><span class="s2">&quot;gene1&quot;</span><span class="p">,</span> <span class="s2">&quot;gene2&quot;</span><span class="p">,</span> <span class="mf">0.8</span><span class="p">),</span> <span class="p">(</span><span class="s2">&quot;gene1&quot;</span><span class="p">,</span> <span class="s2">&quot;gene2&quot;</span><span class="p">,</span> <span class="mf">0.5</span><span class="p">),</span> <span class="p">(</span><span class="s2">&quot;gene2&quot;</span><span class="p">,</span> <span class="s2">&quot;gene3&quot;</span><span class="p">,</span> <span class="mf">0.7</span><span class="p">)],</span> <span class="p">[</span><span class="s2">&quot;targetA&quot;</span><span class="p">,</span> <span class="s2">&quot;targetB&quot;</span><span class="p">,</span> <span class="s2">&quot;scoring&quot;</span><span class="p">])</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">L2GGoldStandard</span><span class="o">.</span><span class="n">process_gene_interactions</span><span class="p">(</span><span class="n">interactions</span><span class="p">)</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
 <span class="go">+-------+-------+-----+</span>
@@ -3974,7 +4016,7 @@ <h3 id="gentropy.dataset.l2g_gold_standard.L2GGoldStandard.remove_false_negative
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3985,7 +4027,7 @@ <h3 id="gentropy.dataset.l2g_gold_standard.L2GGoldStandard.remove_false_negative
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>interactions_df</code></td>
           <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
@@ -4004,7 +4046,7 @@ <h3 id="gentropy.dataset.l2g_gold_standard.L2GGoldStandard.remove_false_negative
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -4013,7 +4055,7 @@ <h3 id="gentropy.dataset.l2g_gold_standard.L2GGoldStandard.remove_false_negative
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>L2GGoldStandard</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.l2g_gold_standard.L2GGoldStandard" href="#gentropy.dataset.l2g_gold_standard.L2GGoldStandard">L2GGoldStandard</a></code>
           </td>
diff --git a/python_api/datasets/l2g_prediction/index.html b/python_api/datasets/l2g_prediction/index.html
index d4c592225..18a026d0d 100644
--- a/python_api/datasets/l2g_prediction/index.html
+++ b/python_api/datasets/l2g_prediction/index.html
@@ -2275,6 +2275,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../methods/susie_inf/" class="md-nav__link">
         
@@ -2306,7 +2348,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2322,14 +2364,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3063,7 +3105,7 @@ <h2 id="gentropy.dataset.l2g_prediction.L2GPrediction" class="doc doc-heading">
         <span class="bp">cls</span><span class="p">:</span> <span class="n">Type</span><span class="p">[</span><span class="n">L2GPrediction</span><span class="p">],</span>
         <span class="n">model_path</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
         <span class="n">features_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span>
-        <span class="n">study_locus</span><span class="p">:</span> <span class="n">StudyLocus</span><span class="p">,</span>
+        <span class="n">credible_set</span><span class="p">:</span> <span class="n">StudyLocus</span><span class="p">,</span>
         <span class="n">study_index</span><span class="p">:</span> <span class="n">StudyIndex</span><span class="p">,</span>
         <span class="n">v2g</span><span class="p">:</span> <span class="n">V2G</span><span class="p">,</span>
         <span class="n">coloc</span><span class="p">:</span> <span class="n">Colocalisation</span><span class="p">,</span>
@@ -3073,7 +3115,7 @@ <h2 id="gentropy.dataset.l2g_prediction.L2GPrediction" class="doc doc-heading">
 <span class="sd">        Args:</span>
 <span class="sd">            model_path (str): Path to the fitted model</span>
 <span class="sd">            features_list (list[str]): List of features to use for the model</span>
-<span class="sd">            study_locus (StudyLocus): Study locus dataset</span>
+<span class="sd">            credible_set (StudyLocus): Credible set dataset</span>
 <span class="sd">            study_index (StudyIndex): Study index dataset</span>
 <span class="sd">            v2g (V2G): Variant to gene dataset</span>
 <span class="sd">            coloc (Colocalisation): Colocalisation dataset</span>
@@ -3083,7 +3125,7 @@ <h2 id="gentropy.dataset.l2g_prediction.L2GPrediction" class="doc doc-heading">
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="n">fm</span> <span class="o">=</span> <span class="n">L2GFeatureMatrix</span><span class="o">.</span><span class="n">generate_features</span><span class="p">(</span>
             <span class="n">features_list</span><span class="o">=</span><span class="n">features_list</span><span class="p">,</span>
-            <span class="n">study_locus</span><span class="o">=</span><span class="n">study_locus</span><span class="p">,</span>
+            <span class="n">credible_set</span><span class="o">=</span><span class="n">credible_set</span><span class="p">,</span>
             <span class="n">study_index</span><span class="o">=</span><span class="n">study_index</span><span class="p">,</span>
             <span class="n">variant_gene</span><span class="o">=</span><span class="n">v2g</span><span class="p">,</span>
             <span class="n">colocalisation</span><span class="o">=</span><span class="n">coloc</span><span class="p">,</span>
@@ -3092,7 +3134,7 @@ <h2 id="gentropy.dataset.l2g_prediction.L2GPrediction" class="doc doc-heading">
         <span class="n">gwas_fm</span> <span class="o">=</span> <span class="n">L2GFeatureMatrix</span><span class="p">(</span>
             <span class="n">_df</span><span class="o">=</span><span class="p">(</span>
                 <span class="n">fm</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
-                    <span class="n">study_locus</span><span class="o">.</span><span class="n">filter_by_study_type</span><span class="p">(</span><span class="s2">&quot;gwas&quot;</span><span class="p">,</span> <span class="n">study_index</span><span class="p">)</span><span class="o">.</span><span class="n">df</span><span class="p">,</span>
+                    <span class="n">credible_set</span><span class="o">.</span><span class="n">filter_by_study_type</span><span class="p">(</span><span class="s2">&quot;gwas&quot;</span><span class="p">,</span> <span class="n">study_index</span><span class="p">)</span><span class="o">.</span><span class="n">df</span><span class="p">,</span>
                     <span class="n">on</span><span class="o">=</span><span class="s2">&quot;studyLocusId&quot;</span><span class="p">,</span>
                 <span class="p">)</span>
             <span class="p">),</span>
@@ -3137,7 +3179,7 @@ <h2 id="gentropy.dataset.l2g_prediction.L2GPrediction" class="doc doc-heading">
 
 
 <h3 id="gentropy.dataset.l2g_prediction.L2GPrediction.from_credible_set" class="doc doc-heading">
-          <code class="highlight language-python"><span class="n">from_credible_set</span><span class="p">(</span><span class="n">model_path</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">features_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="n">study_locus</span><span class="p">:</span> <span class="n">StudyLocus</span><span class="p">,</span> <span class="n">study_index</span><span class="p">:</span> <span class="n">StudyIndex</span><span class="p">,</span> <span class="n">v2g</span><span class="p">:</span> <span class="n">V2G</span><span class="p">,</span> <span class="n">coloc</span><span class="p">:</span> <span class="n">Colocalisation</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">L2GPrediction</span></code>
+          <code class="highlight language-python"><span class="n">from_credible_set</span><span class="p">(</span><span class="n">model_path</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">features_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="n">credible_set</span><span class="p">:</span> <span class="n">StudyLocus</span><span class="p">,</span> <span class="n">study_index</span><span class="p">:</span> <span class="n">StudyIndex</span><span class="p">,</span> <span class="n">v2g</span><span class="p">:</span> <span class="n">V2G</span><span class="p">,</span> <span class="n">coloc</span><span class="p">:</span> <span class="n">Colocalisation</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">L2GPrediction</span></code>
   
   <span class="doc doc-labels">
       <small class="doc doc-label doc-label-classmethod"><code>classmethod</code></small>
@@ -3152,7 +3194,7 @@ <h3 id="gentropy.dataset.l2g_prediction.L2GPrediction.from_credible_set" class="
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3163,7 +3205,7 @@ <h3 id="gentropy.dataset.l2g_prediction.L2GPrediction.from_credible_set" class="
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>model_path</code></td>
           <td>
                 <code>str</code>
@@ -3177,7 +3219,7 @@ <h3 id="gentropy.dataset.l2g_prediction.L2GPrediction.from_credible_set" class="
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>features_list</code></td>
           <td>
                 <code>list[str]</code>
@@ -3191,21 +3233,21 @@ <h3 id="gentropy.dataset.l2g_prediction.L2GPrediction.from_credible_set" class="
               <em>required</em>
           </td>
         </tr>
-        <tr>
-          <td><code>study_locus</code></td>
+        <tr class="doc-section-item">
+          <td><code>credible_set</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_locus.StudyLocus" href="../study_locus/#gentropy.dataset.study_locus.StudyLocus">StudyLocus</a></code>
           </td>
           <td>
             <div class="doc-md-description">
-              <p>Study locus dataset</p>
+              <p>Credible set dataset</p>
             </div>
           </td>
           <td>
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>study_index</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_index.StudyIndex" href="../study_index/#gentropy.dataset.study_index.StudyIndex">StudyIndex</a></code>
@@ -3219,7 +3261,7 @@ <h3 id="gentropy.dataset.l2g_prediction.L2GPrediction.from_credible_set" class="
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>v2g</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.v2g.V2G" href="../variant_to_gene/#gentropy.dataset.v2g.V2G">V2G</a></code>
@@ -3233,7 +3275,7 @@ <h3 id="gentropy.dataset.l2g_prediction.L2GPrediction.from_credible_set" class="
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>coloc</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.colocalisation.Colocalisation" href="../colocalisation/#gentropy.dataset.colocalisation.Colocalisation">Colocalisation</a></code>
@@ -3252,7 +3294,7 @@ <h3 id="gentropy.dataset.l2g_prediction.L2GPrediction.from_credible_set" class="
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3261,7 +3303,7 @@ <h3 id="gentropy.dataset.l2g_prediction.L2GPrediction.from_credible_set" class="
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>L2GPrediction</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.l2g_prediction.L2GPrediction" href="#gentropy.dataset.l2g_prediction.L2GPrediction">L2GPrediction</a></code>
           </td>
@@ -3338,7 +3380,7 @@ <h3 id="gentropy.dataset.l2g_prediction.L2GPrediction.from_credible_set" class="
     <span class="bp">cls</span><span class="p">:</span> <span class="n">Type</span><span class="p">[</span><span class="n">L2GPrediction</span><span class="p">],</span>
     <span class="n">model_path</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
     <span class="n">features_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span>
-    <span class="n">study_locus</span><span class="p">:</span> <span class="n">StudyLocus</span><span class="p">,</span>
+    <span class="n">credible_set</span><span class="p">:</span> <span class="n">StudyLocus</span><span class="p">,</span>
     <span class="n">study_index</span><span class="p">:</span> <span class="n">StudyIndex</span><span class="p">,</span>
     <span class="n">v2g</span><span class="p">:</span> <span class="n">V2G</span><span class="p">,</span>
     <span class="n">coloc</span><span class="p">:</span> <span class="n">Colocalisation</span><span class="p">,</span>
@@ -3348,7 +3390,7 @@ <h3 id="gentropy.dataset.l2g_prediction.L2GPrediction.from_credible_set" class="
 <span class="sd">    Args:</span>
 <span class="sd">        model_path (str): Path to the fitted model</span>
 <span class="sd">        features_list (list[str]): List of features to use for the model</span>
-<span class="sd">        study_locus (StudyLocus): Study locus dataset</span>
+<span class="sd">        credible_set (StudyLocus): Credible set dataset</span>
 <span class="sd">        study_index (StudyIndex): Study index dataset</span>
 <span class="sd">        v2g (V2G): Variant to gene dataset</span>
 <span class="sd">        coloc (Colocalisation): Colocalisation dataset</span>
@@ -3358,7 +3400,7 @@ <h3 id="gentropy.dataset.l2g_prediction.L2GPrediction.from_credible_set" class="
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="n">fm</span> <span class="o">=</span> <span class="n">L2GFeatureMatrix</span><span class="o">.</span><span class="n">generate_features</span><span class="p">(</span>
         <span class="n">features_list</span><span class="o">=</span><span class="n">features_list</span><span class="p">,</span>
-        <span class="n">study_locus</span><span class="o">=</span><span class="n">study_locus</span><span class="p">,</span>
+        <span class="n">credible_set</span><span class="o">=</span><span class="n">credible_set</span><span class="p">,</span>
         <span class="n">study_index</span><span class="o">=</span><span class="n">study_index</span><span class="p">,</span>
         <span class="n">variant_gene</span><span class="o">=</span><span class="n">v2g</span><span class="p">,</span>
         <span class="n">colocalisation</span><span class="o">=</span><span class="n">coloc</span><span class="p">,</span>
@@ -3367,7 +3409,7 @@ <h3 id="gentropy.dataset.l2g_prediction.L2GPrediction.from_credible_set" class="
     <span class="n">gwas_fm</span> <span class="o">=</span> <span class="n">L2GFeatureMatrix</span><span class="p">(</span>
         <span class="n">_df</span><span class="o">=</span><span class="p">(</span>
             <span class="n">fm</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
-                <span class="n">study_locus</span><span class="o">.</span><span class="n">filter_by_study_type</span><span class="p">(</span><span class="s2">&quot;gwas&quot;</span><span class="p">,</span> <span class="n">study_index</span><span class="p">)</span><span class="o">.</span><span class="n">df</span><span class="p">,</span>
+                <span class="n">credible_set</span><span class="o">.</span><span class="n">filter_by_study_type</span><span class="p">(</span><span class="s2">&quot;gwas&quot;</span><span class="p">,</span> <span class="n">study_index</span><span class="p">)</span><span class="o">.</span><span class="n">df</span><span class="p">,</span>
                 <span class="n">on</span><span class="o">=</span><span class="s2">&quot;studyLocusId&quot;</span><span class="p">,</span>
             <span class="p">)</span>
         <span class="p">),</span>
@@ -3418,7 +3460,7 @@ <h3 id="gentropy.dataset.l2g_prediction.L2GPrediction.get_schema" class="doc doc
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3427,7 +3469,7 @@ <h3 id="gentropy.dataset.l2g_prediction.L2GPrediction.get_schema" class="doc doc
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StructType</code></td>          <td>
                 <code><span title="pyspark.sql.types.StructType">StructType</span></code>
           </td>
diff --git a/python_api/datasets/ld_index/index.html b/python_api/datasets/ld_index/index.html
index c2329a2e6..64346f883 100644
--- a/python_api/datasets/ld_index/index.html
+++ b/python_api/datasets/ld_index/index.html
@@ -2266,6 +2266,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../methods/susie_inf/" class="md-nav__link">
         
@@ -2297,7 +2339,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2313,14 +2355,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3003,7 +3045,7 @@ <h3 id="gentropy.dataset.ld_index.LDIndex.get_schema" class="doc doc-heading">
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3012,7 +3054,7 @@ <h3 id="gentropy.dataset.ld_index.LDIndex.get_schema" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StructType</code></td>          <td>
                 <code><span title="pyspark.sql.types.StructType">StructType</span></code>
           </td>
diff --git a/python_api/datasets/study_index/index.html b/python_api/datasets/study_index/index.html
index 5ac1dbe48..a26e7dd19 100644
--- a/python_api/datasets/study_index/index.html
+++ b/python_api/datasets/study_index/index.html
@@ -2329,6 +2329,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../methods/susie_inf/" class="md-nav__link">
         
@@ -2360,7 +2402,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2376,14 +2418,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3436,7 +3478,7 @@ <h3 id="gentropy.dataset.study_index.StudyIndex.aggregate_and_map_ancestries" cl
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3447,7 +3489,7 @@ <h3 id="gentropy.dataset.study_index.StudyIndex.aggregate_and_map_ancestries" cl
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>discovery_samples</code></td>
           <td>
                 <code><span title="pyspark.sql.Column">Column</span></code>
@@ -3466,7 +3508,7 @@ <h3 id="gentropy.dataset.study_index.StudyIndex.aggregate_and_map_ancestries" cl
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3475,7 +3517,7 @@ <h3 id="gentropy.dataset.study_index.StudyIndex.aggregate_and_map_ancestries" cl
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>Column</code></td>          <td>
                 <code><span title="pyspark.sql.Column">Column</span></code>
           </td>
@@ -3606,7 +3648,7 @@ <h3 id="gentropy.dataset.study_index.StudyIndex.get_schema" class="doc doc-headi
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3615,7 +3657,7 @@ <h3 id="gentropy.dataset.study_index.StudyIndex.get_schema" class="doc doc-headi
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StructType</code></td>          <td>
                 <code><span title="pyspark.sql.types.StructType">StructType</span></code>
           </td>
@@ -3668,7 +3710,7 @@ <h3 id="gentropy.dataset.study_index.StudyIndex.has_mapped_trait" class="doc doc
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3677,7 +3719,7 @@ <h3 id="gentropy.dataset.study_index.StudyIndex.has_mapped_trait" class="doc doc
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>Column</code></td>          <td>
                 <code><span title="pyspark.sql.Column">Column</span></code>
           </td>
@@ -3728,7 +3770,7 @@ <h3 id="gentropy.dataset.study_index.StudyIndex.has_summarystats" class="doc doc
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3737,7 +3779,7 @@ <h3 id="gentropy.dataset.study_index.StudyIndex.has_summarystats" class="doc doc
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>Column</code></td>          <td>
                 <code><span title="pyspark.sql.Column">Column</span></code>
           </td>
@@ -3788,7 +3830,7 @@ <h3 id="gentropy.dataset.study_index.StudyIndex.is_gwas" class="doc doc-heading"
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3797,7 +3839,7 @@ <h3 id="gentropy.dataset.study_index.StudyIndex.is_gwas" class="doc doc-heading"
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>Column</code></td>          <td>
                 <code><span title="pyspark.sql.Column">Column</span></code>
           </td>
@@ -3848,7 +3890,7 @@ <h3 id="gentropy.dataset.study_index.StudyIndex.is_qtl" class="doc doc-heading">
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3857,7 +3899,7 @@ <h3 id="gentropy.dataset.study_index.StudyIndex.is_qtl" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>Column</code></td>          <td>
                 <code><span title="pyspark.sql.Column">Column</span></code>
           </td>
@@ -3908,7 +3950,7 @@ <h3 id="gentropy.dataset.study_index.StudyIndex.is_quality_flagged" class="doc d
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3917,7 +3959,7 @@ <h3 id="gentropy.dataset.study_index.StudyIndex.is_quality_flagged" class="doc d
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>Column</code></td>          <td>
                 <code><span title="pyspark.sql.Column">Column</span></code>
           </td>
@@ -3976,7 +4018,7 @@ <h3 id="gentropy.dataset.study_index.StudyIndex.study_type_lut" class="doc doc-h
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3985,7 +4027,7 @@ <h3 id="gentropy.dataset.study_index.StudyIndex.study_type_lut" class="doc doc-h
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>DataFrame</code></td>          <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
           </td>
diff --git a/python_api/datasets/study_locus/index.html b/python_api/datasets/study_locus/index.html
index e94552901..e73f00ba1 100644
--- a/python_api/datasets/study_locus/index.html
+++ b/python_api/datasets/study_locus/index.html
@@ -1211,6 +1211,15 @@
     </span>
   </a>
   
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#gentropy.dataset.study_locus.StudyLocus.annotate_locus_statistics" class="md-nav__link">
+    <span class="md-ellipsis">
+      annotate_locus_statistics
+    </span>
+  </a>
+  
 </li>
         
           <li class="md-nav__item">
@@ -2383,6 +2392,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../methods/susie_inf/" class="md-nav__link">
         
@@ -2414,7 +2465,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2430,14 +2481,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3010,6 +3061,15 @@
     </span>
   </a>
   
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#gentropy.dataset.study_locus.StudyLocus.annotate_locus_statistics" class="md-nav__link">
+    <span class="md-ellipsis">
+      annotate_locus_statistics
+    </span>
+  </a>
+  
 </li>
         
           <li class="md-nav__item">
@@ -3179,9 +3239,7 @@ <h2 id="gentropy.dataset.study_locus.StudyLocus" class="doc doc-heading">
 
             <details class="quote">
               <summary>Source code in <code>src/gentropy/dataset/study_locus.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 75</span>
-<span class="normal"> 76</span>
-<span class="normal"> 77</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 77</span>
 <span class="normal"> 78</span>
 <span class="normal"> 79</span>
 <span class="normal"> 80</span>
@@ -3603,7 +3661,103 @@ <h2 id="gentropy.dataset.study_locus.StudyLocus" class="doc doc-heading">
 <span class="normal">496</span>
 <span class="normal">497</span>
 <span class="normal">498</span>
-<span class="normal">499</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@dataclass</span>
+<span class="normal">499</span>
+<span class="normal">500</span>
+<span class="normal">501</span>
+<span class="normal">502</span>
+<span class="normal">503</span>
+<span class="normal">504</span>
+<span class="normal">505</span>
+<span class="normal">506</span>
+<span class="normal">507</span>
+<span class="normal">508</span>
+<span class="normal">509</span>
+<span class="normal">510</span>
+<span class="normal">511</span>
+<span class="normal">512</span>
+<span class="normal">513</span>
+<span class="normal">514</span>
+<span class="normal">515</span>
+<span class="normal">516</span>
+<span class="normal">517</span>
+<span class="normal">518</span>
+<span class="normal">519</span>
+<span class="normal">520</span>
+<span class="normal">521</span>
+<span class="normal">522</span>
+<span class="normal">523</span>
+<span class="normal">524</span>
+<span class="normal">525</span>
+<span class="normal">526</span>
+<span class="normal">527</span>
+<span class="normal">528</span>
+<span class="normal">529</span>
+<span class="normal">530</span>
+<span class="normal">531</span>
+<span class="normal">532</span>
+<span class="normal">533</span>
+<span class="normal">534</span>
+<span class="normal">535</span>
+<span class="normal">536</span>
+<span class="normal">537</span>
+<span class="normal">538</span>
+<span class="normal">539</span>
+<span class="normal">540</span>
+<span class="normal">541</span>
+<span class="normal">542</span>
+<span class="normal">543</span>
+<span class="normal">544</span>
+<span class="normal">545</span>
+<span class="normal">546</span>
+<span class="normal">547</span>
+<span class="normal">548</span>
+<span class="normal">549</span>
+<span class="normal">550</span>
+<span class="normal">551</span>
+<span class="normal">552</span>
+<span class="normal">553</span>
+<span class="normal">554</span>
+<span class="normal">555</span>
+<span class="normal">556</span>
+<span class="normal">557</span>
+<span class="normal">558</span>
+<span class="normal">559</span>
+<span class="normal">560</span>
+<span class="normal">561</span>
+<span class="normal">562</span>
+<span class="normal">563</span>
+<span class="normal">564</span>
+<span class="normal">565</span>
+<span class="normal">566</span>
+<span class="normal">567</span>
+<span class="normal">568</span>
+<span class="normal">569</span>
+<span class="normal">570</span>
+<span class="normal">571</span>
+<span class="normal">572</span>
+<span class="normal">573</span>
+<span class="normal">574</span>
+<span class="normal">575</span>
+<span class="normal">576</span>
+<span class="normal">577</span>
+<span class="normal">578</span>
+<span class="normal">579</span>
+<span class="normal">580</span>
+<span class="normal">581</span>
+<span class="normal">582</span>
+<span class="normal">583</span>
+<span class="normal">584</span>
+<span class="normal">585</span>
+<span class="normal">586</span>
+<span class="normal">587</span>
+<span class="normal">588</span>
+<span class="normal">589</span>
+<span class="normal">590</span>
+<span class="normal">591</span>
+<span class="normal">592</span>
+<span class="normal">593</span>
+<span class="normal">594</span>
+<span class="normal">595</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@dataclass</span>
 <span class="k">class</span> <span class="nc">StudyLocus</span><span class="p">(</span><span class="n">Dataset</span><span class="p">):</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Study-Locus dataset.</span>
 
@@ -3611,31 +3765,52 @@ <h2 id="gentropy.dataset.study_locus.StudyLocus" class="doc doc-heading">
 <span class="sd">    &quot;&quot;&quot;</span>
 
     <span class="nd">@staticmethod</span>
-    <span class="k">def</span> <span class="nf">_overlapping_peaks</span><span class="p">(</span><span class="n">credset_to_overlap</span><span class="p">:</span> <span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">DataFrame</span><span class="p">:</span>
+    <span class="k">def</span> <span class="nf">_overlapping_peaks</span><span class="p">(</span>
+        <span class="n">credset_to_overlap</span><span class="p">:</span> <span class="n">DataFrame</span><span class="p">,</span> <span class="n">intra_study_overlap</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">DataFrame</span><span class="p">:</span>
 <span class="w">        </span><span class="sd">&quot;&quot;&quot;Calculate overlapping signals (study-locus) between GWAS-GWAS and GWAS-Molecular trait.</span>
 
 <span class="sd">        Args:</span>
 <span class="sd">            credset_to_overlap (DataFrame): DataFrame containing at least `studyLocusId`, `studyType`, `chromosome` and `tagVariantId` columns.</span>
+<span class="sd">            intra_study_overlap (bool): When True, finds intra-study overlaps for credible set deduplication. Default is False.</span>
 
 <span class="sd">        Returns:</span>
 <span class="sd">            DataFrame: containing `leftStudyLocusId`, `rightStudyLocusId` and `chromosome` columns.</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="c1"># Reduce columns to the minimum to reduce the size of the dataframe</span>
         <span class="n">credset_to_overlap</span> <span class="o">=</span> <span class="n">credset_to_overlap</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
-            <span class="s2">&quot;studyLocusId&quot;</span><span class="p">,</span> <span class="s2">&quot;studyType&quot;</span><span class="p">,</span> <span class="s2">&quot;chromosome&quot;</span><span class="p">,</span> <span class="s2">&quot;tagVariantId&quot;</span>
+            <span class="s2">&quot;studyLocusId&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;studyId&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;studyType&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;chromosome&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;region&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;tagVariantId&quot;</span><span class="p">,</span>
         <span class="p">)</span>
+        <span class="c1"># Define join condition - if intra_study_overlap is True, finds overlaps within the same study. Otherwise finds gwas vs everything overlaps for coloc.</span>
+        <span class="n">join_condition</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="p">[</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;left.studyId&quot;</span><span class="p">)</span> <span class="o">==</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;right.studyId&quot;</span><span class="p">),</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;left.chromosome&quot;</span><span class="p">)</span> <span class="o">==</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;right.chromosome&quot;</span><span class="p">),</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;left.tagVariantId&quot;</span><span class="p">)</span> <span class="o">==</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;right.tagVariantId&quot;</span><span class="p">),</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;left.studyLocusId&quot;</span><span class="p">)</span> <span class="o">&gt;</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;right.studyLocusId&quot;</span><span class="p">),</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;left.region&quot;</span><span class="p">)</span> <span class="o">!=</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;right.region&quot;</span><span class="p">),</span>
+            <span class="p">]</span>
+            <span class="k">if</span> <span class="n">intra_study_overlap</span>
+            <span class="k">else</span> <span class="p">[</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;left.chromosome&quot;</span><span class="p">)</span> <span class="o">==</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;right.chromosome&quot;</span><span class="p">),</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;left.tagVariantId&quot;</span><span class="p">)</span> <span class="o">==</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;right.tagVariantId&quot;</span><span class="p">),</span>
+                <span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;right.studyType&quot;</span><span class="p">)</span> <span class="o">!=</span> <span class="s2">&quot;gwas&quot;</span><span class="p">)</span>
+                <span class="o">|</span> <span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;left.studyLocusId&quot;</span><span class="p">)</span> <span class="o">&gt;</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;right.studyLocusId&quot;</span><span class="p">)),</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;left.studyType&quot;</span><span class="p">)</span> <span class="o">==</span> <span class="n">f</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">&quot;gwas&quot;</span><span class="p">),</span>
+            <span class="p">]</span>
+        <span class="p">)</span>
+
         <span class="k">return</span> <span class="p">(</span>
             <span class="n">credset_to_overlap</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;left&quot;</span><span class="p">)</span>
-            <span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;studyType&quot;</span><span class="p">)</span> <span class="o">==</span> <span class="s2">&quot;gwas&quot;</span><span class="p">)</span>
-            <span class="c1"># Self join with complex condition. Left it&#39;s all gwas and right can be gwas or molecular trait</span>
+            <span class="c1"># Self join with complex condition.</span>
             <span class="o">.</span><span class="n">join</span><span class="p">(</span>
                 <span class="n">credset_to_overlap</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;right&quot;</span><span class="p">),</span>
-                <span class="n">on</span><span class="o">=</span><span class="p">[</span>
-                    <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;left.chromosome&quot;</span><span class="p">)</span> <span class="o">==</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;right.chromosome&quot;</span><span class="p">),</span>
-                    <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;left.tagVariantId&quot;</span><span class="p">)</span> <span class="o">==</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;right.tagVariantId&quot;</span><span class="p">),</span>
-                    <span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;right.studyType&quot;</span><span class="p">)</span> <span class="o">!=</span> <span class="s2">&quot;gwas&quot;</span><span class="p">)</span>
-                    <span class="o">|</span> <span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;left.studyLocusId&quot;</span><span class="p">)</span> <span class="o">&gt;</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;right.studyLocusId&quot;</span><span class="p">)),</span>
-                <span class="p">],</span>
+                <span class="n">on</span><span class="o">=</span><span class="n">join_condition</span><span class="p">,</span>
                 <span class="n">how</span><span class="o">=</span><span class="s2">&quot;inner&quot;</span><span class="p">,</span>
             <span class="p">)</span>
             <span class="o">.</span><span class="n">select</span><span class="p">(</span>
@@ -3834,7 +4009,9 @@ <h2 id="gentropy.dataset.study_locus.StudyLocus" class="doc doc-heading">
         <span class="p">)</span>
         <span class="k">return</span> <span class="bp">self</span>
 
-    <span class="k">def</span> <span class="nf">find_overlaps</span><span class="p">(</span><span class="bp">self</span><span class="p">:</span> <span class="n">StudyLocus</span><span class="p">,</span> <span class="n">study_index</span><span class="p">:</span> <span class="n">StudyIndex</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyLocusOverlap</span><span class="p">:</span>
+    <span class="k">def</span> <span class="nf">find_overlaps</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">:</span> <span class="n">StudyLocus</span><span class="p">,</span> <span class="n">study_index</span><span class="p">:</span> <span class="n">StudyIndex</span><span class="p">,</span> <span class="n">intra_study_overlap</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyLocusOverlap</span><span class="p">:</span>
 <span class="w">        </span><span class="sd">&quot;&quot;&quot;Calculate overlapping study-locus.</span>
 
 <span class="sd">        Find overlapping study-locus that share at least one tagging variant. All GWAS-GWAS and all GWAS-Molecular traits are computed with the Molecular traits always</span>
@@ -3842,6 +4019,7 @@ <h2 id="gentropy.dataset.study_locus.StudyLocus" class="doc doc-heading">
 
 <span class="sd">        Args:</span>
 <span class="sd">            study_index (StudyIndex): Study index to resolve study types.</span>
+<span class="sd">            intra_study_overlap (bool): If True, finds intra-study overlaps for credible set deduplication. Default is False.</span>
 
 <span class="sd">        Returns:</span>
 <span class="sd">            StudyLocusOverlap: Pairs of overlapping study-locus with aligned tags.</span>
@@ -3851,8 +4029,10 @@ <h2 id="gentropy.dataset.study_locus.StudyLocus" class="doc doc-heading">
             <span class="o">.</span><span class="n">withColumn</span><span class="p">(</span><span class="s2">&quot;locus&quot;</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">explode</span><span class="p">(</span><span class="s2">&quot;locus&quot;</span><span class="p">))</span>
             <span class="o">.</span><span class="n">select</span><span class="p">(</span>
                 <span class="s2">&quot;studyLocusId&quot;</span><span class="p">,</span>
+                <span class="s2">&quot;studyId&quot;</span><span class="p">,</span>
                 <span class="s2">&quot;studyType&quot;</span><span class="p">,</span>
                 <span class="s2">&quot;chromosome&quot;</span><span class="p">,</span>
+                <span class="s2">&quot;region&quot;</span><span class="p">,</span>
                 <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;locus.variantId&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;tagVariantId&quot;</span><span class="p">),</span>
                 <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;locus.logBF&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;logBF&quot;</span><span class="p">),</span>
                 <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;locus.posteriorProbability&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;posteriorProbability&quot;</span><span class="p">),</span>
@@ -3864,7 +4044,7 @@ <h2 id="gentropy.dataset.study_locus.StudyLocus" class="doc doc-heading">
         <span class="p">)</span>
 
         <span class="c1"># overlapping study-locus</span>
-        <span class="n">peak_overlaps</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_overlapping_peaks</span><span class="p">(</span><span class="n">loci_to_overlap</span><span class="p">)</span>
+        <span class="n">peak_overlaps</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_overlapping_peaks</span><span class="p">(</span><span class="n">loci_to_overlap</span><span class="p">,</span> <span class="n">intra_study_overlap</span><span class="p">)</span>
 
         <span class="c1"># study-locus overlap by aligning overlapping variants</span>
         <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_align_overlapping_tags</span><span class="p">(</span><span class="n">loci_to_overlap</span><span class="p">,</span> <span class="n">peak_overlaps</span><span class="p">)</span>
@@ -3958,6 +4138,74 @@ <h2 id="gentropy.dataset.study_locus.StudyLocus" class="doc doc-heading">
         <span class="p">)</span>
         <span class="k">return</span> <span class="bp">self</span>
 
+    <span class="k">def</span> <span class="nf">annotate_locus_statistics</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">:</span> <span class="n">StudyLocus</span><span class="p">,</span>
+        <span class="n">summary_statistics</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">,</span>
+        <span class="n">collect_locus_distance</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyLocus</span><span class="p">:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Annotates study locus with summary statistics in the specified distance around the position.</span>
+
+<span class="sd">        Args:</span>
+<span class="sd">            summary_statistics (SummaryStatistics): Summary statistics to be used for annotation.</span>
+<span class="sd">            collect_locus_distance (int): distance from variant defining window for inclusion of variants in locus.</span>
+
+<span class="sd">        Returns:</span>
+<span class="sd">            StudyLocus: Study locus annotated with summary statistics in `locus` column. If no statistics are found, the `locus` column will be empty.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="c1"># The clumps will be used several times (persisting)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">persist</span><span class="p">()</span>
+        <span class="c1"># Renaming columns:</span>
+        <span class="n">sumstats_renamed</span> <span class="o">=</span> <span class="n">summary_statistics</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">selectExpr</span><span class="p">(</span>
+            <span class="o">*</span><span class="p">[</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">col</span><span class="si">}</span><span class="s2"> as tag_</span><span class="si">{</span><span class="n">col</span><span class="si">}</span><span class="s2">&quot;</span> <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">summary_statistics</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">columns</span><span class="p">]</span>
+        <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;sumstat&quot;</span><span class="p">)</span>
+
+        <span class="n">locus_df</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="n">sumstats_renamed</span>
+            <span class="c1"># Joining the two datasets together:</span>
+            <span class="o">.</span><span class="n">join</span><span class="p">(</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">broadcast</span><span class="p">(</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;clumped&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+                        <span class="s2">&quot;position&quot;</span><span class="p">,</span> <span class="s2">&quot;chromosome&quot;</span><span class="p">,</span> <span class="s2">&quot;studyId&quot;</span><span class="p">,</span> <span class="s2">&quot;studyLocusId&quot;</span>
+                    <span class="p">)</span>
+                <span class="p">),</span>
+                <span class="n">on</span><span class="o">=</span><span class="p">[</span>
+                    <span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;sumstat.tag_studyId&quot;</span><span class="p">)</span> <span class="o">==</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;clumped.studyId&quot;</span><span class="p">))</span>
+                    <span class="o">&amp;</span> <span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;sumstat.tag_chromosome&quot;</span><span class="p">)</span> <span class="o">==</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;clumped.chromosome&quot;</span><span class="p">))</span>
+                    <span class="o">&amp;</span> <span class="p">(</span>
+                        <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;sumstat.tag_position&quot;</span><span class="p">)</span>
+                        <span class="o">&gt;=</span> <span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;clumped.position&quot;</span><span class="p">)</span> <span class="o">-</span> <span class="n">collect_locus_distance</span><span class="p">)</span>
+                    <span class="p">)</span>
+                    <span class="o">&amp;</span> <span class="p">(</span>
+                        <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;sumstat.tag_position&quot;</span><span class="p">)</span>
+                        <span class="o">&lt;=</span> <span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;clumped.position&quot;</span><span class="p">)</span> <span class="o">+</span> <span class="n">collect_locus_distance</span><span class="p">)</span>
+                    <span class="p">)</span>
+                <span class="p">],</span>
+                <span class="n">how</span><span class="o">=</span><span class="s2">&quot;inner&quot;</span><span class="p">,</span>
+            <span class="p">)</span>
+            <span class="o">.</span><span class="n">withColumn</span><span class="p">(</span>
+                <span class="s2">&quot;locus&quot;</span><span class="p">,</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">struct</span><span class="p">(</span>
+                    <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;tag_variantId&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;variantId&quot;</span><span class="p">),</span>
+                    <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;tag_beta&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;beta&quot;</span><span class="p">),</span>
+                    <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;tag_pValueMantissa&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;pValueMantissa&quot;</span><span class="p">),</span>
+                    <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;tag_pValueExponent&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;pValueExponent&quot;</span><span class="p">),</span>
+                    <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;tag_standardError&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;standardError&quot;</span><span class="p">),</span>
+                <span class="p">),</span>
+            <span class="p">)</span>
+            <span class="o">.</span><span class="n">groupBy</span><span class="p">(</span><span class="s2">&quot;studyLocusId&quot;</span><span class="p">)</span>
+            <span class="o">.</span><span class="n">agg</span><span class="p">(</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">collect_list</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;locus&quot;</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;locus&quot;</span><span class="p">),</span>
+            <span class="p">)</span>
+        <span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">df</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="s2">&quot;locus&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
+            <span class="n">locus_df</span><span class="p">,</span>
+            <span class="n">on</span><span class="o">=</span><span class="s2">&quot;studyLocusId&quot;</span><span class="p">,</span>
+            <span class="n">how</span><span class="o">=</span><span class="s2">&quot;left&quot;</span><span class="p">,</span>
+        <span class="p">)</span>
+
+        <span class="k">return</span> <span class="bp">self</span>
+
     <span class="k">def</span> <span class="nf">annotate_ld</span><span class="p">(</span>
         <span class="bp">self</span><span class="p">:</span> <span class="n">StudyLocus</span><span class="p">,</span> <span class="n">study_index</span><span class="p">:</span> <span class="n">StudyIndex</span><span class="p">,</span> <span class="n">ld_index</span><span class="p">:</span> <span class="n">LDIndex</span>
     <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyLocus</span><span class="p">:</span>
@@ -4063,7 +4311,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.annotate_credible_sets" class="d
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -4072,7 +4320,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.annotate_credible_sets" class="d
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StudyLocus</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_locus.StudyLocus" href="#gentropy.dataset.study_locus.StudyLocus">StudyLocus</a></code>
           </td>
@@ -4087,7 +4335,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.annotate_credible_sets" class="d
 
 
 
-  <p><strong>Raises:</strong></p>
+  <p><span class="doc-section-title">Raises:</span></p>
   <table>
     <thead>
       <tr>
@@ -4096,7 +4344,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.annotate_credible_sets" class="d
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td>
                 <code>ValueError</code>
           </td>
@@ -4111,35 +4359,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.annotate_credible_sets" class="d
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/dataset/study_locus.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">375</span>
-<span class="normal">376</span>
-<span class="normal">377</span>
-<span class="normal">378</span>
-<span class="normal">379</span>
-<span class="normal">380</span>
-<span class="normal">381</span>
-<span class="normal">382</span>
-<span class="normal">383</span>
-<span class="normal">384</span>
-<span class="normal">385</span>
-<span class="normal">386</span>
-<span class="normal">387</span>
-<span class="normal">388</span>
-<span class="normal">389</span>
-<span class="normal">390</span>
-<span class="normal">391</span>
-<span class="normal">392</span>
-<span class="normal">393</span>
-<span class="normal">394</span>
-<span class="normal">395</span>
-<span class="normal">396</span>
-<span class="normal">397</span>
-<span class="normal">398</span>
-<span class="normal">399</span>
-<span class="normal">400</span>
-<span class="normal">401</span>
-<span class="normal">402</span>
-<span class="normal">403</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">403</span>
 <span class="normal">404</span>
 <span class="normal">405</span>
 <span class="normal">406</span>
@@ -4164,7 +4384,35 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.annotate_credible_sets" class="d
 <span class="normal">425</span>
 <span class="normal">426</span>
 <span class="normal">427</span>
-<span class="normal">428</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">annotate_credible_sets</span><span class="p">(</span><span class="bp">self</span><span class="p">:</span> <span class="n">StudyLocus</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyLocus</span><span class="p">:</span>
+<span class="normal">428</span>
+<span class="normal">429</span>
+<span class="normal">430</span>
+<span class="normal">431</span>
+<span class="normal">432</span>
+<span class="normal">433</span>
+<span class="normal">434</span>
+<span class="normal">435</span>
+<span class="normal">436</span>
+<span class="normal">437</span>
+<span class="normal">438</span>
+<span class="normal">439</span>
+<span class="normal">440</span>
+<span class="normal">441</span>
+<span class="normal">442</span>
+<span class="normal">443</span>
+<span class="normal">444</span>
+<span class="normal">445</span>
+<span class="normal">446</span>
+<span class="normal">447</span>
+<span class="normal">448</span>
+<span class="normal">449</span>
+<span class="normal">450</span>
+<span class="normal">451</span>
+<span class="normal">452</span>
+<span class="normal">453</span>
+<span class="normal">454</span>
+<span class="normal">455</span>
+<span class="normal">456</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">annotate_credible_sets</span><span class="p">(</span><span class="bp">self</span><span class="p">:</span> <span class="n">StudyLocus</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyLocus</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Annotate study-locus dataset with credible set flags.</span>
 
 <span class="sd">    Sorts the array in the `locus` column elements by their `posteriorProbability` values in descending order and adds</span>
@@ -4241,7 +4489,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.annotate_ld" class="doc doc-head
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -4252,7 +4500,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.annotate_ld" class="doc doc-head
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>study_index</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_index.StudyIndex" href="../study_index/#gentropy.dataset.study_index.StudyIndex">StudyIndex</a></code>
@@ -4266,7 +4514,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.annotate_ld" class="doc doc-head
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>ld_index</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.ld_index.LDIndex" href="../ld_index/#gentropy.dataset.ld_index.LDIndex">LDIndex</a></code>
@@ -4285,7 +4533,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.annotate_ld" class="doc doc-head
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -4294,7 +4542,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.annotate_ld" class="doc doc-head
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StudyLocus</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_locus.StudyLocus" href="#gentropy.dataset.study_locus.StudyLocus">StudyLocus</a></code>
           </td>
@@ -4309,21 +4557,21 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.annotate_ld" class="doc doc-head
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/dataset/study_locus.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">430</span>
-<span class="normal">431</span>
-<span class="normal">432</span>
-<span class="normal">433</span>
-<span class="normal">434</span>
-<span class="normal">435</span>
-<span class="normal">436</span>
-<span class="normal">437</span>
-<span class="normal">438</span>
-<span class="normal">439</span>
-<span class="normal">440</span>
-<span class="normal">441</span>
-<span class="normal">442</span>
-<span class="normal">443</span>
-<span class="normal">444</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">annotate_ld</span><span class="p">(</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">526</span>
+<span class="normal">527</span>
+<span class="normal">528</span>
+<span class="normal">529</span>
+<span class="normal">530</span>
+<span class="normal">531</span>
+<span class="normal">532</span>
+<span class="normal">533</span>
+<span class="normal">534</span>
+<span class="normal">535</span>
+<span class="normal">536</span>
+<span class="normal">537</span>
+<span class="normal">538</span>
+<span class="normal">539</span>
+<span class="normal">540</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">annotate_ld</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">:</span> <span class="n">StudyLocus</span><span class="p">,</span> <span class="n">study_index</span><span class="p">:</span> <span class="n">StudyIndex</span><span class="p">,</span> <span class="n">ld_index</span><span class="p">:</span> <span class="n">LDIndex</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyLocus</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Annotate LD information to study-locus.</span>
@@ -4349,23 +4597,19 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.annotate_ld" class="doc doc-head
 
 
 
-<h3 id="gentropy.dataset.study_locus.StudyLocus.assign_study_locus_id" class="doc doc-heading">
-          <code class="highlight language-python"><span class="n">assign_study_locus_id</span><span class="p">(</span><span class="n">study_id_col</span><span class="p">:</span> <span class="n">Column</span><span class="p">,</span> <span class="n">variant_id_col</span><span class="p">:</span> <span class="n">Column</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Column</span></code>
-  
-  <span class="doc doc-labels">
-      <small class="doc doc-label doc-label-staticmethod"><code>staticmethod</code></small>
-  </span>
+<h3 id="gentropy.dataset.study_locus.StudyLocus.annotate_locus_statistics" class="doc doc-heading">
+          <code class="highlight language-python"><span class="n">annotate_locus_statistics</span><span class="p">(</span><span class="n">summary_statistics</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">,</span> <span class="n">collect_locus_distance</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyLocus</span></code>
 
-<a href="#gentropy.dataset.study_locus.StudyLocus.assign_study_locus_id" class="headerlink" title="Permanent link">&para;</a></h3>
+<a href="#gentropy.dataset.study_locus.StudyLocus.annotate_locus_statistics" class="headerlink" title="Permanent link">&para;</a></h3>
 
 
   <div class="doc doc-contents ">
   
-      <p>Hashes a column with a variant ID and a study ID to extract a consistent studyLocusId.</p>
+      <p>Annotates study locus with summary statistics in the specified distance around the position.</p>
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -4376,28 +4620,28 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.assign_study_locus_id" class="do
       </tr>
     </thead>
     <tbody>
-        <tr>
-          <td><code>study_id_col</code></td>
+        <tr class="doc-section-item">
+          <td><code>summary_statistics</code></td>
           <td>
-                <code><span title="pyspark.sql.Column">Column</span></code>
+                <code><a class="autorefs autorefs-internal" title="gentropy.dataset.summary_statistics.SummaryStatistics" href="../summary_statistics/#gentropy.dataset.summary_statistics.SummaryStatistics">SummaryStatistics</a></code>
           </td>
           <td>
             <div class="doc-md-description">
-              <p>column name with a study ID</p>
+              <p>Summary statistics to be used for annotation.</p>
             </div>
           </td>
           <td>
               <em>required</em>
           </td>
         </tr>
-        <tr>
-          <td><code>variant_id_col</code></td>
+        <tr class="doc-section-item">
+          <td><code>collect_locus_distance</code></td>
           <td>
-                <code><span title="pyspark.sql.Column">Column</span></code>
+                <code>int</code>
           </td>
           <td>
             <div class="doc-md-description">
-              <p>column name with a variant ID</p>
+              <p>distance from variant defining window for inclusion of variants in locus.</p>
             </div>
           </td>
           <td>
@@ -4409,7 +4653,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.assign_study_locus_id" class="do
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -4418,67 +4662,295 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.assign_study_locus_id" class="do
       </tr>
     </thead>
     <tbody>
-        <tr>
-<td><code>Column</code></td>          <td>
-                <code><span title="pyspark.sql.Column">Column</span></code>
+        <tr class="doc-section-item">
+<td><code>StudyLocus</code></td>          <td>
+                <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_locus.StudyLocus" href="#gentropy.dataset.study_locus.StudyLocus">StudyLocus</a></code>
           </td>
           <td>
             <div class="doc-md-description">
-              <p>column with a study locus ID</p>
+              <p>Study locus annotated with summary statistics in <code>locus</code> column. If no statistics are found, the <code>locus</code> column will be empty.</p>
             </div>
           </td>
         </tr>
     </tbody>
   </table>
 
-
-
-<p><strong>Examples:</strong></p>
-    <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="n">df</span> <span class="o">=</span> <span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">([(</span><span class="s2">&quot;GCST000001&quot;</span><span class="p">,</span> <span class="s2">&quot;1_1000_A_C&quot;</span><span class="p">),</span> <span class="p">(</span><span class="s2">&quot;GCST000002&quot;</span><span class="p">,</span> <span class="s2">&quot;1_1000_A_C&quot;</span><span class="p">)])</span><span class="o">.</span><span class="n">toDF</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">,</span> <span class="s2">&quot;variantId&quot;</span><span class="p">)</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="n">df</span><span class="o">.</span><span class="n">withColumn</span><span class="p">(</span><span class="s2">&quot;study_locus_id&quot;</span><span class="p">,</span> <span class="n">StudyLocus</span><span class="o">.</span><span class="n">assign_study_locus_id</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">),</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;variantId&quot;</span><span class="p">)))</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
-<span class="go">+----------+----------+-------------------+</span>
-<span class="go">|   studyId| variantId|     study_locus_id|</span>
-<span class="go">+----------+----------+-------------------+</span>
-<span class="go">|GCST000001|1_1000_A_C|1553357789130151995|</span>
-<span class="go">|GCST000002|1_1000_A_C|-415050894682709184|</span>
-<span class="go">+----------+----------+-------------------+</span>
-</code></pre></div>
-
           <details class="quote">
             <summary>Source code in <code>src/gentropy/dataset/study_locus.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">200</span>
-<span class="normal">201</span>
-<span class="normal">202</span>
-<span class="normal">203</span>
-<span class="normal">204</span>
-<span class="normal">205</span>
-<span class="normal">206</span>
-<span class="normal">207</span>
-<span class="normal">208</span>
-<span class="normal">209</span>
-<span class="normal">210</span>
-<span class="normal">211</span>
-<span class="normal">212</span>
-<span class="normal">213</span>
-<span class="normal">214</span>
-<span class="normal">215</span>
-<span class="normal">216</span>
-<span class="normal">217</span>
-<span class="normal">218</span>
-<span class="normal">219</span>
-<span class="normal">220</span>
-<span class="normal">221</span>
-<span class="normal">222</span>
-<span class="normal">223</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@staticmethod</span>
-<span class="k">def</span> <span class="nf">assign_study_locus_id</span><span class="p">(</span><span class="n">study_id_col</span><span class="p">:</span> <span class="n">Column</span><span class="p">,</span> <span class="n">variant_id_col</span><span class="p">:</span> <span class="n">Column</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Column</span><span class="p">:</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;Hashes a column with a variant ID and a study ID to extract a consistent studyLocusId.</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">458</span>
+<span class="normal">459</span>
+<span class="normal">460</span>
+<span class="normal">461</span>
+<span class="normal">462</span>
+<span class="normal">463</span>
+<span class="normal">464</span>
+<span class="normal">465</span>
+<span class="normal">466</span>
+<span class="normal">467</span>
+<span class="normal">468</span>
+<span class="normal">469</span>
+<span class="normal">470</span>
+<span class="normal">471</span>
+<span class="normal">472</span>
+<span class="normal">473</span>
+<span class="normal">474</span>
+<span class="normal">475</span>
+<span class="normal">476</span>
+<span class="normal">477</span>
+<span class="normal">478</span>
+<span class="normal">479</span>
+<span class="normal">480</span>
+<span class="normal">481</span>
+<span class="normal">482</span>
+<span class="normal">483</span>
+<span class="normal">484</span>
+<span class="normal">485</span>
+<span class="normal">486</span>
+<span class="normal">487</span>
+<span class="normal">488</span>
+<span class="normal">489</span>
+<span class="normal">490</span>
+<span class="normal">491</span>
+<span class="normal">492</span>
+<span class="normal">493</span>
+<span class="normal">494</span>
+<span class="normal">495</span>
+<span class="normal">496</span>
+<span class="normal">497</span>
+<span class="normal">498</span>
+<span class="normal">499</span>
+<span class="normal">500</span>
+<span class="normal">501</span>
+<span class="normal">502</span>
+<span class="normal">503</span>
+<span class="normal">504</span>
+<span class="normal">505</span>
+<span class="normal">506</span>
+<span class="normal">507</span>
+<span class="normal">508</span>
+<span class="normal">509</span>
+<span class="normal">510</span>
+<span class="normal">511</span>
+<span class="normal">512</span>
+<span class="normal">513</span>
+<span class="normal">514</span>
+<span class="normal">515</span>
+<span class="normal">516</span>
+<span class="normal">517</span>
+<span class="normal">518</span>
+<span class="normal">519</span>
+<span class="normal">520</span>
+<span class="normal">521</span>
+<span class="normal">522</span>
+<span class="normal">523</span>
+<span class="normal">524</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">annotate_locus_statistics</span><span class="p">(</span>
+    <span class="bp">self</span><span class="p">:</span> <span class="n">StudyLocus</span><span class="p">,</span>
+    <span class="n">summary_statistics</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">,</span>
+    <span class="n">collect_locus_distance</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
+<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyLocus</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;Annotates study locus with summary statistics in the specified distance around the position.</span>
 
 <span class="sd">    Args:</span>
-<span class="sd">        study_id_col (Column): column name with a study ID</span>
-<span class="sd">        variant_id_col (Column): column name with a variant ID</span>
+<span class="sd">        summary_statistics (SummaryStatistics): Summary statistics to be used for annotation.</span>
+<span class="sd">        collect_locus_distance (int): distance from variant defining window for inclusion of variants in locus.</span>
 
 <span class="sd">    Returns:</span>
-<span class="sd">        Column: column with a study locus ID</span>
+<span class="sd">        StudyLocus: Study locus annotated with summary statistics in `locus` column. If no statistics are found, the `locus` column will be empty.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="c1"># The clumps will be used several times (persisting)</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">persist</span><span class="p">()</span>
+    <span class="c1"># Renaming columns:</span>
+    <span class="n">sumstats_renamed</span> <span class="o">=</span> <span class="n">summary_statistics</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">selectExpr</span><span class="p">(</span>
+        <span class="o">*</span><span class="p">[</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">col</span><span class="si">}</span><span class="s2"> as tag_</span><span class="si">{</span><span class="n">col</span><span class="si">}</span><span class="s2">&quot;</span> <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">summary_statistics</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">columns</span><span class="p">]</span>
+    <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;sumstat&quot;</span><span class="p">)</span>
+
+    <span class="n">locus_df</span> <span class="o">=</span> <span class="p">(</span>
+        <span class="n">sumstats_renamed</span>
+        <span class="c1"># Joining the two datasets together:</span>
+        <span class="o">.</span><span class="n">join</span><span class="p">(</span>
+            <span class="n">f</span><span class="o">.</span><span class="n">broadcast</span><span class="p">(</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;clumped&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+                    <span class="s2">&quot;position&quot;</span><span class="p">,</span> <span class="s2">&quot;chromosome&quot;</span><span class="p">,</span> <span class="s2">&quot;studyId&quot;</span><span class="p">,</span> <span class="s2">&quot;studyLocusId&quot;</span>
+                <span class="p">)</span>
+            <span class="p">),</span>
+            <span class="n">on</span><span class="o">=</span><span class="p">[</span>
+                <span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;sumstat.tag_studyId&quot;</span><span class="p">)</span> <span class="o">==</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;clumped.studyId&quot;</span><span class="p">))</span>
+                <span class="o">&amp;</span> <span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;sumstat.tag_chromosome&quot;</span><span class="p">)</span> <span class="o">==</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;clumped.chromosome&quot;</span><span class="p">))</span>
+                <span class="o">&amp;</span> <span class="p">(</span>
+                    <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;sumstat.tag_position&quot;</span><span class="p">)</span>
+                    <span class="o">&gt;=</span> <span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;clumped.position&quot;</span><span class="p">)</span> <span class="o">-</span> <span class="n">collect_locus_distance</span><span class="p">)</span>
+                <span class="p">)</span>
+                <span class="o">&amp;</span> <span class="p">(</span>
+                    <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;sumstat.tag_position&quot;</span><span class="p">)</span>
+                    <span class="o">&lt;=</span> <span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;clumped.position&quot;</span><span class="p">)</span> <span class="o">+</span> <span class="n">collect_locus_distance</span><span class="p">)</span>
+                <span class="p">)</span>
+            <span class="p">],</span>
+            <span class="n">how</span><span class="o">=</span><span class="s2">&quot;inner&quot;</span><span class="p">,</span>
+        <span class="p">)</span>
+        <span class="o">.</span><span class="n">withColumn</span><span class="p">(</span>
+            <span class="s2">&quot;locus&quot;</span><span class="p">,</span>
+            <span class="n">f</span><span class="o">.</span><span class="n">struct</span><span class="p">(</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;tag_variantId&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;variantId&quot;</span><span class="p">),</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;tag_beta&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;beta&quot;</span><span class="p">),</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;tag_pValueMantissa&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;pValueMantissa&quot;</span><span class="p">),</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;tag_pValueExponent&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;pValueExponent&quot;</span><span class="p">),</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;tag_standardError&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;standardError&quot;</span><span class="p">),</span>
+            <span class="p">),</span>
+        <span class="p">)</span>
+        <span class="o">.</span><span class="n">groupBy</span><span class="p">(</span><span class="s2">&quot;studyLocusId&quot;</span><span class="p">)</span>
+        <span class="o">.</span><span class="n">agg</span><span class="p">(</span>
+            <span class="n">f</span><span class="o">.</span><span class="n">collect_list</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;locus&quot;</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;locus&quot;</span><span class="p">),</span>
+        <span class="p">)</span>
+    <span class="p">)</span>
+
+    <span class="bp">self</span><span class="o">.</span><span class="n">df</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="s2">&quot;locus&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
+        <span class="n">locus_df</span><span class="p">,</span>
+        <span class="n">on</span><span class="o">=</span><span class="s2">&quot;studyLocusId&quot;</span><span class="p">,</span>
+        <span class="n">how</span><span class="o">=</span><span class="s2">&quot;left&quot;</span><span class="p">,</span>
+    <span class="p">)</span>
+
+    <span class="k">return</span> <span class="bp">self</span>
+</code></pre></div></td></tr></table></div>
+          </details>
+  </div>
+
+</div>
+
+
+<div class="doc doc-object doc-function">
+
+
+
+<h3 id="gentropy.dataset.study_locus.StudyLocus.assign_study_locus_id" class="doc doc-heading">
+          <code class="highlight language-python"><span class="n">assign_study_locus_id</span><span class="p">(</span><span class="n">study_id_col</span><span class="p">:</span> <span class="n">Column</span><span class="p">,</span> <span class="n">variant_id_col</span><span class="p">:</span> <span class="n">Column</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Column</span></code>
+  
+  <span class="doc doc-labels">
+      <small class="doc doc-label doc-label-staticmethod"><code>staticmethod</code></small>
+  </span>
+
+<a href="#gentropy.dataset.study_locus.StudyLocus.assign_study_locus_id" class="headerlink" title="Permanent link">&para;</a></h3>
+
+
+  <div class="doc doc-contents ">
+  
+      <p>Hashes a column with a variant ID and a study ID to extract a consistent studyLocusId.</p>
+
+
+
+  <p><span class="doc-section-title">Parameters:</span></p>
+  <table>
+    <thead>
+      <tr>
+        <th>Name</th>
+        <th>Type</th>
+        <th>Description</th>
+        <th>Default</th>
+      </tr>
+    </thead>
+    <tbody>
+        <tr class="doc-section-item">
+          <td><code>study_id_col</code></td>
+          <td>
+                <code><span title="pyspark.sql.Column">Column</span></code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>column name with a study ID</p>
+            </div>
+          </td>
+          <td>
+              <em>required</em>
+          </td>
+        </tr>
+        <tr class="doc-section-item">
+          <td><code>variant_id_col</code></td>
+          <td>
+                <code><span title="pyspark.sql.Column">Column</span></code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>column name with a variant ID</p>
+            </div>
+          </td>
+          <td>
+              <em>required</em>
+          </td>
+        </tr>
+    </tbody>
+  </table>
+
+
+
+  <p><span class="doc-section-title">Returns:</span></p>
+  <table>
+    <thead>
+      <tr>
+<th>Name</th>        <th>Type</th>
+        <th>Description</th>
+      </tr>
+    </thead>
+    <tbody>
+        <tr class="doc-section-item">
+<td><code>Column</code></td>          <td>
+                <code><span title="pyspark.sql.Column">Column</span></code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>column with a study locus ID</p>
+            </div>
+          </td>
+        </tr>
+    </tbody>
+  </table>
+
+
+
+<p><span class="doc-section-title">Examples:</span></p>
+    <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="n">df</span> <span class="o">=</span> <span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">([(</span><span class="s2">&quot;GCST000001&quot;</span><span class="p">,</span> <span class="s2">&quot;1_1000_A_C&quot;</span><span class="p">),</span> <span class="p">(</span><span class="s2">&quot;GCST000002&quot;</span><span class="p">,</span> <span class="s2">&quot;1_1000_A_C&quot;</span><span class="p">)])</span><span class="o">.</span><span class="n">toDF</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">,</span> <span class="s2">&quot;variantId&quot;</span><span class="p">)</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">df</span><span class="o">.</span><span class="n">withColumn</span><span class="p">(</span><span class="s2">&quot;study_locus_id&quot;</span><span class="p">,</span> <span class="n">StudyLocus</span><span class="o">.</span><span class="n">assign_study_locus_id</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">),</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;variantId&quot;</span><span class="p">)))</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
+<span class="go">+----------+----------+-------------------+</span>
+<span class="go">|   studyId| variantId|     study_locus_id|</span>
+<span class="go">+----------+----------+-------------------+</span>
+<span class="go">|GCST000001|1_1000_A_C|1553357789130151995|</span>
+<span class="go">|GCST000002|1_1000_A_C|-415050894682709184|</span>
+<span class="go">+----------+----------+-------------------+</span>
+</code></pre></div>
+
+          <details class="quote">
+            <summary>Source code in <code>src/gentropy/dataset/study_locus.py</code></summary>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">223</span>
+<span class="normal">224</span>
+<span class="normal">225</span>
+<span class="normal">226</span>
+<span class="normal">227</span>
+<span class="normal">228</span>
+<span class="normal">229</span>
+<span class="normal">230</span>
+<span class="normal">231</span>
+<span class="normal">232</span>
+<span class="normal">233</span>
+<span class="normal">234</span>
+<span class="normal">235</span>
+<span class="normal">236</span>
+<span class="normal">237</span>
+<span class="normal">238</span>
+<span class="normal">239</span>
+<span class="normal">240</span>
+<span class="normal">241</span>
+<span class="normal">242</span>
+<span class="normal">243</span>
+<span class="normal">244</span>
+<span class="normal">245</span>
+<span class="normal">246</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@staticmethod</span>
+<span class="k">def</span> <span class="nf">assign_study_locus_id</span><span class="p">(</span><span class="n">study_id_col</span><span class="p">:</span> <span class="n">Column</span><span class="p">,</span> <span class="n">variant_id_col</span><span class="p">:</span> <span class="n">Column</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Column</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;Hashes a column with a variant ID and a study ID to extract a consistent studyLocusId.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        study_id_col (Column): column name with a study ID</span>
+<span class="sd">        variant_id_col (Column): column name with a variant ID</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">        Column: column with a study locus ID</span>
 
 <span class="sd">    Examples:</span>
 <span class="sd">        &gt;&gt;&gt; df = spark.createDataFrame([(&quot;GCST000001&quot;, &quot;1_1000_A_C&quot;), (&quot;GCST000002&quot;, &quot;1_1000_A_C&quot;)]).toDF(&quot;studyId&quot;, &quot;variantId&quot;)</span>
@@ -4520,7 +4992,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.calculate_credible_set_log10bf"
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -4531,7 +5003,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.calculate_credible_set_log10bf"
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>logbfs</code></td>
           <td>
                 <code><span title="pyspark.sql.Column">Column</span></code>
@@ -4550,7 +5022,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.calculate_credible_set_log10bf"
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -4559,7 +5031,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.calculate_credible_set_log10bf"
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>Column</code></td>          <td>
                 <code><span title="pyspark.sql.Column">Column</span></code>
           </td>
@@ -4574,7 +5046,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.calculate_credible_set_log10bf"
 
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">([([</span><span class="mf">0.2</span><span class="p">,</span> <span class="mf">0.1</span><span class="p">,</span> <span class="mf">0.05</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">],)])</span><span class="o">.</span><span class="n">toDF</span><span class="p">(</span><span class="s2">&quot;logBF&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">round</span><span class="p">(</span><span class="n">StudyLocus</span><span class="o">.</span><span class="n">calculate_credible_set_log10bf</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;logBF&quot;</span><span class="p">)),</span> <span class="mi">7</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;credibleSetlog10BF&quot;</span><span class="p">))</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
 <span class="go">+------------------+</span>
 <span class="go">|credibleSetlog10BF|</span>
@@ -4585,27 +5057,27 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.calculate_credible_set_log10bf"
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/dataset/study_locus.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">225</span>
-<span class="normal">226</span>
-<span class="normal">227</span>
-<span class="normal">228</span>
-<span class="normal">229</span>
-<span class="normal">230</span>
-<span class="normal">231</span>
-<span class="normal">232</span>
-<span class="normal">233</span>
-<span class="normal">234</span>
-<span class="normal">235</span>
-<span class="normal">236</span>
-<span class="normal">237</span>
-<span class="normal">238</span>
-<span class="normal">239</span>
-<span class="normal">240</span>
-<span class="normal">241</span>
-<span class="normal">242</span>
-<span class="normal">243</span>
-<span class="normal">244</span>
-<span class="normal">245</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@classmethod</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">248</span>
+<span class="normal">249</span>
+<span class="normal">250</span>
+<span class="normal">251</span>
+<span class="normal">252</span>
+<span class="normal">253</span>
+<span class="normal">254</span>
+<span class="normal">255</span>
+<span class="normal">256</span>
+<span class="normal">257</span>
+<span class="normal">258</span>
+<span class="normal">259</span>
+<span class="normal">260</span>
+<span class="normal">261</span>
+<span class="normal">262</span>
+<span class="normal">263</span>
+<span class="normal">264</span>
+<span class="normal">265</span>
+<span class="normal">266</span>
+<span class="normal">267</span>
+<span class="normal">268</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@classmethod</span>
 <span class="k">def</span> <span class="nf">calculate_credible_set_log10bf</span><span class="p">(</span><span class="bp">cls</span><span class="p">:</span> <span class="nb">type</span><span class="p">[</span><span class="n">StudyLocus</span><span class="p">],</span> <span class="n">logbfs</span><span class="p">:</span> <span class="n">Column</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Column</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Calculate Bayes factor for the entire credible set. The Bayes factor is calculated as the logsumexp of the logBF values of the variants in the locus.</span>
 
@@ -4650,7 +5122,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.clump" class="doc doc-heading">
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -4659,7 +5131,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.clump" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StudyLocus</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_locus.StudyLocus" href="#gentropy.dataset.study_locus.StudyLocus">StudyLocus</a></code>
           </td>
@@ -4674,40 +5146,40 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.clump" class="doc doc-heading">
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/dataset/study_locus.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">446</span>
-<span class="normal">447</span>
-<span class="normal">448</span>
-<span class="normal">449</span>
-<span class="normal">450</span>
-<span class="normal">451</span>
-<span class="normal">452</span>
-<span class="normal">453</span>
-<span class="normal">454</span>
-<span class="normal">455</span>
-<span class="normal">456</span>
-<span class="normal">457</span>
-<span class="normal">458</span>
-<span class="normal">459</span>
-<span class="normal">460</span>
-<span class="normal">461</span>
-<span class="normal">462</span>
-<span class="normal">463</span>
-<span class="normal">464</span>
-<span class="normal">465</span>
-<span class="normal">466</span>
-<span class="normal">467</span>
-<span class="normal">468</span>
-<span class="normal">469</span>
-<span class="normal">470</span>
-<span class="normal">471</span>
-<span class="normal">472</span>
-<span class="normal">473</span>
-<span class="normal">474</span>
-<span class="normal">475</span>
-<span class="normal">476</span>
-<span class="normal">477</span>
-<span class="normal">478</span>
-<span class="normal">479</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">clump</span><span class="p">(</span><span class="bp">self</span><span class="p">:</span> <span class="n">StudyLocus</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyLocus</span><span class="p">:</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">542</span>
+<span class="normal">543</span>
+<span class="normal">544</span>
+<span class="normal">545</span>
+<span class="normal">546</span>
+<span class="normal">547</span>
+<span class="normal">548</span>
+<span class="normal">549</span>
+<span class="normal">550</span>
+<span class="normal">551</span>
+<span class="normal">552</span>
+<span class="normal">553</span>
+<span class="normal">554</span>
+<span class="normal">555</span>
+<span class="normal">556</span>
+<span class="normal">557</span>
+<span class="normal">558</span>
+<span class="normal">559</span>
+<span class="normal">560</span>
+<span class="normal">561</span>
+<span class="normal">562</span>
+<span class="normal">563</span>
+<span class="normal">564</span>
+<span class="normal">565</span>
+<span class="normal">566</span>
+<span class="normal">567</span>
+<span class="normal">568</span>
+<span class="normal">569</span>
+<span class="normal">570</span>
+<span class="normal">571</span>
+<span class="normal">572</span>
+<span class="normal">573</span>
+<span class="normal">574</span>
+<span class="normal">575</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">clump</span><span class="p">(</span><span class="bp">self</span><span class="p">:</span> <span class="n">StudyLocus</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyLocus</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Perform LD clumping of the studyLocus.</span>
 
 <span class="sd">    Evaluates whether a lead variant is linked to a tag (with lowest p-value) in the same studyLocus dataset.</span>
@@ -4764,7 +5236,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.filter_by_study_type" class="doc
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -4775,7 +5247,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.filter_by_study_type" class="doc
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>study_type</code></td>
           <td>
                 <code>str</code>
@@ -4789,7 +5261,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.filter_by_study_type" class="doc
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>study_index</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_index.StudyIndex" href="../study_index/#gentropy.dataset.study_index.StudyIndex">StudyIndex</a></code>
@@ -4808,7 +5280,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.filter_by_study_type" class="doc
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -4817,7 +5289,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.filter_by_study_type" class="doc
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StudyLocus</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_locus.StudyLocus" href="#gentropy.dataset.study_locus.StudyLocus">StudyLocus</a></code>
           </td>
@@ -4832,7 +5304,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.filter_by_study_type" class="doc
 
 
 
-  <p><strong>Raises:</strong></p>
+  <p><span class="doc-section-title">Raises:</span></p>
   <table>
     <thead>
       <tr>
@@ -4841,7 +5313,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.filter_by_study_type" class="doc
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td>
                 <code>ValueError</code>
           </td>
@@ -4856,34 +5328,34 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.filter_by_study_type" class="doc
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/dataset/study_locus.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">256</span>
-<span class="normal">257</span>
-<span class="normal">258</span>
-<span class="normal">259</span>
-<span class="normal">260</span>
-<span class="normal">261</span>
-<span class="normal">262</span>
-<span class="normal">263</span>
-<span class="normal">264</span>
-<span class="normal">265</span>
-<span class="normal">266</span>
-<span class="normal">267</span>
-<span class="normal">268</span>
-<span class="normal">269</span>
-<span class="normal">270</span>
-<span class="normal">271</span>
-<span class="normal">272</span>
-<span class="normal">273</span>
-<span class="normal">274</span>
-<span class="normal">275</span>
-<span class="normal">276</span>
-<span class="normal">277</span>
-<span class="normal">278</span>
-<span class="normal">279</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">279</span>
 <span class="normal">280</span>
 <span class="normal">281</span>
 <span class="normal">282</span>
-<span class="normal">283</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">filter_by_study_type</span><span class="p">(</span>
+<span class="normal">283</span>
+<span class="normal">284</span>
+<span class="normal">285</span>
+<span class="normal">286</span>
+<span class="normal">287</span>
+<span class="normal">288</span>
+<span class="normal">289</span>
+<span class="normal">290</span>
+<span class="normal">291</span>
+<span class="normal">292</span>
+<span class="normal">293</span>
+<span class="normal">294</span>
+<span class="normal">295</span>
+<span class="normal">296</span>
+<span class="normal">297</span>
+<span class="normal">298</span>
+<span class="normal">299</span>
+<span class="normal">300</span>
+<span class="normal">301</span>
+<span class="normal">302</span>
+<span class="normal">303</span>
+<span class="normal">304</span>
+<span class="normal">305</span>
+<span class="normal">306</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">filter_by_study_type</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">:</span> <span class="n">StudyLocus</span><span class="p">,</span> <span class="n">study_type</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">study_index</span><span class="p">:</span> <span class="n">StudyIndex</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyLocus</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Creates a new StudyLocus dataset filtered by study type.</span>
@@ -4934,7 +5406,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.filter_credible_set" class="doc
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -4945,7 +5417,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.filter_credible_set" class="doc
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>credible_interval</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_locus.CredibleInterval" href="#gentropy.dataset.study_locus.CredibleInterval">CredibleInterval</a></code>
@@ -4964,7 +5436,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.filter_credible_set" class="doc
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -4973,7 +5445,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.filter_credible_set" class="doc
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StudyLocus</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_locus.StudyLocus" href="#gentropy.dataset.study_locus.StudyLocus">StudyLocus</a></code>
           </td>
@@ -4988,26 +5460,26 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.filter_credible_set" class="doc
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/dataset/study_locus.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">285</span>
-<span class="normal">286</span>
-<span class="normal">287</span>
-<span class="normal">288</span>
-<span class="normal">289</span>
-<span class="normal">290</span>
-<span class="normal">291</span>
-<span class="normal">292</span>
-<span class="normal">293</span>
-<span class="normal">294</span>
-<span class="normal">295</span>
-<span class="normal">296</span>
-<span class="normal">297</span>
-<span class="normal">298</span>
-<span class="normal">299</span>
-<span class="normal">300</span>
-<span class="normal">301</span>
-<span class="normal">302</span>
-<span class="normal">303</span>
-<span class="normal">304</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">filter_credible_set</span><span class="p">(</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">308</span>
+<span class="normal">309</span>
+<span class="normal">310</span>
+<span class="normal">311</span>
+<span class="normal">312</span>
+<span class="normal">313</span>
+<span class="normal">314</span>
+<span class="normal">315</span>
+<span class="normal">316</span>
+<span class="normal">317</span>
+<span class="normal">318</span>
+<span class="normal">319</span>
+<span class="normal">320</span>
+<span class="normal">321</span>
+<span class="normal">322</span>
+<span class="normal">323</span>
+<span class="normal">324</span>
+<span class="normal">325</span>
+<span class="normal">326</span>
+<span class="normal">327</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">filter_credible_set</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">:</span> <span class="n">StudyLocus</span><span class="p">,</span>
     <span class="n">credible_interval</span><span class="p">:</span> <span class="n">CredibleInterval</span><span class="p">,</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyLocus</span><span class="p">:</span>
@@ -5039,7 +5511,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.filter_credible_set" class="doc
 
 
 <h3 id="gentropy.dataset.study_locus.StudyLocus.find_overlaps" class="doc doc-heading">
-          <code class="highlight language-python"><span class="n">find_overlaps</span><span class="p">(</span><span class="n">study_index</span><span class="p">:</span> <span class="n">StudyIndex</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyLocusOverlap</span></code>
+          <code class="highlight language-python"><span class="n">find_overlaps</span><span class="p">(</span><span class="n">study_index</span><span class="p">:</span> <span class="n">StudyIndex</span><span class="p">,</span> <span class="n">intra_study_overlap</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyLocusOverlap</span></code>
 
 <a href="#gentropy.dataset.study_locus.StudyLocus.find_overlaps" class="headerlink" title="Permanent link">&para;</a></h3>
 
@@ -5052,7 +5524,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.find_overlaps" class="doc doc-he
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -5063,7 +5535,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.find_overlaps" class="doc doc-he
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>study_index</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_index.StudyIndex" href="../study_index/#gentropy.dataset.study_index.StudyIndex">StudyIndex</a></code>
@@ -5077,12 +5549,26 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.find_overlaps" class="doc doc-he
               <em>required</em>
           </td>
         </tr>
+        <tr class="doc-section-item">
+          <td><code>intra_study_overlap</code></td>
+          <td>
+                <code>bool</code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>If True, finds intra-study overlaps for credible set deduplication. Default is False.</p>
+            </div>
+          </td>
+          <td>
+                <code>False</code>
+          </td>
+        </tr>
     </tbody>
   </table>
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -5091,7 +5577,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.find_overlaps" class="doc doc-he
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StudyLocusOverlap</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_locus_overlap.StudyLocusOverlap" href="../study_locus_overlap/#gentropy.dataset.study_locus_overlap.StudyLocusOverlap">StudyLocusOverlap</a></code>
           </td>
@@ -5106,30 +5592,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.find_overlaps" class="doc doc-he
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/dataset/study_locus.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">306</span>
-<span class="normal">307</span>
-<span class="normal">308</span>
-<span class="normal">309</span>
-<span class="normal">310</span>
-<span class="normal">311</span>
-<span class="normal">312</span>
-<span class="normal">313</span>
-<span class="normal">314</span>
-<span class="normal">315</span>
-<span class="normal">316</span>
-<span class="normal">317</span>
-<span class="normal">318</span>
-<span class="normal">319</span>
-<span class="normal">320</span>
-<span class="normal">321</span>
-<span class="normal">322</span>
-<span class="normal">323</span>
-<span class="normal">324</span>
-<span class="normal">325</span>
-<span class="normal">326</span>
-<span class="normal">327</span>
-<span class="normal">328</span>
-<span class="normal">329</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">329</span>
 <span class="normal">330</span>
 <span class="normal">331</span>
 <span class="normal">332</span>
@@ -5139,7 +5602,37 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.find_overlaps" class="doc doc-he
 <span class="normal">336</span>
 <span class="normal">337</span>
 <span class="normal">338</span>
-<span class="normal">339</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">find_overlaps</span><span class="p">(</span><span class="bp">self</span><span class="p">:</span> <span class="n">StudyLocus</span><span class="p">,</span> <span class="n">study_index</span><span class="p">:</span> <span class="n">StudyIndex</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyLocusOverlap</span><span class="p">:</span>
+<span class="normal">339</span>
+<span class="normal">340</span>
+<span class="normal">341</span>
+<span class="normal">342</span>
+<span class="normal">343</span>
+<span class="normal">344</span>
+<span class="normal">345</span>
+<span class="normal">346</span>
+<span class="normal">347</span>
+<span class="normal">348</span>
+<span class="normal">349</span>
+<span class="normal">350</span>
+<span class="normal">351</span>
+<span class="normal">352</span>
+<span class="normal">353</span>
+<span class="normal">354</span>
+<span class="normal">355</span>
+<span class="normal">356</span>
+<span class="normal">357</span>
+<span class="normal">358</span>
+<span class="normal">359</span>
+<span class="normal">360</span>
+<span class="normal">361</span>
+<span class="normal">362</span>
+<span class="normal">363</span>
+<span class="normal">364</span>
+<span class="normal">365</span>
+<span class="normal">366</span>
+<span class="normal">367</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">find_overlaps</span><span class="p">(</span>
+    <span class="bp">self</span><span class="p">:</span> <span class="n">StudyLocus</span><span class="p">,</span> <span class="n">study_index</span><span class="p">:</span> <span class="n">StudyIndex</span><span class="p">,</span> <span class="n">intra_study_overlap</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
+<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyLocusOverlap</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Calculate overlapping study-locus.</span>
 
 <span class="sd">    Find overlapping study-locus that share at least one tagging variant. All GWAS-GWAS and all GWAS-Molecular traits are computed with the Molecular traits always</span>
@@ -5147,6 +5640,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.find_overlaps" class="doc doc-he
 
 <span class="sd">    Args:</span>
 <span class="sd">        study_index (StudyIndex): Study index to resolve study types.</span>
+<span class="sd">        intra_study_overlap (bool): If True, finds intra-study overlaps for credible set deduplication. Default is False.</span>
 
 <span class="sd">    Returns:</span>
 <span class="sd">        StudyLocusOverlap: Pairs of overlapping study-locus with aligned tags.</span>
@@ -5156,8 +5650,10 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.find_overlaps" class="doc doc-he
         <span class="o">.</span><span class="n">withColumn</span><span class="p">(</span><span class="s2">&quot;locus&quot;</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">explode</span><span class="p">(</span><span class="s2">&quot;locus&quot;</span><span class="p">))</span>
         <span class="o">.</span><span class="n">select</span><span class="p">(</span>
             <span class="s2">&quot;studyLocusId&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;studyId&quot;</span><span class="p">,</span>
             <span class="s2">&quot;studyType&quot;</span><span class="p">,</span>
             <span class="s2">&quot;chromosome&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;region&quot;</span><span class="p">,</span>
             <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;locus.variantId&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;tagVariantId&quot;</span><span class="p">),</span>
             <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;locus.logBF&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;logBF&quot;</span><span class="p">),</span>
             <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;locus.posteriorProbability&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;posteriorProbability&quot;</span><span class="p">),</span>
@@ -5169,7 +5665,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.find_overlaps" class="doc doc-he
     <span class="p">)</span>
 
     <span class="c1"># overlapping study-locus</span>
-    <span class="n">peak_overlaps</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_overlapping_peaks</span><span class="p">(</span><span class="n">loci_to_overlap</span><span class="p">)</span>
+    <span class="n">peak_overlaps</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_overlapping_peaks</span><span class="p">(</span><span class="n">loci_to_overlap</span><span class="p">,</span> <span class="n">intra_study_overlap</span><span class="p">)</span>
 
     <span class="c1"># study-locus overlap by aligning overlapping variants</span>
     <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_align_overlapping_tags</span><span class="p">(</span><span class="n">loci_to_overlap</span><span class="p">,</span> <span class="n">peak_overlaps</span><span class="p">)</span>
@@ -5200,7 +5696,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.get_schema" class="doc doc-headi
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -5209,7 +5705,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.get_schema" class="doc doc-headi
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StructType</code></td>          <td>
                 <code><span title="pyspark.sql.types.StructType">StructType</span></code>
           </td>
@@ -5224,14 +5720,14 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.get_schema" class="doc doc-headi
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/dataset/study_locus.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">247</span>
-<span class="normal">248</span>
-<span class="normal">249</span>
-<span class="normal">250</span>
-<span class="normal">251</span>
-<span class="normal">252</span>
-<span class="normal">253</span>
-<span class="normal">254</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@classmethod</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">270</span>
+<span class="normal">271</span>
+<span class="normal">272</span>
+<span class="normal">273</span>
+<span class="normal">274</span>
+<span class="normal">275</span>
+<span class="normal">276</span>
+<span class="normal">277</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@classmethod</span>
 <span class="k">def</span> <span class="nf">get_schema</span><span class="p">(</span><span class="bp">cls</span><span class="p">:</span> <span class="nb">type</span><span class="p">[</span><span class="n">StudyLocus</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">StructType</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Provides the schema for the StudyLocus dataset.</span>
 
@@ -5262,7 +5758,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.neglog_pvalue" class="doc doc-he
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -5271,7 +5767,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.neglog_pvalue" class="doc doc-he
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>Column</code></td>          <td>
                 <code><span title="pyspark.sql.Column">Column</span></code>
           </td>
@@ -5286,16 +5782,16 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.neglog_pvalue" class="doc doc-he
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/dataset/study_locus.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">364</span>
-<span class="normal">365</span>
-<span class="normal">366</span>
-<span class="normal">367</span>
-<span class="normal">368</span>
-<span class="normal">369</span>
-<span class="normal">370</span>
-<span class="normal">371</span>
-<span class="normal">372</span>
-<span class="normal">373</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">neglog_pvalue</span><span class="p">(</span><span class="bp">self</span><span class="p">:</span> <span class="n">StudyLocus</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Column</span><span class="p">:</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">392</span>
+<span class="normal">393</span>
+<span class="normal">394</span>
+<span class="normal">395</span>
+<span class="normal">396</span>
+<span class="normal">397</span>
+<span class="normal">398</span>
+<span class="normal">399</span>
+<span class="normal">400</span>
+<span class="normal">401</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">neglog_pvalue</span><span class="p">(</span><span class="bp">self</span><span class="p">:</span> <span class="n">StudyLocus</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Column</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Returns the negative log p-value.</span>
 
 <span class="sd">    Returns:</span>
@@ -5328,7 +5824,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.unique_variants_in_locus" class=
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -5337,7 +5833,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.unique_variants_in_locus" class=
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>DataFrame</code></td>          <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
           </td>
@@ -5352,28 +5848,28 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.unique_variants_in_locus" class=
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/dataset/study_locus.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">341</span>
-<span class="normal">342</span>
-<span class="normal">343</span>
-<span class="normal">344</span>
-<span class="normal">345</span>
-<span class="normal">346</span>
-<span class="normal">347</span>
-<span class="normal">348</span>
-<span class="normal">349</span>
-<span class="normal">350</span>
-<span class="normal">351</span>
-<span class="normal">352</span>
-<span class="normal">353</span>
-<span class="normal">354</span>
-<span class="normal">355</span>
-<span class="normal">356</span>
-<span class="normal">357</span>
-<span class="normal">358</span>
-<span class="normal">359</span>
-<span class="normal">360</span>
-<span class="normal">361</span>
-<span class="normal">362</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">unique_variants_in_locus</span><span class="p">(</span><span class="bp">self</span><span class="p">:</span> <span class="n">StudyLocus</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">DataFrame</span><span class="p">:</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">369</span>
+<span class="normal">370</span>
+<span class="normal">371</span>
+<span class="normal">372</span>
+<span class="normal">373</span>
+<span class="normal">374</span>
+<span class="normal">375</span>
+<span class="normal">376</span>
+<span class="normal">377</span>
+<span class="normal">378</span>
+<span class="normal">379</span>
+<span class="normal">380</span>
+<span class="normal">381</span>
+<span class="normal">382</span>
+<span class="normal">383</span>
+<span class="normal">384</span>
+<span class="normal">385</span>
+<span class="normal">386</span>
+<span class="normal">387</span>
+<span class="normal">388</span>
+<span class="normal">389</span>
+<span class="normal">390</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">unique_variants_in_locus</span><span class="p">(</span><span class="bp">self</span><span class="p">:</span> <span class="n">StudyLocus</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">DataFrame</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;All unique variants collected in a `StudyLocus` dataframe.</span>
 
 <span class="sd">    Returns:</span>
@@ -5422,7 +5918,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.update_quality_flag" class="doc
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -5433,7 +5929,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.update_quality_flag" class="doc
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>qc</code></td>
           <td>
                 <code><span title="pyspark.sql.Column">Column</span></code>
@@ -5447,7 +5943,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.update_quality_flag" class="doc
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>flag_condition</code></td>
           <td>
                 <code><span title="pyspark.sql.Column">Column</span></code>
@@ -5461,7 +5957,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.update_quality_flag" class="doc
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>flag_text</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_locus.StudyLocusQualityCheck" href="#gentropy.dataset.study_locus.StudyLocusQualityCheck">StudyLocusQualityCheck</a></code>
@@ -5480,7 +5976,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.update_quality_flag" class="doc
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -5489,7 +5985,7 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.update_quality_flag" class="doc
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>Column</code></td>          <td>
                 <code><span title="pyspark.sql.Column">Column</span></code>
           </td>
@@ -5504,25 +6000,25 @@ <h3 id="gentropy.dataset.study_locus.StudyLocus.update_quality_flag" class="doc
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/dataset/study_locus.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">180</span>
-<span class="normal">181</span>
-<span class="normal">182</span>
-<span class="normal">183</span>
-<span class="normal">184</span>
-<span class="normal">185</span>
-<span class="normal">186</span>
-<span class="normal">187</span>
-<span class="normal">188</span>
-<span class="normal">189</span>
-<span class="normal">190</span>
-<span class="normal">191</span>
-<span class="normal">192</span>
-<span class="normal">193</span>
-<span class="normal">194</span>
-<span class="normal">195</span>
-<span class="normal">196</span>
-<span class="normal">197</span>
-<span class="normal">198</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@staticmethod</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">203</span>
+<span class="normal">204</span>
+<span class="normal">205</span>
+<span class="normal">206</span>
+<span class="normal">207</span>
+<span class="normal">208</span>
+<span class="normal">209</span>
+<span class="normal">210</span>
+<span class="normal">211</span>
+<span class="normal">212</span>
+<span class="normal">213</span>
+<span class="normal">214</span>
+<span class="normal">215</span>
+<span class="normal">216</span>
+<span class="normal">217</span>
+<span class="normal">218</span>
+<span class="normal">219</span>
+<span class="normal">220</span>
+<span class="normal">221</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@staticmethod</span>
 <span class="k">def</span> <span class="nf">update_quality_flag</span><span class="p">(</span>
     <span class="n">qc</span><span class="p">:</span> <span class="n">Column</span><span class="p">,</span> <span class="n">flag_condition</span><span class="p">:</span> <span class="n">Column</span><span class="p">,</span> <span class="n">flag_text</span><span class="p">:</span> <span class="n">StudyLocusQualityCheck</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Column</span><span class="p">:</span>
@@ -5577,7 +6073,7 @@ <h2 id="gentropy.dataset.study_locus.StudyLocusQualityCheck" class="doc doc-head
 
 
 
-  <p><strong>Attributes:</strong></p>
+  <p><span class="doc-section-title">Attributes:</span></p>
   <table>
     <thead>
       <tr>
@@ -5587,7 +6083,7 @@ <h2 id="gentropy.dataset.study_locus.StudyLocusQualityCheck" class="doc doc-head
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code><span title="gentropy.dataset.study_locus.StudyLocusQualityCheck.SUBSIGNIFICANT_FLAG">SUBSIGNIFICANT_FLAG</span></code></td>
           <td>
                 <code>str</code>
@@ -5598,7 +6094,7 @@ <h2 id="gentropy.dataset.study_locus.StudyLocusQualityCheck" class="doc doc-head
             </div>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code><span title="gentropy.dataset.study_locus.StudyLocusQualityCheck.NO_GENOMIC_LOCATION_FLAG">NO_GENOMIC_LOCATION_FLAG</span></code></td>
           <td>
                 <code>str</code>
@@ -5609,7 +6105,7 @@ <h2 id="gentropy.dataset.study_locus.StudyLocusQualityCheck" class="doc doc-head
             </div>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code><span title="gentropy.dataset.study_locus.StudyLocusQualityCheck.COMPOSITE_FLAG">COMPOSITE_FLAG</span></code></td>
           <td>
                 <code>str</code>
@@ -5620,7 +6116,7 @@ <h2 id="gentropy.dataset.study_locus.StudyLocusQualityCheck" class="doc doc-head
             </div>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code><span title="gentropy.dataset.study_locus.StudyLocusQualityCheck.INCONSISTENCY_FLAG">INCONSISTENCY_FLAG</span></code></td>
           <td>
                 <code>str</code>
@@ -5631,7 +6127,7 @@ <h2 id="gentropy.dataset.study_locus.StudyLocusQualityCheck" class="doc doc-head
             </div>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code><span title="gentropy.dataset.study_locus.StudyLocusQualityCheck.NON_MAPPED_VARIANT_FLAG">NON_MAPPED_VARIANT_FLAG</span></code></td>
           <td>
                 <code>str</code>
@@ -5642,7 +6138,7 @@ <h2 id="gentropy.dataset.study_locus.StudyLocusQualityCheck" class="doc doc-head
             </div>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code><span title="gentropy.dataset.study_locus.StudyLocusQualityCheck.PALINDROMIC_ALLELE_FLAG">PALINDROMIC_ALLELE_FLAG</span></code></td>
           <td>
                 <code>str</code>
@@ -5653,7 +6149,7 @@ <h2 id="gentropy.dataset.study_locus.StudyLocusQualityCheck" class="doc doc-head
             </div>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code><span title="gentropy.dataset.study_locus.StudyLocusQualityCheck.AMBIGUOUS_STUDY">AMBIGUOUS_STUDY</span></code></td>
           <td>
                 <code>str</code>
@@ -5664,7 +6160,7 @@ <h2 id="gentropy.dataset.study_locus.StudyLocusQualityCheck" class="doc doc-head
             </div>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code><span title="gentropy.dataset.study_locus.StudyLocusQualityCheck.UNRESOLVED_LD">UNRESOLVED_LD</span></code></td>
           <td>
                 <code>str</code>
@@ -5675,7 +6171,7 @@ <h2 id="gentropy.dataset.study_locus.StudyLocusQualityCheck" class="doc doc-head
             </div>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code><span title="gentropy.dataset.study_locus.StudyLocusQualityCheck.LD_CLUMPED">LD_CLUMPED</span></code></td>
           <td>
                 <code>str</code>
@@ -5686,7 +6182,7 @@ <h2 id="gentropy.dataset.study_locus.StudyLocusQualityCheck" class="doc doc-head
             </div>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code><span title="gentropy.dataset.study_locus.StudyLocusQualityCheck.NO_POPULATION">NO_POPULATION</span></code></td>
           <td>
                 <code>str</code>
@@ -5697,7 +6193,7 @@ <h2 id="gentropy.dataset.study_locus.StudyLocusQualityCheck" class="doc doc-head
             </div>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code><span title="gentropy.dataset.study_locus.StudyLocusQualityCheck.NOT_QUALIFYING_LD_BLOCK">NOT_QUALIFYING_LD_BLOCK</span></code></td>
           <td>
                 <code>str</code>
@@ -5713,9 +6209,7 @@ <h2 id="gentropy.dataset.study_locus.StudyLocusQualityCheck" class="doc doc-head
 
             <details class="quote">
               <summary>Source code in <code>src/gentropy/dataset/study_locus.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">29</span>
-<span class="normal">30</span>
-<span class="normal">31</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">31</span>
 <span class="normal">32</span>
 <span class="normal">33</span>
 <span class="normal">34</span>
@@ -5742,7 +6236,9 @@ <h2 id="gentropy.dataset.study_locus.StudyLocusQualityCheck" class="doc doc-head
 <span class="normal">55</span>
 <span class="normal">56</span>
 <span class="normal">57</span>
-<span class="normal">58</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">StudyLocusQualityCheck</span><span class="p">(</span><span class="n">Enum</span><span class="p">):</span>
+<span class="normal">58</span>
+<span class="normal">59</span>
+<span class="normal">60</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">StudyLocusQualityCheck</span><span class="p">(</span><span class="n">Enum</span><span class="p">):</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Study-Locus quality control options listing concerns on the quality of the association.</span>
 
 <span class="sd">    Attributes:</span>
@@ -5818,7 +6314,7 @@ <h2 id="gentropy.dataset.study_locus.CredibleInterval" class="doc doc-heading">
 
 
 
-  <p><strong>Attributes:</strong></p>
+  <p><span class="doc-section-title">Attributes:</span></p>
   <table>
     <thead>
       <tr>
@@ -5828,7 +6324,7 @@ <h2 id="gentropy.dataset.study_locus.CredibleInterval" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code><span title="gentropy.dataset.study_locus.CredibleInterval.IS95">IS95</span></code></td>
           <td>
                 <code>str</code>
@@ -5839,7 +6335,7 @@ <h2 id="gentropy.dataset.study_locus.CredibleInterval" class="doc doc-heading">
             </div>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code><span title="gentropy.dataset.study_locus.CredibleInterval.IS99">IS99</span></code></td>
           <td>
                 <code>str</code>
@@ -5855,9 +6351,7 @@ <h2 id="gentropy.dataset.study_locus.CredibleInterval" class="doc doc-heading">
 
             <details class="quote">
               <summary>Source code in <code>src/gentropy/dataset/study_locus.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">61</span>
-<span class="normal">62</span>
-<span class="normal">63</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">63</span>
 <span class="normal">64</span>
 <span class="normal">65</span>
 <span class="normal">66</span>
@@ -5866,7 +6360,9 @@ <h2 id="gentropy.dataset.study_locus.CredibleInterval" class="doc doc-heading">
 <span class="normal">69</span>
 <span class="normal">70</span>
 <span class="normal">71</span>
-<span class="normal">72</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">CredibleInterval</span><span class="p">(</span><span class="n">Enum</span><span class="p">):</span>
+<span class="normal">72</span>
+<span class="normal">73</span>
+<span class="normal">74</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">CredibleInterval</span><span class="p">(</span><span class="n">Enum</span><span class="p">):</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Credible interval enum.</span>
 
 <span class="sd">    Interval within which an unobserved parameter value falls with a particular probability.</span>
@@ -5909,6 +6405,7 @@ <h2 id="gentropy.dataset.study_locus.CredibleInterval" class="doc doc-heading">
  |-- region: string (nullable = true)
  |-- studyId: string (nullable = false)
  |-- beta: double (nullable = true)
+ |-- zScore: double (nullable = true)
  |-- pValueMantissa: float (nullable = true)
  |-- pValueExponent: integer (nullable = true)
  |-- effectAlleleFrequencyFromSource: float (nullable = true)
@@ -5919,6 +6416,8 @@ <h2 id="gentropy.dataset.study_locus.CredibleInterval" class="doc doc-heading">
  |-- finemappingMethod: string (nullable = true)
  |-- credibleSetIndex: integer (nullable = true)
  |-- credibleSetlog10BF: double (nullable = true)
+ |-- purityMeanR2: double (nullable = true)
+ |-- purityMinR2: double (nullable = true)
  |-- sampleSize: integer (nullable = true)
  |-- ldSet: array (nullable = true)
  |    |-- element: struct (containsNull = true)
diff --git a/python_api/datasets/study_locus_overlap/index.html b/python_api/datasets/study_locus_overlap/index.html
index 7a37396b6..b2337cdcb 100644
--- a/python_api/datasets/study_locus_overlap/index.html
+++ b/python_api/datasets/study_locus_overlap/index.html
@@ -2275,6 +2275,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../methods/susie_inf/" class="md-nav__link">
         
@@ -2306,7 +2348,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2322,14 +2364,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3104,7 +3146,7 @@ <h3 id="gentropy.dataset.study_locus_overlap.StudyLocusOverlap.from_associations
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3115,7 +3157,7 @@ <h3 id="gentropy.dataset.study_locus_overlap.StudyLocusOverlap.from_associations
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>study_locus</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_locus.StudyLocus" href="../study_locus/#gentropy.dataset.study_locus.StudyLocus">StudyLocus</a></code>
@@ -3129,7 +3171,7 @@ <h3 id="gentropy.dataset.study_locus_overlap.StudyLocusOverlap.from_associations
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>study_index</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_index.StudyIndex" href="../study_index/#gentropy.dataset.study_index.StudyIndex">StudyIndex</a></code>
@@ -3148,7 +3190,7 @@ <h3 id="gentropy.dataset.study_locus_overlap.StudyLocusOverlap.from_associations
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3157,7 +3199,7 @@ <h3 id="gentropy.dataset.study_locus_overlap.StudyLocusOverlap.from_associations
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StudyLocusOverlap</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_locus_overlap.StudyLocusOverlap" href="#gentropy.dataset.study_locus_overlap.StudyLocusOverlap">StudyLocusOverlap</a></code>
           </td>
@@ -3226,7 +3268,7 @@ <h3 id="gentropy.dataset.study_locus_overlap.StudyLocusOverlap.get_schema" class
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3235,7 +3277,7 @@ <h3 id="gentropy.dataset.study_locus_overlap.StudyLocusOverlap.get_schema" class
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StructType</code></td>          <td>
                 <code><span title="pyspark.sql.types.StructType">StructType</span></code>
           </td>
diff --git a/python_api/datasets/summary_statistics/index.html b/python_api/datasets/summary_statistics/index.html
index 736f0e26b..4e7602c83 100644
--- a/python_api/datasets/summary_statistics/index.html
+++ b/python_api/datasets/summary_statistics/index.html
@@ -1262,6 +1262,15 @@
     </span>
   </a>
   
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#gentropy.dataset.summary_statistics.SummaryStatistics.sanity_filter" class="md-nav__link">
+    <span class="md-ellipsis">
+      sanity_filter
+    </span>
+  </a>
+  
 </li>
         
           <li class="md-nav__item">
@@ -2293,6 +2302,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../methods/susie_inf/" class="md-nav__link">
         
@@ -2324,7 +2375,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2340,14 +2391,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -2929,6 +2980,15 @@
     </span>
   </a>
   
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#gentropy.dataset.summary_statistics.SummaryStatistics.sanity_filter" class="md-nav__link">
+    <span class="md-ellipsis">
+      sanity_filter
+    </span>
+  </a>
+  
 </li>
         
           <li class="md-nav__item">
@@ -2999,7 +3059,8 @@ <h2 id="gentropy.dataset.summary_statistics.SummaryStatistics" class="doc doc-he
 
             <details class="quote">
               <summary>Source code in <code>src/gentropy/dataset/summary_statistics.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 20</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 19</span>
+<span class="normal"> 20</span>
 <span class="normal"> 21</span>
 <span class="normal"> 22</span>
 <span class="normal"> 23</span>
@@ -3095,7 +3156,22 @@ <h2 id="gentropy.dataset.summary_statistics.SummaryStatistics" class="doc doc-he
 <span class="normal">113</span>
 <span class="normal">114</span>
 <span class="normal">115</span>
-<span class="normal">116</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@dataclass</span>
+<span class="normal">116</span>
+<span class="normal">117</span>
+<span class="normal">118</span>
+<span class="normal">119</span>
+<span class="normal">120</span>
+<span class="normal">121</span>
+<span class="normal">122</span>
+<span class="normal">123</span>
+<span class="normal">124</span>
+<span class="normal">125</span>
+<span class="normal">126</span>
+<span class="normal">127</span>
+<span class="normal">128</span>
+<span class="normal">129</span>
+<span class="normal">130</span>
+<span class="normal">131</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@dataclass</span>
 <span class="k">class</span> <span class="nc">SummaryStatistics</span><span class="p">(</span><span class="n">Dataset</span><span class="p">):</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Summary Statistics dataset.</span>
 
@@ -3137,34 +3213,24 @@ <h2 id="gentropy.dataset.summary_statistics.SummaryStatistics" class="doc doc-he
         <span class="bp">self</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">,</span>
         <span class="n">distance</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">500_000</span><span class="p">,</span>
         <span class="n">gwas_significance</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">5e-8</span><span class="p">,</span>
-        <span class="n">baseline_significance</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.05</span><span class="p">,</span>
-        <span class="n">locus_collect_distance</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
     <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyLocus</span><span class="p">:</span>
-<span class="w">        </span><span class="sd">&quot;&quot;&quot;Generate study-locus from summary statistics by distance based clumping + collect locus.</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Generate study-locus from summary statistics using window-based clumping.</span>
+
+<span class="sd">        For more info, see [`WindowBasedClumping`][gentropy.method.window_based_clumping.WindowBasedClumping]</span>
 
 <span class="sd">        Args:</span>
 <span class="sd">            distance (int): Distance in base pairs to be used for clumping. Defaults to 500_000.</span>
 <span class="sd">            gwas_significance (float, optional): GWAS significance threshold. Defaults to 5e-8.</span>
-<span class="sd">            baseline_significance (float, optional): Baseline significance threshold for inclusion in the locus. Defaults to 0.05.</span>
-<span class="sd">            locus_collect_distance (int | None): The distance to collect locus around semi-indices. If not provided, locus is not collected.</span>
 
 <span class="sd">        Returns:</span>
-<span class="sd">            StudyLocus: Clumped study-locus containing variants based on window.</span>
+<span class="sd">            StudyLocus: Clumped study-locus optionally containing variants based on window.</span>
 <span class="sd">        &quot;&quot;&quot;</span>
-        <span class="k">return</span> <span class="p">(</span>
-            <span class="n">WindowBasedClumping</span><span class="o">.</span><span class="n">clump_with_locus</span><span class="p">(</span>
-                <span class="bp">self</span><span class="p">,</span>
-                <span class="n">window_length</span><span class="o">=</span><span class="n">distance</span><span class="p">,</span>
-                <span class="n">p_value_significance</span><span class="o">=</span><span class="n">gwas_significance</span><span class="p">,</span>
-                <span class="n">p_value_baseline</span><span class="o">=</span><span class="n">baseline_significance</span><span class="p">,</span>
-                <span class="n">locus_window_length</span><span class="o">=</span><span class="n">locus_collect_distance</span><span class="p">,</span>
-            <span class="p">)</span>
-            <span class="k">if</span> <span class="n">locus_collect_distance</span>
-            <span class="k">else</span> <span class="n">WindowBasedClumping</span><span class="o">.</span><span class="n">clump</span><span class="p">(</span>
-                <span class="bp">self</span><span class="p">,</span>
-                <span class="n">window_length</span><span class="o">=</span><span class="n">distance</span><span class="p">,</span>
-                <span class="n">p_value_significance</span><span class="o">=</span><span class="n">gwas_significance</span><span class="p">,</span>
-            <span class="p">)</span>
+        <span class="kn">from</span> <span class="nn">gentropy.method.window_based_clumping</span> <span class="kn">import</span> <span class="n">WindowBasedClumping</span>
+
+        <span class="k">return</span> <span class="n">WindowBasedClumping</span><span class="o">.</span><span class="n">clump</span><span class="p">(</span>
+            <span class="bp">self</span><span class="p">,</span>
+            <span class="n">distance</span><span class="o">=</span><span class="n">distance</span><span class="p">,</span>
+            <span class="n">gwas_significance</span><span class="o">=</span><span class="n">gwas_significance</span><span class="p">,</span>
         <span class="p">)</span>
 
     <span class="k">def</span> <span class="nf">exclude_region</span><span class="p">(</span><span class="bp">self</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">,</span> <span class="n">region</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">SummaryStatistics</span><span class="p">:</span>
@@ -3192,6 +3258,32 @@ <h2 id="gentropy.dataset.summary_statistics.SummaryStatistics" class="doc doc-he
             <span class="p">),</span>
             <span class="n">_schema</span><span class="o">=</span><span class="n">SummaryStatistics</span><span class="o">.</span><span class="n">get_schema</span><span class="p">(),</span>
         <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">sanity_filter</span><span class="p">(</span><span class="bp">self</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">SummaryStatistics</span><span class="p">:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;The function filters the summary statistics by sanity filters.</span>
+
+<span class="sd">        The function filters the summary statistics by the following filters:</span>
+<span class="sd">            - The p-value should not be eqaul 1.</span>
+<span class="sd">            - The beta and se should not be equal 0.</span>
+<span class="sd">            - The p-value, beta and se should not be NaN.</span>
+
+<span class="sd">        Returns:</span>
+<span class="sd">            SummaryStatistics: The filtered summary statistics.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">gwas_df</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_df</span>
+        <span class="n">gwas_df</span> <span class="o">=</span> <span class="n">gwas_df</span><span class="o">.</span><span class="n">dropna</span><span class="p">(</span>
+            <span class="n">subset</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;beta&quot;</span><span class="p">,</span> <span class="s2">&quot;standardError&quot;</span><span class="p">,</span> <span class="s2">&quot;pValueMantissa&quot;</span><span class="p">,</span> <span class="s2">&quot;pValueExponent&quot;</span><span class="p">]</span>
+        <span class="p">)</span>
+
+        <span class="n">gwas_df</span> <span class="o">=</span> <span class="n">gwas_df</span><span class="o">.</span><span class="n">filter</span><span class="p">((</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;beta&quot;</span><span class="p">)</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">)</span> <span class="o">&amp;</span> <span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;standardError&quot;</span><span class="p">)</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">))</span>
+        <span class="n">gwas_df</span> <span class="o">=</span> <span class="n">gwas_df</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span>
+            <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;pValueMantissa&quot;</span><span class="p">)</span> <span class="o">*</span> <span class="mi">10</span> <span class="o">**</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;pValueExponent&quot;</span><span class="p">)</span> <span class="o">!=</span> <span class="mi">1</span>
+        <span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">SummaryStatistics</span><span class="p">(</span>
+            <span class="n">_df</span><span class="o">=</span><span class="n">gwas_df</span><span class="p">,</span>
+            <span class="n">_schema</span><span class="o">=</span><span class="n">SummaryStatistics</span><span class="o">.</span><span class="n">get_schema</span><span class="p">(),</span>
+        <span class="p">)</span>
 </code></pre></div></td></tr></table></div>
             </details>
 
@@ -3224,7 +3316,7 @@ <h3 id="gentropy.dataset.summary_statistics.SummaryStatistics.exclude_region" cl
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3235,7 +3327,7 @@ <h3 id="gentropy.dataset.summary_statistics.SummaryStatistics.exclude_region" cl
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>region</code></td>
           <td>
                 <code>str</code>
@@ -3254,7 +3346,7 @@ <h3 id="gentropy.dataset.summary_statistics.SummaryStatistics.exclude_region" cl
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3263,7 +3355,7 @@ <h3 id="gentropy.dataset.summary_statistics.SummaryStatistics.exclude_region" cl
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>SummaryStatistics</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.summary_statistics.SummaryStatistics" href="#gentropy.dataset.summary_statistics.SummaryStatistics">SummaryStatistics</a></code>
           </td>
@@ -3278,7 +3370,18 @@ <h3 id="gentropy.dataset.summary_statistics.SummaryStatistics.exclude_region" cl
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/dataset/summary_statistics.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 92</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 81</span>
+<span class="normal"> 82</span>
+<span class="normal"> 83</span>
+<span class="normal"> 84</span>
+<span class="normal"> 85</span>
+<span class="normal"> 86</span>
+<span class="normal"> 87</span>
+<span class="normal"> 88</span>
+<span class="normal"> 89</span>
+<span class="normal"> 90</span>
+<span class="normal"> 91</span>
+<span class="normal"> 92</span>
 <span class="normal"> 93</span>
 <span class="normal"> 94</span>
 <span class="normal"> 95</span>
@@ -3291,18 +3394,7 @@ <h3 id="gentropy.dataset.summary_statistics.SummaryStatistics.exclude_region" cl
 <span class="normal">102</span>
 <span class="normal">103</span>
 <span class="normal">104</span>
-<span class="normal">105</span>
-<span class="normal">106</span>
-<span class="normal">107</span>
-<span class="normal">108</span>
-<span class="normal">109</span>
-<span class="normal">110</span>
-<span class="normal">111</span>
-<span class="normal">112</span>
-<span class="normal">113</span>
-<span class="normal">114</span>
-<span class="normal">115</span>
-<span class="normal">116</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">exclude_region</span><span class="p">(</span><span class="bp">self</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">,</span> <span class="n">region</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">SummaryStatistics</span><span class="p">:</span>
+<span class="normal">105</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">exclude_region</span><span class="p">(</span><span class="bp">self</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">,</span> <span class="n">region</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">SummaryStatistics</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Exclude a region from the summary stats dataset.</span>
 
 <span class="sd">    Args:</span>
@@ -3354,7 +3446,7 @@ <h3 id="gentropy.dataset.summary_statistics.SummaryStatistics.get_schema" class=
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3363,7 +3455,7 @@ <h3 id="gentropy.dataset.summary_statistics.SummaryStatistics.get_schema" class=
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StructType</code></td>          <td>
                 <code><span title="pyspark.sql.types.StructType">StructType</span></code>
           </td>
@@ -3378,14 +3470,14 @@ <h3 id="gentropy.dataset.summary_statistics.SummaryStatistics.get_schema" class=
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/dataset/summary_statistics.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">27</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">26</span>
+<span class="normal">27</span>
 <span class="normal">28</span>
 <span class="normal">29</span>
 <span class="normal">30</span>
 <span class="normal">31</span>
 <span class="normal">32</span>
-<span class="normal">33</span>
-<span class="normal">34</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@classmethod</span>
+<span class="normal">33</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@classmethod</span>
 <span class="k">def</span> <span class="nf">get_schema</span><span class="p">(</span><span class="bp">cls</span><span class="p">:</span> <span class="nb">type</span><span class="p">[</span><span class="n">SummaryStatistics</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">StructType</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Provides the schema for the SummaryStatistics dataset.</span>
 
@@ -3416,7 +3508,7 @@ <h3 id="gentropy.dataset.summary_statistics.SummaryStatistics.pvalue_filter" cla
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3427,7 +3519,7 @@ <h3 id="gentropy.dataset.summary_statistics.SummaryStatistics.pvalue_filter" cla
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>pvalue</code></td>
           <td>
                 <code>float</code>
@@ -3446,7 +3538,7 @@ <h3 id="gentropy.dataset.summary_statistics.SummaryStatistics.pvalue_filter" cla
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3455,7 +3547,7 @@ <h3 id="gentropy.dataset.summary_statistics.SummaryStatistics.pvalue_filter" cla
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>SummaryStatistics</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.summary_statistics.SummaryStatistics" href="#gentropy.dataset.summary_statistics.SummaryStatistics">SummaryStatistics</a></code>
           </td>
@@ -3470,7 +3562,8 @@ <h3 id="gentropy.dataset.summary_statistics.SummaryStatistics.pvalue_filter" cla
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/dataset/summary_statistics.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">36</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">35</span>
+<span class="normal">36</span>
 <span class="normal">37</span>
 <span class="normal">38</span>
 <span class="normal">39</span>
@@ -3489,8 +3582,7 @@ <h3 id="gentropy.dataset.summary_statistics.SummaryStatistics.pvalue_filter" cla
 <span class="normal">52</span>
 <span class="normal">53</span>
 <span class="normal">54</span>
-<span class="normal">55</span>
-<span class="normal">56</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">pvalue_filter</span><span class="p">(</span><span class="bp">self</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">,</span> <span class="n">pvalue</span><span class="p">:</span> <span class="nb">float</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">SummaryStatistics</span><span class="p">:</span>
+<span class="normal">55</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">pvalue_filter</span><span class="p">(</span><span class="bp">self</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">,</span> <span class="n">pvalue</span><span class="p">:</span> <span class="nb">float</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">SummaryStatistics</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Filter summary statistics based on the provided p-value threshold.</span>
 
 <span class="sd">    Args:</span>
@@ -3522,19 +3614,124 @@ <h3 id="gentropy.dataset.summary_statistics.SummaryStatistics.pvalue_filter" cla
 
 
 
+<h3 id="gentropy.dataset.summary_statistics.SummaryStatistics.sanity_filter" class="doc doc-heading">
+          <code class="highlight language-python"><span class="n">sanity_filter</span><span class="p">()</span> <span class="o">-&gt;</span> <span class="n">SummaryStatistics</span></code>
+
+<a href="#gentropy.dataset.summary_statistics.SummaryStatistics.sanity_filter" class="headerlink" title="Permanent link">&para;</a></h3>
+
+
+  <div class="doc doc-contents ">
+  
+      <p>The function filters the summary statistics by sanity filters.</p>
+
+<details class="the-function-filters-the-summary-statistics-by-the-following-filters" open>
+  <summary>The function filters the summary statistics by the following filters</summary>
+  <ul>
+<li>The p-value should not be eqaul 1.</li>
+<li>The beta and se should not be equal 0.</li>
+<li>The p-value, beta and se should not be NaN.</li>
+</ul>
+</details>
+
+
+  <p><span class="doc-section-title">Returns:</span></p>
+  <table>
+    <thead>
+      <tr>
+<th>Name</th>        <th>Type</th>
+        <th>Description</th>
+      </tr>
+    </thead>
+    <tbody>
+        <tr class="doc-section-item">
+<td><code>SummaryStatistics</code></td>          <td>
+                <code><a class="autorefs autorefs-internal" title="gentropy.dataset.summary_statistics.SummaryStatistics" href="#gentropy.dataset.summary_statistics.SummaryStatistics">SummaryStatistics</a></code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>The filtered summary statistics.</p>
+            </div>
+          </td>
+        </tr>
+    </tbody>
+  </table>
+
+          <details class="quote">
+            <summary>Source code in <code>src/gentropy/dataset/summary_statistics.py</code></summary>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">107</span>
+<span class="normal">108</span>
+<span class="normal">109</span>
+<span class="normal">110</span>
+<span class="normal">111</span>
+<span class="normal">112</span>
+<span class="normal">113</span>
+<span class="normal">114</span>
+<span class="normal">115</span>
+<span class="normal">116</span>
+<span class="normal">117</span>
+<span class="normal">118</span>
+<span class="normal">119</span>
+<span class="normal">120</span>
+<span class="normal">121</span>
+<span class="normal">122</span>
+<span class="normal">123</span>
+<span class="normal">124</span>
+<span class="normal">125</span>
+<span class="normal">126</span>
+<span class="normal">127</span>
+<span class="normal">128</span>
+<span class="normal">129</span>
+<span class="normal">130</span>
+<span class="normal">131</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">sanity_filter</span><span class="p">(</span><span class="bp">self</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">SummaryStatistics</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;The function filters the summary statistics by sanity filters.</span>
+
+<span class="sd">    The function filters the summary statistics by the following filters:</span>
+<span class="sd">        - The p-value should not be eqaul 1.</span>
+<span class="sd">        - The beta and se should not be equal 0.</span>
+<span class="sd">        - The p-value, beta and se should not be NaN.</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">        SummaryStatistics: The filtered summary statistics.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">gwas_df</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_df</span>
+    <span class="n">gwas_df</span> <span class="o">=</span> <span class="n">gwas_df</span><span class="o">.</span><span class="n">dropna</span><span class="p">(</span>
+        <span class="n">subset</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;beta&quot;</span><span class="p">,</span> <span class="s2">&quot;standardError&quot;</span><span class="p">,</span> <span class="s2">&quot;pValueMantissa&quot;</span><span class="p">,</span> <span class="s2">&quot;pValueExponent&quot;</span><span class="p">]</span>
+    <span class="p">)</span>
+
+    <span class="n">gwas_df</span> <span class="o">=</span> <span class="n">gwas_df</span><span class="o">.</span><span class="n">filter</span><span class="p">((</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;beta&quot;</span><span class="p">)</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">)</span> <span class="o">&amp;</span> <span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;standardError&quot;</span><span class="p">)</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">))</span>
+    <span class="n">gwas_df</span> <span class="o">=</span> <span class="n">gwas_df</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span>
+        <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;pValueMantissa&quot;</span><span class="p">)</span> <span class="o">*</span> <span class="mi">10</span> <span class="o">**</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;pValueExponent&quot;</span><span class="p">)</span> <span class="o">!=</span> <span class="mi">1</span>
+    <span class="p">)</span>
+
+    <span class="k">return</span> <span class="n">SummaryStatistics</span><span class="p">(</span>
+        <span class="n">_df</span><span class="o">=</span><span class="n">gwas_df</span><span class="p">,</span>
+        <span class="n">_schema</span><span class="o">=</span><span class="n">SummaryStatistics</span><span class="o">.</span><span class="n">get_schema</span><span class="p">(),</span>
+    <span class="p">)</span>
+</code></pre></div></td></tr></table></div>
+          </details>
+  </div>
+
+</div>
+
+
+<div class="doc doc-object doc-function">
+
+
+
 <h3 id="gentropy.dataset.summary_statistics.SummaryStatistics.window_based_clumping" class="doc doc-heading">
-          <code class="highlight language-python"><span class="n">window_based_clumping</span><span class="p">(</span><span class="n">distance</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">500000</span><span class="p">,</span> <span class="n">gwas_significance</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">5e-08</span><span class="p">,</span> <span class="n">baseline_significance</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.05</span><span class="p">,</span> <span class="n">locus_collect_distance</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyLocus</span></code>
+          <code class="highlight language-python"><span class="n">window_based_clumping</span><span class="p">(</span><span class="n">distance</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">500000</span><span class="p">,</span> <span class="n">gwas_significance</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">5e-08</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyLocus</span></code>
 
 <a href="#gentropy.dataset.summary_statistics.SummaryStatistics.window_based_clumping" class="headerlink" title="Permanent link">&para;</a></h3>
 
 
   <div class="doc doc-contents ">
   
-      <p>Generate study-locus from summary statistics by distance based clumping + collect locus.</p>
+      <p>Generate study-locus from summary statistics using window-based clumping.</p>
+<p>For more info, see <a class="autorefs autorefs-internal" href="../../methods/clumping/#gentropy.method.window_based_clumping.WindowBasedClumping"><code>WindowBasedClumping</code></a></p>
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3545,7 +3742,7 @@ <h3 id="gentropy.dataset.summary_statistics.SummaryStatistics.window_based_clump
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>distance</code></td>
           <td>
                 <code>int</code>
@@ -3559,7 +3756,7 @@ <h3 id="gentropy.dataset.summary_statistics.SummaryStatistics.window_based_clump
                 <code>500000</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>gwas_significance</code></td>
           <td>
                 <code>float</code>
@@ -3573,40 +3770,12 @@ <h3 id="gentropy.dataset.summary_statistics.SummaryStatistics.window_based_clump
                 <code>5e-08</code>
           </td>
         </tr>
-        <tr>
-          <td><code>baseline_significance</code></td>
-          <td>
-                <code>float</code>
-          </td>
-          <td>
-            <div class="doc-md-description">
-              <p>Baseline significance threshold for inclusion in the locus. Defaults to 0.05.</p>
-            </div>
-          </td>
-          <td>
-                <code>0.05</code>
-          </td>
-        </tr>
-        <tr>
-          <td><code>locus_collect_distance</code></td>
-          <td>
-                <code>int | None</code>
-          </td>
-          <td>
-            <div class="doc-md-description">
-              <p>The distance to collect locus around semi-indices. If not provided, locus is not collected.</p>
-            </div>
-          </td>
-          <td>
-                <code>None</code>
-          </td>
-        </tr>
     </tbody>
   </table>
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3615,13 +3784,13 @@ <h3 id="gentropy.dataset.summary_statistics.SummaryStatistics.window_based_clump
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StudyLocus</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_locus.StudyLocus" href="../study_locus/#gentropy.dataset.study_locus.StudyLocus">StudyLocus</a></code>
           </td>
           <td>
             <div class="doc-md-description">
-              <p>Clumped study-locus containing variants based on window.</p>
+              <p>Clumped study-locus optionally containing variants based on window.</p>
             </div>
           </td>
         </tr>
@@ -3630,7 +3799,8 @@ <h3 id="gentropy.dataset.summary_statistics.SummaryStatistics.window_based_clump
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/dataset/summary_statistics.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">58</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">57</span>
+<span class="normal">58</span>
 <span class="normal">59</span>
 <span class="normal">60</span>
 <span class="normal">61</span>
@@ -3651,49 +3821,28 @@ <h3 id="gentropy.dataset.summary_statistics.SummaryStatistics.window_based_clump
 <span class="normal">76</span>
 <span class="normal">77</span>
 <span class="normal">78</span>
-<span class="normal">79</span>
-<span class="normal">80</span>
-<span class="normal">81</span>
-<span class="normal">82</span>
-<span class="normal">83</span>
-<span class="normal">84</span>
-<span class="normal">85</span>
-<span class="normal">86</span>
-<span class="normal">87</span>
-<span class="normal">88</span>
-<span class="normal">89</span>
-<span class="normal">90</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">window_based_clumping</span><span class="p">(</span>
+<span class="normal">79</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">window_based_clumping</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">,</span>
     <span class="n">distance</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">500_000</span><span class="p">,</span>
     <span class="n">gwas_significance</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">5e-8</span><span class="p">,</span>
-    <span class="n">baseline_significance</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.05</span><span class="p">,</span>
-    <span class="n">locus_collect_distance</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyLocus</span><span class="p">:</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;Generate study-locus from summary statistics by distance based clumping + collect locus.</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;Generate study-locus from summary statistics using window-based clumping.</span>
+
+<span class="sd">    For more info, see [`WindowBasedClumping`][gentropy.method.window_based_clumping.WindowBasedClumping]</span>
 
 <span class="sd">    Args:</span>
 <span class="sd">        distance (int): Distance in base pairs to be used for clumping. Defaults to 500_000.</span>
 <span class="sd">        gwas_significance (float, optional): GWAS significance threshold. Defaults to 5e-8.</span>
-<span class="sd">        baseline_significance (float, optional): Baseline significance threshold for inclusion in the locus. Defaults to 0.05.</span>
-<span class="sd">        locus_collect_distance (int | None): The distance to collect locus around semi-indices. If not provided, locus is not collected.</span>
 
 <span class="sd">    Returns:</span>
-<span class="sd">        StudyLocus: Clumped study-locus containing variants based on window.</span>
+<span class="sd">        StudyLocus: Clumped study-locus optionally containing variants based on window.</span>
 <span class="sd">    &quot;&quot;&quot;</span>
-    <span class="k">return</span> <span class="p">(</span>
-        <span class="n">WindowBasedClumping</span><span class="o">.</span><span class="n">clump_with_locus</span><span class="p">(</span>
-            <span class="bp">self</span><span class="p">,</span>
-            <span class="n">window_length</span><span class="o">=</span><span class="n">distance</span><span class="p">,</span>
-            <span class="n">p_value_significance</span><span class="o">=</span><span class="n">gwas_significance</span><span class="p">,</span>
-            <span class="n">p_value_baseline</span><span class="o">=</span><span class="n">baseline_significance</span><span class="p">,</span>
-            <span class="n">locus_window_length</span><span class="o">=</span><span class="n">locus_collect_distance</span><span class="p">,</span>
-        <span class="p">)</span>
-        <span class="k">if</span> <span class="n">locus_collect_distance</span>
-        <span class="k">else</span> <span class="n">WindowBasedClumping</span><span class="o">.</span><span class="n">clump</span><span class="p">(</span>
-            <span class="bp">self</span><span class="p">,</span>
-            <span class="n">window_length</span><span class="o">=</span><span class="n">distance</span><span class="p">,</span>
-            <span class="n">p_value_significance</span><span class="o">=</span><span class="n">gwas_significance</span><span class="p">,</span>
-        <span class="p">)</span>
+    <span class="kn">from</span> <span class="nn">gentropy.method.window_based_clumping</span> <span class="kn">import</span> <span class="n">WindowBasedClumping</span>
+
+    <span class="k">return</span> <span class="n">WindowBasedClumping</span><span class="o">.</span><span class="n">clump</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">distance</span><span class="o">=</span><span class="n">distance</span><span class="p">,</span>
+        <span class="n">gwas_significance</span><span class="o">=</span><span class="n">gwas_significance</span><span class="p">,</span>
     <span class="p">)</span>
 </code></pre></div></td></tr></table></div>
           </details>
diff --git a/python_api/datasets/variant_annotation/index.html b/python_api/datasets/variant_annotation/index.html
index d90edc880..7ee7e3a8e 100644
--- a/python_api/datasets/variant_annotation/index.html
+++ b/python_api/datasets/variant_annotation/index.html
@@ -2320,6 +2320,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../methods/susie_inf/" class="md-nav__link">
         
@@ -2351,7 +2393,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2367,14 +2409,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3491,7 +3533,7 @@ <h3 id="gentropy.dataset.variant_annotation.VariantAnnotation.filter_by_variant_
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3502,7 +3544,7 @@ <h3 id="gentropy.dataset.variant_annotation.VariantAnnotation.filter_by_variant_
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>df</code></td>
           <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
@@ -3521,7 +3563,7 @@ <h3 id="gentropy.dataset.variant_annotation.VariantAnnotation.filter_by_variant_
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3530,7 +3572,7 @@ <h3 id="gentropy.dataset.variant_annotation.VariantAnnotation.filter_by_variant_
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>VariantAnnotation</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.variant_annotation.VariantAnnotation" href="#gentropy.dataset.variant_annotation.VariantAnnotation">VariantAnnotation</a></code>
           </td>
@@ -3601,7 +3643,7 @@ <h3 id="gentropy.dataset.variant_annotation.VariantAnnotation.get_distance_to_ts
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3612,7 +3654,7 @@ <h3 id="gentropy.dataset.variant_annotation.VariantAnnotation.get_distance_to_ts
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>gene_index</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.gene_index.GeneIndex" href="../gene_index/#gentropy.dataset.gene_index.GeneIndex">GeneIndex</a></code>
@@ -3626,7 +3668,7 @@ <h3 id="gentropy.dataset.variant_annotation.VariantAnnotation.get_distance_to_ts
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>max_distance</code></td>
           <td>
                 <code>int</code>
@@ -3645,7 +3687,7 @@ <h3 id="gentropy.dataset.variant_annotation.VariantAnnotation.get_distance_to_ts
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3654,7 +3696,7 @@ <h3 id="gentropy.dataset.variant_annotation.VariantAnnotation.get_distance_to_ts
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>V2G</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.v2g.V2G" href="../variant_to_gene/#gentropy.dataset.v2g.V2G">V2G</a></code>
           </td>
@@ -3784,7 +3826,7 @@ <h3 id="gentropy.dataset.variant_annotation.VariantAnnotation.get_most_severe_ve
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3795,7 +3837,7 @@ <h3 id="gentropy.dataset.variant_annotation.VariantAnnotation.get_most_severe_ve
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>vep_consequences</code></td>
           <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
@@ -3809,7 +3851,7 @@ <h3 id="gentropy.dataset.variant_annotation.VariantAnnotation.get_most_severe_ve
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>gene_index</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.gene_index.GeneIndex" href="../gene_index/#gentropy.dataset.gene_index.GeneIndex">GeneIndex</a></code>
@@ -3828,7 +3870,7 @@ <h3 id="gentropy.dataset.variant_annotation.VariantAnnotation.get_most_severe_ve
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3837,7 +3879,7 @@ <h3 id="gentropy.dataset.variant_annotation.VariantAnnotation.get_most_severe_ve
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>V2G</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.v2g.V2G" href="../variant_to_gene/#gentropy.dataset.v2g.V2G">V2G</a></code>
           </td>
@@ -3957,7 +3999,7 @@ <h3 id="gentropy.dataset.variant_annotation.VariantAnnotation.get_plof_v2g" clas
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3968,7 +4010,7 @@ <h3 id="gentropy.dataset.variant_annotation.VariantAnnotation.get_plof_v2g" clas
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>gene_index</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.gene_index.GeneIndex" href="../gene_index/#gentropy.dataset.gene_index.GeneIndex">GeneIndex</a></code>
@@ -3987,7 +4029,7 @@ <h3 id="gentropy.dataset.variant_annotation.VariantAnnotation.get_plof_v2g" clas
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3996,7 +4038,7 @@ <h3 id="gentropy.dataset.variant_annotation.VariantAnnotation.get_plof_v2g" clas
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>V2G</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.v2g.V2G" href="../variant_to_gene/#gentropy.dataset.v2g.V2G">V2G</a></code>
           </td>
@@ -4115,7 +4157,7 @@ <h3 id="gentropy.dataset.variant_annotation.VariantAnnotation.get_schema" class=
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -4124,7 +4166,7 @@ <h3 id="gentropy.dataset.variant_annotation.VariantAnnotation.get_schema" class=
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StructType</code></td>          <td>
                 <code><span title="pyspark.sql.types.StructType">StructType</span></code>
           </td>
@@ -4178,7 +4220,7 @@ <h3 id="gentropy.dataset.variant_annotation.VariantAnnotation.get_transcript_con
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -4189,7 +4231,7 @@ <h3 id="gentropy.dataset.variant_annotation.VariantAnnotation.get_transcript_con
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>gene_index</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.gene_index.GeneIndex" href="../gene_index/#gentropy.dataset.gene_index.GeneIndex">GeneIndex</a> | None</code>
@@ -4208,7 +4250,7 @@ <h3 id="gentropy.dataset.variant_annotation.VariantAnnotation.get_transcript_con
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -4217,7 +4259,7 @@ <h3 id="gentropy.dataset.variant_annotation.VariantAnnotation.get_transcript_con
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>DataFrame</code></td>          <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
           </td>
@@ -4312,7 +4354,7 @@ <h3 id="gentropy.dataset.variant_annotation.VariantAnnotation.max_maf" class="do
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -4321,7 +4363,7 @@ <h3 id="gentropy.dataset.variant_annotation.VariantAnnotation.max_maf" class="do
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>Column</code></td>          <td>
                 <code><span title="pyspark.sql.Column">Column</span></code>
           </td>
@@ -4381,7 +4423,6 @@ <h3 id="gentropy.dataset.variant_annotation.VariantAnnotation.max_maf" class="do
  |-- variantId: string (nullable = false)
  |-- chromosome: string (nullable = false)
  |-- position: integer (nullable = false)
- |-- gnomadVariantId: string (nullable = false)
  |-- referenceAllele: string (nullable = false)
  |-- alternateAllele: string (nullable = false)
  |-- chromosomeB37: string (nullable = true)
diff --git a/python_api/datasets/variant_index/index.html b/python_api/datasets/variant_index/index.html
index 413a16ea9..f5623412d 100644
--- a/python_api/datasets/variant_index/index.html
+++ b/python_api/datasets/variant_index/index.html
@@ -2275,6 +2275,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../methods/susie_inf/" class="md-nav__link">
         
@@ -2306,7 +2348,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2322,14 +2364,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3116,7 +3158,7 @@ <h3 id="gentropy.dataset.variant_index.VariantIndex.from_variant_annotation" cla
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3127,7 +3169,7 @@ <h3 id="gentropy.dataset.variant_index.VariantIndex.from_variant_annotation" cla
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>variant_annotation</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.variant_annotation.VariantAnnotation" href="../variant_annotation/#gentropy.dataset.variant_annotation.VariantAnnotation">VariantAnnotation</a></code>
@@ -3141,7 +3183,7 @@ <h3 id="gentropy.dataset.variant_index.VariantIndex.from_variant_annotation" cla
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>study_locus</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_locus.StudyLocus" href="../study_locus/#gentropy.dataset.study_locus.StudyLocus">StudyLocus</a></code>
@@ -3160,7 +3202,7 @@ <h3 id="gentropy.dataset.variant_index.VariantIndex.from_variant_annotation" cla
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3169,7 +3211,7 @@ <h3 id="gentropy.dataset.variant_index.VariantIndex.from_variant_annotation" cla
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>VariantIndex</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.variant_index.VariantIndex" href="#gentropy.dataset.variant_index.VariantIndex">VariantIndex</a></code>
           </td>
@@ -3296,7 +3338,7 @@ <h3 id="gentropy.dataset.variant_index.VariantIndex.get_schema" class="doc doc-h
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3305,7 +3347,7 @@ <h3 id="gentropy.dataset.variant_index.VariantIndex.get_schema" class="doc doc-h
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StructType</code></td>          <td>
                 <code><span title="pyspark.sql.types.StructType">StructType</span></code>
           </td>
diff --git a/python_api/datasets/variant_to_gene/index.html b/python_api/datasets/variant_to_gene/index.html
index d62e36ee5..6451714b0 100644
--- a/python_api/datasets/variant_to_gene/index.html
+++ b/python_api/datasets/variant_to_gene/index.html
@@ -2284,6 +2284,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../methods/susie_inf/" class="md-nav__link">
         
@@ -2315,7 +2357,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2331,14 +2373,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3114,7 +3156,7 @@ <h3 id="gentropy.dataset.v2g.V2G.filter_by_genes" class="doc doc-heading">
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3125,7 +3167,7 @@ <h3 id="gentropy.dataset.v2g.V2G.filter_by_genes" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>genes</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.gene_index.GeneIndex" href="../gene_index/#gentropy.dataset.gene_index.GeneIndex">GeneIndex</a></code>
@@ -3144,7 +3186,7 @@ <h3 id="gentropy.dataset.v2g.V2G.filter_by_genes" class="doc doc-heading">
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3153,7 +3195,7 @@ <h3 id="gentropy.dataset.v2g.V2G.filter_by_genes" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>V2G</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.v2g.V2G" href="#gentropy.dataset.v2g.V2G">V2G</a></code>
           </td>
@@ -3216,7 +3258,7 @@ <h3 id="gentropy.dataset.v2g.V2G.get_schema" class="doc doc-heading">
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3225,7 +3267,7 @@ <h3 id="gentropy.dataset.v2g.V2G.get_schema" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StructType</code></td>          <td>
                 <code><span title="pyspark.sql.types.StructType">StructType</span></code>
           </td>
diff --git a/python_api/datasources/_datasources/index.html b/python_api/datasources/_datasources/index.html
index 2b0177e35..69d614df3 100644
--- a/python_api/datasources/_datasources/index.html
+++ b/python_api/datasources/_datasources/index.html
@@ -2197,6 +2197,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../methods/susie_inf/" class="md-nav__link">
         
@@ -2228,7 +2270,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2244,14 +2286,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
diff --git a/python_api/datasources/eqtl_catalogue/_eqtl_catalogue/index.html b/python_api/datasources/eqtl_catalogue/_eqtl_catalogue/index.html
index 623d6e2d9..0475bad53 100644
--- a/python_api/datasources/eqtl_catalogue/_eqtl_catalogue/index.html
+++ b/python_api/datasources/eqtl_catalogue/_eqtl_catalogue/index.html
@@ -2194,6 +2194,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../methods/susie_inf/" class="md-nav__link">
         
@@ -2225,7 +2267,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2241,14 +2283,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
diff --git a/python_api/datasources/eqtl_catalogue/finemapping/index.html b/python_api/datasources/eqtl_catalogue/finemapping/index.html
index 09b669b38..777af2b9d 100644
--- a/python_api/datasources/eqtl_catalogue/finemapping/index.html
+++ b/python_api/datasources/eqtl_catalogue/finemapping/index.html
@@ -2286,6 +2286,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../methods/susie_inf/" class="md-nav__link">
         
@@ -2317,7 +2359,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2333,14 +2375,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3591,7 +3633,7 @@ <h3 id="gentropy.datasource.eqtl_catalogue.finemapping.EqtlCatalogueFinemapping.
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3602,7 +3644,7 @@ <h3 id="gentropy.datasource.eqtl_catalogue.finemapping.EqtlCatalogueFinemapping.
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>processed_finemapping_df</code></td>
           <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
@@ -3621,7 +3663,7 @@ <h3 id="gentropy.datasource.eqtl_catalogue.finemapping.EqtlCatalogueFinemapping.
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3630,7 +3672,7 @@ <h3 id="gentropy.datasource.eqtl_catalogue.finemapping.EqtlCatalogueFinemapping.
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StudyLocus</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_locus.StudyLocus" href="../../../datasets/study_locus/#gentropy.dataset.study_locus.StudyLocus">StudyLocus</a></code>
           </td>
@@ -3789,7 +3831,7 @@ <h3 id="gentropy.datasource.eqtl_catalogue.finemapping.EqtlCatalogueFinemapping.
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3800,7 +3842,7 @@ <h3 id="gentropy.datasource.eqtl_catalogue.finemapping.EqtlCatalogueFinemapping.
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>credible_sets</code></td>
           <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
@@ -3814,7 +3856,7 @@ <h3 id="gentropy.datasource.eqtl_catalogue.finemapping.EqtlCatalogueFinemapping.
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>lbf</code></td>
           <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
@@ -3828,7 +3870,7 @@ <h3 id="gentropy.datasource.eqtl_catalogue.finemapping.EqtlCatalogueFinemapping.
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>studies_metadata</code></td>
           <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
@@ -3847,7 +3889,7 @@ <h3 id="gentropy.datasource.eqtl_catalogue.finemapping.EqtlCatalogueFinemapping.
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3856,7 +3898,7 @@ <h3 id="gentropy.datasource.eqtl_catalogue.finemapping.EqtlCatalogueFinemapping.
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>DataFrame</code></td>          <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
           </td>
@@ -4075,7 +4117,7 @@ <h3 id="gentropy.datasource.eqtl_catalogue.finemapping.EqtlCatalogueFinemapping.
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -4086,7 +4128,7 @@ <h3 id="gentropy.datasource.eqtl_catalogue.finemapping.EqtlCatalogueFinemapping.
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>session</code></td>
           <td>
                 <code><span title="gentropy.common.session.Session">Session</span></code>
@@ -4100,7 +4142,7 @@ <h3 id="gentropy.datasource.eqtl_catalogue.finemapping.EqtlCatalogueFinemapping.
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>credible_set_path</code></td>
           <td>
                 <code>str | list[str]</code>
@@ -4119,7 +4161,7 @@ <h3 id="gentropy.datasource.eqtl_catalogue.finemapping.EqtlCatalogueFinemapping.
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -4128,7 +4170,7 @@ <h3 id="gentropy.datasource.eqtl_catalogue.finemapping.EqtlCatalogueFinemapping.
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>DataFrame</code></td>          <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
           </td>
@@ -4211,7 +4253,7 @@ <h3 id="gentropy.datasource.eqtl_catalogue.finemapping.EqtlCatalogueFinemapping.
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -4222,7 +4264,7 @@ <h3 id="gentropy.datasource.eqtl_catalogue.finemapping.EqtlCatalogueFinemapping.
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>session</code></td>
           <td>
                 <code><span title="gentropy.common.session.Session">Session</span></code>
@@ -4236,7 +4278,7 @@ <h3 id="gentropy.datasource.eqtl_catalogue.finemapping.EqtlCatalogueFinemapping.
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>lbf_path</code></td>
           <td>
                 <code>str | list[str]</code>
@@ -4255,7 +4297,7 @@ <h3 id="gentropy.datasource.eqtl_catalogue.finemapping.EqtlCatalogueFinemapping.
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -4264,7 +4306,7 @@ <h3 id="gentropy.datasource.eqtl_catalogue.finemapping.EqtlCatalogueFinemapping.
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>DataFrame</code></td>          <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
           </td>
diff --git a/python_api/datasources/eqtl_catalogue/study_index/index.html b/python_api/datasources/eqtl_catalogue/study_index/index.html
index dde42559b..af5527c97 100644
--- a/python_api/datasources/eqtl_catalogue/study_index/index.html
+++ b/python_api/datasources/eqtl_catalogue/study_index/index.html
@@ -2277,6 +2277,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../methods/susie_inf/" class="md-nav__link">
         
@@ -2308,7 +2350,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2324,14 +2366,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3255,7 +3297,7 @@ <h3 id="gentropy.datasource.eqtl_catalogue.study_index.EqtlCatalogueStudyIndex.f
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3266,7 +3308,7 @@ <h3 id="gentropy.datasource.eqtl_catalogue.study_index.EqtlCatalogueStudyIndex.f
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>processed_finemapping_df</code></td>
           <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
@@ -3285,7 +3327,7 @@ <h3 id="gentropy.datasource.eqtl_catalogue.study_index.EqtlCatalogueStudyIndex.f
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3294,7 +3336,7 @@ <h3 id="gentropy.datasource.eqtl_catalogue.study_index.EqtlCatalogueStudyIndex.f
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StudyIndex</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_index.StudyIndex" href="../../../datasets/study_index/#gentropy.dataset.study_index.StudyIndex">StudyIndex</a></code>
           </td>
@@ -3379,7 +3421,7 @@ <h3 id="gentropy.datasource.eqtl_catalogue.study_index.EqtlCatalogueStudyIndex.g
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3390,7 +3432,7 @@ <h3 id="gentropy.datasource.eqtl_catalogue.study_index.EqtlCatalogueStudyIndex.g
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>studies_metadata</code></td>
           <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
@@ -3409,7 +3451,7 @@ <h3 id="gentropy.datasource.eqtl_catalogue.study_index.EqtlCatalogueStudyIndex.g
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3418,7 +3460,7 @@ <h3 id="gentropy.datasource.eqtl_catalogue.study_index.EqtlCatalogueStudyIndex.g
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td>
                 <code>list[str]</code>
           </td>
@@ -3495,7 +3537,7 @@ <h3 id="gentropy.datasource.eqtl_catalogue.study_index.EqtlCatalogueStudyIndex.r
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3506,7 +3548,7 @@ <h3 id="gentropy.datasource.eqtl_catalogue.study_index.EqtlCatalogueStudyIndex.r
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>session</code></td>
           <td>
                 <code><span title="gentropy.common.session.Session">Session</span></code>
@@ -3520,7 +3562,7 @@ <h3 id="gentropy.datasource.eqtl_catalogue.study_index.EqtlCatalogueStudyIndex.r
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>mqtl_quantification_methods_blacklist</code></td>
           <td>
                 <code>list[str]</code>
@@ -3539,7 +3581,7 @@ <h3 id="gentropy.datasource.eqtl_catalogue.study_index.EqtlCatalogueStudyIndex.r
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3548,7 +3590,7 @@ <h3 id="gentropy.datasource.eqtl_catalogue.study_index.EqtlCatalogueStudyIndex.r
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>DataFrame</code></td>          <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
           </td>
diff --git a/python_api/datasources/eqtl_catalogue/summary_stats/index.html b/python_api/datasources/eqtl_catalogue/summary_stats/index.html
index 9f37ca81e..b2924daae 100644
--- a/python_api/datasources/eqtl_catalogue/summary_stats/index.html
+++ b/python_api/datasources/eqtl_catalogue/summary_stats/index.html
@@ -2259,6 +2259,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../methods/susie_inf/" class="md-nav__link">
         
@@ -2290,7 +2332,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2306,14 +2348,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3111,7 +3153,7 @@ <h3 id="gentropy.datasource.eqtl_catalogue.summary_stats.EqtlCatalogueSummarySta
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3122,7 +3164,7 @@ <h3 id="gentropy.datasource.eqtl_catalogue.summary_stats.EqtlCatalogueSummarySta
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>summary_stats_df</code></td>
           <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
@@ -3141,7 +3183,7 @@ <h3 id="gentropy.datasource.eqtl_catalogue.summary_stats.EqtlCatalogueSummarySta
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3150,7 +3192,7 @@ <h3 id="gentropy.datasource.eqtl_catalogue.summary_stats.EqtlCatalogueSummarySta
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>SummaryStatistics</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.summary_statistics.SummaryStatistics" href="../../../datasets/summary_statistics/#gentropy.dataset.summary_statistics.SummaryStatistics">SummaryStatistics</a></code>
           </td>
diff --git a/python_api/datasources/finngen/_finngen/index.html b/python_api/datasources/finngen/_finngen/index.html
index 79c5df01a..36ad48642 100644
--- a/python_api/datasources/finngen/_finngen/index.html
+++ b/python_api/datasources/finngen/_finngen/index.html
@@ -2194,6 +2194,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../methods/susie_inf/" class="md-nav__link">
         
@@ -2225,7 +2267,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2241,14 +2283,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
diff --git a/python_api/datasources/finngen/finemapping/index.html b/python_api/datasources/finngen/finemapping/index.html
index 62718b437..b10015457 100644
--- a/python_api/datasources/finngen/finemapping/index.html
+++ b/python_api/datasources/finngen/finemapping/index.html
@@ -2259,6 +2259,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../methods/susie_inf/" class="md-nav__link">
         
@@ -2290,7 +2332,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2306,14 +2348,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3559,7 +3601,7 @@ <h3 id="gentropy.datasource.finngen.finemapping.FinnGenFinemapping.from_finngen_
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3570,7 +3612,7 @@ <h3 id="gentropy.datasource.finngen.finemapping.FinnGenFinemapping.from_finngen_
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>spark</code></td>
           <td>
                 <code><span title="pyspark.sql.SparkSession">SparkSession</span></code>
@@ -3584,7 +3626,7 @@ <h3 id="gentropy.datasource.finngen.finemapping.FinnGenFinemapping.from_finngen_
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>finngen_finemapping_df</code></td>
           <td>
                 <code>str | list[str]</code>
@@ -3598,7 +3640,7 @@ <h3 id="gentropy.datasource.finngen.finemapping.FinnGenFinemapping.from_finngen_
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>finngen_finemapping_summaries</code></td>
           <td>
                 <code>str | list[str]</code>
@@ -3612,7 +3654,7 @@ <h3 id="gentropy.datasource.finngen.finemapping.FinnGenFinemapping.from_finngen_
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>finngen_release_prefix</code></td>
           <td>
                 <code>str</code>
@@ -3626,7 +3668,7 @@ <h3 id="gentropy.datasource.finngen.finemapping.FinnGenFinemapping.from_finngen_
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>credset_lbf_threshold</code></td>
           <td>
                 <code>float</code>
@@ -3645,7 +3687,7 @@ <h3 id="gentropy.datasource.finngen.finemapping.FinnGenFinemapping.from_finngen_
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3654,7 +3696,7 @@ <h3 id="gentropy.datasource.finngen.finemapping.FinnGenFinemapping.from_finngen_
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StudyLocus</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_locus.StudyLocus" href="../../../datasets/study_locus/#gentropy.dataset.study_locus.StudyLocus">StudyLocus</a></code>
           </td>
diff --git a/python_api/datasources/finngen/study_index/index.html b/python_api/datasources/finngen/study_index/index.html
index b11475733..7acaafe75 100644
--- a/python_api/datasources/finngen/study_index/index.html
+++ b/python_api/datasources/finngen/study_index/index.html
@@ -2259,6 +2259,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../methods/susie_inf/" class="md-nav__link">
         
@@ -2290,7 +2332,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2306,14 +2348,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3109,7 +3151,7 @@ <h3 id="gentropy.datasource.finngen.study_index.FinnGenStudyIndex.from_source" c
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3120,7 +3162,7 @@ <h3 id="gentropy.datasource.finngen.study_index.FinnGenStudyIndex.from_source" c
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>spark</code></td>
           <td>
                 <code><span title="pyspark.sql.SparkSession">SparkSession</span></code>
@@ -3139,7 +3181,7 @@ <h3 id="gentropy.datasource.finngen.study_index.FinnGenStudyIndex.from_source" c
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3148,7 +3190,7 @@ <h3 id="gentropy.datasource.finngen.study_index.FinnGenStudyIndex.from_source" c
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StudyIndex</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_index.StudyIndex" href="../../../datasets/study_index/#gentropy.dataset.study_index.StudyIndex">StudyIndex</a></code>
           </td>
diff --git a/python_api/datasources/finngen/summary_stats/index.html b/python_api/datasources/finngen/summary_stats/index.html
index 13f289538..d6d561e28 100644
--- a/python_api/datasources/finngen/summary_stats/index.html
+++ b/python_api/datasources/finngen/summary_stats/index.html
@@ -2259,6 +2259,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../methods/susie_inf/" class="md-nav__link">
         
@@ -2290,7 +2332,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2306,14 +2348,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3003,7 +3045,10 @@ <h2 id="gentropy.datasource.finngen.summary_stats.FinnGenSummaryStats" class="do
 <span class="normal">90</span>
 <span class="normal">91</span>
 <span class="normal">92</span>
-<span class="normal">93</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@dataclass</span>
+<span class="normal">93</span>
+<span class="normal">94</span>
+<span class="normal">95</span>
+<span class="normal">96</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@dataclass</span>
 <span class="k">class</span> <span class="nc">FinnGenSummaryStats</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Summary statistics dataset for FinnGen.&quot;&quot;&quot;</span>
 
@@ -3040,7 +3085,6 @@ <h2 id="gentropy.datasource.finngen.summary_stats.FinnGenSummaryStats" class="do
 <span class="sd">        Returns:</span>
 <span class="sd">            SummaryStatistics: Processed summary statistics dataset</span>
 <span class="sd">        &quot;&quot;&quot;</span>
-        <span class="n">study_id</span> <span class="o">=</span> <span class="n">raw_file</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;/&quot;</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;.&quot;</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span>
         <span class="n">processed_summary_stats_df</span> <span class="o">=</span> <span class="p">(</span>
             <span class="n">spark</span><span class="o">.</span><span class="n">read</span><span class="o">.</span><span class="n">schema</span><span class="p">(</span><span class="bp">cls</span><span class="o">.</span><span class="n">raw_schema</span><span class="p">)</span>
             <span class="o">.</span><span class="n">option</span><span class="p">(</span><span class="s2">&quot;delimiter&quot;</span><span class="p">,</span> <span class="s2">&quot;</span><span class="se">\t</span><span class="s2">&quot;</span><span class="p">)</span>
@@ -3049,7 +3093,11 @@ <h2 id="gentropy.datasource.finngen.summary_stats.FinnGenSummaryStats" class="do
             <span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;pos&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">cast</span><span class="p">(</span><span class="n">t</span><span class="o">.</span><span class="n">IntegerType</span><span class="p">())</span><span class="o">.</span><span class="n">isNotNull</span><span class="p">())</span>
             <span class="o">.</span><span class="n">select</span><span class="p">(</span>
                 <span class="c1"># From the full path, extracts just the filename, and converts to upper case to get the study ID.</span>
-                <span class="n">f</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="n">study_id</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">),</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">upper</span><span class="p">(</span>
+                    <span class="n">f</span><span class="o">.</span><span class="n">regexp_extract</span><span class="p">(</span>
+                        <span class="n">f</span><span class="o">.</span><span class="n">input_file_name</span><span class="p">(),</span> <span class="sa">r</span><span class="s2">&quot;([^/]+)(\.tsv\.gz|\.gz|\.tsv)&quot;</span><span class="p">,</span> <span class="mi">1</span>
+                    <span class="p">)</span>
+                <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">),</span>
                 <span class="c1"># Add variant information.</span>
                 <span class="n">f</span><span class="o">.</span><span class="n">concat_ws</span><span class="p">(</span>
                     <span class="s2">&quot;_&quot;</span><span class="p">,</span>
@@ -3117,7 +3165,7 @@ <h3 id="gentropy.datasource.finngen.summary_stats.FinnGenSummaryStats.from_sourc
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3128,7 +3176,7 @@ <h3 id="gentropy.datasource.finngen.summary_stats.FinnGenSummaryStats.from_sourc
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>spark</code></td>
           <td>
                 <code><span title="pyspark.sql.SparkSession">SparkSession</span></code>
@@ -3142,7 +3190,7 @@ <h3 id="gentropy.datasource.finngen.summary_stats.FinnGenSummaryStats.from_sourc
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>raw_file</code></td>
           <td>
                 <code>str</code>
@@ -3161,7 +3209,7 @@ <h3 id="gentropy.datasource.finngen.summary_stats.FinnGenSummaryStats.from_sourc
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3170,7 +3218,7 @@ <h3 id="gentropy.datasource.finngen.summary_stats.FinnGenSummaryStats.from_sourc
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>SummaryStatistics</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.summary_statistics.SummaryStatistics" href="../../../datasets/summary_statistics/#gentropy.dataset.summary_statistics.SummaryStatistics">SummaryStatistics</a></code>
           </td>
@@ -3240,7 +3288,10 @@ <h3 id="gentropy.datasource.finngen.summary_stats.FinnGenSummaryStats.from_sourc
 <span class="normal">90</span>
 <span class="normal">91</span>
 <span class="normal">92</span>
-<span class="normal">93</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@classmethod</span>
+<span class="normal">93</span>
+<span class="normal">94</span>
+<span class="normal">95</span>
+<span class="normal">96</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@classmethod</span>
 <span class="k">def</span> <span class="nf">from_source</span><span class="p">(</span>
     <span class="bp">cls</span><span class="p">:</span> <span class="nb">type</span><span class="p">[</span><span class="n">FinnGenSummaryStats</span><span class="p">],</span>
     <span class="n">spark</span><span class="p">:</span> <span class="n">SparkSession</span><span class="p">,</span>
@@ -3255,7 +3306,6 @@ <h3 id="gentropy.datasource.finngen.summary_stats.FinnGenSummaryStats.from_sourc
 <span class="sd">    Returns:</span>
 <span class="sd">        SummaryStatistics: Processed summary statistics dataset</span>
 <span class="sd">    &quot;&quot;&quot;</span>
-    <span class="n">study_id</span> <span class="o">=</span> <span class="n">raw_file</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;/&quot;</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;.&quot;</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span>
     <span class="n">processed_summary_stats_df</span> <span class="o">=</span> <span class="p">(</span>
         <span class="n">spark</span><span class="o">.</span><span class="n">read</span><span class="o">.</span><span class="n">schema</span><span class="p">(</span><span class="bp">cls</span><span class="o">.</span><span class="n">raw_schema</span><span class="p">)</span>
         <span class="o">.</span><span class="n">option</span><span class="p">(</span><span class="s2">&quot;delimiter&quot;</span><span class="p">,</span> <span class="s2">&quot;</span><span class="se">\t</span><span class="s2">&quot;</span><span class="p">)</span>
@@ -3264,7 +3314,11 @@ <h3 id="gentropy.datasource.finngen.summary_stats.FinnGenSummaryStats.from_sourc
         <span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;pos&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">cast</span><span class="p">(</span><span class="n">t</span><span class="o">.</span><span class="n">IntegerType</span><span class="p">())</span><span class="o">.</span><span class="n">isNotNull</span><span class="p">())</span>
         <span class="o">.</span><span class="n">select</span><span class="p">(</span>
             <span class="c1"># From the full path, extracts just the filename, and converts to upper case to get the study ID.</span>
-            <span class="n">f</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="n">study_id</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">),</span>
+            <span class="n">f</span><span class="o">.</span><span class="n">upper</span><span class="p">(</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">regexp_extract</span><span class="p">(</span>
+                    <span class="n">f</span><span class="o">.</span><span class="n">input_file_name</span><span class="p">(),</span> <span class="sa">r</span><span class="s2">&quot;([^/]+)(\.tsv\.gz|\.gz|\.tsv)&quot;</span><span class="p">,</span> <span class="mi">1</span>
+                <span class="p">)</span>
+            <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">),</span>
             <span class="c1"># Add variant information.</span>
             <span class="n">f</span><span class="o">.</span><span class="n">concat_ws</span><span class="p">(</span>
                 <span class="s2">&quot;_&quot;</span><span class="p">,</span>
diff --git a/python_api/datasources/gnomad/_gnomad/index.html b/python_api/datasources/gnomad/_gnomad/index.html
index 67b415c38..47cbbfb6e 100644
--- a/python_api/datasources/gnomad/_gnomad/index.html
+++ b/python_api/datasources/gnomad/_gnomad/index.html
@@ -2194,6 +2194,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../methods/susie_inf/" class="md-nav__link">
         
@@ -2225,7 +2267,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2241,14 +2283,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
diff --git a/python_api/datasources/gnomad/gnomad_ld/index.html b/python_api/datasources/gnomad/gnomad_ld/index.html
index d480d9783..b83a4c56c 100644
--- a/python_api/datasources/gnomad/gnomad_ld/index.html
+++ b/python_api/datasources/gnomad/gnomad_ld/index.html
@@ -1678,6 +1678,24 @@
     </span>
   </a>
   
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#gentropy.datasource.gnomad.ld.GnomADLDMatrix.get_locus_index" class="md-nav__link">
+    <span class="md-ellipsis">
+      get_locus_index
+    </span>
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#gentropy.datasource.gnomad.ld.GnomADLDMatrix.get_numpy_matrix" class="md-nav__link">
+    <span class="md-ellipsis">
+      get_numpy_matrix
+    </span>
+  </a>
+  
 </li>
         
       </ul>
@@ -2277,6 +2295,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../methods/susie_inf/" class="md-nav__link">
         
@@ -2308,7 +2368,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2324,14 +2384,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -2913,6 +2973,24 @@
     </span>
   </a>
   
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#gentropy.datasource.gnomad.ld.GnomADLDMatrix.get_locus_index" class="md-nav__link">
+    <span class="md-ellipsis">
+      get_locus_index
+    </span>
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#gentropy.datasource.gnomad.ld.GnomADLDMatrix.get_numpy_matrix" class="md-nav__link">
+    <span class="md-ellipsis">
+      get_numpy_matrix
+    </span>
+  </a>
+  
 </li>
         
       </ul>
@@ -2963,7 +3041,7 @@ <h2 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix" class="doc doc-heading">
 
 
 
-  <p><strong>Attributes:</strong></p>
+  <p><span class="doc-section-title">Attributes:</span></p>
   <table>
     <thead>
       <tr>
@@ -2973,7 +3051,7 @@ <h2 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code><span title="gentropy.datasource.gnomad.ld.GnomADLDMatrix.ld_matrix_template">ld_matrix_template</span></code></td>
           <td>
                 <code>str</code>
@@ -2984,7 +3062,7 @@ <h2 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix" class="doc doc-heading">
             </div>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code><span title="gentropy.datasource.gnomad.ld.GnomADLDMatrix.ld_index_raw_template">ld_index_raw_template</span></code></td>
           <td>
                 <code>str</code>
@@ -2995,7 +3073,7 @@ <h2 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix" class="doc doc-heading">
             </div>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code><span title="gentropy.datasource.gnomad.ld.GnomADLDMatrix.grch37_to_grch38_chain_path">grch37_to_grch38_chain_path</span></code></td>
           <td>
                 <code>str</code>
@@ -3006,7 +3084,7 @@ <h2 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix" class="doc doc-heading">
             </div>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code><span title="gentropy.datasource.gnomad.ld.GnomADLDMatrix.ld_populations">ld_populations</span></code></td>
           <td>
                 <code>list[str]</code>
@@ -3022,9 +3100,7 @@ <h2 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix" class="doc doc-heading">
 
             <details class="quote">
               <summary>Source code in <code>src/gentropy/datasource/gnomad/ld.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 22</span>
-<span class="normal"> 23</span>
-<span class="normal"> 24</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 24</span>
 <span class="normal"> 25</span>
 <span class="normal"> 26</span>
 <span class="normal"> 27</span>
@@ -3450,7 +3526,70 @@ <h2 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix" class="doc doc-heading">
 <span class="normal">447</span>
 <span class="normal">448</span>
 <span class="normal">449</span>
-<span class="normal">450</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@dataclass</span>
+<span class="normal">450</span>
+<span class="normal">451</span>
+<span class="normal">452</span>
+<span class="normal">453</span>
+<span class="normal">454</span>
+<span class="normal">455</span>
+<span class="normal">456</span>
+<span class="normal">457</span>
+<span class="normal">458</span>
+<span class="normal">459</span>
+<span class="normal">460</span>
+<span class="normal">461</span>
+<span class="normal">462</span>
+<span class="normal">463</span>
+<span class="normal">464</span>
+<span class="normal">465</span>
+<span class="normal">466</span>
+<span class="normal">467</span>
+<span class="normal">468</span>
+<span class="normal">469</span>
+<span class="normal">470</span>
+<span class="normal">471</span>
+<span class="normal">472</span>
+<span class="normal">473</span>
+<span class="normal">474</span>
+<span class="normal">475</span>
+<span class="normal">476</span>
+<span class="normal">477</span>
+<span class="normal">478</span>
+<span class="normal">479</span>
+<span class="normal">480</span>
+<span class="normal">481</span>
+<span class="normal">482</span>
+<span class="normal">483</span>
+<span class="normal">484</span>
+<span class="normal">485</span>
+<span class="normal">486</span>
+<span class="normal">487</span>
+<span class="normal">488</span>
+<span class="normal">489</span>
+<span class="normal">490</span>
+<span class="normal">491</span>
+<span class="normal">492</span>
+<span class="normal">493</span>
+<span class="normal">494</span>
+<span class="normal">495</span>
+<span class="normal">496</span>
+<span class="normal">497</span>
+<span class="normal">498</span>
+<span class="normal">499</span>
+<span class="normal">500</span>
+<span class="normal">501</span>
+<span class="normal">502</span>
+<span class="normal">503</span>
+<span class="normal">504</span>
+<span class="normal">505</span>
+<span class="normal">506</span>
+<span class="normal">507</span>
+<span class="normal">508</span>
+<span class="normal">509</span>
+<span class="normal">510</span>
+<span class="normal">511</span>
+<span class="normal">512</span>
+<span class="normal">513</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@dataclass</span>
 <span class="k">class</span> <span class="nc">GnomADLDMatrix</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Toolset ot interact with GnomAD LD dataset (version: r2.1.1).</span>
 
@@ -3465,6 +3604,7 @@ <h2 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix" class="doc doc-heading">
 
     <span class="n">ld_matrix_template</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;gs://gcp-public-data--gnomad/release/2.1.1/ld/gnomad.genomes.r2.1.1.</span><span class="si">{POP}</span><span class="s2">.common.adj.ld.bm&quot;</span>
     <span class="n">ld_index_raw_template</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;gs://gcp-public-data--gnomad/release/2.1.1/ld/gnomad.genomes.r2.1.1.</span><span class="si">{POP}</span><span class="s2">.common.ld.variant_indices.ht&quot;</span>
+    <span class="n">liftover_ht_path</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;gs://gcp-public-data--gnomad/release/2.1.1/liftover_grch38/ht/genomes/gnomad.genomes.r2.1.1.sites.liftover_grch38.ht&quot;</span>
     <span class="n">grch37_to_grch38_chain_path</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="p">(</span>
         <span class="s2">&quot;gs://hail-common/references/grch37_to_grch38.over.chain.gz&quot;</span>
     <span class="p">)</span>
@@ -3606,24 +3746,15 @@ <h2 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix" class="doc doc-heading">
             <span class="n">ld_index_38</span><span class="o">.</span><span class="n">to_spark</span><span class="p">()</span>
             <span class="c1"># Filter out variants where the liftover failed</span>
             <span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;`locus_GRCh38.position`&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">isNotNull</span><span class="p">())</span>
-            <span class="o">.</span><span class="n">withColumn</span><span class="p">(</span>
-                <span class="s2">&quot;chromosome&quot;</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">regexp_replace</span><span class="p">(</span><span class="s2">&quot;`locus_GRCh38.contig`&quot;</span><span class="p">,</span> <span class="s2">&quot;chr&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span>
-            <span class="p">)</span>
-            <span class="o">.</span><span class="n">withColumn</span><span class="p">(</span>
-                <span class="s2">&quot;position&quot;</span><span class="p">,</span>
-                <span class="n">convert_gnomad_position_to_ensembl</span><span class="p">(</span>
-                    <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;`locus_GRCh38.position`&quot;</span><span class="p">),</span>
-                    <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;`alleles`&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">getItem</span><span class="p">(</span><span class="mi">0</span><span class="p">),</span>
-                    <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;`alleles`&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">getItem</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span>
-                <span class="p">),</span>
-            <span class="p">)</span>
             <span class="o">.</span><span class="n">select</span><span class="p">(</span>
-                <span class="s2">&quot;chromosome&quot;</span><span class="p">,</span>
-                <span class="s2">&quot;position&quot;</span><span class="p">,</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">regexp_replace</span><span class="p">(</span><span class="s2">&quot;`locus_GRCh38.contig`&quot;</span><span class="p">,</span> <span class="s2">&quot;chr&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span>
+                    <span class="s2">&quot;chromosome&quot;</span>
+                <span class="p">),</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;`locus_GRCh38.position`&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;position&quot;</span><span class="p">),</span>
                 <span class="n">f</span><span class="o">.</span><span class="n">concat_ws</span><span class="p">(</span>
                     <span class="s2">&quot;_&quot;</span><span class="p">,</span>
-                    <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;chromosome&quot;</span><span class="p">),</span>
-                    <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;position&quot;</span><span class="p">),</span>
+                    <span class="n">f</span><span class="o">.</span><span class="n">regexp_replace</span><span class="p">(</span><span class="s2">&quot;`locus_GRCh38.contig`&quot;</span><span class="p">,</span> <span class="s2">&quot;chr&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">),</span>
+                    <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;`locus_GRCh38.position`&quot;</span><span class="p">),</span>
                     <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;`alleles`&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">getItem</span><span class="p">(</span><span class="mi">0</span><span class="p">),</span>
                     <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;`alleles`&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">getItem</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span>
                 <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;variantId&quot;</span><span class="p">),</span>
@@ -3649,9 +3780,9 @@ <h2 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix" class="doc doc-heading">
 <span class="sd">            DataFrame: Dataframe with variant IDs instead of `i` and `j` indices</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="n">ld_index_i</span> <span class="o">=</span> <span class="n">ld_index</span><span class="o">.</span><span class="n">selectExpr</span><span class="p">(</span>
-            <span class="s2">&quot;idx as i&quot;</span><span class="p">,</span> <span class="s2">&quot;variantId as variantId_i&quot;</span><span class="p">,</span> <span class="s2">&quot;chromosome&quot;</span>
+            <span class="s2">&quot;idx as i&quot;</span><span class="p">,</span> <span class="s2">&quot;variantId as variantIdI&quot;</span><span class="p">,</span> <span class="s2">&quot;chromosome&quot;</span>
         <span class="p">)</span>
-        <span class="n">ld_index_j</span> <span class="o">=</span> <span class="n">ld_index</span><span class="o">.</span><span class="n">selectExpr</span><span class="p">(</span><span class="s2">&quot;idx as j&quot;</span><span class="p">,</span> <span class="s2">&quot;variantId as variantId_j&quot;</span><span class="p">)</span>
+        <span class="n">ld_index_j</span> <span class="o">=</span> <span class="n">ld_index</span><span class="o">.</span><span class="n">selectExpr</span><span class="p">(</span><span class="s2">&quot;idx as j&quot;</span><span class="p">,</span> <span class="s2">&quot;variantId as variantIdJ&quot;</span><span class="p">)</span>
         <span class="k">return</span> <span class="p">(</span>
             <span class="n">ld_matrix</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">ld_index_i</span><span class="p">,</span> <span class="n">on</span><span class="o">=</span><span class="s2">&quot;i&quot;</span><span class="p">,</span> <span class="n">how</span><span class="o">=</span><span class="s2">&quot;inner&quot;</span><span class="p">)</span>
             <span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">ld_index_j</span><span class="p">,</span> <span class="n">on</span><span class="o">=</span><span class="s2">&quot;j&quot;</span><span class="p">,</span> <span class="n">how</span><span class="o">=</span><span class="s2">&quot;inner&quot;</span><span class="p">)</span>
@@ -3669,35 +3800,35 @@ <h2 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix" class="doc doc-heading">
 <span class="sd">            DataFrame: Square LD matrix without diagonal duplicates</span>
 
 <span class="sd">        Examples:</span>
-<span class="sd">            &gt;&gt;&gt; df = spark.createDataFrame(</span>
-<span class="sd">            ...     [</span>
-<span class="sd">            ...         (1, 1, 1.0, &quot;1&quot;, &quot;AFR&quot;),</span>
-<span class="sd">            ...         (1, 2, 0.5, &quot;1&quot;, &quot;AFR&quot;),</span>
-<span class="sd">            ...         (2, 2, 1.0, &quot;1&quot;, &quot;AFR&quot;),</span>
-<span class="sd">            ...     ],</span>
-<span class="sd">            ...     [&quot;variantId_i&quot;, &quot;variantId_j&quot;, &quot;r&quot;, &quot;chromosome&quot;, &quot;population&quot;],</span>
-<span class="sd">            ... )</span>
-<span class="sd">            &gt;&gt;&gt; GnomADLDMatrix._transpose_ld_matrix(df).show()</span>
-<span class="sd">            +-----------+-----------+---+----------+----------+</span>
-<span class="sd">            |variantId_i|variantId_j|  r|chromosome|population|</span>
-<span class="sd">            +-----------+-----------+---+----------+----------+</span>
-<span class="sd">            |          1|          2|0.5|         1|       AFR|</span>
-<span class="sd">            |          1|          1|1.0|         1|       AFR|</span>
-<span class="sd">            |          2|          1|0.5|         1|       AFR|</span>
-<span class="sd">            |          2|          2|1.0|         1|       AFR|</span>
-<span class="sd">            +-----------+-----------+---+----------+----------+</span>
-<span class="sd">            &lt;BLANKLINE&gt;</span>
+<span class="sd">        &gt;&gt;&gt; df = spark.createDataFrame(</span>
+<span class="sd">        ...     [</span>
+<span class="sd">        ...         (1, 1, 1.0, &quot;1&quot;, &quot;AFR&quot;),</span>
+<span class="sd">        ...         (1, 2, 0.5, &quot;1&quot;, &quot;AFR&quot;),</span>
+<span class="sd">        ...         (2, 2, 1.0, &quot;1&quot;, &quot;AFR&quot;),</span>
+<span class="sd">        ...     ],</span>
+<span class="sd">        ...     [&quot;variantIdI&quot;, &quot;variantIdJ&quot;, &quot;r&quot;, &quot;chromosome&quot;, &quot;population&quot;],</span>
+<span class="sd">        ... )</span>
+<span class="sd">        &gt;&gt;&gt; GnomADLDMatrix._transpose_ld_matrix(df).show()</span>
+<span class="sd">        +----------+----------+---+----------+----------+</span>
+<span class="sd">        |variantIdI|variantIdJ|  r|chromosome|population|</span>
+<span class="sd">        +----------+----------+---+----------+----------+</span>
+<span class="sd">        |         1|         2|0.5|         1|       AFR|</span>
+<span class="sd">        |         1|         1|1.0|         1|       AFR|</span>
+<span class="sd">        |         2|         1|0.5|         1|       AFR|</span>
+<span class="sd">        |         2|         2|1.0|         1|       AFR|</span>
+<span class="sd">        +----------+----------+---+----------+----------+</span>
+<span class="sd">        &lt;BLANKLINE&gt;</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="n">ld_matrix_transposed</span> <span class="o">=</span> <span class="n">ld_matrix</span><span class="o">.</span><span class="n">selectExpr</span><span class="p">(</span>
-            <span class="s2">&quot;variantId_i as variantId_j&quot;</span><span class="p">,</span>
-            <span class="s2">&quot;variantId_j as variantId_i&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;variantIdI as variantIdJ&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;variantIdJ as variantIdI&quot;</span><span class="p">,</span>
             <span class="s2">&quot;r&quot;</span><span class="p">,</span>
             <span class="s2">&quot;chromosome&quot;</span><span class="p">,</span>
             <span class="s2">&quot;population&quot;</span><span class="p">,</span>
         <span class="p">)</span>
-        <span class="k">return</span> <span class="n">ld_matrix</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span>
-            <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;variantId_i&quot;</span><span class="p">)</span> <span class="o">!=</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;variantId_j&quot;</span><span class="p">)</span>
-        <span class="p">)</span><span class="o">.</span><span class="n">unionByName</span><span class="p">(</span><span class="n">ld_matrix_transposed</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">ld_matrix</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;variantIdI&quot;</span><span class="p">)</span> <span class="o">!=</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;variantIdJ&quot;</span><span class="p">))</span><span class="o">.</span><span class="n">unionByName</span><span class="p">(</span>
+            <span class="n">ld_matrix_transposed</span>
+        <span class="p">)</span>
 
     <span class="k">def</span> <span class="nf">as_ld_index</span><span class="p">(</span>
         <span class="bp">self</span><span class="p">:</span> <span class="n">GnomADLDMatrix</span><span class="p">,</span>
@@ -3738,8 +3869,8 @@ <h2 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix" class="doc doc-heading">
             <span class="n">GnomADLDMatrix</span><span class="o">.</span><span class="n">_transpose_ld_matrix</span><span class="p">(</span>
                 <span class="n">reduce</span><span class="p">(</span><span class="k">lambda</span> <span class="n">df1</span><span class="p">,</span> <span class="n">df2</span><span class="p">:</span> <span class="n">df1</span><span class="o">.</span><span class="n">unionByName</span><span class="p">(</span><span class="n">df2</span><span class="p">),</span> <span class="n">ld_indices_unaggregated</span><span class="p">)</span>
             <span class="p">)</span>
-            <span class="o">.</span><span class="n">withColumnRenamed</span><span class="p">(</span><span class="s2">&quot;variantId_i&quot;</span><span class="p">,</span> <span class="s2">&quot;variantId&quot;</span><span class="p">)</span>
-            <span class="o">.</span><span class="n">withColumnRenamed</span><span class="p">(</span><span class="s2">&quot;variantId_j&quot;</span><span class="p">,</span> <span class="s2">&quot;tagVariantId&quot;</span><span class="p">)</span>
+            <span class="o">.</span><span class="n">withColumnRenamed</span><span class="p">(</span><span class="s2">&quot;variantIdI&quot;</span><span class="p">,</span> <span class="s2">&quot;variantId&quot;</span><span class="p">)</span>
+            <span class="o">.</span><span class="n">withColumnRenamed</span><span class="p">(</span><span class="s2">&quot;variantIdJ&quot;</span><span class="p">,</span> <span class="s2">&quot;tagVariantId&quot;</span><span class="p">)</span>
         <span class="p">)</span>
         <span class="k">return</span> <span class="n">LDIndex</span><span class="p">(</span>
             <span class="n">_df</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_aggregate_ld_index_across_populations</span><span class="p">(</span><span class="n">ld_index_unaggregated</span><span class="p">),</span>
@@ -3776,7 +3907,6 @@ <h2 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix" class="doc doc-heading">
                 <span class="o">&amp;</span> <span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;position&quot;</span><span class="p">)</span> <span class="o">&lt;=</span> <span class="n">end</span><span class="p">)</span>
             <span class="p">)</span>
             <span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">&quot;chromosome&quot;</span><span class="p">,</span> <span class="s2">&quot;position&quot;</span><span class="p">,</span> <span class="s2">&quot;variantId&quot;</span><span class="p">,</span> <span class="s2">&quot;idx&quot;</span><span class="p">)</span>
-            <span class="o">.</span><span class="n">persist</span><span class="p">()</span>
         <span class="p">)</span>
 
         <span class="k">if</span> <span class="n">ld_index_df</span><span class="o">.</span><span class="n">limit</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">count</span><span class="p">()</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
@@ -3826,7 +3956,7 @@ <h2 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix" class="doc doc-heading">
             <span class="o">.</span><span class="n">join</span><span class="p">(</span>
                 <span class="n">ld_index_df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
                     <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;idx&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;idx_i&quot;</span><span class="p">),</span>
-                    <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;variantId&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;variantId_i&quot;</span><span class="p">),</span>
+                    <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;variantId&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;variantIdI&quot;</span><span class="p">),</span>
                 <span class="p">),</span>
                 <span class="n">on</span><span class="o">=</span><span class="s2">&quot;idx_i&quot;</span><span class="p">,</span>
                 <span class="n">how</span><span class="o">=</span><span class="s2">&quot;inner&quot;</span><span class="p">,</span>
@@ -3834,12 +3964,12 @@ <h2 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix" class="doc doc-heading">
             <span class="o">.</span><span class="n">join</span><span class="p">(</span>
                 <span class="n">ld_index_df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
                     <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;idx&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;idx_j&quot;</span><span class="p">),</span>
-                    <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;variantId&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;variantId_j&quot;</span><span class="p">),</span>
+                    <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;variantId&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;variantIdJ&quot;</span><span class="p">),</span>
                 <span class="p">),</span>
                 <span class="n">on</span><span class="o">=</span><span class="s2">&quot;idx_j&quot;</span><span class="p">,</span>
                 <span class="n">how</span><span class="o">=</span><span class="s2">&quot;inner&quot;</span><span class="p">,</span>
             <span class="p">)</span>
-            <span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">&quot;variantId_i&quot;</span><span class="p">,</span> <span class="s2">&quot;variantId_j&quot;</span><span class="p">,</span> <span class="s2">&quot;r&quot;</span><span class="p">)</span>
+            <span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">&quot;variantIdI&quot;</span><span class="p">,</span> <span class="s2">&quot;variantIdJ&quot;</span><span class="p">,</span> <span class="s2">&quot;r&quot;</span><span class="p">)</span>
         <span class="p">)</span>
 
     <span class="k">def</span> <span class="nf">get_ld_matrix_slice</span><span class="p">(</span>
@@ -3879,6 +4009,76 @@ <h2 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix" class="doc doc-heading">
                 <span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;r&quot;</span><span class="p">),</span>
             <span class="p">)</span>
         <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">get_locus_index</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">:</span> <span class="n">GnomADLDMatrix</span><span class="p">,</span>
+        <span class="n">study_locus_row</span><span class="p">:</span> <span class="n">Row</span><span class="p">,</span>
+        <span class="n">radius</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">500_000</span><span class="p">,</span>
+        <span class="n">major_population</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;nfe&quot;</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">DataFrame</span><span class="p">:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Extract hail matrix index from StudyLocus rows.</span>
+
+<span class="sd">        Args:</span>
+<span class="sd">            study_locus_row (Row): Study-locus row</span>
+<span class="sd">            radius (int): Locus radius to extract from gnomad matrix</span>
+<span class="sd">            major_population (str): Major population to extract from gnomad matrix, default is &quot;nfe&quot;</span>
+
+<span class="sd">        Returns:</span>
+<span class="sd">            DataFrame: Returns the index of the gnomad matrix for the locus</span>
+
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">chromosome</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="s2">&quot;chr&quot;</span> <span class="o">+</span> <span class="n">study_locus_row</span><span class="p">[</span><span class="s2">&quot;chromosome&quot;</span><span class="p">])</span>
+        <span class="n">start</span> <span class="o">=</span> <span class="n">study_locus_row</span><span class="p">[</span><span class="s2">&quot;position&quot;</span><span class="p">]</span> <span class="o">-</span> <span class="n">radius</span>
+        <span class="n">end</span> <span class="o">=</span> <span class="n">study_locus_row</span><span class="p">[</span><span class="s2">&quot;position&quot;</span><span class="p">]</span> <span class="o">+</span> <span class="n">radius</span>
+
+        <span class="n">liftover_ht</span> <span class="o">=</span> <span class="n">hl</span><span class="o">.</span><span class="n">read_table</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">liftover_ht_path</span><span class="p">)</span>
+        <span class="n">liftover_ht</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="n">liftover_ht</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span>
+                <span class="p">(</span><span class="n">liftover_ht</span><span class="o">.</span><span class="n">locus</span><span class="o">.</span><span class="n">contig</span> <span class="o">==</span> <span class="n">chromosome</span><span class="p">)</span>
+                <span class="o">&amp;</span> <span class="p">(</span><span class="n">liftover_ht</span><span class="o">.</span><span class="n">locus</span><span class="o">.</span><span class="n">position</span> <span class="o">&gt;=</span> <span class="n">start</span><span class="p">)</span>
+                <span class="o">&amp;</span> <span class="p">(</span><span class="n">liftover_ht</span><span class="o">.</span><span class="n">locus</span><span class="o">.</span><span class="n">position</span> <span class="o">&lt;=</span> <span class="n">end</span><span class="p">)</span>
+            <span class="p">)</span>
+            <span class="o">.</span><span class="n">key_by</span><span class="p">()</span>
+            <span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">&quot;locus&quot;</span><span class="p">,</span> <span class="s2">&quot;alleles&quot;</span><span class="p">,</span> <span class="s2">&quot;original_locus&quot;</span><span class="p">)</span>
+            <span class="o">.</span><span class="n">key_by</span><span class="p">(</span><span class="s2">&quot;original_locus&quot;</span><span class="p">,</span> <span class="s2">&quot;alleles&quot;</span><span class="p">)</span>
+            <span class="o">.</span><span class="n">naive_coalesce</span><span class="p">(</span><span class="mi">20</span><span class="p">)</span>
+        <span class="p">)</span>
+
+        <span class="n">hail_index</span> <span class="o">=</span> <span class="n">hl</span><span class="o">.</span><span class="n">read_table</span><span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">ld_index_raw_template</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">POP</span><span class="o">=</span><span class="n">major_population</span><span class="p">)</span>
+        <span class="p">)</span>
+
+        <span class="n">joined_index</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="n">liftover_ht</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">hail_index</span><span class="p">,</span> <span class="n">how</span><span class="o">=</span><span class="s2">&quot;inner&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">order_by</span><span class="p">(</span><span class="s2">&quot;idx&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">to_spark</span><span class="p">()</span>
+        <span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">joined_index</span>
+
+    <span class="nd">@staticmethod</span>
+    <span class="k">def</span> <span class="nf">get_numpy_matrix</span><span class="p">(</span>
+        <span class="n">locus_index</span><span class="p">:</span> <span class="n">DataFrame</span><span class="p">,</span>
+        <span class="n">gnomad_ancestry</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;nfe&quot;</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Extract the LD block matrix for a locus.</span>
+
+<span class="sd">        Args:</span>
+<span class="sd">            locus_index (DataFrame): hail matrix variant index table</span>
+<span class="sd">            gnomad_ancestry (str): GnomAD major ancestry label eg. `nfe`</span>
+
+<span class="sd">        Returns:</span>
+<span class="sd">            np.ndarray: LD block matrix for the locus</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">idx</span> <span class="o">=</span> <span class="p">[</span><span class="n">row</span><span class="p">[</span><span class="s2">&quot;idx&quot;</span><span class="p">]</span> <span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">locus_index</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">&quot;idx&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()]</span>
+
+        <span class="n">half_matrix</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="n">BlockMatrix</span><span class="o">.</span><span class="n">read</span><span class="p">(</span>
+                <span class="n">GnomADLDMatrix</span><span class="o">.</span><span class="n">ld_matrix_template</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">POP</span><span class="o">=</span><span class="n">gnomad_ancestry</span><span class="p">)</span>
+            <span class="p">)</span>
+            <span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">idx</span><span class="p">,</span> <span class="n">idx</span><span class="p">)</span>
+            <span class="o">.</span><span class="n">to_numpy</span><span class="p">()</span>
+        <span class="p">)</span>
+
+        <span class="k">return</span> <span class="p">(</span><span class="n">half_matrix</span> <span class="o">+</span> <span class="n">half_matrix</span><span class="o">.</span><span class="n">T</span><span class="p">)</span> <span class="o">-</span> <span class="n">np</span><span class="o">.</span><span class="n">diag</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">diag</span><span class="p">(</span><span class="n">half_matrix</span><span class="p">))</span>
 </code></pre></div></td></tr></table></div>
             </details>
 
@@ -3917,7 +4117,7 @@ <h3 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix.as_ld_index" class="doc doc
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3928,7 +4128,7 @@ <h3 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix.as_ld_index" class="doc doc
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>min_r2</code></td>
           <td>
                 <code>float</code>
@@ -3947,7 +4147,7 @@ <h3 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix.as_ld_index" class="doc doc
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3956,7 +4156,7 @@ <h3 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix.as_ld_index" class="doc doc
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>LDIndex</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.ld_index.LDIndex" href="../../../datasets/ld_index/#gentropy.dataset.ld_index.LDIndex">LDIndex</a></code>
           </td>
@@ -3971,7 +4171,13 @@ <h3 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix.as_ld_index" class="doc doc
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/datasource/gnomad/ld.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">271</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">265</span>
+<span class="normal">266</span>
+<span class="normal">267</span>
+<span class="normal">268</span>
+<span class="normal">269</span>
+<span class="normal">270</span>
+<span class="normal">271</span>
 <span class="normal">272</span>
 <span class="normal">273</span>
 <span class="normal">274</span>
@@ -4010,13 +4216,7 @@ <h3 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix.as_ld_index" class="doc doc
 <span class="normal">307</span>
 <span class="normal">308</span>
 <span class="normal">309</span>
-<span class="normal">310</span>
-<span class="normal">311</span>
-<span class="normal">312</span>
-<span class="normal">313</span>
-<span class="normal">314</span>
-<span class="normal">315</span>
-<span class="normal">316</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">as_ld_index</span><span class="p">(</span>
+<span class="normal">310</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">as_ld_index</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">:</span> <span class="n">GnomADLDMatrix</span><span class="p">,</span>
     <span class="n">min_r2</span><span class="p">:</span> <span class="nb">float</span><span class="p">,</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">LDIndex</span><span class="p">:</span>
@@ -4055,8 +4255,8 @@ <h3 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix.as_ld_index" class="doc doc
         <span class="n">GnomADLDMatrix</span><span class="o">.</span><span class="n">_transpose_ld_matrix</span><span class="p">(</span>
             <span class="n">reduce</span><span class="p">(</span><span class="k">lambda</span> <span class="n">df1</span><span class="p">,</span> <span class="n">df2</span><span class="p">:</span> <span class="n">df1</span><span class="o">.</span><span class="n">unionByName</span><span class="p">(</span><span class="n">df2</span><span class="p">),</span> <span class="n">ld_indices_unaggregated</span><span class="p">)</span>
         <span class="p">)</span>
-        <span class="o">.</span><span class="n">withColumnRenamed</span><span class="p">(</span><span class="s2">&quot;variantId_i&quot;</span><span class="p">,</span> <span class="s2">&quot;variantId&quot;</span><span class="p">)</span>
-        <span class="o">.</span><span class="n">withColumnRenamed</span><span class="p">(</span><span class="s2">&quot;variantId_j&quot;</span><span class="p">,</span> <span class="s2">&quot;tagVariantId&quot;</span><span class="p">)</span>
+        <span class="o">.</span><span class="n">withColumnRenamed</span><span class="p">(</span><span class="s2">&quot;variantIdI&quot;</span><span class="p">,</span> <span class="s2">&quot;variantId&quot;</span><span class="p">)</span>
+        <span class="o">.</span><span class="n">withColumnRenamed</span><span class="p">(</span><span class="s2">&quot;variantIdJ&quot;</span><span class="p">,</span> <span class="s2">&quot;tagVariantId&quot;</span><span class="p">)</span>
     <span class="p">)</span>
     <span class="k">return</span> <span class="n">LDIndex</span><span class="p">(</span>
         <span class="n">_df</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_aggregate_ld_index_across_populations</span><span class="p">(</span><span class="n">ld_index_unaggregated</span><span class="p">),</span>
@@ -4089,7 +4289,7 @@ <h3 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix.get_ld_matrix_slice" class=
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -4100,7 +4300,7 @@ <h3 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix.get_ld_matrix_slice" class=
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>gnomad_ancestry</code></td>
           <td>
                 <code>str</code>
@@ -4114,7 +4314,7 @@ <h3 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix.get_ld_matrix_slice" class=
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>start_index</code></td>
           <td>
                 <code>int</code>
@@ -4128,7 +4328,7 @@ <h3 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix.get_ld_matrix_slice" class=
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>end_index</code></td>
           <td>
                 <code>int</code>
@@ -4147,7 +4347,7 @@ <h3 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix.get_ld_matrix_slice" class=
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -4156,7 +4356,7 @@ <h3 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix.get_ld_matrix_slice" class=
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>DataFrame</code></td>          <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
           </td>
@@ -4171,7 +4371,14 @@ <h3 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix.get_ld_matrix_slice" class=
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/datasource/gnomad/ld.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">414</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">407</span>
+<span class="normal">408</span>
+<span class="normal">409</span>
+<span class="normal">410</span>
+<span class="normal">411</span>
+<span class="normal">412</span>
+<span class="normal">413</span>
+<span class="normal">414</span>
 <span class="normal">415</span>
 <span class="normal">416</span>
 <span class="normal">417</span>
@@ -4200,14 +4407,7 @@ <h3 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix.get_ld_matrix_slice" class=
 <span class="normal">440</span>
 <span class="normal">441</span>
 <span class="normal">442</span>
-<span class="normal">443</span>
-<span class="normal">444</span>
-<span class="normal">445</span>
-<span class="normal">446</span>
-<span class="normal">447</span>
-<span class="normal">448</span>
-<span class="normal">449</span>
-<span class="normal">450</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">get_ld_matrix_slice</span><span class="p">(</span>
+<span class="normal">443</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">get_ld_matrix_slice</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">:</span> <span class="n">GnomADLDMatrix</span><span class="p">,</span>
     <span class="n">gnomad_ancestry</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
     <span class="n">start_index</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
@@ -4267,7 +4467,7 @@ <h3 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix.get_ld_variants" class="doc
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -4278,7 +4478,7 @@ <h3 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix.get_ld_variants" class="doc
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>gnomad_ancestry</code></td>
           <td>
                 <code>str</code>
@@ -4292,7 +4492,7 @@ <h3 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix.get_ld_variants" class="doc
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>chromosome</code></td>
           <td>
                 <code>str</code>
@@ -4306,7 +4506,7 @@ <h3 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix.get_ld_variants" class="doc
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>start</code></td>
           <td>
                 <code>int</code>
@@ -4320,7 +4520,7 @@ <h3 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix.get_ld_variants" class="doc
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>end</code></td>
           <td>
                 <code>int</code>
@@ -4339,7 +4539,7 @@ <h3 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix.get_ld_variants" class="doc
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -4348,7 +4548,7 @@ <h3 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix.get_ld_variants" class="doc
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span> | None</code>
           </td>
@@ -4363,7 +4563,13 @@ <h3 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix.get_ld_variants" class="doc
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/datasource/gnomad/ld.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">318</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">312</span>
+<span class="normal">313</span>
+<span class="normal">314</span>
+<span class="normal">315</span>
+<span class="normal">316</span>
+<span class="normal">317</span>
+<span class="normal">318</span>
 <span class="normal">319</span>
 <span class="normal">320</span>
 <span class="normal">321</span>
@@ -4409,14 +4615,7 @@ <h3 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix.get_ld_variants" class="doc
 <span class="normal">361</span>
 <span class="normal">362</span>
 <span class="normal">363</span>
-<span class="normal">364</span>
-<span class="normal">365</span>
-<span class="normal">366</span>
-<span class="normal">367</span>
-<span class="normal">368</span>
-<span class="normal">369</span>
-<span class="normal">370</span>
-<span class="normal">371</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">get_ld_variants</span><span class="p">(</span>
+<span class="normal">364</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">get_ld_variants</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">:</span> <span class="n">GnomADLDMatrix</span><span class="p">,</span>
     <span class="n">gnomad_ancestry</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
     <span class="n">chromosome</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
@@ -4446,7 +4645,6 @@ <h3 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix.get_ld_variants" class="doc
             <span class="o">&amp;</span> <span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;position&quot;</span><span class="p">)</span> <span class="o">&lt;=</span> <span class="n">end</span><span class="p">)</span>
         <span class="p">)</span>
         <span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">&quot;chromosome&quot;</span><span class="p">,</span> <span class="s2">&quot;position&quot;</span><span class="p">,</span> <span class="s2">&quot;variantId&quot;</span><span class="p">,</span> <span class="s2">&quot;idx&quot;</span><span class="p">)</span>
-        <span class="o">.</span><span class="n">persist</span><span class="p">()</span>
     <span class="p">)</span>
 
     <span class="k">if</span> <span class="n">ld_index_df</span><span class="o">.</span><span class="n">limit</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">count</span><span class="p">()</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
@@ -4477,6 +4675,340 @@ <h3 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix.get_ld_variants" class="doc
 </div>
 
 
+<div class="doc doc-object doc-function">
+
+
+
+<h3 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix.get_locus_index" class="doc doc-heading">
+          <code class="highlight language-python"><span class="n">get_locus_index</span><span class="p">(</span><span class="n">study_locus_row</span><span class="p">:</span> <span class="n">Row</span><span class="p">,</span> <span class="n">radius</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">500000</span><span class="p">,</span> <span class="n">major_population</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s1">&#39;nfe&#39;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">DataFrame</span></code>
+
+<a href="#gentropy.datasource.gnomad.ld.GnomADLDMatrix.get_locus_index" class="headerlink" title="Permanent link">&para;</a></h3>
+
+
+  <div class="doc doc-contents ">
+  
+      <p>Extract hail matrix index from StudyLocus rows.</p>
+
+
+
+  <p><span class="doc-section-title">Parameters:</span></p>
+  <table>
+    <thead>
+      <tr>
+        <th>Name</th>
+        <th>Type</th>
+        <th>Description</th>
+        <th>Default</th>
+      </tr>
+    </thead>
+    <tbody>
+        <tr class="doc-section-item">
+          <td><code>study_locus_row</code></td>
+          <td>
+                <code><span title="pyspark.sql.Row">Row</span></code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>Study-locus row</p>
+            </div>
+          </td>
+          <td>
+              <em>required</em>
+          </td>
+        </tr>
+        <tr class="doc-section-item">
+          <td><code>radius</code></td>
+          <td>
+                <code>int</code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>Locus radius to extract from gnomad matrix</p>
+            </div>
+          </td>
+          <td>
+                <code>500000</code>
+          </td>
+        </tr>
+        <tr class="doc-section-item">
+          <td><code>major_population</code></td>
+          <td>
+                <code>str</code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>Major population to extract from gnomad matrix, default is "nfe"</p>
+            </div>
+          </td>
+          <td>
+                <code>&#39;nfe&#39;</code>
+          </td>
+        </tr>
+    </tbody>
+  </table>
+
+
+
+  <p><span class="doc-section-title">Returns:</span></p>
+  <table>
+    <thead>
+      <tr>
+<th>Name</th>        <th>Type</th>
+        <th>Description</th>
+      </tr>
+    </thead>
+    <tbody>
+        <tr class="doc-section-item">
+<td><code>DataFrame</code></td>          <td>
+                <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>Returns the index of the gnomad matrix for the locus</p>
+            </div>
+          </td>
+        </tr>
+    </tbody>
+  </table>
+
+          <details class="quote">
+            <summary>Source code in <code>src/gentropy/datasource/gnomad/ld.py</code></summary>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">445</span>
+<span class="normal">446</span>
+<span class="normal">447</span>
+<span class="normal">448</span>
+<span class="normal">449</span>
+<span class="normal">450</span>
+<span class="normal">451</span>
+<span class="normal">452</span>
+<span class="normal">453</span>
+<span class="normal">454</span>
+<span class="normal">455</span>
+<span class="normal">456</span>
+<span class="normal">457</span>
+<span class="normal">458</span>
+<span class="normal">459</span>
+<span class="normal">460</span>
+<span class="normal">461</span>
+<span class="normal">462</span>
+<span class="normal">463</span>
+<span class="normal">464</span>
+<span class="normal">465</span>
+<span class="normal">466</span>
+<span class="normal">467</span>
+<span class="normal">468</span>
+<span class="normal">469</span>
+<span class="normal">470</span>
+<span class="normal">471</span>
+<span class="normal">472</span>
+<span class="normal">473</span>
+<span class="normal">474</span>
+<span class="normal">475</span>
+<span class="normal">476</span>
+<span class="normal">477</span>
+<span class="normal">478</span>
+<span class="normal">479</span>
+<span class="normal">480</span>
+<span class="normal">481</span>
+<span class="normal">482</span>
+<span class="normal">483</span>
+<span class="normal">484</span>
+<span class="normal">485</span>
+<span class="normal">486</span>
+<span class="normal">487</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">get_locus_index</span><span class="p">(</span>
+    <span class="bp">self</span><span class="p">:</span> <span class="n">GnomADLDMatrix</span><span class="p">,</span>
+    <span class="n">study_locus_row</span><span class="p">:</span> <span class="n">Row</span><span class="p">,</span>
+    <span class="n">radius</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">500_000</span><span class="p">,</span>
+    <span class="n">major_population</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;nfe&quot;</span><span class="p">,</span>
+<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">DataFrame</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;Extract hail matrix index from StudyLocus rows.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        study_locus_row (Row): Study-locus row</span>
+<span class="sd">        radius (int): Locus radius to extract from gnomad matrix</span>
+<span class="sd">        major_population (str): Major population to extract from gnomad matrix, default is &quot;nfe&quot;</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">        DataFrame: Returns the index of the gnomad matrix for the locus</span>
+
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">chromosome</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="s2">&quot;chr&quot;</span> <span class="o">+</span> <span class="n">study_locus_row</span><span class="p">[</span><span class="s2">&quot;chromosome&quot;</span><span class="p">])</span>
+    <span class="n">start</span> <span class="o">=</span> <span class="n">study_locus_row</span><span class="p">[</span><span class="s2">&quot;position&quot;</span><span class="p">]</span> <span class="o">-</span> <span class="n">radius</span>
+    <span class="n">end</span> <span class="o">=</span> <span class="n">study_locus_row</span><span class="p">[</span><span class="s2">&quot;position&quot;</span><span class="p">]</span> <span class="o">+</span> <span class="n">radius</span>
+
+    <span class="n">liftover_ht</span> <span class="o">=</span> <span class="n">hl</span><span class="o">.</span><span class="n">read_table</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">liftover_ht_path</span><span class="p">)</span>
+    <span class="n">liftover_ht</span> <span class="o">=</span> <span class="p">(</span>
+        <span class="n">liftover_ht</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span>
+            <span class="p">(</span><span class="n">liftover_ht</span><span class="o">.</span><span class="n">locus</span><span class="o">.</span><span class="n">contig</span> <span class="o">==</span> <span class="n">chromosome</span><span class="p">)</span>
+            <span class="o">&amp;</span> <span class="p">(</span><span class="n">liftover_ht</span><span class="o">.</span><span class="n">locus</span><span class="o">.</span><span class="n">position</span> <span class="o">&gt;=</span> <span class="n">start</span><span class="p">)</span>
+            <span class="o">&amp;</span> <span class="p">(</span><span class="n">liftover_ht</span><span class="o">.</span><span class="n">locus</span><span class="o">.</span><span class="n">position</span> <span class="o">&lt;=</span> <span class="n">end</span><span class="p">)</span>
+        <span class="p">)</span>
+        <span class="o">.</span><span class="n">key_by</span><span class="p">()</span>
+        <span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">&quot;locus&quot;</span><span class="p">,</span> <span class="s2">&quot;alleles&quot;</span><span class="p">,</span> <span class="s2">&quot;original_locus&quot;</span><span class="p">)</span>
+        <span class="o">.</span><span class="n">key_by</span><span class="p">(</span><span class="s2">&quot;original_locus&quot;</span><span class="p">,</span> <span class="s2">&quot;alleles&quot;</span><span class="p">)</span>
+        <span class="o">.</span><span class="n">naive_coalesce</span><span class="p">(</span><span class="mi">20</span><span class="p">)</span>
+    <span class="p">)</span>
+
+    <span class="n">hail_index</span> <span class="o">=</span> <span class="n">hl</span><span class="o">.</span><span class="n">read_table</span><span class="p">(</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">ld_index_raw_template</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">POP</span><span class="o">=</span><span class="n">major_population</span><span class="p">)</span>
+    <span class="p">)</span>
+
+    <span class="n">joined_index</span> <span class="o">=</span> <span class="p">(</span>
+        <span class="n">liftover_ht</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">hail_index</span><span class="p">,</span> <span class="n">how</span><span class="o">=</span><span class="s2">&quot;inner&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">order_by</span><span class="p">(</span><span class="s2">&quot;idx&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">to_spark</span><span class="p">()</span>
+    <span class="p">)</span>
+
+    <span class="k">return</span> <span class="n">joined_index</span>
+</code></pre></div></td></tr></table></div>
+          </details>
+  </div>
+
+</div>
+
+
+<div class="doc doc-object doc-function">
+
+
+
+<h3 id="gentropy.datasource.gnomad.ld.GnomADLDMatrix.get_numpy_matrix" class="doc doc-heading">
+          <code class="highlight language-python"><span class="n">get_numpy_matrix</span><span class="p">(</span><span class="n">locus_index</span><span class="p">:</span> <span class="n">DataFrame</span><span class="p">,</span> <span class="n">gnomad_ancestry</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s1">&#39;nfe&#39;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span></code>
+  
+  <span class="doc doc-labels">
+      <small class="doc doc-label doc-label-staticmethod"><code>staticmethod</code></small>
+  </span>
+
+<a href="#gentropy.datasource.gnomad.ld.GnomADLDMatrix.get_numpy_matrix" class="headerlink" title="Permanent link">&para;</a></h3>
+
+
+  <div class="doc doc-contents ">
+  
+      <p>Extract the LD block matrix for a locus.</p>
+
+
+
+  <p><span class="doc-section-title">Parameters:</span></p>
+  <table>
+    <thead>
+      <tr>
+        <th>Name</th>
+        <th>Type</th>
+        <th>Description</th>
+        <th>Default</th>
+      </tr>
+    </thead>
+    <tbody>
+        <tr class="doc-section-item">
+          <td><code>locus_index</code></td>
+          <td>
+                <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>hail matrix variant index table</p>
+            </div>
+          </td>
+          <td>
+              <em>required</em>
+          </td>
+        </tr>
+        <tr class="doc-section-item">
+          <td><code>gnomad_ancestry</code></td>
+          <td>
+                <code>str</code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>GnomAD major ancestry label eg. <code>nfe</code></p>
+            </div>
+          </td>
+          <td>
+                <code>&#39;nfe&#39;</code>
+          </td>
+        </tr>
+    </tbody>
+  </table>
+
+
+
+  <p><span class="doc-section-title">Returns:</span></p>
+  <table>
+    <thead>
+      <tr>
+        <th>Type</th>
+        <th>Description</th>
+      </tr>
+    </thead>
+    <tbody>
+        <tr class="doc-section-item">
+          <td>
+                <code><span title="numpy.ndarray">ndarray</span></code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>np.ndarray: LD block matrix for the locus</p>
+            </div>
+          </td>
+        </tr>
+    </tbody>
+  </table>
+
+          <details class="quote">
+            <summary>Source code in <code>src/gentropy/datasource/gnomad/ld.py</code></summary>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">489</span>
+<span class="normal">490</span>
+<span class="normal">491</span>
+<span class="normal">492</span>
+<span class="normal">493</span>
+<span class="normal">494</span>
+<span class="normal">495</span>
+<span class="normal">496</span>
+<span class="normal">497</span>
+<span class="normal">498</span>
+<span class="normal">499</span>
+<span class="normal">500</span>
+<span class="normal">501</span>
+<span class="normal">502</span>
+<span class="normal">503</span>
+<span class="normal">504</span>
+<span class="normal">505</span>
+<span class="normal">506</span>
+<span class="normal">507</span>
+<span class="normal">508</span>
+<span class="normal">509</span>
+<span class="normal">510</span>
+<span class="normal">511</span>
+<span class="normal">512</span>
+<span class="normal">513</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@staticmethod</span>
+<span class="k">def</span> <span class="nf">get_numpy_matrix</span><span class="p">(</span>
+    <span class="n">locus_index</span><span class="p">:</span> <span class="n">DataFrame</span><span class="p">,</span>
+    <span class="n">gnomad_ancestry</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;nfe&quot;</span><span class="p">,</span>
+<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;Extract the LD block matrix for a locus.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        locus_index (DataFrame): hail matrix variant index table</span>
+<span class="sd">        gnomad_ancestry (str): GnomAD major ancestry label eg. `nfe`</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">        np.ndarray: LD block matrix for the locus</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">idx</span> <span class="o">=</span> <span class="p">[</span><span class="n">row</span><span class="p">[</span><span class="s2">&quot;idx&quot;</span><span class="p">]</span> <span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">locus_index</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">&quot;idx&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()]</span>
+
+    <span class="n">half_matrix</span> <span class="o">=</span> <span class="p">(</span>
+        <span class="n">BlockMatrix</span><span class="o">.</span><span class="n">read</span><span class="p">(</span>
+            <span class="n">GnomADLDMatrix</span><span class="o">.</span><span class="n">ld_matrix_template</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">POP</span><span class="o">=</span><span class="n">gnomad_ancestry</span><span class="p">)</span>
+        <span class="p">)</span>
+        <span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">idx</span><span class="p">,</span> <span class="n">idx</span><span class="p">)</span>
+        <span class="o">.</span><span class="n">to_numpy</span><span class="p">()</span>
+    <span class="p">)</span>
+
+    <span class="k">return</span> <span class="p">(</span><span class="n">half_matrix</span> <span class="o">+</span> <span class="n">half_matrix</span><span class="o">.</span><span class="n">T</span><span class="p">)</span> <span class="o">-</span> <span class="n">np</span><span class="o">.</span><span class="n">diag</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">diag</span><span class="p">(</span><span class="n">half_matrix</span><span class="p">))</span>
+</code></pre></div></td></tr></table></div>
+          </details>
+  </div>
+
+</div>
+
+
 
   </div>
 
diff --git a/python_api/datasources/gnomad/gnomad_variants/index.html b/python_api/datasources/gnomad/gnomad_variants/index.html
index 868849117..f0b42112b 100644
--- a/python_api/datasources/gnomad/gnomad_variants/index.html
+++ b/python_api/datasources/gnomad/gnomad_variants/index.html
@@ -2259,6 +2259,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../methods/susie_inf/" class="md-nav__link">
         
@@ -2290,7 +2332,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2306,14 +2348,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -2926,7 +2968,7 @@ <h2 id="gentropy.datasource.gnomad.variants.GnomADVariants" class="doc doc-headi
 
 
 
-  <p><strong>Attributes:</strong></p>
+  <p><span class="doc-section-title">Attributes:</span></p>
   <table>
     <thead>
       <tr>
@@ -2936,7 +2978,7 @@ <h2 id="gentropy.datasource.gnomad.variants.GnomADVariants" class="doc doc-headi
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code><span title="gentropy.datasource.gnomad.variants.GnomADVariants.gnomad_genomes">gnomad_genomes</span></code></td>
           <td>
                 <code>str</code>
@@ -2947,7 +2989,7 @@ <h2 id="gentropy.datasource.gnomad.variants.GnomADVariants" class="doc doc-headi
             </div>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code><span title="gentropy.datasource.gnomad.variants.GnomADVariants.chain_hail_38_37">chain_hail_38_37</span></code></td>
           <td>
                 <code>str</code>
@@ -2958,7 +3000,7 @@ <h2 id="gentropy.datasource.gnomad.variants.GnomADVariants" class="doc doc-headi
             </div>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code><span title="gentropy.datasource.gnomad.variants.GnomADVariants.populations">populations</span></code></td>
           <td>
                 <code>list[str]</code>
@@ -2974,8 +3016,7 @@ <h2 id="gentropy.datasource.gnomad.variants.GnomADVariants" class="doc doc-headi
 
             <details class="quote">
               <summary>Source code in <code>src/gentropy/datasource/gnomad/variants.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 15</span>
-<span class="normal"> 16</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 16</span>
 <span class="normal"> 17</span>
 <span class="normal"> 18</span>
 <span class="normal"> 19</span>
@@ -3092,43 +3133,7 @@ <h2 id="gentropy.datasource.gnomad.variants.GnomADVariants" class="doc doc-headi
 <span class="normal">130</span>
 <span class="normal">131</span>
 <span class="normal">132</span>
-<span class="normal">133</span>
-<span class="normal">134</span>
-<span class="normal">135</span>
-<span class="normal">136</span>
-<span class="normal">137</span>
-<span class="normal">138</span>
-<span class="normal">139</span>
-<span class="normal">140</span>
-<span class="normal">141</span>
-<span class="normal">142</span>
-<span class="normal">143</span>
-<span class="normal">144</span>
-<span class="normal">145</span>
-<span class="normal">146</span>
-<span class="normal">147</span>
-<span class="normal">148</span>
-<span class="normal">149</span>
-<span class="normal">150</span>
-<span class="normal">151</span>
-<span class="normal">152</span>
-<span class="normal">153</span>
-<span class="normal">154</span>
-<span class="normal">155</span>
-<span class="normal">156</span>
-<span class="normal">157</span>
-<span class="normal">158</span>
-<span class="normal">159</span>
-<span class="normal">160</span>
-<span class="normal">161</span>
-<span class="normal">162</span>
-<span class="normal">163</span>
-<span class="normal">164</span>
-<span class="normal">165</span>
-<span class="normal">166</span>
-<span class="normal">167</span>
-<span class="normal">168</span>
-<span class="normal">169</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@dataclass</span>
+<span class="normal">133</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@dataclass</span>
 <span class="k">class</span> <span class="nc">GnomADVariants</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;GnomAD variants included in the GnomAD genomes dataset.</span>
 
@@ -3155,29 +3160,6 @@ <h2 id="gentropy.datasource.gnomad.variants.GnomADVariants" class="doc doc-headi
         <span class="p">]</span>
     <span class="p">)</span>
 
-    <span class="nd">@staticmethod</span>
-    <span class="k">def</span> <span class="nf">_convert_gnomad_position_to_ensembl_hail</span><span class="p">(</span>
-        <span class="n">position</span><span class="p">:</span> <span class="n">Int32Expression</span><span class="p">,</span>
-        <span class="n">reference</span><span class="p">:</span> <span class="n">StringExpression</span><span class="p">,</span>
-        <span class="n">alternate</span><span class="p">:</span> <span class="n">StringExpression</span><span class="p">,</span>
-    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Int32Expression</span><span class="p">:</span>
-<span class="w">        </span><span class="sd">&quot;&quot;&quot;Convert GnomAD variant position to Ensembl variant position in hail table.</span>
-
-<span class="sd">        For indels (the reference or alternate allele is longer than 1), then adding 1 to the position, for SNPs, the position is unchanged.</span>
-<span class="sd">        More info about the problem: https://www.biostars.org/p/84686/</span>
-
-<span class="sd">        Args:</span>
-<span class="sd">            position (Int32Expression): Position of the variant in the GnomAD genome.</span>
-<span class="sd">            reference (StringExpression): The reference allele.</span>
-<span class="sd">            alternate (StringExpression): The alternate allele</span>
-
-<span class="sd">        Returns:</span>
-<span class="sd">            Int32Expression: The position of the variant according to Ensembl genome.</span>
-<span class="sd">        &quot;&quot;&quot;</span>
-        <span class="k">return</span> <span class="n">hl</span><span class="o">.</span><span class="n">if_else</span><span class="p">(</span>
-            <span class="p">(</span><span class="n">reference</span><span class="o">.</span><span class="n">length</span><span class="p">()</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">)</span> <span class="o">|</span> <span class="p">(</span><span class="n">alternate</span><span class="o">.</span><span class="n">length</span><span class="p">()</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">),</span> <span class="n">position</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="n">position</span>
-        <span class="p">)</span>
-
     <span class="k">def</span> <span class="nf">as_variant_annotation</span><span class="p">(</span><span class="bp">self</span><span class="p">:</span> <span class="n">GnomADVariants</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">VariantAnnotation</span><span class="p">:</span>
 <span class="w">        </span><span class="sd">&quot;&quot;&quot;Generate variant annotation dataset from gnomAD.</span>
 
@@ -3209,7 +3191,7 @@ <h2 id="gentropy.datasource.gnomad.variants.GnomADVariants" class="doc doc-headi
         <span class="k">return</span> <span class="n">VariantAnnotation</span><span class="p">(</span>
             <span class="n">_df</span><span class="o">=</span><span class="p">(</span>
                 <span class="n">ht</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
-                    <span class="n">gnomadVariantId</span><span class="o">=</span><span class="n">hl</span><span class="o">.</span><span class="n">str</span><span class="p">(</span><span class="s2">&quot;-&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
+                    <span class="n">variantId</span><span class="o">=</span><span class="n">hl</span><span class="o">.</span><span class="n">str</span><span class="p">(</span><span class="s2">&quot;_&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
                         <span class="p">[</span>
                             <span class="n">ht</span><span class="o">.</span><span class="n">locus</span><span class="o">.</span><span class="n">contig</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;chr&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">),</span>
                             <span class="n">hl</span><span class="o">.</span><span class="n">str</span><span class="p">(</span><span class="n">ht</span><span class="o">.</span><span class="n">locus</span><span class="o">.</span><span class="n">position</span><span class="p">),</span>
@@ -3218,21 +3200,7 @@ <h2 id="gentropy.datasource.gnomad.variants.GnomADVariants" class="doc doc-headi
                         <span class="p">]</span>
                     <span class="p">),</span>
                     <span class="n">chromosome</span><span class="o">=</span><span class="n">ht</span><span class="o">.</span><span class="n">locus</span><span class="o">.</span><span class="n">contig</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;chr&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">),</span>
-                    <span class="n">position</span><span class="o">=</span><span class="n">GnomADVariants</span><span class="o">.</span><span class="n">_convert_gnomad_position_to_ensembl_hail</span><span class="p">(</span>
-                        <span class="n">ht</span><span class="o">.</span><span class="n">locus</span><span class="o">.</span><span class="n">position</span><span class="p">,</span> <span class="n">ht</span><span class="o">.</span><span class="n">alleles</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">ht</span><span class="o">.</span><span class="n">alleles</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
-                    <span class="p">),</span>
-                    <span class="n">variantId</span><span class="o">=</span><span class="n">hl</span><span class="o">.</span><span class="n">str</span><span class="p">(</span><span class="s2">&quot;_&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
-                        <span class="p">[</span>
-                            <span class="n">ht</span><span class="o">.</span><span class="n">locus</span><span class="o">.</span><span class="n">contig</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;chr&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">),</span>
-                            <span class="n">hl</span><span class="o">.</span><span class="n">str</span><span class="p">(</span>
-                                <span class="n">GnomADVariants</span><span class="o">.</span><span class="n">_convert_gnomad_position_to_ensembl_hail</span><span class="p">(</span>
-                                    <span class="n">ht</span><span class="o">.</span><span class="n">locus</span><span class="o">.</span><span class="n">position</span><span class="p">,</span> <span class="n">ht</span><span class="o">.</span><span class="n">alleles</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">ht</span><span class="o">.</span><span class="n">alleles</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
-                                <span class="p">)</span>
-                            <span class="p">),</span>
-                            <span class="n">ht</span><span class="o">.</span><span class="n">alleles</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span>
-                            <span class="n">ht</span><span class="o">.</span><span class="n">alleles</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span>
-                        <span class="p">]</span>
-                    <span class="p">),</span>
+                    <span class="n">position</span><span class="o">=</span><span class="n">ht</span><span class="o">.</span><span class="n">locus</span><span class="o">.</span><span class="n">position</span><span class="p">,</span>
                     <span class="n">chromosomeB37</span><span class="o">=</span><span class="n">ht</span><span class="o">.</span><span class="n">locus_GRCh37</span><span class="o">.</span><span class="n">contig</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;chr&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">),</span>
                     <span class="n">positionB37</span><span class="o">=</span><span class="n">ht</span><span class="o">.</span><span class="n">locus_GRCh37</span><span class="o">.</span><span class="n">position</span><span class="p">,</span>
                     <span class="n">referenceAllele</span><span class="o">=</span><span class="n">ht</span><span class="o">.</span><span class="n">alleles</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span>
@@ -3321,7 +3289,7 @@ <h3 id="gentropy.datasource.gnomad.variants.GnomADVariants.as_variant_annotation
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3330,7 +3298,7 @@ <h3 id="gentropy.datasource.gnomad.variants.GnomADVariants.as_variant_annotation
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>VariantAnnotation</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.variant_annotation.VariantAnnotation" href="../../../datasets/variant_annotation/#gentropy.dataset.variant_annotation.VariantAnnotation">VariantAnnotation</a></code>
           </td>
@@ -3345,7 +3313,29 @@ <h3 id="gentropy.datasource.gnomad.variants.GnomADVariants.as_variant_annotation
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/datasource/gnomad/variants.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 65</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 43</span>
+<span class="normal"> 44</span>
+<span class="normal"> 45</span>
+<span class="normal"> 46</span>
+<span class="normal"> 47</span>
+<span class="normal"> 48</span>
+<span class="normal"> 49</span>
+<span class="normal"> 50</span>
+<span class="normal"> 51</span>
+<span class="normal"> 52</span>
+<span class="normal"> 53</span>
+<span class="normal"> 54</span>
+<span class="normal"> 55</span>
+<span class="normal"> 56</span>
+<span class="normal"> 57</span>
+<span class="normal"> 58</span>
+<span class="normal"> 59</span>
+<span class="normal"> 60</span>
+<span class="normal"> 61</span>
+<span class="normal"> 62</span>
+<span class="normal"> 63</span>
+<span class="normal"> 64</span>
+<span class="normal"> 65</span>
 <span class="normal"> 66</span>
 <span class="normal"> 67</span>
 <span class="normal"> 68</span>
@@ -3413,43 +3403,7 @@ <h3 id="gentropy.datasource.gnomad.variants.GnomADVariants.as_variant_annotation
 <span class="normal">130</span>
 <span class="normal">131</span>
 <span class="normal">132</span>
-<span class="normal">133</span>
-<span class="normal">134</span>
-<span class="normal">135</span>
-<span class="normal">136</span>
-<span class="normal">137</span>
-<span class="normal">138</span>
-<span class="normal">139</span>
-<span class="normal">140</span>
-<span class="normal">141</span>
-<span class="normal">142</span>
-<span class="normal">143</span>
-<span class="normal">144</span>
-<span class="normal">145</span>
-<span class="normal">146</span>
-<span class="normal">147</span>
-<span class="normal">148</span>
-<span class="normal">149</span>
-<span class="normal">150</span>
-<span class="normal">151</span>
-<span class="normal">152</span>
-<span class="normal">153</span>
-<span class="normal">154</span>
-<span class="normal">155</span>
-<span class="normal">156</span>
-<span class="normal">157</span>
-<span class="normal">158</span>
-<span class="normal">159</span>
-<span class="normal">160</span>
-<span class="normal">161</span>
-<span class="normal">162</span>
-<span class="normal">163</span>
-<span class="normal">164</span>
-<span class="normal">165</span>
-<span class="normal">166</span>
-<span class="normal">167</span>
-<span class="normal">168</span>
-<span class="normal">169</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">as_variant_annotation</span><span class="p">(</span><span class="bp">self</span><span class="p">:</span> <span class="n">GnomADVariants</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">VariantAnnotation</span><span class="p">:</span>
+<span class="normal">133</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">as_variant_annotation</span><span class="p">(</span><span class="bp">self</span><span class="p">:</span> <span class="n">GnomADVariants</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">VariantAnnotation</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Generate variant annotation dataset from gnomAD.</span>
 
 <span class="sd">    Some relevant modifications to the original dataset are:</span>
@@ -3480,7 +3434,7 @@ <h3 id="gentropy.datasource.gnomad.variants.GnomADVariants.as_variant_annotation
     <span class="k">return</span> <span class="n">VariantAnnotation</span><span class="p">(</span>
         <span class="n">_df</span><span class="o">=</span><span class="p">(</span>
             <span class="n">ht</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
-                <span class="n">gnomadVariantId</span><span class="o">=</span><span class="n">hl</span><span class="o">.</span><span class="n">str</span><span class="p">(</span><span class="s2">&quot;-&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
+                <span class="n">variantId</span><span class="o">=</span><span class="n">hl</span><span class="o">.</span><span class="n">str</span><span class="p">(</span><span class="s2">&quot;_&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
                     <span class="p">[</span>
                         <span class="n">ht</span><span class="o">.</span><span class="n">locus</span><span class="o">.</span><span class="n">contig</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;chr&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">),</span>
                         <span class="n">hl</span><span class="o">.</span><span class="n">str</span><span class="p">(</span><span class="n">ht</span><span class="o">.</span><span class="n">locus</span><span class="o">.</span><span class="n">position</span><span class="p">),</span>
@@ -3489,21 +3443,7 @@ <h3 id="gentropy.datasource.gnomad.variants.GnomADVariants.as_variant_annotation
                     <span class="p">]</span>
                 <span class="p">),</span>
                 <span class="n">chromosome</span><span class="o">=</span><span class="n">ht</span><span class="o">.</span><span class="n">locus</span><span class="o">.</span><span class="n">contig</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;chr&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">),</span>
-                <span class="n">position</span><span class="o">=</span><span class="n">GnomADVariants</span><span class="o">.</span><span class="n">_convert_gnomad_position_to_ensembl_hail</span><span class="p">(</span>
-                    <span class="n">ht</span><span class="o">.</span><span class="n">locus</span><span class="o">.</span><span class="n">position</span><span class="p">,</span> <span class="n">ht</span><span class="o">.</span><span class="n">alleles</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">ht</span><span class="o">.</span><span class="n">alleles</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
-                <span class="p">),</span>
-                <span class="n">variantId</span><span class="o">=</span><span class="n">hl</span><span class="o">.</span><span class="n">str</span><span class="p">(</span><span class="s2">&quot;_&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
-                    <span class="p">[</span>
-                        <span class="n">ht</span><span class="o">.</span><span class="n">locus</span><span class="o">.</span><span class="n">contig</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;chr&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">),</span>
-                        <span class="n">hl</span><span class="o">.</span><span class="n">str</span><span class="p">(</span>
-                            <span class="n">GnomADVariants</span><span class="o">.</span><span class="n">_convert_gnomad_position_to_ensembl_hail</span><span class="p">(</span>
-                                <span class="n">ht</span><span class="o">.</span><span class="n">locus</span><span class="o">.</span><span class="n">position</span><span class="p">,</span> <span class="n">ht</span><span class="o">.</span><span class="n">alleles</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">ht</span><span class="o">.</span><span class="n">alleles</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
-                            <span class="p">)</span>
-                        <span class="p">),</span>
-                        <span class="n">ht</span><span class="o">.</span><span class="n">alleles</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span>
-                        <span class="n">ht</span><span class="o">.</span><span class="n">alleles</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span>
-                    <span class="p">]</span>
-                <span class="p">),</span>
+                <span class="n">position</span><span class="o">=</span><span class="n">ht</span><span class="o">.</span><span class="n">locus</span><span class="o">.</span><span class="n">position</span><span class="p">,</span>
                 <span class="n">chromosomeB37</span><span class="o">=</span><span class="n">ht</span><span class="o">.</span><span class="n">locus_GRCh37</span><span class="o">.</span><span class="n">contig</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;chr&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">),</span>
                 <span class="n">positionB37</span><span class="o">=</span><span class="n">ht</span><span class="o">.</span><span class="n">locus_GRCh37</span><span class="o">.</span><span class="n">position</span><span class="p">,</span>
                 <span class="n">referenceAllele</span><span class="o">=</span><span class="n">ht</span><span class="o">.</span><span class="n">alleles</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span>
diff --git a/python_api/datasources/gwas_catalog/_gwas_catalog/index.html b/python_api/datasources/gwas_catalog/_gwas_catalog/index.html
index 69360a8fa..48a9178b8 100644
--- a/python_api/datasources/gwas_catalog/_gwas_catalog/index.html
+++ b/python_api/datasources/gwas_catalog/_gwas_catalog/index.html
@@ -2194,6 +2194,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../methods/susie_inf/" class="md-nav__link">
         
@@ -2225,7 +2267,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2241,14 +2283,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -2856,7 +2898,15 @@ <h1>GWAS Catalog</h1>
         <span class="label"> Contributors </span>
       </li>
       <li class="contributors-holder">
-        <ul class="contributors" data-bi-name="contributors"></ul>
+        <ul class="contributors" data-bi-name="contributors"><li>
+            <a
+              href="https://github.com/d0choa"
+              title="d0choa"
+              data-bi-name="contributorprofile"
+              target="_blank"
+              ><img src="https://avatars.githubusercontent.com/u/5097586?v=4" alt="d0choa"
+            /></a>
+          </li></ul>
       </li>
     </ul>
   </small>
diff --git a/python_api/datasources/gwas_catalog/associations/index.html b/python_api/datasources/gwas_catalog/associations/index.html
index 44197e1bc..449e2c5ed 100644
--- a/python_api/datasources/gwas_catalog/associations/index.html
+++ b/python_api/datasources/gwas_catalog/associations/index.html
@@ -1748,6 +1748,15 @@
     <nav class="md-nav" aria-label="GWASCatalogCuratedAssociationsParser">
       <ul class="md-nav__list">
         
+          <li class="md-nav__item">
+  <a href="#gentropy.datasource.gwas_catalog.associations.GWASCatalogCuratedAssociationsParser.convert_gnomad_position_to_ensembl" class="md-nav__link">
+    <span class="md-ellipsis">
+      convert_gnomad_position_to_ensembl
+    </span>
+  </a>
+  
+</li>
+        
           <li class="md-nav__item">
   <a href="#gentropy.datasource.gwas_catalog.associations.GWASCatalogCuratedAssociationsParser.from_source" class="md-nav__link">
     <span class="md-ellipsis">
@@ -2301,6 +2310,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../methods/susie_inf/" class="md-nav__link">
         
@@ -2332,7 +2383,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2348,14 +2399,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -2912,6 +2963,15 @@
     <nav class="md-nav" aria-label="GWASCatalogCuratedAssociationsParser">
       <ul class="md-nav__list">
         
+          <li class="md-nav__item">
+  <a href="#gentropy.datasource.gwas_catalog.associations.GWASCatalogCuratedAssociationsParser.convert_gnomad_position_to_ensembl" class="md-nav__link">
+    <span class="md-ellipsis">
+      convert_gnomad_position_to_ensembl
+    </span>
+  </a>
+  
+</li>
+        
           <li class="md-nav__item">
   <a href="#gentropy.datasource.gwas_catalog.associations.GWASCatalogCuratedAssociationsParser.from_source" class="md-nav__link">
     <span class="md-ellipsis">
@@ -3010,8 +3070,7 @@ <h2 id="gentropy.datasource.gwas_catalog.associations.GWASCatalogCuratedAssociat
 
             <details class="quote">
               <summary>Source code in <code>src/gentropy/datasource/gwas_catalog/associations.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">  30</span>
-<span class="normal">  31</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">  31</span>
 <span class="normal">  32</span>
 <span class="normal">  33</span>
 <span class="normal">  34</span>
@@ -4047,10 +4106,87 @@ <h2 id="gentropy.datasource.gwas_catalog.associations.GWASCatalogCuratedAssociat
 <span class="normal">1064</span>
 <span class="normal">1065</span>
 <span class="normal">1066</span>
-<span class="normal">1067</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@dataclass</span>
+<span class="normal">1067</span>
+<span class="normal">1068</span>
+<span class="normal">1069</span>
+<span class="normal">1070</span>
+<span class="normal">1071</span>
+<span class="normal">1072</span>
+<span class="normal">1073</span>
+<span class="normal">1074</span>
+<span class="normal">1075</span>
+<span class="normal">1076</span>
+<span class="normal">1077</span>
+<span class="normal">1078</span>
+<span class="normal">1079</span>
+<span class="normal">1080</span>
+<span class="normal">1081</span>
+<span class="normal">1082</span>
+<span class="normal">1083</span>
+<span class="normal">1084</span>
+<span class="normal">1085</span>
+<span class="normal">1086</span>
+<span class="normal">1087</span>
+<span class="normal">1088</span>
+<span class="normal">1089</span>
+<span class="normal">1090</span>
+<span class="normal">1091</span>
+<span class="normal">1092</span>
+<span class="normal">1093</span>
+<span class="normal">1094</span>
+<span class="normal">1095</span>
+<span class="normal">1096</span>
+<span class="normal">1097</span>
+<span class="normal">1098</span>
+<span class="normal">1099</span>
+<span class="normal">1100</span>
+<span class="normal">1101</span>
+<span class="normal">1102</span>
+<span class="normal">1103</span>
+<span class="normal">1104</span>
+<span class="normal">1105</span>
+<span class="normal">1106</span>
+<span class="normal">1107</span>
+<span class="normal">1108</span>
+<span class="normal">1109</span>
+<span class="normal">1110</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@dataclass</span>
 <span class="k">class</span> <span class="nc">GWASCatalogCuratedAssociationsParser</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;GWAS Catalog curated associations parser.&quot;&quot;&quot;</span>
 
+    <span class="nd">@staticmethod</span>
+    <span class="k">def</span> <span class="nf">convert_gnomad_position_to_ensembl</span><span class="p">(</span>
+        <span class="n">position</span><span class="p">:</span> <span class="n">Column</span><span class="p">,</span> <span class="n">reference</span><span class="p">:</span> <span class="n">Column</span><span class="p">,</span> <span class="n">alternate</span><span class="p">:</span> <span class="n">Column</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Column</span><span class="p">:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Convert GnomAD variant position to Ensembl variant position.</span>
+
+<span class="sd">        For indels (the reference or alternate allele is longer than 1), then adding 1 to the position, for SNPs,</span>
+<span class="sd">        the position is unchanged. More info about the problem: https://www.biostars.org/p/84686/</span>
+
+<span class="sd">        Args:</span>
+<span class="sd">            position (Column): Position of the variant in GnomAD&#39;s coordinates system.</span>
+<span class="sd">            reference (Column): The reference allele in GnomAD&#39;s coordinates system.</span>
+<span class="sd">            alternate (Column): The alternate allele in GnomAD&#39;s coordinates system.</span>
+
+<span class="sd">        Returns:</span>
+<span class="sd">            Column: The position of the variant in the Ensembl genome.</span>
+
+<span class="sd">        Examples:</span>
+<span class="sd">            &gt;&gt;&gt; d = [(1, &quot;A&quot;, &quot;C&quot;), (2, &quot;AA&quot;, &quot;C&quot;), (3, &quot;A&quot;, &quot;AA&quot;)]</span>
+<span class="sd">            &gt;&gt;&gt; df = spark.createDataFrame(d).toDF(&quot;position&quot;, &quot;reference&quot;, &quot;alternate&quot;)</span>
+<span class="sd">            &gt;&gt;&gt; df.withColumn(&quot;new_position&quot;, GWASCatalogCuratedAssociationsParser.convert_gnomad_position_to_ensembl(f.col(&quot;position&quot;), f.col(&quot;reference&quot;), f.col(&quot;alternate&quot;))).show()</span>
+<span class="sd">            +--------+---------+---------+------------+</span>
+<span class="sd">            |position|reference|alternate|new_position|</span>
+<span class="sd">            +--------+---------+---------+------------+</span>
+<span class="sd">            |       1|        A|        C|           1|</span>
+<span class="sd">            |       2|       AA|        C|           3|</span>
+<span class="sd">            |       3|        A|       AA|           4|</span>
+<span class="sd">            +--------+---------+---------+------------+</span>
+<span class="sd">            &lt;BLANKLINE&gt;</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="n">f</span><span class="o">.</span><span class="n">when</span><span class="p">(</span>
+            <span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">length</span><span class="p">(</span><span class="n">reference</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">)</span> <span class="o">|</span> <span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">length</span><span class="p">(</span><span class="n">alternate</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">),</span> <span class="n">position</span> <span class="o">+</span> <span class="mi">1</span>
+        <span class="p">)</span><span class="o">.</span><span class="n">otherwise</span><span class="p">(</span><span class="n">position</span><span class="p">)</span>
+
     <span class="nd">@staticmethod</span>
     <span class="k">def</span> <span class="nf">_parse_pvalue</span><span class="p">(</span><span class="n">pvalue</span><span class="p">:</span> <span class="n">Column</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">tuple</span><span class="p">[</span><span class="n">Column</span><span class="p">,</span> <span class="n">Column</span><span class="p">]:</span>
 <span class="w">        </span><span class="sd">&quot;&quot;&quot;Parse p-value column.</span>
@@ -4198,7 +4334,8 @@ <h2 id="gentropy.datasource.gwas_catalog.associations.GWASCatalogCuratedAssociat
         <span class="n">gwas_associations_subset</span> <span class="o">=</span> <span class="n">gwas_associations</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
             <span class="s2">&quot;studyLocusId&quot;</span><span class="p">,</span>
             <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;CHR_ID&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;chromosome&quot;</span><span class="p">),</span>
-            <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;CHR_POS&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">cast</span><span class="p">(</span><span class="n">IntegerType</span><span class="p">())</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;position&quot;</span><span class="p">),</span>
+            <span class="c1"># The positions from GWAS Catalog are from ensembl that causes discrepancy for indels:</span>
+            <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;CHR_POS&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">cast</span><span class="p">(</span><span class="n">IntegerType</span><span class="p">())</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;ensemblPosition&quot;</span><span class="p">),</span>
             <span class="c1"># List of all SNPs associated with the variant</span>
             <span class="n">GWASCatalogCuratedAssociationsParser</span><span class="o">.</span><span class="n">_collect_rsids</span><span class="p">(</span>
                 <span class="n">f</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;SNPS&quot;</span><span class="p">),</span> <span class="s2">&quot;; &quot;</span><span class="p">)</span><span class="o">.</span><span class="n">getItem</span><span class="p">(</span><span class="mi">0</span><span class="p">),</span>
@@ -4214,6 +4351,11 @@ <h2 id="gentropy.datasource.gwas_catalog.associations.GWASCatalogCuratedAssociat
         <span class="n">va_subset</span> <span class="o">=</span> <span class="n">variant_annotation</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
             <span class="s2">&quot;variantId&quot;</span><span class="p">,</span>
             <span class="s2">&quot;chromosome&quot;</span><span class="p">,</span>
+            <span class="c1"># Calculate the position in Ensembl coordinates for indels:</span>
+            <span class="n">GWASCatalogCuratedAssociationsParser</span><span class="o">.</span><span class="n">convert_gnomad_position_to_ensembl</span><span class="p">(</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;position&quot;</span><span class="p">),</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;referenceAllele&quot;</span><span class="p">),</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;alternateAllele&quot;</span><span class="p">)</span>
+            <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;ensemblPosition&quot;</span><span class="p">),</span>
+            <span class="c1"># Keeping GnomAD position:</span>
             <span class="s2">&quot;position&quot;</span><span class="p">,</span>
             <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;rsIds&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;rsIdsGnomad&quot;</span><span class="p">),</span>
             <span class="s2">&quot;referenceAllele&quot;</span><span class="p">,</span>
@@ -4222,9 +4364,11 @@ <h2 id="gentropy.datasource.gwas_catalog.associations.GWASCatalogCuratedAssociat
             <span class="n">variant_annotation</span><span class="o">.</span><span class="n">max_maf</span><span class="p">()</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;maxMaf&quot;</span><span class="p">),</span>
         <span class="p">)</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
             <span class="n">f</span><span class="o">.</span><span class="n">broadcast</span><span class="p">(</span>
-                <span class="n">gwas_associations_subset</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">&quot;chromosome&quot;</span><span class="p">,</span> <span class="s2">&quot;position&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">distinct</span><span class="p">()</span>
+                <span class="n">gwas_associations_subset</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+                    <span class="s2">&quot;chromosome&quot;</span><span class="p">,</span> <span class="s2">&quot;ensemblPosition&quot;</span>
+                <span class="p">)</span><span class="o">.</span><span class="n">distinct</span><span class="p">()</span>
             <span class="p">),</span>
-            <span class="n">on</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;chromosome&quot;</span><span class="p">,</span> <span class="s2">&quot;position&quot;</span><span class="p">],</span>
+            <span class="n">on</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;chromosome&quot;</span><span class="p">,</span> <span class="s2">&quot;ensemblPosition&quot;</span><span class="p">],</span>
             <span class="n">how</span><span class="o">=</span><span class="s2">&quot;inner&quot;</span><span class="p">,</span>
         <span class="p">)</span>
 
@@ -4233,7 +4377,7 @@ <h2 id="gentropy.datasource.gwas_catalog.associations.GWASCatalogCuratedAssociat
         <span class="n">filtered_associations</span> <span class="o">=</span> <span class="p">(</span>
             <span class="n">gwas_associations_subset</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
                 <span class="n">f</span><span class="o">.</span><span class="n">broadcast</span><span class="p">(</span><span class="n">va_subset</span><span class="p">),</span>
-                <span class="n">on</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;chromosome&quot;</span><span class="p">,</span> <span class="s2">&quot;position&quot;</span><span class="p">],</span>
+                <span class="n">on</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;chromosome&quot;</span><span class="p">,</span> <span class="s2">&quot;ensemblPosition&quot;</span><span class="p">],</span>
                 <span class="n">how</span><span class="o">=</span><span class="s2">&quot;left&quot;</span><span class="p">,</span>
             <span class="p">)</span>
             <span class="o">.</span><span class="n">withColumn</span><span class="p">(</span>
@@ -5101,6 +5245,197 @@ <h2 id="gentropy.datasource.gwas_catalog.associations.GWASCatalogCuratedAssociat
 
 
 
+<div class="doc doc-object doc-function">
+
+
+
+<h3 id="gentropy.datasource.gwas_catalog.associations.GWASCatalogCuratedAssociationsParser.convert_gnomad_position_to_ensembl" class="doc doc-heading">
+          <code class="highlight language-python"><span class="n">convert_gnomad_position_to_ensembl</span><span class="p">(</span><span class="n">position</span><span class="p">:</span> <span class="n">Column</span><span class="p">,</span> <span class="n">reference</span><span class="p">:</span> <span class="n">Column</span><span class="p">,</span> <span class="n">alternate</span><span class="p">:</span> <span class="n">Column</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Column</span></code>
+  
+  <span class="doc doc-labels">
+      <small class="doc doc-label doc-label-staticmethod"><code>staticmethod</code></small>
+  </span>
+
+<a href="#gentropy.datasource.gwas_catalog.associations.GWASCatalogCuratedAssociationsParser.convert_gnomad_position_to_ensembl" class="headerlink" title="Permanent link">&para;</a></h3>
+
+
+  <div class="doc doc-contents ">
+  
+      <p>Convert GnomAD variant position to Ensembl variant position.</p>
+<p>For indels (the reference or alternate allele is longer than 1), then adding 1 to the position, for SNPs,
+the position is unchanged. More info about the problem: https://www.biostars.org/p/84686/</p>
+
+
+
+  <p><span class="doc-section-title">Parameters:</span></p>
+  <table>
+    <thead>
+      <tr>
+        <th>Name</th>
+        <th>Type</th>
+        <th>Description</th>
+        <th>Default</th>
+      </tr>
+    </thead>
+    <tbody>
+        <tr class="doc-section-item">
+          <td><code>position</code></td>
+          <td>
+                <code><span title="pyspark.sql.Column">Column</span></code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>Position of the variant in GnomAD's coordinates system.</p>
+            </div>
+          </td>
+          <td>
+              <em>required</em>
+          </td>
+        </tr>
+        <tr class="doc-section-item">
+          <td><code>reference</code></td>
+          <td>
+                <code><span title="pyspark.sql.Column">Column</span></code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>The reference allele in GnomAD's coordinates system.</p>
+            </div>
+          </td>
+          <td>
+              <em>required</em>
+          </td>
+        </tr>
+        <tr class="doc-section-item">
+          <td><code>alternate</code></td>
+          <td>
+                <code><span title="pyspark.sql.Column">Column</span></code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>The alternate allele in GnomAD's coordinates system.</p>
+            </div>
+          </td>
+          <td>
+              <em>required</em>
+          </td>
+        </tr>
+    </tbody>
+  </table>
+
+
+
+  <p><span class="doc-section-title">Returns:</span></p>
+  <table>
+    <thead>
+      <tr>
+<th>Name</th>        <th>Type</th>
+        <th>Description</th>
+      </tr>
+    </thead>
+    <tbody>
+        <tr class="doc-section-item">
+<td><code>Column</code></td>          <td>
+                <code><span title="pyspark.sql.Column">Column</span></code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>The position of the variant in the Ensembl genome.</p>
+            </div>
+          </td>
+        </tr>
+    </tbody>
+  </table>
+
+
+
+<p><span class="doc-section-title">Examples:</span></p>
+    <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="n">d</span> <span class="o">=</span> <span class="p">[(</span><span class="mi">1</span><span class="p">,</span> <span class="s2">&quot;A&quot;</span><span class="p">,</span> <span class="s2">&quot;C&quot;</span><span class="p">),</span> <span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="s2">&quot;AA&quot;</span><span class="p">,</span> <span class="s2">&quot;C&quot;</span><span class="p">),</span> <span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="s2">&quot;A&quot;</span><span class="p">,</span> <span class="s2">&quot;AA&quot;</span><span class="p">)]</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">df</span> <span class="o">=</span> <span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">(</span><span class="n">d</span><span class="p">)</span><span class="o">.</span><span class="n">toDF</span><span class="p">(</span><span class="s2">&quot;position&quot;</span><span class="p">,</span> <span class="s2">&quot;reference&quot;</span><span class="p">,</span> <span class="s2">&quot;alternate&quot;</span><span class="p">)</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">df</span><span class="o">.</span><span class="n">withColumn</span><span class="p">(</span><span class="s2">&quot;new_position&quot;</span><span class="p">,</span> <span class="n">GWASCatalogCuratedAssociationsParser</span><span class="o">.</span><span class="n">convert_gnomad_position_to_ensembl</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;position&quot;</span><span class="p">),</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;reference&quot;</span><span class="p">),</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;alternate&quot;</span><span class="p">)))</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
+<span class="go">+--------+---------+---------+------------+</span>
+<span class="go">|position|reference|alternate|new_position|</span>
+<span class="go">+--------+---------+---------+------------+</span>
+<span class="go">|       1|        A|        C|           1|</span>
+<span class="go">|       2|       AA|        C|           3|</span>
+<span class="go">|       3|        A|       AA|           4|</span>
+<span class="go">+--------+---------+---------+------------+</span>
+</code></pre></div>
+
+          <details class="quote">
+            <summary>Source code in <code>src/gentropy/datasource/gwas_catalog/associations.py</code></summary>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">35</span>
+<span class="normal">36</span>
+<span class="normal">37</span>
+<span class="normal">38</span>
+<span class="normal">39</span>
+<span class="normal">40</span>
+<span class="normal">41</span>
+<span class="normal">42</span>
+<span class="normal">43</span>
+<span class="normal">44</span>
+<span class="normal">45</span>
+<span class="normal">46</span>
+<span class="normal">47</span>
+<span class="normal">48</span>
+<span class="normal">49</span>
+<span class="normal">50</span>
+<span class="normal">51</span>
+<span class="normal">52</span>
+<span class="normal">53</span>
+<span class="normal">54</span>
+<span class="normal">55</span>
+<span class="normal">56</span>
+<span class="normal">57</span>
+<span class="normal">58</span>
+<span class="normal">59</span>
+<span class="normal">60</span>
+<span class="normal">61</span>
+<span class="normal">62</span>
+<span class="normal">63</span>
+<span class="normal">64</span>
+<span class="normal">65</span>
+<span class="normal">66</span>
+<span class="normal">67</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@staticmethod</span>
+<span class="k">def</span> <span class="nf">convert_gnomad_position_to_ensembl</span><span class="p">(</span>
+    <span class="n">position</span><span class="p">:</span> <span class="n">Column</span><span class="p">,</span> <span class="n">reference</span><span class="p">:</span> <span class="n">Column</span><span class="p">,</span> <span class="n">alternate</span><span class="p">:</span> <span class="n">Column</span>
+<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Column</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;Convert GnomAD variant position to Ensembl variant position.</span>
+
+<span class="sd">    For indels (the reference or alternate allele is longer than 1), then adding 1 to the position, for SNPs,</span>
+<span class="sd">    the position is unchanged. More info about the problem: https://www.biostars.org/p/84686/</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        position (Column): Position of the variant in GnomAD&#39;s coordinates system.</span>
+<span class="sd">        reference (Column): The reference allele in GnomAD&#39;s coordinates system.</span>
+<span class="sd">        alternate (Column): The alternate allele in GnomAD&#39;s coordinates system.</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">        Column: The position of the variant in the Ensembl genome.</span>
+
+<span class="sd">    Examples:</span>
+<span class="sd">        &gt;&gt;&gt; d = [(1, &quot;A&quot;, &quot;C&quot;), (2, &quot;AA&quot;, &quot;C&quot;), (3, &quot;A&quot;, &quot;AA&quot;)]</span>
+<span class="sd">        &gt;&gt;&gt; df = spark.createDataFrame(d).toDF(&quot;position&quot;, &quot;reference&quot;, &quot;alternate&quot;)</span>
+<span class="sd">        &gt;&gt;&gt; df.withColumn(&quot;new_position&quot;, GWASCatalogCuratedAssociationsParser.convert_gnomad_position_to_ensembl(f.col(&quot;position&quot;), f.col(&quot;reference&quot;), f.col(&quot;alternate&quot;))).show()</span>
+<span class="sd">        +--------+---------+---------+------------+</span>
+<span class="sd">        |position|reference|alternate|new_position|</span>
+<span class="sd">        +--------+---------+---------+------------+</span>
+<span class="sd">        |       1|        A|        C|           1|</span>
+<span class="sd">        |       2|       AA|        C|           3|</span>
+<span class="sd">        |       3|        A|       AA|           4|</span>
+<span class="sd">        +--------+---------+---------+------------+</span>
+<span class="sd">        &lt;BLANKLINE&gt;</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">return</span> <span class="n">f</span><span class="o">.</span><span class="n">when</span><span class="p">(</span>
+        <span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">length</span><span class="p">(</span><span class="n">reference</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">)</span> <span class="o">|</span> <span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">length</span><span class="p">(</span><span class="n">alternate</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">),</span> <span class="n">position</span> <span class="o">+</span> <span class="mi">1</span>
+    <span class="p">)</span><span class="o">.</span><span class="n">otherwise</span><span class="p">(</span><span class="n">position</span><span class="p">)</span>
+</code></pre></div></td></tr></table></div>
+          </details>
+  </div>
+
+</div>
+
+
 <div class="doc doc-object doc-function">
 
 
@@ -5123,7 +5458,7 @@ <h3 id="gentropy.datasource.gwas_catalog.associations.GWASCatalogCuratedAssociat
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -5134,7 +5469,7 @@ <h3 id="gentropy.datasource.gwas_catalog.associations.GWASCatalogCuratedAssociat
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>gwas_associations</code></td>
           <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
@@ -5148,7 +5483,7 @@ <h3 id="gentropy.datasource.gwas_catalog.associations.GWASCatalogCuratedAssociat
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>variant_annotation</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.variant_annotation.VariantAnnotation" href="../../../datasets/variant_annotation/#gentropy.dataset.variant_annotation.VariantAnnotation">VariantAnnotation</a></code>
@@ -5162,7 +5497,7 @@ <h3 id="gentropy.datasource.gwas_catalog.associations.GWASCatalogCuratedAssociat
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>pvalue_threshold</code></td>
           <td>
                 <code>float</code>
@@ -5181,7 +5516,7 @@ <h3 id="gentropy.datasource.gwas_catalog.associations.GWASCatalogCuratedAssociat
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -5190,7 +5525,7 @@ <h3 id="gentropy.datasource.gwas_catalog.associations.GWASCatalogCuratedAssociat
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StudyLocusGWASCatalog</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog" href="#gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog">StudyLocusGWASCatalog</a></code>
           </td>
@@ -5205,50 +5540,7 @@ <h3 id="gentropy.datasource.gwas_catalog.associations.GWASCatalogCuratedAssociat
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/datasource/gwas_catalog/associations.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 990</span>
-<span class="normal"> 991</span>
-<span class="normal"> 992</span>
-<span class="normal"> 993</span>
-<span class="normal"> 994</span>
-<span class="normal"> 995</span>
-<span class="normal"> 996</span>
-<span class="normal"> 997</span>
-<span class="normal"> 998</span>
-<span class="normal"> 999</span>
-<span class="normal">1000</span>
-<span class="normal">1001</span>
-<span class="normal">1002</span>
-<span class="normal">1003</span>
-<span class="normal">1004</span>
-<span class="normal">1005</span>
-<span class="normal">1006</span>
-<span class="normal">1007</span>
-<span class="normal">1008</span>
-<span class="normal">1009</span>
-<span class="normal">1010</span>
-<span class="normal">1011</span>
-<span class="normal">1012</span>
-<span class="normal">1013</span>
-<span class="normal">1014</span>
-<span class="normal">1015</span>
-<span class="normal">1016</span>
-<span class="normal">1017</span>
-<span class="normal">1018</span>
-<span class="normal">1019</span>
-<span class="normal">1020</span>
-<span class="normal">1021</span>
-<span class="normal">1022</span>
-<span class="normal">1023</span>
-<span class="normal">1024</span>
-<span class="normal">1025</span>
-<span class="normal">1026</span>
-<span class="normal">1027</span>
-<span class="normal">1028</span>
-<span class="normal">1029</span>
-<span class="normal">1030</span>
-<span class="normal">1031</span>
-<span class="normal">1032</span>
-<span class="normal">1033</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">1033</span>
 <span class="normal">1034</span>
 <span class="normal">1035</span>
 <span class="normal">1036</span>
@@ -5282,7 +5574,50 @@ <h3 id="gentropy.datasource.gwas_catalog.associations.GWASCatalogCuratedAssociat
 <span class="normal">1064</span>
 <span class="normal">1065</span>
 <span class="normal">1066</span>
-<span class="normal">1067</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@classmethod</span>
+<span class="normal">1067</span>
+<span class="normal">1068</span>
+<span class="normal">1069</span>
+<span class="normal">1070</span>
+<span class="normal">1071</span>
+<span class="normal">1072</span>
+<span class="normal">1073</span>
+<span class="normal">1074</span>
+<span class="normal">1075</span>
+<span class="normal">1076</span>
+<span class="normal">1077</span>
+<span class="normal">1078</span>
+<span class="normal">1079</span>
+<span class="normal">1080</span>
+<span class="normal">1081</span>
+<span class="normal">1082</span>
+<span class="normal">1083</span>
+<span class="normal">1084</span>
+<span class="normal">1085</span>
+<span class="normal">1086</span>
+<span class="normal">1087</span>
+<span class="normal">1088</span>
+<span class="normal">1089</span>
+<span class="normal">1090</span>
+<span class="normal">1091</span>
+<span class="normal">1092</span>
+<span class="normal">1093</span>
+<span class="normal">1094</span>
+<span class="normal">1095</span>
+<span class="normal">1096</span>
+<span class="normal">1097</span>
+<span class="normal">1098</span>
+<span class="normal">1099</span>
+<span class="normal">1100</span>
+<span class="normal">1101</span>
+<span class="normal">1102</span>
+<span class="normal">1103</span>
+<span class="normal">1104</span>
+<span class="normal">1105</span>
+<span class="normal">1106</span>
+<span class="normal">1107</span>
+<span class="normal">1108</span>
+<span class="normal">1109</span>
+<span class="normal">1110</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@classmethod</span>
 <span class="k">def</span> <span class="nf">from_source</span><span class="p">(</span>
     <span class="bp">cls</span><span class="p">:</span> <span class="nb">type</span><span class="p">[</span><span class="n">GWASCatalogCuratedAssociationsParser</span><span class="p">],</span>
     <span class="n">gwas_associations</span><span class="p">:</span> <span class="n">DataFrame</span><span class="p">,</span>
@@ -5401,50 +5736,7 @@ <h2 id="gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog" cla
 
             <details class="quote">
               <summary>Source code in <code>src/gentropy/datasource/gwas_catalog/associations.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">1070</span>
-<span class="normal">1071</span>
-<span class="normal">1072</span>
-<span class="normal">1073</span>
-<span class="normal">1074</span>
-<span class="normal">1075</span>
-<span class="normal">1076</span>
-<span class="normal">1077</span>
-<span class="normal">1078</span>
-<span class="normal">1079</span>
-<span class="normal">1080</span>
-<span class="normal">1081</span>
-<span class="normal">1082</span>
-<span class="normal">1083</span>
-<span class="normal">1084</span>
-<span class="normal">1085</span>
-<span class="normal">1086</span>
-<span class="normal">1087</span>
-<span class="normal">1088</span>
-<span class="normal">1089</span>
-<span class="normal">1090</span>
-<span class="normal">1091</span>
-<span class="normal">1092</span>
-<span class="normal">1093</span>
-<span class="normal">1094</span>
-<span class="normal">1095</span>
-<span class="normal">1096</span>
-<span class="normal">1097</span>
-<span class="normal">1098</span>
-<span class="normal">1099</span>
-<span class="normal">1100</span>
-<span class="normal">1101</span>
-<span class="normal">1102</span>
-<span class="normal">1103</span>
-<span class="normal">1104</span>
-<span class="normal">1105</span>
-<span class="normal">1106</span>
-<span class="normal">1107</span>
-<span class="normal">1108</span>
-<span class="normal">1109</span>
-<span class="normal">1110</span>
-<span class="normal">1111</span>
-<span class="normal">1112</span>
-<span class="normal">1113</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">1113</span>
 <span class="normal">1114</span>
 <span class="normal">1115</span>
 <span class="normal">1116</span>
@@ -5465,7 +5757,50 @@ <h2 id="gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog" cla
 <span class="normal">1131</span>
 <span class="normal">1132</span>
 <span class="normal">1133</span>
-<span class="normal">1134</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@dataclass</span>
+<span class="normal">1134</span>
+<span class="normal">1135</span>
+<span class="normal">1136</span>
+<span class="normal">1137</span>
+<span class="normal">1138</span>
+<span class="normal">1139</span>
+<span class="normal">1140</span>
+<span class="normal">1141</span>
+<span class="normal">1142</span>
+<span class="normal">1143</span>
+<span class="normal">1144</span>
+<span class="normal">1145</span>
+<span class="normal">1146</span>
+<span class="normal">1147</span>
+<span class="normal">1148</span>
+<span class="normal">1149</span>
+<span class="normal">1150</span>
+<span class="normal">1151</span>
+<span class="normal">1152</span>
+<span class="normal">1153</span>
+<span class="normal">1154</span>
+<span class="normal">1155</span>
+<span class="normal">1156</span>
+<span class="normal">1157</span>
+<span class="normal">1158</span>
+<span class="normal">1159</span>
+<span class="normal">1160</span>
+<span class="normal">1161</span>
+<span class="normal">1162</span>
+<span class="normal">1163</span>
+<span class="normal">1164</span>
+<span class="normal">1165</span>
+<span class="normal">1166</span>
+<span class="normal">1167</span>
+<span class="normal">1168</span>
+<span class="normal">1169</span>
+<span class="normal">1170</span>
+<span class="normal">1171</span>
+<span class="normal">1172</span>
+<span class="normal">1173</span>
+<span class="normal">1174</span>
+<span class="normal">1175</span>
+<span class="normal">1176</span>
+<span class="normal">1177</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@dataclass</span>
 <span class="k">class</span> <span class="nc">StudyLocusGWASCatalog</span><span class="p">(</span><span class="n">StudyLocus</span><span class="p">):</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Study locus Dataset for GWAS Catalog curated associations.</span>
 
@@ -5562,7 +5897,7 @@ <h3 id="gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog.appl
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -5573,7 +5908,7 @@ <h3 id="gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog.appl
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>inclusion_list</code></td>
           <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
@@ -5592,7 +5927,7 @@ <h3 id="gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog.appl
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -5601,7 +5936,7 @@ <h3 id="gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog.appl
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StudyLocusGWASCatalog</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog" href="#gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog">StudyLocusGWASCatalog</a></code>
           </td>
@@ -5616,21 +5951,21 @@ <h3 id="gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog.appl
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/datasource/gwas_catalog/associations.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">1120</span>
-<span class="normal">1121</span>
-<span class="normal">1122</span>
-<span class="normal">1123</span>
-<span class="normal">1124</span>
-<span class="normal">1125</span>
-<span class="normal">1126</span>
-<span class="normal">1127</span>
-<span class="normal">1128</span>
-<span class="normal">1129</span>
-<span class="normal">1130</span>
-<span class="normal">1131</span>
-<span class="normal">1132</span>
-<span class="normal">1133</span>
-<span class="normal">1134</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">apply_inclusion_list</span><span class="p">(</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">1163</span>
+<span class="normal">1164</span>
+<span class="normal">1165</span>
+<span class="normal">1166</span>
+<span class="normal">1167</span>
+<span class="normal">1168</span>
+<span class="normal">1169</span>
+<span class="normal">1170</span>
+<span class="normal">1171</span>
+<span class="normal">1172</span>
+<span class="normal">1173</span>
+<span class="normal">1174</span>
+<span class="normal">1175</span>
+<span class="normal">1176</span>
+<span class="normal">1177</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">apply_inclusion_list</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">:</span> <span class="n">StudyLocusGWASCatalog</span><span class="p">,</span> <span class="n">inclusion_list</span><span class="p">:</span> <span class="n">DataFrame</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyLocusGWASCatalog</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Restricting GWAS Catalog studies based on a list of accpected study ids.</span>
@@ -5668,7 +6003,7 @@ <h3 id="gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog.qc_a
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -5677,7 +6012,7 @@ <h3 id="gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog.qc_a
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StudyLocusGWASCatalog</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog" href="#gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog">StudyLocusGWASCatalog</a></code>
           </td>
@@ -5692,25 +6027,25 @@ <h3 id="gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog.qc_a
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/datasource/gwas_catalog/associations.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">1100</span>
-<span class="normal">1101</span>
-<span class="normal">1102</span>
-<span class="normal">1103</span>
-<span class="normal">1104</span>
-<span class="normal">1105</span>
-<span class="normal">1106</span>
-<span class="normal">1107</span>
-<span class="normal">1108</span>
-<span class="normal">1109</span>
-<span class="normal">1110</span>
-<span class="normal">1111</span>
-<span class="normal">1112</span>
-<span class="normal">1113</span>
-<span class="normal">1114</span>
-<span class="normal">1115</span>
-<span class="normal">1116</span>
-<span class="normal">1117</span>
-<span class="normal">1118</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">qc_ambiguous_study</span><span class="p">(</span><span class="bp">self</span><span class="p">:</span> <span class="n">StudyLocusGWASCatalog</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyLocusGWASCatalog</span><span class="p">:</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">1143</span>
+<span class="normal">1144</span>
+<span class="normal">1145</span>
+<span class="normal">1146</span>
+<span class="normal">1147</span>
+<span class="normal">1148</span>
+<span class="normal">1149</span>
+<span class="normal">1150</span>
+<span class="normal">1151</span>
+<span class="normal">1152</span>
+<span class="normal">1153</span>
+<span class="normal">1154</span>
+<span class="normal">1155</span>
+<span class="normal">1156</span>
+<span class="normal">1157</span>
+<span class="normal">1158</span>
+<span class="normal">1159</span>
+<span class="normal">1160</span>
+<span class="normal">1161</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">qc_ambiguous_study</span><span class="p">(</span><span class="bp">self</span><span class="p">:</span> <span class="n">StudyLocusGWASCatalog</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyLocusGWASCatalog</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Flag associations with variants that can not be unambiguously associated with one study.</span>
 
 <span class="sd">    Returns:</span>
@@ -5752,7 +6087,7 @@ <h3 id="gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog.upda
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -5763,7 +6098,7 @@ <h3 id="gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog.upda
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>study_annotation</code></td>
           <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
@@ -5782,7 +6117,7 @@ <h3 id="gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog.upda
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -5791,7 +6126,7 @@ <h3 id="gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog.upda
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StudyLocusGWASCatalog</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog" href="#gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog">StudyLocusGWASCatalog</a></code>
           </td>
@@ -5806,28 +6141,28 @@ <h3 id="gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog.upda
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/datasource/gwas_catalog/associations.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">1077</span>
-<span class="normal">1078</span>
-<span class="normal">1079</span>
-<span class="normal">1080</span>
-<span class="normal">1081</span>
-<span class="normal">1082</span>
-<span class="normal">1083</span>
-<span class="normal">1084</span>
-<span class="normal">1085</span>
-<span class="normal">1086</span>
-<span class="normal">1087</span>
-<span class="normal">1088</span>
-<span class="normal">1089</span>
-<span class="normal">1090</span>
-<span class="normal">1091</span>
-<span class="normal">1092</span>
-<span class="normal">1093</span>
-<span class="normal">1094</span>
-<span class="normal">1095</span>
-<span class="normal">1096</span>
-<span class="normal">1097</span>
-<span class="normal">1098</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">update_study_id</span><span class="p">(</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">1120</span>
+<span class="normal">1121</span>
+<span class="normal">1122</span>
+<span class="normal">1123</span>
+<span class="normal">1124</span>
+<span class="normal">1125</span>
+<span class="normal">1126</span>
+<span class="normal">1127</span>
+<span class="normal">1128</span>
+<span class="normal">1129</span>
+<span class="normal">1130</span>
+<span class="normal">1131</span>
+<span class="normal">1132</span>
+<span class="normal">1133</span>
+<span class="normal">1134</span>
+<span class="normal">1135</span>
+<span class="normal">1136</span>
+<span class="normal">1137</span>
+<span class="normal">1138</span>
+<span class="normal">1139</span>
+<span class="normal">1140</span>
+<span class="normal">1141</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">update_study_id</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">:</span> <span class="n">StudyLocusGWASCatalog</span><span class="p">,</span> <span class="n">study_annotation</span><span class="p">:</span> <span class="n">DataFrame</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyLocusGWASCatalog</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Update final studyId and studyLocusId with a dataframe containing study annotation.</span>
@@ -5896,7 +6231,15 @@ <h3 id="gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog.upda
         <span class="label"> Contributors </span>
       </li>
       <li class="contributors-holder">
-        <ul class="contributors" data-bi-name="contributors"></ul>
+        <ul class="contributors" data-bi-name="contributors"><li>
+            <a
+              href="https://github.com/d0choa"
+              title="d0choa"
+              data-bi-name="contributorprofile"
+              target="_blank"
+              ><img src="https://avatars.githubusercontent.com/u/5097586?v=4" alt="d0choa"
+            /></a>
+          </li></ul>
       </li>
     </ul>
   </small>
diff --git a/python_api/datasources/gwas_catalog/study_index/index.html b/python_api/datasources/gwas_catalog/study_index/index.html
index ffb629405..4acf5d0b8 100644
--- a/python_api/datasources/gwas_catalog/study_index/index.html
+++ b/python_api/datasources/gwas_catalog/study_index/index.html
@@ -2346,6 +2346,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../methods/susie_inf/" class="md-nav__link">
         
@@ -2377,7 +2419,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2393,14 +2435,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3381,7 +3423,8 @@ <h2 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalogParser
 <span class="normal">332</span>
 <span class="normal">333</span>
 <span class="normal">334</span>
-<span class="normal">335</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@dataclass</span>
+<span class="normal">335</span>
+<span class="normal">336</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@dataclass</span>
 <span class="k">class</span> <span class="nc">StudyIndexGWASCatalogParser</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;GWAS Catalog study index parser.</span>
 
@@ -3625,6 +3668,7 @@ <h2 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalogParser
                 <span class="n">parse_efos</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;MAPPED BACKGROUND TRAIT URI&quot;</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span>
                     <span class="s2">&quot;backgroundTraitFromSourceMappedIds&quot;</span>
                 <span class="p">),</span>
+                <span class="bp">cls</span><span class="o">.</span><span class="n">parse_cohorts</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;COHORT&quot;</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;cohorts&quot;</span><span class="p">),</span>
             <span class="p">),</span>
             <span class="n">_schema</span><span class="o">=</span><span class="n">StudyIndexGWASCatalog</span><span class="o">.</span><span class="n">get_schema</span><span class="p">(),</span>
         <span class="p">)</span>
@@ -3689,7 +3733,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalogParser
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3700,7 +3744,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalogParser
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>catalog_studies</code></td>
           <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
@@ -3714,7 +3758,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalogParser
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>ancestry_file</code></td>
           <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
@@ -3728,7 +3772,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalogParser
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>sumstats_lut</code></td>
           <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
@@ -3747,7 +3791,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalogParser
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3756,7 +3800,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalogParser
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StudyIndexGWASCatalog</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog" href="#gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog">StudyIndexGWASCatalog</a></code>
           </td>
@@ -3771,8 +3815,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalogParser
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/datasource/gwas_catalog/study_index.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">312</span>
-<span class="normal">313</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">313</span>
 <span class="normal">314</span>
 <span class="normal">315</span>
 <span class="normal">316</span>
@@ -3794,7 +3837,8 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalogParser
 <span class="normal">332</span>
 <span class="normal">333</span>
 <span class="normal">334</span>
-<span class="normal">335</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@classmethod</span>
+<span class="normal">335</span>
+<span class="normal">336</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@classmethod</span>
 <span class="k">def</span> <span class="nf">from_source</span><span class="p">(</span>
     <span class="bp">cls</span><span class="p">:</span> <span class="nb">type</span><span class="p">[</span><span class="n">StudyIndexGWASCatalogParser</span><span class="p">],</span>
     <span class="n">catalog_studies</span><span class="p">:</span> <span class="n">DataFrame</span><span class="p">,</span>
@@ -3845,7 +3889,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalogParser
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3856,7 +3900,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalogParser
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>raw_cohort</code></td>
           <td>
                 <code><span title="pyspark.sql.Column">Column</span></code>
@@ -3875,7 +3919,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalogParser
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3884,7 +3928,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalogParser
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>Column</code></td>          <td>
                 <code><span title="pyspark.sql.Column">Column</span></code>
           </td>
@@ -4005,8 +4049,7 @@ <h2 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog" clas
 
             <details class="quote">
               <summary>Source code in <code>src/gentropy/datasource/gwas_catalog/study_index.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">338</span>
-<span class="normal">339</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">339</span>
 <span class="normal">340</span>
 <span class="normal">341</span>
 <span class="normal">342</span>
@@ -4432,16 +4475,7 @@ <h2 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog" clas
 <span class="normal">762</span>
 <span class="normal">763</span>
 <span class="normal">764</span>
-<span class="normal">765</span>
-<span class="normal">766</span>
-<span class="normal">767</span>
-<span class="normal">768</span>
-<span class="normal">769</span>
-<span class="normal">770</span>
-<span class="normal">771</span>
-<span class="normal">772</span>
-<span class="normal">773</span>
-<span class="normal">774</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@dataclass</span>
+<span class="normal">765</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@dataclass</span>
 <span class="k">class</span> <span class="nc">StudyIndexGWASCatalog</span><span class="p">(</span><span class="n">StudyIndex</span><span class="p">):</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Study index dataset from GWAS Catalog.</span>
 
@@ -4654,14 +4688,6 @@ <h2 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog" clas
             <span class="p">)</span>  <span class="c1"># studyId has not been split yet</span>
         <span class="p">)</span>
 
-        <span class="c1"># Parsing cohort information:</span>
-        <span class="n">cohorts</span> <span class="o">=</span> <span class="n">ancestry_lut</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
-            <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;STUDY ACCESSION&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">),</span>
-            <span class="n">GWASCatalogStudyIndexParser</span><span class="o">.</span><span class="n">parse_cohorts</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;COHORT(S)&quot;</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span>
-                <span class="s2">&quot;cohorts&quot;</span>
-            <span class="p">),</span>
-        <span class="p">)</span><span class="o">.</span><span class="n">distinct</span><span class="p">()</span>
-
         <span class="c1"># Get a high resolution dataset on experimental stage:</span>
         <span class="n">ancestry_stages</span> <span class="o">=</span> <span class="p">(</span>
             <span class="n">ancestry</span><span class="o">.</span><span class="n">groupBy</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">)</span>
@@ -4750,9 +4776,7 @@ <h2 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog" clas
         <span class="p">)</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
             <span class="s2">&quot;studyId&quot;</span><span class="p">,</span> <span class="s2">&quot;discoverySamples&quot;</span><span class="p">,</span> <span class="s2">&quot;ldPopulationStructure&quot;</span><span class="p">,</span> <span class="s2">&quot;replicationSamples&quot;</span>
         <span class="p">)</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">df</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">parsed_ancestry_lut</span><span class="p">,</span> <span class="n">on</span><span class="o">=</span><span class="s2">&quot;studyId&quot;</span><span class="p">,</span> <span class="n">how</span><span class="o">=</span><span class="s2">&quot;left&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
-            <span class="n">cohorts</span><span class="p">,</span> <span class="n">on</span><span class="o">=</span><span class="s2">&quot;studyId&quot;</span><span class="p">,</span> <span class="n">how</span><span class="o">=</span><span class="s2">&quot;left&quot;</span>
-        <span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">df</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">parsed_ancestry_lut</span><span class="p">,</span> <span class="n">on</span><span class="o">=</span><span class="s2">&quot;studyId&quot;</span><span class="p">,</span> <span class="n">how</span><span class="o">=</span><span class="s2">&quot;left&quot;</span><span class="p">)</span>
         <span class="k">return</span> <span class="bp">self</span>
 
     <span class="k">def</span> <span class="nf">annotate_sumstats_info</span><span class="p">(</span>
@@ -4912,7 +4936,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.annot
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -4923,7 +4947,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.annot
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>ancestry_lut</code></td>
           <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
@@ -4942,7 +4966,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.annot
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -4951,7 +4975,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.annot
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StudyIndexGWASCatalog</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog" href="#gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog">StudyIndexGWASCatalog</a></code>
           </td>
@@ -4966,8 +4990,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.annot
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/datasource/gwas_catalog/study_index.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">521</span>
-<span class="normal">522</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">522</span>
 <span class="normal">523</span>
 <span class="normal">524</span>
 <span class="normal">525</span>
@@ -5086,16 +5109,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.annot
 <span class="normal">638</span>
 <span class="normal">639</span>
 <span class="normal">640</span>
-<span class="normal">641</span>
-<span class="normal">642</span>
-<span class="normal">643</span>
-<span class="normal">644</span>
-<span class="normal">645</span>
-<span class="normal">646</span>
-<span class="normal">647</span>
-<span class="normal">648</span>
-<span class="normal">649</span>
-<span class="normal">650</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">annotate_ancestries</span><span class="p">(</span>
+<span class="normal">641</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">annotate_ancestries</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">:</span> <span class="n">StudyIndexGWASCatalog</span><span class="p">,</span> <span class="n">ancestry_lut</span><span class="p">:</span> <span class="n">DataFrame</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyIndexGWASCatalog</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Extracting sample sizes and ancestry information.</span>
@@ -5125,14 +5139,6 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.annot
         <span class="p">)</span>  <span class="c1"># studyId has not been split yet</span>
     <span class="p">)</span>
 
-    <span class="c1"># Parsing cohort information:</span>
-    <span class="n">cohorts</span> <span class="o">=</span> <span class="n">ancestry_lut</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
-        <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;STUDY ACCESSION&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">),</span>
-        <span class="n">GWASCatalogStudyIndexParser</span><span class="o">.</span><span class="n">parse_cohorts</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;COHORT(S)&quot;</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span>
-            <span class="s2">&quot;cohorts&quot;</span>
-        <span class="p">),</span>
-    <span class="p">)</span><span class="o">.</span><span class="n">distinct</span><span class="p">()</span>
-
     <span class="c1"># Get a high resolution dataset on experimental stage:</span>
     <span class="n">ancestry_stages</span> <span class="o">=</span> <span class="p">(</span>
         <span class="n">ancestry</span><span class="o">.</span><span class="n">groupBy</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">)</span>
@@ -5221,9 +5227,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.annot
     <span class="p">)</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
         <span class="s2">&quot;studyId&quot;</span><span class="p">,</span> <span class="s2">&quot;discoverySamples&quot;</span><span class="p">,</span> <span class="s2">&quot;ldPopulationStructure&quot;</span><span class="p">,</span> <span class="s2">&quot;replicationSamples&quot;</span>
     <span class="p">)</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">df</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">parsed_ancestry_lut</span><span class="p">,</span> <span class="n">on</span><span class="o">=</span><span class="s2">&quot;studyId&quot;</span><span class="p">,</span> <span class="n">how</span><span class="o">=</span><span class="s2">&quot;left&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
-        <span class="n">cohorts</span><span class="p">,</span> <span class="n">on</span><span class="o">=</span><span class="s2">&quot;studyId&quot;</span><span class="p">,</span> <span class="n">how</span><span class="o">=</span><span class="s2">&quot;left&quot;</span>
-    <span class="p">)</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">df</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">parsed_ancestry_lut</span><span class="p">,</span> <span class="n">on</span><span class="o">=</span><span class="s2">&quot;studyId&quot;</span><span class="p">,</span> <span class="n">how</span><span class="o">=</span><span class="s2">&quot;left&quot;</span><span class="p">)</span>
     <span class="k">return</span> <span class="bp">self</span>
 </code></pre></div></td></tr></table></div>
           </details>
@@ -5249,7 +5253,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.annot
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -5258,7 +5262,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.annot
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StudyIndexGWASCatalog</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog" href="#gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog">StudyIndexGWASCatalog</a></code>
           </td>
@@ -5273,7 +5277,16 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.annot
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/datasource/gwas_catalog/study_index.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">693</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">684</span>
+<span class="normal">685</span>
+<span class="normal">686</span>
+<span class="normal">687</span>
+<span class="normal">688</span>
+<span class="normal">689</span>
+<span class="normal">690</span>
+<span class="normal">691</span>
+<span class="normal">692</span>
+<span class="normal">693</span>
 <span class="normal">694</span>
 <span class="normal">695</span>
 <span class="normal">696</span>
@@ -5307,16 +5320,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.annot
 <span class="normal">724</span>
 <span class="normal">725</span>
 <span class="normal">726</span>
-<span class="normal">727</span>
-<span class="normal">728</span>
-<span class="normal">729</span>
-<span class="normal">730</span>
-<span class="normal">731</span>
-<span class="normal">732</span>
-<span class="normal">733</span>
-<span class="normal">734</span>
-<span class="normal">735</span>
-<span class="normal">736</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">annotate_discovery_sample_sizes</span><span class="p">(</span>
+<span class="normal">727</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">annotate_discovery_sample_sizes</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">:</span> <span class="n">StudyIndexGWASCatalog</span><span class="p">,</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyIndexGWASCatalog</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Extract the sample size of the discovery stage of the study as annotated in the GWAS Catalog.</span>
@@ -5383,7 +5387,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.annot
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -5394,7 +5398,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.annot
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>curation_table</code></td>
           <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span> | None</code>
@@ -5413,7 +5417,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.annot
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -5422,7 +5426,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.annot
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StudyIndexGWASCatalog</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog" href="#gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog">StudyIndexGWASCatalog</a></code>
           </td>
@@ -5437,8 +5441,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.annot
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/datasource/gwas_catalog/study_index.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">389</span>
-<span class="normal">390</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">390</span>
 <span class="normal">391</span>
 <span class="normal">392</span>
 <span class="normal">393</span>
@@ -5507,7 +5510,8 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.annot
 <span class="normal">456</span>
 <span class="normal">457</span>
 <span class="normal">458</span>
-<span class="normal">459</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">annotate_from_study_curation</span><span class="p">(</span>
+<span class="normal">459</span>
+<span class="normal">460</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">annotate_from_study_curation</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">:</span> <span class="n">StudyIndexGWASCatalog</span><span class="p">,</span> <span class="n">curation_table</span><span class="p">:</span> <span class="n">DataFrame</span> <span class="o">|</span> <span class="kc">None</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyIndexGWASCatalog</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Annotating study index with curation.</span>
@@ -5601,7 +5605,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.annot
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -5612,7 +5616,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.annot
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>sumstats_lut</code></td>
           <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
@@ -5631,7 +5635,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.annot
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -5640,7 +5644,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.annot
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StudyIndexGWASCatalog</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog" href="#gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog">StudyIndexGWASCatalog</a></code>
           </td>
@@ -5655,7 +5659,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.annot
 
 
 
-  <p><strong>Raises:</strong></p>
+  <p><span class="doc-section-title">Raises:</span></p>
   <table>
     <thead>
       <tr>
@@ -5664,7 +5668,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.annot
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td>
                 <code>ValueError</code>
           </td>
@@ -5679,7 +5683,16 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.annot
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/datasource/gwas_catalog/study_index.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">652</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">643</span>
+<span class="normal">644</span>
+<span class="normal">645</span>
+<span class="normal">646</span>
+<span class="normal">647</span>
+<span class="normal">648</span>
+<span class="normal">649</span>
+<span class="normal">650</span>
+<span class="normal">651</span>
+<span class="normal">652</span>
 <span class="normal">653</span>
 <span class="normal">654</span>
 <span class="normal">655</span>
@@ -5709,16 +5722,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.annot
 <span class="normal">679</span>
 <span class="normal">680</span>
 <span class="normal">681</span>
-<span class="normal">682</span>
-<span class="normal">683</span>
-<span class="normal">684</span>
-<span class="normal">685</span>
-<span class="normal">686</span>
-<span class="normal">687</span>
-<span class="normal">688</span>
-<span class="normal">689</span>
-<span class="normal">690</span>
-<span class="normal">691</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">annotate_sumstats_info</span><span class="p">(</span>
+<span class="normal">682</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">annotate_sumstats_info</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">:</span> <span class="n">StudyIndexGWASCatalog</span><span class="p">,</span> <span class="n">sumstats_lut</span><span class="p">:</span> <span class="n">DataFrame</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyIndexGWASCatalog</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Annotate summary stat locations.</span>
@@ -5781,7 +5785,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.apply
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -5792,7 +5796,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.apply
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>inclusion_list</code></td>
           <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
@@ -5811,7 +5815,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.apply
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -5820,7 +5824,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.apply
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StudyIndexGWASCatalog</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog" href="#gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog">StudyIndexGWASCatalog</a></code>
           </td>
@@ -5835,21 +5839,21 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.apply
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/datasource/gwas_catalog/study_index.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">738</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">729</span>
+<span class="normal">730</span>
+<span class="normal">731</span>
+<span class="normal">732</span>
+<span class="normal">733</span>
+<span class="normal">734</span>
+<span class="normal">735</span>
+<span class="normal">736</span>
+<span class="normal">737</span>
+<span class="normal">738</span>
 <span class="normal">739</span>
 <span class="normal">740</span>
 <span class="normal">741</span>
 <span class="normal">742</span>
-<span class="normal">743</span>
-<span class="normal">744</span>
-<span class="normal">745</span>
-<span class="normal">746</span>
-<span class="normal">747</span>
-<span class="normal">748</span>
-<span class="normal">749</span>
-<span class="normal">750</span>
-<span class="normal">751</span>
-<span class="normal">752</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">apply_inclusion_list</span><span class="p">(</span>
+<span class="normal">743</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">apply_inclusion_list</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">:</span> <span class="n">StudyIndexGWASCatalog</span><span class="p">,</span> <span class="n">inclusion_list</span><span class="p">:</span> <span class="n">DataFrame</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyIndexGWASCatalog</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Restricting GWAS Catalog studies based on a list of accepted study identifiers.</span>
@@ -5887,7 +5891,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.extra
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -5898,7 +5902,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.extra
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>curation</code></td>
           <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span> | None</code>
@@ -5917,7 +5921,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.extra
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -5926,7 +5930,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.extra
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>DataFrame</code></td>          <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
           </td>
@@ -5941,8 +5945,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.extra
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/datasource/gwas_catalog/study_index.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">461</span>
-<span class="normal">462</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">462</span>
 <span class="normal">463</span>
 <span class="normal">464</span>
 <span class="normal">465</span>
@@ -5999,7 +6002,8 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.extra
 <span class="normal">516</span>
 <span class="normal">517</span>
 <span class="normal">518</span>
-<span class="normal">519</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">extract_studies_for_curation</span><span class="p">(</span>
+<span class="normal">519</span>
+<span class="normal">520</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">extract_studies_for_curation</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">:</span> <span class="n">StudyIndexGWASCatalog</span><span class="p">,</span> <span class="n">curation</span><span class="p">:</span> <span class="n">DataFrame</span> <span class="o">|</span> <span class="kc">None</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">DataFrame</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Extract studies for curation.</span>
@@ -6081,7 +6085,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.updat
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -6092,7 +6096,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.updat
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>study_annotation</code></td>
           <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
@@ -6111,7 +6115,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.updat
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -6120,7 +6124,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.updat
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StudyIndexGWASCatalog</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog" href="#gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog">StudyIndexGWASCatalog</a></code>
           </td>
@@ -6135,8 +6139,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.updat
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/datasource/gwas_catalog/study_index.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">345</span>
-<span class="normal">346</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">346</span>
 <span class="normal">347</span>
 <span class="normal">348</span>
 <span class="normal">349</span>
@@ -6177,7 +6180,8 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.updat
 <span class="normal">384</span>
 <span class="normal">385</span>
 <span class="normal">386</span>
-<span class="normal">387</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">update_study_id</span><span class="p">(</span>
+<span class="normal">387</span>
+<span class="normal">388</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">update_study_id</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">:</span> <span class="n">StudyIndexGWASCatalog</span><span class="p">,</span> <span class="n">study_annotation</span><span class="p">:</span> <span class="n">DataFrame</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyIndexGWASCatalog</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Update studyId with a dataframe containing study.</span>
@@ -6267,7 +6271,15 @@ <h3 id="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.updat
         <span class="label"> Contributors </span>
       </li>
       <li class="contributors-holder">
-        <ul class="contributors" data-bi-name="contributors"></ul>
+        <ul class="contributors" data-bi-name="contributors"><li>
+            <a
+              href="https://github.com/d0choa"
+              title="d0choa"
+              data-bi-name="contributorprofile"
+              target="_blank"
+              ><img src="https://avatars.githubusercontent.com/u/5097586?v=4" alt="d0choa"
+            /></a>
+          </li></ul>
       </li>
     </ul>
   </small>
diff --git a/python_api/datasources/gwas_catalog/study_splitter/index.html b/python_api/datasources/gwas_catalog/study_splitter/index.html
index 36ee94142..4feface66 100644
--- a/python_api/datasources/gwas_catalog/study_splitter/index.html
+++ b/python_api/datasources/gwas_catalog/study_splitter/index.html
@@ -2259,6 +2259,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../methods/susie_inf/" class="md-nav__link">
         
@@ -2290,7 +2332,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2306,14 +2348,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3199,7 +3241,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_splitter.GWASCatalogStudySplitter
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3210,7 +3252,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_splitter.GWASCatalogStudySplitter
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>studies</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog" href="../study_index/#gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog">StudyIndexGWASCatalog</a></code>
@@ -3224,7 +3266,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_splitter.GWASCatalogStudySplitter
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>associations</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog" href="../associations/#gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog">StudyLocusGWASCatalog</a></code>
@@ -3243,7 +3285,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_splitter.GWASCatalogStudySplitter
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3252,7 +3294,7 @@ <h3 id="gentropy.datasource.gwas_catalog.study_splitter.GWASCatalogStudySplitter
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td>
                 <code><span title="typing.Tuple">Tuple</span>[<a class="autorefs autorefs-internal" title="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog" href="../study_index/#gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog">StudyIndexGWASCatalog</a>, <a class="autorefs autorefs-internal" title="gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog" href="../associations/#gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog">StudyLocusGWASCatalog</a>]</code>
           </td>
@@ -3423,7 +3465,15 @@ <h3 id="gentropy.datasource.gwas_catalog.study_splitter.GWASCatalogStudySplitter
         <span class="label"> Contributors </span>
       </li>
       <li class="contributors-holder">
-        <ul class="contributors" data-bi-name="contributors"></ul>
+        <ul class="contributors" data-bi-name="contributors"><li>
+            <a
+              href="https://github.com/d0choa"
+              title="d0choa"
+              data-bi-name="contributorprofile"
+              target="_blank"
+              ><img src="https://avatars.githubusercontent.com/u/5097586?v=4" alt="d0choa"
+            /></a>
+          </li></ul>
       </li>
     </ul>
   </small>
diff --git a/python_api/datasources/gwas_catalog/summary_statistics/index.html b/python_api/datasources/gwas_catalog/summary_statistics/index.html
index 6da4d2a03..18470a486 100644
--- a/python_api/datasources/gwas_catalog/summary_statistics/index.html
+++ b/python_api/datasources/gwas_catalog/summary_statistics/index.html
@@ -2259,6 +2259,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../methods/susie_inf/" class="md-nav__link">
         
@@ -2290,7 +2332,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2306,14 +2348,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3249,7 +3291,7 @@ <h3 id="gentropy.datasource.gwas_catalog.summary_statistics.GWASCatalogSummarySt
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3260,7 +3302,7 @@ <h3 id="gentropy.datasource.gwas_catalog.summary_statistics.GWASCatalogSummarySt
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>spark</code></td>
           <td>
                 <code><span title="pyspark.sql.SparkSession">SparkSession</span></code>
@@ -3274,7 +3316,7 @@ <h3 id="gentropy.datasource.gwas_catalog.summary_statistics.GWASCatalogSummarySt
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>sumstats_file</code></td>
           <td>
                 <code>str</code>
@@ -3293,7 +3335,7 @@ <h3 id="gentropy.datasource.gwas_catalog.summary_statistics.GWASCatalogSummarySt
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3302,7 +3344,7 @@ <h3 id="gentropy.datasource.gwas_catalog.summary_statistics.GWASCatalogSummarySt
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>GWASCatalogSummaryStatistics</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.datasource.gwas_catalog.summary_statistics.GWASCatalogSummaryStatistics" href="#gentropy.datasource.gwas_catalog.summary_statistics.GWASCatalogSummaryStatistics">GWASCatalogSummaryStatistics</a></code>
           </td>
@@ -3639,7 +3681,15 @@ <h3 id="gentropy.datasource.gwas_catalog.summary_statistics.GWASCatalogSummarySt
         <span class="label"> Contributors </span>
       </li>
       <li class="contributors-holder">
-        <ul class="contributors" data-bi-name="contributors"></ul>
+        <ul class="contributors" data-bi-name="contributors"><li>
+            <a
+              href="https://github.com/d0choa"
+              title="d0choa"
+              data-bi-name="contributorprofile"
+              target="_blank"
+              ><img src="https://avatars.githubusercontent.com/u/5097586?v=4" alt="d0choa"
+            /></a>
+          </li></ul>
       </li>
     </ul>
   </small>
diff --git a/python_api/datasources/intervals/_intervals/index.html b/python_api/datasources/intervals/_intervals/index.html
index f7eaf24cf..47fe2a051 100644
--- a/python_api/datasources/intervals/_intervals/index.html
+++ b/python_api/datasources/intervals/_intervals/index.html
@@ -2199,6 +2199,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../methods/susie_inf/" class="md-nav__link">
         
@@ -2230,7 +2272,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2246,14 +2288,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -2870,7 +2912,15 @@ <h1 id="list-of-interaction-and-interval-based-studies">List of Interaction and
         <span class="label"> Contributors </span>
       </li>
       <li class="contributors-holder">
-        <ul class="contributors" data-bi-name="contributors"></ul>
+        <ul class="contributors" data-bi-name="contributors"><li>
+            <a
+              href="https://github.com/d0choa"
+              title="d0choa"
+              data-bi-name="contributorprofile"
+              target="_blank"
+              ><img src="https://avatars.githubusercontent.com/u/5097586?v=4" alt="d0choa"
+            /></a>
+          </li></ul>
       </li>
     </ul>
   </small>
diff --git a/python_api/datasources/intervals/andersson/index.html b/python_api/datasources/intervals/andersson/index.html
index f9b48c6a1..4deeba72e 100644
--- a/python_api/datasources/intervals/andersson/index.html
+++ b/python_api/datasources/intervals/andersson/index.html
@@ -2268,6 +2268,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../methods/susie_inf/" class="md-nav__link">
         
@@ -2299,7 +2341,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2315,14 +2357,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3215,7 +3257,7 @@ <h3 id="gentropy.datasource.intervals.andersson.IntervalsAndersson.parse" class=
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3226,7 +3268,7 @@ <h3 id="gentropy.datasource.intervals.andersson.IntervalsAndersson.parse" class=
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>raw_anderson_df</code></td>
           <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
@@ -3240,7 +3282,7 @@ <h3 id="gentropy.datasource.intervals.andersson.IntervalsAndersson.parse" class=
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>gene_index</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.gene_index.GeneIndex" href="../../../datasets/gene_index/#gentropy.dataset.gene_index.GeneIndex">GeneIndex</a></code>
@@ -3254,7 +3296,7 @@ <h3 id="gentropy.datasource.intervals.andersson.IntervalsAndersson.parse" class=
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>lift</code></td>
           <td>
                 <code><span title="gentropy.common.Liftover.LiftOverSpark">LiftOverSpark</span></code>
@@ -3273,7 +3315,7 @@ <h3 id="gentropy.datasource.intervals.andersson.IntervalsAndersson.parse" class=
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3282,7 +3324,7 @@ <h3 id="gentropy.datasource.intervals.andersson.IntervalsAndersson.parse" class=
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>Intervals</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.intervals.Intervals" href="../../../datasets/intervals/#gentropy.dataset.intervals.Intervals">Intervals</a></code>
           </td>
@@ -3509,7 +3551,7 @@ <h3 id="gentropy.datasource.intervals.andersson.IntervalsAndersson.read" class="
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3520,7 +3562,7 @@ <h3 id="gentropy.datasource.intervals.andersson.IntervalsAndersson.read" class="
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>spark</code></td>
           <td>
                 <code><span title="pyspark.sql.SparkSession">SparkSession</span></code>
@@ -3534,7 +3576,7 @@ <h3 id="gentropy.datasource.intervals.andersson.IntervalsAndersson.read" class="
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>path</code></td>
           <td>
                 <code>str</code>
@@ -3553,7 +3595,7 @@ <h3 id="gentropy.datasource.intervals.andersson.IntervalsAndersson.read" class="
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3562,7 +3604,7 @@ <h3 id="gentropy.datasource.intervals.andersson.IntervalsAndersson.read" class="
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>DataFrame</code></td>          <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
           </td>
@@ -3669,7 +3711,15 @@ <h3 id="gentropy.datasource.intervals.andersson.IntervalsAndersson.read" class="
         <span class="label"> Contributors </span>
       </li>
       <li class="contributors-holder">
-        <ul class="contributors" data-bi-name="contributors"></ul>
+        <ul class="contributors" data-bi-name="contributors"><li>
+            <a
+              href="https://github.com/d0choa"
+              title="d0choa"
+              data-bi-name="contributorprofile"
+              target="_blank"
+              ><img src="https://avatars.githubusercontent.com/u/5097586?v=4" alt="d0choa"
+            /></a>
+          </li></ul>
       </li>
     </ul>
   </small>
diff --git a/python_api/datasources/intervals/javierre/index.html b/python_api/datasources/intervals/javierre/index.html
index 59178c841..34e5d7df5 100644
--- a/python_api/datasources/intervals/javierre/index.html
+++ b/python_api/datasources/intervals/javierre/index.html
@@ -2268,6 +2268,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../methods/susie_inf/" class="md-nav__link">
         
@@ -2299,7 +2341,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2315,14 +2357,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3283,7 +3325,7 @@ <h3 id="gentropy.datasource.intervals.javierre.IntervalsJavierre.parse" class="d
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3294,7 +3336,7 @@ <h3 id="gentropy.datasource.intervals.javierre.IntervalsJavierre.parse" class="d
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>javierre_raw</code></td>
           <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
@@ -3308,7 +3350,7 @@ <h3 id="gentropy.datasource.intervals.javierre.IntervalsJavierre.parse" class="d
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>gene_index</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.gene_index.GeneIndex" href="../../../datasets/gene_index/#gentropy.dataset.gene_index.GeneIndex">GeneIndex</a></code>
@@ -3322,7 +3364,7 @@ <h3 id="gentropy.datasource.intervals.javierre.IntervalsJavierre.parse" class="d
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>lift</code></td>
           <td>
                 <code><span title="gentropy.common.Liftover.LiftOverSpark">LiftOverSpark</span></code>
@@ -3341,7 +3383,7 @@ <h3 id="gentropy.datasource.intervals.javierre.IntervalsJavierre.parse" class="d
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3350,7 +3392,7 @@ <h3 id="gentropy.datasource.intervals.javierre.IntervalsJavierre.parse" class="d
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>Intervals</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.intervals.Intervals" href="../../../datasets/intervals/#gentropy.dataset.intervals.Intervals">Intervals</a></code>
           </td>
@@ -3667,7 +3709,7 @@ <h3 id="gentropy.datasource.intervals.javierre.IntervalsJavierre.read" class="do
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3678,7 +3720,7 @@ <h3 id="gentropy.datasource.intervals.javierre.IntervalsJavierre.read" class="do
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>spark</code></td>
           <td>
                 <code><span title="pyspark.sql.SparkSession">SparkSession</span></code>
@@ -3692,7 +3734,7 @@ <h3 id="gentropy.datasource.intervals.javierre.IntervalsJavierre.read" class="do
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>path</code></td>
           <td>
                 <code>str</code>
@@ -3711,7 +3753,7 @@ <h3 id="gentropy.datasource.intervals.javierre.IntervalsJavierre.read" class="do
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3720,7 +3762,7 @@ <h3 id="gentropy.datasource.intervals.javierre.IntervalsJavierre.read" class="do
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>DataFrame</code></td>          <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
           </td>
@@ -3805,7 +3847,15 @@ <h3 id="gentropy.datasource.intervals.javierre.IntervalsJavierre.read" class="do
         <span class="label"> Contributors </span>
       </li>
       <li class="contributors-holder">
-        <ul class="contributors" data-bi-name="contributors"></ul>
+        <ul class="contributors" data-bi-name="contributors"><li>
+            <a
+              href="https://github.com/d0choa"
+              title="d0choa"
+              data-bi-name="contributorprofile"
+              target="_blank"
+              ><img src="https://avatars.githubusercontent.com/u/5097586?v=4" alt="d0choa"
+            /></a>
+          </li></ul>
       </li>
     </ul>
   </small>
diff --git a/python_api/datasources/intervals/jung/index.html b/python_api/datasources/intervals/jung/index.html
index ee6b037d9..c34063e02 100644
--- a/python_api/datasources/intervals/jung/index.html
+++ b/python_api/datasources/intervals/jung/index.html
@@ -2268,6 +2268,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../methods/susie_inf/" class="md-nav__link">
         
@@ -2299,7 +2341,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2315,14 +2357,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3147,7 +3189,7 @@ <h3 id="gentropy.datasource.intervals.jung.IntervalsJung.parse" class="doc doc-h
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3158,7 +3200,7 @@ <h3 id="gentropy.datasource.intervals.jung.IntervalsJung.parse" class="doc doc-h
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>jung_raw</code></td>
           <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
@@ -3172,7 +3214,7 @@ <h3 id="gentropy.datasource.intervals.jung.IntervalsJung.parse" class="doc doc-h
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>gene_index</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.gene_index.GeneIndex" href="../../../datasets/gene_index/#gentropy.dataset.gene_index.GeneIndex">GeneIndex</a></code>
@@ -3186,7 +3228,7 @@ <h3 id="gentropy.datasource.intervals.jung.IntervalsJung.parse" class="doc doc-h
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>lift</code></td>
           <td>
                 <code><span title="gentropy.common.Liftover.LiftOverSpark">LiftOverSpark</span></code>
@@ -3205,7 +3247,7 @@ <h3 id="gentropy.datasource.intervals.jung.IntervalsJung.parse" class="doc doc-h
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3214,7 +3256,7 @@ <h3 id="gentropy.datasource.intervals.jung.IntervalsJung.parse" class="doc doc-h
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>Intervals</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.intervals.Intervals" href="../../../datasets/intervals/#gentropy.dataset.intervals.Intervals">Intervals</a></code>
           </td>
@@ -3395,7 +3437,7 @@ <h3 id="gentropy.datasource.intervals.jung.IntervalsJung.read" class="doc doc-he
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3406,7 +3448,7 @@ <h3 id="gentropy.datasource.intervals.jung.IntervalsJung.read" class="doc doc-he
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>spark</code></td>
           <td>
                 <code><span title="pyspark.sql.SparkSession">SparkSession</span></code>
@@ -3420,7 +3462,7 @@ <h3 id="gentropy.datasource.intervals.jung.IntervalsJung.read" class="doc doc-he
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>path</code></td>
           <td>
                 <code>str</code>
@@ -3439,7 +3481,7 @@ <h3 id="gentropy.datasource.intervals.jung.IntervalsJung.read" class="doc doc-he
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3448,7 +3490,7 @@ <h3 id="gentropy.datasource.intervals.jung.IntervalsJung.read" class="doc doc-he
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>DataFrame</code></td>          <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
           </td>
@@ -3533,7 +3575,15 @@ <h3 id="gentropy.datasource.intervals.jung.IntervalsJung.read" class="doc doc-he
         <span class="label"> Contributors </span>
       </li>
       <li class="contributors-holder">
-        <ul class="contributors" data-bi-name="contributors"></ul>
+        <ul class="contributors" data-bi-name="contributors"><li>
+            <a
+              href="https://github.com/d0choa"
+              title="d0choa"
+              data-bi-name="contributorprofile"
+              target="_blank"
+              ><img src="https://avatars.githubusercontent.com/u/5097586?v=4" alt="d0choa"
+            /></a>
+          </li></ul>
       </li>
     </ul>
   </small>
diff --git a/python_api/datasources/intervals/thurman/index.html b/python_api/datasources/intervals/thurman/index.html
index db73612c6..ef48438e2 100644
--- a/python_api/datasources/intervals/thurman/index.html
+++ b/python_api/datasources/intervals/thurman/index.html
@@ -2268,6 +2268,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../methods/susie_inf/" class="md-nav__link">
         
@@ -2299,7 +2341,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2315,14 +2357,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3052,7 +3094,7 @@ <h2 id="gentropy.datasource.intervals.thurman.IntervalsThurman" class="doc doc-h
                 <span class="n">t</span><span class="o">.</span><span class="n">StructField</span><span class="p">(</span><span class="s2">&quot;score&quot;</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">FloatType</span><span class="p">(),</span> <span class="kc">False</span><span class="p">),</span>
             <span class="p">]</span>
         <span class="p">)</span>
-        <span class="k">return</span> <span class="n">spark</span><span class="o">.</span><span class="n">read</span><span class="o">.</span><span class="n">csv</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s2">&quot;</span><span class="se">\t</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">header</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="n">thurman_schema</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">spark</span><span class="o">.</span><span class="n">read</span><span class="o">.</span><span class="n">csv</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s2">&quot;</span><span class="se">\t</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">header</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="n">thurman_schema</span><span class="p">)</span>
 
     <span class="nd">@classmethod</span>
     <span class="k">def</span> <span class="nf">parse</span><span class="p">(</span>
@@ -3149,7 +3191,7 @@ <h3 id="gentropy.datasource.intervals.thurman.IntervalsThurman.parse" class="doc
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3160,7 +3202,7 @@ <h3 id="gentropy.datasource.intervals.thurman.IntervalsThurman.parse" class="doc
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>thurman_raw</code></td>
           <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
@@ -3174,7 +3216,7 @@ <h3 id="gentropy.datasource.intervals.thurman.IntervalsThurman.parse" class="doc
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>gene_index</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.gene_index.GeneIndex" href="../../../datasets/gene_index/#gentropy.dataset.gene_index.GeneIndex">GeneIndex</a></code>
@@ -3188,7 +3230,7 @@ <h3 id="gentropy.datasource.intervals.thurman.IntervalsThurman.parse" class="doc
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>lift</code></td>
           <td>
                 <code><span title="gentropy.common.Liftover.LiftOverSpark">LiftOverSpark</span></code>
@@ -3207,7 +3249,7 @@ <h3 id="gentropy.datasource.intervals.thurman.IntervalsThurman.parse" class="doc
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3216,7 +3258,7 @@ <h3 id="gentropy.datasource.intervals.thurman.IntervalsThurman.parse" class="doc
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>Intervals</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.intervals.Intervals" href="../../../datasets/intervals/#gentropy.dataset.intervals.Intervals">Intervals</a></code>
           </td>
@@ -3375,7 +3417,7 @@ <h3 id="gentropy.datasource.intervals.thurman.IntervalsThurman.read" class="doc
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3386,7 +3428,7 @@ <h3 id="gentropy.datasource.intervals.thurman.IntervalsThurman.read" class="doc
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>spark</code></td>
           <td>
                 <code><span title="pyspark.sql.SparkSession">SparkSession</span></code>
@@ -3400,7 +3442,7 @@ <h3 id="gentropy.datasource.intervals.thurman.IntervalsThurman.read" class="doc
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>path</code></td>
           <td>
                 <code>str</code>
@@ -3419,7 +3461,7 @@ <h3 id="gentropy.datasource.intervals.thurman.IntervalsThurman.read" class="doc
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3428,7 +3470,7 @@ <h3 id="gentropy.datasource.intervals.thurman.IntervalsThurman.read" class="doc
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>DataFrame</code></td>          <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
           </td>
@@ -3489,7 +3531,7 @@ <h3 id="gentropy.datasource.intervals.thurman.IntervalsThurman.read" class="doc
             <span class="n">t</span><span class="o">.</span><span class="n">StructField</span><span class="p">(</span><span class="s2">&quot;score&quot;</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">FloatType</span><span class="p">(),</span> <span class="kc">False</span><span class="p">),</span>
         <span class="p">]</span>
     <span class="p">)</span>
-    <span class="k">return</span> <span class="n">spark</span><span class="o">.</span><span class="n">read</span><span class="o">.</span><span class="n">csv</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s2">&quot;</span><span class="se">\t</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">header</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="n">thurman_schema</span><span class="p">)</span>
+    <span class="k">return</span> <span class="n">spark</span><span class="o">.</span><span class="n">read</span><span class="o">.</span><span class="n">csv</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s2">&quot;</span><span class="se">\t</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">header</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="n">thurman_schema</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -3537,7 +3579,15 @@ <h3 id="gentropy.datasource.intervals.thurman.IntervalsThurman.read" class="doc
         <span class="label"> Contributors </span>
       </li>
       <li class="contributors-holder">
-        <ul class="contributors" data-bi-name="contributors"></ul>
+        <ul class="contributors" data-bi-name="contributors"><li>
+            <a
+              href="https://github.com/d0choa"
+              title="d0choa"
+              data-bi-name="contributorprofile"
+              target="_blank"
+              ><img src="https://avatars.githubusercontent.com/u/5097586?v=4" alt="d0choa"
+            /></a>
+          </li></ul>
       </li>
     </ul>
   </small>
diff --git a/python_api/datasources/open_targets/_open_targets/index.html b/python_api/datasources/open_targets/_open_targets/index.html
index 5ffad0f35..b9ddd2d55 100644
--- a/python_api/datasources/open_targets/_open_targets/index.html
+++ b/python_api/datasources/open_targets/_open_targets/index.html
@@ -2194,6 +2194,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../methods/susie_inf/" class="md-nav__link">
         
@@ -2225,7 +2267,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2241,14 +2283,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -2862,7 +2904,15 @@ <h1>Open targets</h1>
         <span class="label"> Contributors </span>
       </li>
       <li class="contributors-holder">
-        <ul class="contributors" data-bi-name="contributors"></ul>
+        <ul class="contributors" data-bi-name="contributors"><li>
+            <a
+              href="https://github.com/d0choa"
+              title="d0choa"
+              data-bi-name="contributorprofile"
+              target="_blank"
+              ><img src="https://avatars.githubusercontent.com/u/5097586?v=4" alt="d0choa"
+            /></a>
+          </li></ul>
       </li>
     </ul>
   </small>
diff --git a/python_api/datasources/open_targets/l2g_gold_standard/index.html b/python_api/datasources/open_targets/l2g_gold_standard/index.html
index 71cb0a671..edcd11fcb 100644
--- a/python_api/datasources/open_targets/l2g_gold_standard/index.html
+++ b/python_api/datasources/open_targets/l2g_gold_standard/index.html
@@ -2277,6 +2277,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../methods/susie_inf/" class="md-nav__link">
         
@@ -2308,7 +2350,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2324,14 +2366,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3220,7 +3262,7 @@ <h3 id="gentropy.datasource.open_targets.l2g_gold_standard.OpenTargetsL2GGoldSta
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3231,7 +3273,7 @@ <h3 id="gentropy.datasource.open_targets.l2g_gold_standard.OpenTargetsL2GGoldSta
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>gold_standard_curation</code></td>
           <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
@@ -3245,7 +3287,7 @@ <h3 id="gentropy.datasource.open_targets.l2g_gold_standard.OpenTargetsL2GGoldSta
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>v2g</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.v2g.V2G" href="../../../datasets/variant_to_gene/#gentropy.dataset.v2g.V2G">V2G</a></code>
@@ -3264,7 +3306,7 @@ <h3 id="gentropy.datasource.open_targets.l2g_gold_standard.OpenTargetsL2GGoldSta
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3273,7 +3315,7 @@ <h3 id="gentropy.datasource.open_targets.l2g_gold_standard.OpenTargetsL2GGoldSta
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>L2GGoldStandard</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.l2g_gold_standard.L2GGoldStandard" href="../../../datasets/l2g_gold_standard/#gentropy.dataset.l2g_gold_standard.L2GGoldStandard">L2GGoldStandard</a></code>
           </td>
@@ -3357,7 +3399,7 @@ <h3 id="gentropy.datasource.open_targets.l2g_gold_standard.OpenTargetsL2GGoldSta
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3368,7 +3410,7 @@ <h3 id="gentropy.datasource.open_targets.l2g_gold_standard.OpenTargetsL2GGoldSta
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>positive_set</code></td>
           <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
@@ -3382,7 +3424,7 @@ <h3 id="gentropy.datasource.open_targets.l2g_gold_standard.OpenTargetsL2GGoldSta
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>v2g</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.v2g.V2G" href="../../../datasets/variant_to_gene/#gentropy.dataset.v2g.V2G">V2G</a></code>
@@ -3401,7 +3443,7 @@ <h3 id="gentropy.datasource.open_targets.l2g_gold_standard.OpenTargetsL2GGoldSta
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3410,7 +3452,7 @@ <h3 id="gentropy.datasource.open_targets.l2g_gold_standard.OpenTargetsL2GGoldSta
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>DataFrame</code></td>          <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
           </td>
@@ -3535,7 +3577,7 @@ <h3 id="gentropy.datasource.open_targets.l2g_gold_standard.OpenTargetsL2GGoldSta
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3546,7 +3588,7 @@ <h3 id="gentropy.datasource.open_targets.l2g_gold_standard.OpenTargetsL2GGoldSta
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>gold_standard_curation</code></td>
           <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
@@ -3565,7 +3607,7 @@ <h3 id="gentropy.datasource.open_targets.l2g_gold_standard.OpenTargetsL2GGoldSta
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3574,7 +3616,7 @@ <h3 id="gentropy.datasource.open_targets.l2g_gold_standard.OpenTargetsL2GGoldSta
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>DataFrame</code></td>          <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
           </td>
@@ -3705,7 +3747,15 @@ <h3 id="gentropy.datasource.open_targets.l2g_gold_standard.OpenTargetsL2GGoldSta
         <span class="label"> Contributors </span>
       </li>
       <li class="contributors-holder">
-        <ul class="contributors" data-bi-name="contributors"></ul>
+        <ul class="contributors" data-bi-name="contributors"><li>
+            <a
+              href="https://github.com/d0choa"
+              title="d0choa"
+              data-bi-name="contributorprofile"
+              target="_blank"
+              ><img src="https://avatars.githubusercontent.com/u/5097586?v=4" alt="d0choa"
+            /></a>
+          </li></ul>
       </li>
     </ul>
   </small>
diff --git a/python_api/datasources/open_targets/target/index.html b/python_api/datasources/open_targets/target/index.html
index 05455ddc8..09b9895b9 100644
--- a/python_api/datasources/open_targets/target/index.html
+++ b/python_api/datasources/open_targets/target/index.html
@@ -2259,6 +2259,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../methods/susie_inf/" class="md-nav__link">
         
@@ -2290,7 +2332,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2306,14 +2348,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3099,7 +3141,7 @@ <h3 id="gentropy.datasource.open_targets.target.OpenTargetsTarget.as_gene_index"
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3110,7 +3152,7 @@ <h3 id="gentropy.datasource.open_targets.target.OpenTargetsTarget.as_gene_index"
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>target_index</code></td>
           <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
@@ -3129,7 +3171,7 @@ <h3 id="gentropy.datasource.open_targets.target.OpenTargetsTarget.as_gene_index"
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3138,7 +3180,7 @@ <h3 id="gentropy.datasource.open_targets.target.OpenTargetsTarget.as_gene_index"
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>GeneIndex</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.gene_index.GeneIndex" href="../../../datasets/gene_index/#gentropy.dataset.gene_index.GeneIndex">GeneIndex</a></code>
           </td>
@@ -3265,7 +3307,15 @@ <h3 id="gentropy.datasource.open_targets.target.OpenTargetsTarget.as_gene_index"
         <span class="label"> Contributors </span>
       </li>
       <li class="contributors-holder">
-        <ul class="contributors" data-bi-name="contributors"></ul>
+        <ul class="contributors" data-bi-name="contributors"><li>
+            <a
+              href="https://github.com/d0choa"
+              title="d0choa"
+              data-bi-name="contributorprofile"
+              target="_blank"
+              ><img src="https://avatars.githubusercontent.com/u/5097586?v=4" alt="d0choa"
+            /></a>
+          </li></ul>
       </li>
     </ul>
   </small>
diff --git a/python_api/methods/_methods/index.html b/python_api/methods/_methods/index.html
index bde5b9648..b17f67cfb 100644
--- a/python_api/methods/_methods/index.html
+++ b/python_api/methods/_methods/index.html
@@ -2192,6 +2192,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../susie_inf/" class="md-nav__link">
         
@@ -2223,7 +2265,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2239,14 +2281,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -2839,7 +2881,15 @@ <h1>Methods</h1>
         <span class="label"> Contributors </span>
       </li>
       <li class="contributors-holder">
-        <ul class="contributors" data-bi-name="contributors"></ul>
+        <ul class="contributors" data-bi-name="contributors"><li>
+            <a
+              href="https://github.com/d0choa"
+              title="d0choa"
+              data-bi-name="contributorprofile"
+              target="_blank"
+              ><img src="https://avatars.githubusercontent.com/u/5097586?v=4" alt="d0choa"
+            /></a>
+          </li></ul>
       </li>
     </ul>
   </small>
diff --git a/python_api/methods/carma/index.html b/python_api/methods/carma/index.html
index 5e41b3266..dc6a47127 100644
--- a/python_api/methods/carma/index.html
+++ b/python_api/methods/carma/index.html
@@ -2266,6 +2266,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../susie_inf/" class="md-nav__link">
         
@@ -2297,7 +2339,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2313,14 +2355,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -2953,8 +2995,7 @@ <h2 id="gentropy.method.carma.CARMA" class="doc doc-heading">
 
             <details class="quote">
               <summary>Source code in <code>src/gentropy/method/carma.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 15</span>
-<span class="normal"> 16</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 16</span>
 <span class="normal"> 17</span>
 <span class="normal"> 18</span>
 <span class="normal"> 19</span>
@@ -3837,7 +3878,20 @@ <h2 id="gentropy.method.carma.CARMA" class="doc doc-heading">
 <span class="normal">896</span>
 <span class="normal">897</span>
 <span class="normal">898</span>
-<span class="normal">899</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">CARMA</span><span class="p">:</span>
+<span class="normal">899</span>
+<span class="normal">900</span>
+<span class="normal">901</span>
+<span class="normal">902</span>
+<span class="normal">903</span>
+<span class="normal">904</span>
+<span class="normal">905</span>
+<span class="normal">906</span>
+<span class="normal">907</span>
+<span class="normal">908</span>
+<span class="normal">909</span>
+<span class="normal">910</span>
+<span class="normal">911</span>
+<span class="normal">912</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">CARMA</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Implementation of CARMA outlier detection method.&quot;&quot;&quot;</span>
 
     <span class="nd">@staticmethod</span>
@@ -3857,6 +3911,8 @@ <h2 id="gentropy.method.carma.CARMA" class="doc doc-heading">
 <span class="sd">                - B_list: A dataframe containing the marginal likelihoods and the corresponding model space or None.</span>
 <span class="sd">                - Outliers: A list of outlier SNPs or None.</span>
 <span class="sd">        &quot;&quot;&quot;</span>
+        <span class="c1"># Ignore pandas future warnings</span>
+        <span class="n">warnings</span><span class="o">.</span><span class="n">simplefilter</span><span class="p">(</span><span class="n">action</span><span class="o">=</span><span class="s2">&quot;ignore&quot;</span><span class="p">,</span> <span class="n">category</span><span class="o">=</span><span class="ne">FutureWarning</span><span class="p">)</span>
         <span class="k">try</span><span class="p">:</span>
             <span class="c1"># Execute CARMA.CARMA_spike_slab_noEM with a timeout</span>
             <span class="k">with</span> <span class="n">concurrent</span><span class="o">.</span><span class="n">futures</span><span class="o">.</span><span class="n">ThreadPoolExecutor</span><span class="p">(</span><span class="n">max_workers</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span> <span class="k">as</span> <span class="n">executor</span><span class="p">:</span>
@@ -4679,9 +4735,19 @@ <h2 id="gentropy.method.carma.CARMA" class="doc doc-heading">
                         <span class="n">sec_sample</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">choice</span><span class="p">(</span>
                             <span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">3</span><span class="p">),</span> <span class="mi">1</span><span class="p">,</span> <span class="n">p</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="n">aa</span><span class="p">)</span> <span class="o">/</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="n">aa</span><span class="p">))</span>
                         <span class="p">)</span>
-                        <span class="n">S</span> <span class="o">=</span> <span class="n">set_gamma</span><span class="p">[</span><span class="n">sec_sample</span><span class="p">[</span><span class="mi">0</span><span class="p">]][</span>
-                            <span class="nb">int</span><span class="p">(</span><span class="n">set_star</span><span class="p">[</span><span class="s2">&quot;gamma_set_index&quot;</span><span class="p">][</span><span class="n">sec_sample</span><span class="p">[</span><span class="mi">0</span><span class="p">]])</span>
-                        <span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
+                        <span class="k">if</span> <span class="n">set_gamma</span><span class="p">[</span><span class="n">sec_sample</span><span class="p">[</span><span class="mi">0</span><span class="p">]]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+                            <span class="n">S</span> <span class="o">=</span> <span class="n">set_gamma</span><span class="p">[</span><span class="n">sec_sample</span><span class="p">[</span><span class="mi">0</span><span class="p">]][</span>
+                                <span class="nb">int</span><span class="p">(</span><span class="n">set_star</span><span class="p">[</span><span class="s2">&quot;gamma_set_index&quot;</span><span class="p">][</span><span class="n">sec_sample</span><span class="p">[</span><span class="mi">0</span><span class="p">]])</span>
+                            <span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
+                        <span class="k">else</span><span class="p">:</span>
+                            <span class="n">sec_sample</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">choice</span><span class="p">(</span>
+                                <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">3</span><span class="p">),</span>
+                                <span class="mi">1</span><span class="p">,</span>
+                                <span class="n">p</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="n">aa</span><span class="p">)[[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">]]</span> <span class="o">/</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="n">aa</span><span class="p">)[[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">]]),</span>
+                            <span class="p">)</span>
+                            <span class="n">S</span> <span class="o">=</span> <span class="n">set_gamma</span><span class="p">[</span><span class="n">sec_sample</span><span class="p">[</span><span class="mi">0</span><span class="p">]][</span>
+                                <span class="nb">int</span><span class="p">(</span><span class="n">set_star</span><span class="p">[</span><span class="s2">&quot;gamma_set_index&quot;</span><span class="p">][</span><span class="n">sec_sample</span><span class="p">[</span><span class="mi">0</span><span class="p">]])</span>
+                            <span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
 
                 <span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">conditional_S</span><span class="p">:</span>
                     <span class="k">if</span> <span class="n">item</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">S</span><span class="p">:</span>
@@ -4758,7 +4824,7 @@ <h3 id="gentropy.method.carma.CARMA.CARMA_spike_slab_noEM" class="doc doc-headin
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -4769,7 +4835,7 @@ <h3 id="gentropy.method.carma.CARMA.CARMA_spike_slab_noEM" class="doc doc-headin
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>z</code></td>
           <td>
                 <code><span title="numpy.ndarray">ndarray</span></code>
@@ -4783,7 +4849,7 @@ <h3 id="gentropy.method.carma.CARMA.CARMA_spike_slab_noEM" class="doc doc-headin
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>ld</code></td>
           <td>
                 <code><span title="numpy.ndarray">ndarray</span></code>
@@ -4797,7 +4863,7 @@ <h3 id="gentropy.method.carma.CARMA.CARMA_spike_slab_noEM" class="doc doc-headin
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>lambda_val</code></td>
           <td>
                 <code>float</code>
@@ -4811,7 +4877,7 @@ <h3 id="gentropy.method.carma.CARMA.CARMA_spike_slab_noEM" class="doc doc-headin
                 <code>1</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>Max_Model_Dim</code></td>
           <td>
                 <code>int</code>
@@ -4825,7 +4891,7 @@ <h3 id="gentropy.method.carma.CARMA.CARMA_spike_slab_noEM" class="doc doc-headin
                 <code>200000</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>all_iter</code></td>
           <td>
                 <code>int</code>
@@ -4839,7 +4905,7 @@ <h3 id="gentropy.method.carma.CARMA.CARMA_spike_slab_noEM" class="doc doc-headin
                 <code>1</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>all_inner_iter</code></td>
           <td>
                 <code>int</code>
@@ -4853,7 +4919,7 @@ <h3 id="gentropy.method.carma.CARMA.CARMA_spike_slab_noEM" class="doc doc-headin
                 <code>10</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>epsilon_threshold</code></td>
           <td>
                 <code>float</code>
@@ -4867,7 +4933,7 @@ <h3 id="gentropy.method.carma.CARMA.CARMA_spike_slab_noEM" class="doc doc-headin
                 <code>1e-05</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>num_causal</code></td>
           <td>
                 <code>int</code>
@@ -4881,7 +4947,7 @@ <h3 id="gentropy.method.carma.CARMA.CARMA_spike_slab_noEM" class="doc doc-headin
                 <code>10</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>tau</code></td>
           <td>
                 <code>float</code>
@@ -4895,7 +4961,7 @@ <h3 id="gentropy.method.carma.CARMA.CARMA_spike_slab_noEM" class="doc doc-headin
                 <code>0.04</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>outlier_switch</code></td>
           <td>
                 <code>bool</code>
@@ -4909,7 +4975,7 @@ <h3 id="gentropy.method.carma.CARMA.CARMA_spike_slab_noEM" class="doc doc-headin
                 <code>True</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>outlier_BF_index</code></td>
           <td>
                 <code>float</code>
@@ -4928,7 +4994,7 @@ <h3 id="gentropy.method.carma.CARMA.CARMA_spike_slab_noEM" class="doc doc-headin
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -4937,7 +5003,7 @@ <h3 id="gentropy.method.carma.CARMA.CARMA_spike_slab_noEM" class="doc doc-headin
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td>
                 <code>dict[str, <span title="typing.Any">Any</span>]</code>
           </td>
@@ -4955,10 +5021,7 @@ <h3 id="gentropy.method.carma.CARMA.CARMA_spike_slab_noEM" class="doc doc-headin
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/method/carma.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 46</span>
-<span class="normal"> 47</span>
-<span class="normal"> 48</span>
-<span class="normal"> 49</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 49</span>
 <span class="normal"> 50</span>
 <span class="normal"> 51</span>
 <span class="normal"> 52</span>
@@ -5046,7 +5109,10 @@ <h3 id="gentropy.method.carma.CARMA.CARMA_spike_slab_noEM" class="doc doc-headin
 <span class="normal">134</span>
 <span class="normal">135</span>
 <span class="normal">136</span>
-<span class="normal">137</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@staticmethod</span>
+<span class="normal">137</span>
+<span class="normal">138</span>
+<span class="normal">139</span>
+<span class="normal">140</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@staticmethod</span>
 <span class="k">def</span> <span class="nf">CARMA_spike_slab_noEM</span><span class="p">(</span>
     <span class="n">z</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span>
     <span class="n">ld</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span>
@@ -5165,7 +5231,7 @@ <h3 id="gentropy.method.carma.CARMA.time_limited_CARMA_spike_slab_noEM" class="d
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -5176,7 +5242,7 @@ <h3 id="gentropy.method.carma.CARMA.time_limited_CARMA_spike_slab_noEM" class="d
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>z</code></td>
           <td>
                 <code><span title="numpy.ndarray">ndarray</span></code>
@@ -5190,7 +5256,7 @@ <h3 id="gentropy.method.carma.CARMA.time_limited_CARMA_spike_slab_noEM" class="d
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>ld</code></td>
           <td>
                 <code><span title="numpy.ndarray">ndarray</span></code>
@@ -5204,7 +5270,7 @@ <h3 id="gentropy.method.carma.CARMA.time_limited_CARMA_spike_slab_noEM" class="d
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>sec_threshold</code></td>
           <td>
                 <code>float</code>
@@ -5223,7 +5289,7 @@ <h3 id="gentropy.method.carma.CARMA.time_limited_CARMA_spike_slab_noEM" class="d
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -5232,7 +5298,7 @@ <h3 id="gentropy.method.carma.CARMA.time_limited_CARMA_spike_slab_noEM" class="d
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td>
                 <code>dict[str, <span title="typing.Any">Any</span>]</code>
           </td>
@@ -5250,8 +5316,7 @@ <h3 id="gentropy.method.carma.CARMA.time_limited_CARMA_spike_slab_noEM" class="d
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/method/carma.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">18</span>
-<span class="normal">19</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">19</span>
 <span class="normal">20</span>
 <span class="normal">21</span>
 <span class="normal">22</span>
@@ -5276,7 +5341,10 @@ <h3 id="gentropy.method.carma.CARMA.time_limited_CARMA_spike_slab_noEM" class="d
 <span class="normal">41</span>
 <span class="normal">42</span>
 <span class="normal">43</span>
-<span class="normal">44</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@staticmethod</span>
+<span class="normal">44</span>
+<span class="normal">45</span>
+<span class="normal">46</span>
+<span class="normal">47</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@staticmethod</span>
 <span class="k">def</span> <span class="nf">time_limited_CARMA_spike_slab_noEM</span><span class="p">(</span>
     <span class="n">z</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">ld</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">sec_threshold</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mi">600</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]:</span>
@@ -5293,6 +5361,8 @@ <h3 id="gentropy.method.carma.CARMA.time_limited_CARMA_spike_slab_noEM" class="d
 <span class="sd">            - B_list: A dataframe containing the marginal likelihoods and the corresponding model space or None.</span>
 <span class="sd">            - Outliers: A list of outlier SNPs or None.</span>
 <span class="sd">    &quot;&quot;&quot;</span>
+    <span class="c1"># Ignore pandas future warnings</span>
+    <span class="n">warnings</span><span class="o">.</span><span class="n">simplefilter</span><span class="p">(</span><span class="n">action</span><span class="o">=</span><span class="s2">&quot;ignore&quot;</span><span class="p">,</span> <span class="n">category</span><span class="o">=</span><span class="ne">FutureWarning</span><span class="p">)</span>
     <span class="k">try</span><span class="p">:</span>
         <span class="c1"># Execute CARMA.CARMA_spike_slab_noEM with a timeout</span>
         <span class="k">with</span> <span class="n">concurrent</span><span class="o">.</span><span class="n">futures</span><span class="o">.</span><span class="n">ThreadPoolExecutor</span><span class="p">(</span><span class="n">max_workers</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span> <span class="k">as</span> <span class="n">executor</span><span class="p">:</span>
diff --git a/python_api/methods/clumping/index.html b/python_api/methods/clumping/index.html
index 1093aae3d..46cda1a2a 100644
--- a/python_api/methods/clumping/index.html
+++ b/python_api/methods/clumping/index.html
@@ -2207,6 +2207,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../susie_inf/" class="md-nav__link">
         
@@ -2238,7 +2280,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2254,14 +2296,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3072,96 +3114,7 @@ <h2 id="gentropy.method.window_based_clumping.WindowBasedClumping" class="doc do
 <span class="normal">243</span>
 <span class="normal">244</span>
 <span class="normal">245</span>
-<span class="normal">246</span>
-<span class="normal">247</span>
-<span class="normal">248</span>
-<span class="normal">249</span>
-<span class="normal">250</span>
-<span class="normal">251</span>
-<span class="normal">252</span>
-<span class="normal">253</span>
-<span class="normal">254</span>
-<span class="normal">255</span>
-<span class="normal">256</span>
-<span class="normal">257</span>
-<span class="normal">258</span>
-<span class="normal">259</span>
-<span class="normal">260</span>
-<span class="normal">261</span>
-<span class="normal">262</span>
-<span class="normal">263</span>
-<span class="normal">264</span>
-<span class="normal">265</span>
-<span class="normal">266</span>
-<span class="normal">267</span>
-<span class="normal">268</span>
-<span class="normal">269</span>
-<span class="normal">270</span>
-<span class="normal">271</span>
-<span class="normal">272</span>
-<span class="normal">273</span>
-<span class="normal">274</span>
-<span class="normal">275</span>
-<span class="normal">276</span>
-<span class="normal">277</span>
-<span class="normal">278</span>
-<span class="normal">279</span>
-<span class="normal">280</span>
-<span class="normal">281</span>
-<span class="normal">282</span>
-<span class="normal">283</span>
-<span class="normal">284</span>
-<span class="normal">285</span>
-<span class="normal">286</span>
-<span class="normal">287</span>
-<span class="normal">288</span>
-<span class="normal">289</span>
-<span class="normal">290</span>
-<span class="normal">291</span>
-<span class="normal">292</span>
-<span class="normal">293</span>
-<span class="normal">294</span>
-<span class="normal">295</span>
-<span class="normal">296</span>
-<span class="normal">297</span>
-<span class="normal">298</span>
-<span class="normal">299</span>
-<span class="normal">300</span>
-<span class="normal">301</span>
-<span class="normal">302</span>
-<span class="normal">303</span>
-<span class="normal">304</span>
-<span class="normal">305</span>
-<span class="normal">306</span>
-<span class="normal">307</span>
-<span class="normal">308</span>
-<span class="normal">309</span>
-<span class="normal">310</span>
-<span class="normal">311</span>
-<span class="normal">312</span>
-<span class="normal">313</span>
-<span class="normal">314</span>
-<span class="normal">315</span>
-<span class="normal">316</span>
-<span class="normal">317</span>
-<span class="normal">318</span>
-<span class="normal">319</span>
-<span class="normal">320</span>
-<span class="normal">321</span>
-<span class="normal">322</span>
-<span class="normal">323</span>
-<span class="normal">324</span>
-<span class="normal">325</span>
-<span class="normal">326</span>
-<span class="normal">327</span>
-<span class="normal">328</span>
-<span class="normal">329</span>
-<span class="normal">330</span>
-<span class="normal">331</span>
-<span class="normal">332</span>
-<span class="normal">333</span>
-<span class="normal">334</span>
-<span class="normal">335</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">WindowBasedClumping</span><span class="p">:</span>
+<span class="normal">246</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">WindowBasedClumping</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Get semi-lead snps from summary statistics using a window based function.&quot;&quot;&quot;</span>
 
     <span class="nd">@staticmethod</span>
@@ -3292,22 +3245,21 @@ <h2 id="gentropy.method.window_based_clumping.WindowBasedClumping" class="doc do
 
         <span class="k">return</span> <span class="n">DenseVector</span><span class="p">(</span><span class="n">is_lead</span><span class="p">)</span>
 
-    <span class="nd">@classmethod</span>
+    <span class="nd">@staticmethod</span>
     <span class="k">def</span> <span class="nf">clump</span><span class="p">(</span>
-        <span class="bp">cls</span><span class="p">:</span> <span class="nb">type</span><span class="p">[</span><span class="n">WindowBasedClumping</span><span class="p">],</span>
-        <span class="n">summary_stats</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">,</span>
-        <span class="n">window_length</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
-        <span class="n">p_value_significance</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">5e-8</span><span class="p">,</span>
+        <span class="n">summary_statistics</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">,</span>
+        <span class="n">distance</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">500_000</span><span class="p">,</span>
+        <span class="n">gwas_significance</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">5e-8</span><span class="p">,</span>
     <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyLocus</span><span class="p">:</span>
-<span class="w">        </span><span class="sd">&quot;&quot;&quot;Clump summary statistics by distance.</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Clump significant signals from summary statistics based on window.</span>
 
 <span class="sd">        Args:</span>
-<span class="sd">            summary_stats (SummaryStatistics): summary statistics to clump</span>
-<span class="sd">            window_length (int): window length in basepair</span>
-<span class="sd">            p_value_significance (float): only more significant variants are considered</span>
+<span class="sd">            summary_statistics (SummaryStatistics): Summary statistics to be used for clumping.</span>
+<span class="sd">            distance (int): Distance in base pairs to be used for clumping. Defaults to 500_000.</span>
+<span class="sd">            gwas_significance (float): GWAS significance threshold. Defaults to 5e-8.</span>
 
 <span class="sd">        Returns:</span>
-<span class="sd">            StudyLocus: clumped summary statistics</span>
+<span class="sd">            StudyLocus: clumped summary statistics (without locus collection)</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="c1"># Create window for locus clusters</span>
         <span class="c1"># - variants where the distance between subsequent variants is below the defined threshold.</span>
@@ -3318,9 +3270,9 @@ <h2 id="gentropy.method.window_based_clumping.WindowBasedClumping" class="doc do
 
         <span class="k">return</span> <span class="n">StudyLocus</span><span class="p">(</span>
             <span class="n">_df</span><span class="o">=</span><span class="p">(</span>
-                <span class="n">summary_stats</span>
+                <span class="n">summary_statistics</span>
                 <span class="c1"># Dropping snps below significance - all subsequent steps are done on significant variants:</span>
-                <span class="o">.</span><span class="n">pvalue_filter</span><span class="p">(</span><span class="n">p_value_significance</span><span class="p">)</span>
+                <span class="o">.</span><span class="n">pvalue_filter</span><span class="p">(</span><span class="n">gwas_significance</span><span class="p">)</span>
                 <span class="o">.</span><span class="n">df</span>
                 <span class="c1"># Clustering summary variants for efficient windowing (complexity reduction):</span>
                 <span class="o">.</span><span class="n">withColumn</span><span class="p">(</span>
@@ -3329,7 +3281,7 @@ <h2 id="gentropy.method.window_based_clumping.WindowBasedClumping" class="doc do
                         <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">),</span>
                         <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;chromosome&quot;</span><span class="p">),</span>
                         <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;position&quot;</span><span class="p">),</span>
-                        <span class="n">window_length</span><span class="p">,</span>
+                        <span class="n">distance</span><span class="p">,</span>
                     <span class="p">),</span>
                 <span class="p">)</span>
                 <span class="c1"># Within each cluster variants are ranked by significance:</span>
@@ -3354,7 +3306,7 @@ <h2 id="gentropy.method.window_based_clumping.WindowBasedClumping" class="doc do
                         <span class="n">fml</span><span class="o">.</span><span class="n">vector_to_array</span><span class="p">(</span>
                             <span class="n">f</span><span class="o">.</span><span class="n">udf</span><span class="p">(</span><span class="n">WindowBasedClumping</span><span class="o">.</span><span class="n">_prune_peak</span><span class="p">,</span> <span class="n">VectorUDT</span><span class="p">())(</span>
                                 <span class="n">fml</span><span class="o">.</span><span class="n">array_to_vector</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;collectedPositions&quot;</span><span class="p">)),</span>
-                                <span class="n">f</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="n">window_length</span><span class="p">),</span>
+                                <span class="n">f</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="n">distance</span><span class="p">),</span>
                             <span class="p">)</span>
                         <span class="p">),</span>
                     <span class="p">),</span>
@@ -3386,94 +3338,6 @@ <h2 id="gentropy.method.window_based_clumping.WindowBasedClumping" class="doc do
             <span class="p">),</span>
             <span class="n">_schema</span><span class="o">=</span><span class="n">StudyLocus</span><span class="o">.</span><span class="n">get_schema</span><span class="p">(),</span>
         <span class="p">)</span>
-
-    <span class="nd">@classmethod</span>
-    <span class="k">def</span> <span class="nf">clump_with_locus</span><span class="p">(</span>
-        <span class="bp">cls</span><span class="p">:</span> <span class="nb">type</span><span class="p">[</span><span class="n">WindowBasedClumping</span><span class="p">],</span>
-        <span class="n">summary_stats</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">,</span>
-        <span class="n">window_length</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
-        <span class="n">p_value_significance</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">5e-8</span><span class="p">,</span>
-        <span class="n">p_value_baseline</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.05</span><span class="p">,</span>
-        <span class="n">locus_window_length</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyLocus</span><span class="p">:</span>
-<span class="w">        </span><span class="sd">&quot;&quot;&quot;Clump significant associations while collecting locus around them.</span>
-
-<span class="sd">        Args:</span>
-<span class="sd">            summary_stats (SummaryStatistics): Input summary statistics dataset</span>
-<span class="sd">            window_length (int): Window size in  bp, used for distance based clumping.</span>
-<span class="sd">            p_value_significance (float): GWAS significance threshold used to filter peaks. Defaults to 5e-8.</span>
-<span class="sd">            p_value_baseline (float): Least significant threshold. Below this, all snps are dropped. Defaults to 0.05.</span>
-<span class="sd">            locus_window_length (int | None): The distance for collecting locus around the semi indices. Defaults to None.</span>
-
-<span class="sd">        Returns:</span>
-<span class="sd">            StudyLocus: StudyLocus after clumping with information about the `locus`</span>
-<span class="sd">        &quot;&quot;&quot;</span>
-        <span class="c1"># If no locus window provided, using the same value:</span>
-        <span class="k">if</span> <span class="n">locus_window_length</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
-            <span class="n">locus_window_length</span> <span class="o">=</span> <span class="n">window_length</span>
-
-        <span class="c1"># Run distance based clumping on the summary stats:</span>
-        <span class="n">clumped_dataframe</span> <span class="o">=</span> <span class="n">WindowBasedClumping</span><span class="o">.</span><span class="n">clump</span><span class="p">(</span>
-            <span class="n">summary_stats</span><span class="p">,</span>
-            <span class="n">window_length</span><span class="o">=</span><span class="n">window_length</span><span class="p">,</span>
-            <span class="n">p_value_significance</span><span class="o">=</span><span class="n">p_value_significance</span><span class="p">,</span>
-        <span class="p">)</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;clumped&quot;</span><span class="p">)</span>
-
-        <span class="c1"># Get list of columns from clumped dataset for further propagation:</span>
-        <span class="n">clumped_columns</span> <span class="o">=</span> <span class="n">clumped_dataframe</span><span class="o">.</span><span class="n">columns</span>
-
-        <span class="c1"># Dropping variants not meeting the baseline criteria:</span>
-        <span class="n">sumstats_baseline</span> <span class="o">=</span> <span class="n">summary_stats</span><span class="o">.</span><span class="n">pvalue_filter</span><span class="p">(</span><span class="n">p_value_baseline</span><span class="p">)</span><span class="o">.</span><span class="n">df</span>
-
-        <span class="c1"># Renaming columns:</span>
-        <span class="n">sumstats_baseline_renamed</span> <span class="o">=</span> <span class="n">sumstats_baseline</span><span class="o">.</span><span class="n">selectExpr</span><span class="p">(</span>
-            <span class="o">*</span><span class="p">[</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">col</span><span class="si">}</span><span class="s2"> as tag_</span><span class="si">{</span><span class="n">col</span><span class="si">}</span><span class="s2">&quot;</span> <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">sumstats_baseline</span><span class="o">.</span><span class="n">columns</span><span class="p">]</span>
-        <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;sumstat&quot;</span><span class="p">)</span>
-
-        <span class="n">study_locus_df</span> <span class="o">=</span> <span class="p">(</span>
-            <span class="n">sumstats_baseline_renamed</span>
-            <span class="c1"># Joining the two datasets together:</span>
-            <span class="o">.</span><span class="n">join</span><span class="p">(</span>
-                <span class="n">f</span><span class="o">.</span><span class="n">broadcast</span><span class="p">(</span><span class="n">clumped_dataframe</span><span class="p">),</span>
-                <span class="n">on</span><span class="o">=</span><span class="p">[</span>
-                    <span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;sumstat.tag_studyId&quot;</span><span class="p">)</span> <span class="o">==</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;clumped.studyId&quot;</span><span class="p">))</span>
-                    <span class="o">&amp;</span> <span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;sumstat.tag_chromosome&quot;</span><span class="p">)</span> <span class="o">==</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;clumped.chromosome&quot;</span><span class="p">))</span>
-                    <span class="o">&amp;</span> <span class="p">(</span>
-                        <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;sumstat.tag_position&quot;</span><span class="p">)</span>
-                        <span class="o">&gt;=</span> <span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;clumped.position&quot;</span><span class="p">)</span> <span class="o">-</span> <span class="n">locus_window_length</span><span class="p">)</span>
-                    <span class="p">)</span>
-                    <span class="o">&amp;</span> <span class="p">(</span>
-                        <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;sumstat.tag_position&quot;</span><span class="p">)</span>
-                        <span class="o">&lt;=</span> <span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;clumped.position&quot;</span><span class="p">)</span> <span class="o">+</span> <span class="n">locus_window_length</span><span class="p">)</span>
-                    <span class="p">)</span>
-                <span class="p">],</span>
-                <span class="n">how</span><span class="o">=</span><span class="s2">&quot;right&quot;</span><span class="p">,</span>
-            <span class="p">)</span>
-            <span class="o">.</span><span class="n">withColumn</span><span class="p">(</span>
-                <span class="s2">&quot;locus&quot;</span><span class="p">,</span>
-                <span class="n">f</span><span class="o">.</span><span class="n">struct</span><span class="p">(</span>
-                    <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;tag_variantId&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;variantId&quot;</span><span class="p">),</span>
-                    <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;tag_beta&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;beta&quot;</span><span class="p">),</span>
-                    <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;tag_pValueMantissa&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;pValueMantissa&quot;</span><span class="p">),</span>
-                    <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;tag_pValueExponent&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;pValueExponent&quot;</span><span class="p">),</span>
-                    <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;tag_standardError&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;standardError&quot;</span><span class="p">),</span>
-                <span class="p">),</span>
-            <span class="p">)</span>
-            <span class="o">.</span><span class="n">groupby</span><span class="p">(</span><span class="s2">&quot;studyLocusId&quot;</span><span class="p">)</span>
-            <span class="o">.</span><span class="n">agg</span><span class="p">(</span>
-                <span class="o">*</span><span class="p">[</span>
-                    <span class="n">f</span><span class="o">.</span><span class="n">first</span><span class="p">(</span><span class="n">col</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="n">col</span><span class="p">)</span>
-                    <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">clumped_columns</span>
-                    <span class="k">if</span> <span class="n">col</span> <span class="o">!=</span> <span class="s2">&quot;studyLocusId&quot;</span>
-                <span class="p">],</span>
-                <span class="n">f</span><span class="o">.</span><span class="n">collect_list</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;locus&quot;</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;locus&quot;</span><span class="p">),</span>
-            <span class="p">)</span>
-        <span class="p">)</span>
-
-        <span class="k">return</span> <span class="n">StudyLocus</span><span class="p">(</span>
-            <span class="n">_df</span><span class="o">=</span><span class="n">study_locus_df</span><span class="p">,</span>
-            <span class="n">_schema</span><span class="o">=</span><span class="n">StudyLocus</span><span class="o">.</span><span class="n">get_schema</span><span class="p">(),</span>
-        <span class="p">)</span>
 </code></pre></div></td></tr></table></div>
             </details>
 
@@ -3495,10 +3359,10 @@ <h2 id="gentropy.method.window_based_clumping.WindowBasedClumping" class="doc do
 
 
 <h3 id="gentropy.method.window_based_clumping.WindowBasedClumping.clump" class="doc doc-heading">
-          <code class="highlight language-python"><span class="n">clump</span><span class="p">(</span><span class="n">summary_stats</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">,</span> <span class="n">window_length</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">p_value_significance</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">5e-08</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyLocus</span></code>
+          <code class="highlight language-python"><span class="n">clump</span><span class="p">(</span><span class="n">summary_statistics</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">,</span> <span class="n">distance</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">500000</span><span class="p">,</span> <span class="n">gwas_significance</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">5e-08</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyLocus</span></code>
   
   <span class="doc doc-labels">
-      <small class="doc doc-label doc-label-classmethod"><code>classmethod</code></small>
+      <small class="doc doc-label doc-label-staticmethod"><code>staticmethod</code></small>
   </span>
 
 <a href="#gentropy.method.window_based_clumping.WindowBasedClumping.clump" class="headerlink" title="Permanent link">&para;</a></h3>
@@ -3506,11 +3370,11 @@ <h3 id="gentropy.method.window_based_clumping.WindowBasedClumping.clump" class="
 
   <div class="doc doc-contents ">
   
-      <p>Clump summary statistics by distance.</p>
+      <p>Clump significant signals from summary statistics based on window.</p>
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3521,42 +3385,42 @@ <h3 id="gentropy.method.window_based_clumping.WindowBasedClumping.clump" class="
       </tr>
     </thead>
     <tbody>
-        <tr>
-          <td><code>summary_stats</code></td>
+        <tr class="doc-section-item">
+          <td><code>summary_statistics</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.summary_statistics.SummaryStatistics" href="../../datasets/summary_statistics/#gentropy.dataset.summary_statistics.SummaryStatistics">SummaryStatistics</a></code>
           </td>
           <td>
             <div class="doc-md-description">
-              <p>summary statistics to clump</p>
+              <p>Summary statistics to be used for clumping.</p>
             </div>
           </td>
           <td>
               <em>required</em>
           </td>
         </tr>
-        <tr>
-          <td><code>window_length</code></td>
+        <tr class="doc-section-item">
+          <td><code>distance</code></td>
           <td>
                 <code>int</code>
           </td>
           <td>
             <div class="doc-md-description">
-              <p>window length in basepair</p>
+              <p>Distance in base pairs to be used for clumping. Defaults to 500_000.</p>
             </div>
           </td>
           <td>
-              <em>required</em>
+                <code>500000</code>
           </td>
         </tr>
-        <tr>
-          <td><code>p_value_significance</code></td>
+        <tr class="doc-section-item">
+          <td><code>gwas_significance</code></td>
           <td>
                 <code>float</code>
           </td>
           <td>
             <div class="doc-md-description">
-              <p>only more significant variants are considered</p>
+              <p>GWAS significance threshold. Defaults to 5e-8.</p>
             </div>
           </td>
           <td>
@@ -3568,7 +3432,7 @@ <h3 id="gentropy.method.window_based_clumping.WindowBasedClumping.clump" class="
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3577,13 +3441,13 @@ <h3 id="gentropy.method.window_based_clumping.WindowBasedClumping.clump" class="
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StudyLocus</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_locus.StudyLocus" href="../../datasets/study_locus/#gentropy.dataset.study_locus.StudyLocus">StudyLocus</a></code>
           </td>
           <td>
             <div class="doc-md-description">
-              <p>clumped summary statistics</p>
+              <p>clumped summary statistics (without locus collection)</p>
             </div>
           </td>
         </tr>
@@ -3684,23 +3548,21 @@ <h3 id="gentropy.method.window_based_clumping.WindowBasedClumping.clump" class="
 <span class="normal">243</span>
 <span class="normal">244</span>
 <span class="normal">245</span>
-<span class="normal">246</span>
-<span class="normal">247</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@classmethod</span>
+<span class="normal">246</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@staticmethod</span>
 <span class="k">def</span> <span class="nf">clump</span><span class="p">(</span>
-    <span class="bp">cls</span><span class="p">:</span> <span class="nb">type</span><span class="p">[</span><span class="n">WindowBasedClumping</span><span class="p">],</span>
-    <span class="n">summary_stats</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">,</span>
-    <span class="n">window_length</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
-    <span class="n">p_value_significance</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">5e-8</span><span class="p">,</span>
+    <span class="n">summary_statistics</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">,</span>
+    <span class="n">distance</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">500_000</span><span class="p">,</span>
+    <span class="n">gwas_significance</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">5e-8</span><span class="p">,</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyLocus</span><span class="p">:</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;Clump summary statistics by distance.</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;Clump significant signals from summary statistics based on window.</span>
 
 <span class="sd">    Args:</span>
-<span class="sd">        summary_stats (SummaryStatistics): summary statistics to clump</span>
-<span class="sd">        window_length (int): window length in basepair</span>
-<span class="sd">        p_value_significance (float): only more significant variants are considered</span>
+<span class="sd">        summary_statistics (SummaryStatistics): Summary statistics to be used for clumping.</span>
+<span class="sd">        distance (int): Distance in base pairs to be used for clumping. Defaults to 500_000.</span>
+<span class="sd">        gwas_significance (float): GWAS significance threshold. Defaults to 5e-8.</span>
 
 <span class="sd">    Returns:</span>
-<span class="sd">        StudyLocus: clumped summary statistics</span>
+<span class="sd">        StudyLocus: clumped summary statistics (without locus collection)</span>
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="c1"># Create window for locus clusters</span>
     <span class="c1"># - variants where the distance between subsequent variants is below the defined threshold.</span>
@@ -3711,9 +3573,9 @@ <h3 id="gentropy.method.window_based_clumping.WindowBasedClumping.clump" class="
 
     <span class="k">return</span> <span class="n">StudyLocus</span><span class="p">(</span>
         <span class="n">_df</span><span class="o">=</span><span class="p">(</span>
-            <span class="n">summary_stats</span>
+            <span class="n">summary_statistics</span>
             <span class="c1"># Dropping snps below significance - all subsequent steps are done on significant variants:</span>
-            <span class="o">.</span><span class="n">pvalue_filter</span><span class="p">(</span><span class="n">p_value_significance</span><span class="p">)</span>
+            <span class="o">.</span><span class="n">pvalue_filter</span><span class="p">(</span><span class="n">gwas_significance</span><span class="p">)</span>
             <span class="o">.</span><span class="n">df</span>
             <span class="c1"># Clustering summary variants for efficient windowing (complexity reduction):</span>
             <span class="o">.</span><span class="n">withColumn</span><span class="p">(</span>
@@ -3722,7 +3584,7 @@ <h3 id="gentropy.method.window_based_clumping.WindowBasedClumping.clump" class="
                     <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">),</span>
                     <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;chromosome&quot;</span><span class="p">),</span>
                     <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;position&quot;</span><span class="p">),</span>
-                    <span class="n">window_length</span><span class="p">,</span>
+                    <span class="n">distance</span><span class="p">,</span>
                 <span class="p">),</span>
             <span class="p">)</span>
             <span class="c1"># Within each cluster variants are ranked by significance:</span>
@@ -3747,7 +3609,7 @@ <h3 id="gentropy.method.window_based_clumping.WindowBasedClumping.clump" class="
                     <span class="n">fml</span><span class="o">.</span><span class="n">vector_to_array</span><span class="p">(</span>
                         <span class="n">f</span><span class="o">.</span><span class="n">udf</span><span class="p">(</span><span class="n">WindowBasedClumping</span><span class="o">.</span><span class="n">_prune_peak</span><span class="p">,</span> <span class="n">VectorUDT</span><span class="p">())(</span>
                             <span class="n">fml</span><span class="o">.</span><span class="n">array_to_vector</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;collectedPositions&quot;</span><span class="p">)),</span>
-                            <span class="n">f</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="n">window_length</span><span class="p">),</span>
+                            <span class="n">f</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="n">distance</span><span class="p">),</span>
                         <span class="p">)</span>
                     <span class="p">),</span>
                 <span class="p">),</span>
@@ -3786,316 +3648,6 @@ <h3 id="gentropy.method.window_based_clumping.WindowBasedClumping.clump" class="
 </div>
 
 
-<div class="doc doc-object doc-function">
-
-
-
-<h3 id="gentropy.method.window_based_clumping.WindowBasedClumping.clump_with_locus" class="doc doc-heading">
-          <code class="highlight language-python"><span class="n">clump_with_locus</span><span class="p">(</span><span class="n">summary_stats</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">,</span> <span class="n">window_length</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">p_value_significance</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">5e-08</span><span class="p">,</span> <span class="n">p_value_baseline</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.05</span><span class="p">,</span> <span class="n">locus_window_length</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyLocus</span></code>
-  
-  <span class="doc doc-labels">
-      <small class="doc doc-label doc-label-classmethod"><code>classmethod</code></small>
-  </span>
-
-<a href="#gentropy.method.window_based_clumping.WindowBasedClumping.clump_with_locus" class="headerlink" title="Permanent link">&para;</a></h3>
-
-
-  <div class="doc doc-contents ">
-  
-      <p>Clump significant associations while collecting locus around them.</p>
-
-
-
-  <p><strong>Parameters:</strong></p>
-  <table>
-    <thead>
-      <tr>
-        <th>Name</th>
-        <th>Type</th>
-        <th>Description</th>
-        <th>Default</th>
-      </tr>
-    </thead>
-    <tbody>
-        <tr>
-          <td><code>summary_stats</code></td>
-          <td>
-                <code><a class="autorefs autorefs-internal" title="gentropy.dataset.summary_statistics.SummaryStatistics" href="../../datasets/summary_statistics/#gentropy.dataset.summary_statistics.SummaryStatistics">SummaryStatistics</a></code>
-          </td>
-          <td>
-            <div class="doc-md-description">
-              <p>Input summary statistics dataset</p>
-            </div>
-          </td>
-          <td>
-              <em>required</em>
-          </td>
-        </tr>
-        <tr>
-          <td><code>window_length</code></td>
-          <td>
-                <code>int</code>
-          </td>
-          <td>
-            <div class="doc-md-description">
-              <p>Window size in  bp, used for distance based clumping.</p>
-            </div>
-          </td>
-          <td>
-              <em>required</em>
-          </td>
-        </tr>
-        <tr>
-          <td><code>p_value_significance</code></td>
-          <td>
-                <code>float</code>
-          </td>
-          <td>
-            <div class="doc-md-description">
-              <p>GWAS significance threshold used to filter peaks. Defaults to 5e-8.</p>
-            </div>
-          </td>
-          <td>
-                <code>5e-08</code>
-          </td>
-        </tr>
-        <tr>
-          <td><code>p_value_baseline</code></td>
-          <td>
-                <code>float</code>
-          </td>
-          <td>
-            <div class="doc-md-description">
-              <p>Least significant threshold. Below this, all snps are dropped. Defaults to 0.05.</p>
-            </div>
-          </td>
-          <td>
-                <code>0.05</code>
-          </td>
-        </tr>
-        <tr>
-          <td><code>locus_window_length</code></td>
-          <td>
-                <code>int | None</code>
-          </td>
-          <td>
-            <div class="doc-md-description">
-              <p>The distance for collecting locus around the semi indices. Defaults to None.</p>
-            </div>
-          </td>
-          <td>
-                <code>None</code>
-          </td>
-        </tr>
-    </tbody>
-  </table>
-
-
-
-  <p><strong>Returns:</strong></p>
-  <table>
-    <thead>
-      <tr>
-<th>Name</th>        <th>Type</th>
-        <th>Description</th>
-      </tr>
-    </thead>
-    <tbody>
-        <tr>
-<td><code>StudyLocus</code></td>          <td>
-                <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_locus.StudyLocus" href="../../datasets/study_locus/#gentropy.dataset.study_locus.StudyLocus">StudyLocus</a></code>
-          </td>
-          <td>
-            <div class="doc-md-description">
-              <p>StudyLocus after clumping with information about the <code>locus</code></p>
-            </div>
-          </td>
-        </tr>
-    </tbody>
-  </table>
-
-          <details class="quote">
-            <summary>Source code in <code>src/gentropy/method/window_based_clumping.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">249</span>
-<span class="normal">250</span>
-<span class="normal">251</span>
-<span class="normal">252</span>
-<span class="normal">253</span>
-<span class="normal">254</span>
-<span class="normal">255</span>
-<span class="normal">256</span>
-<span class="normal">257</span>
-<span class="normal">258</span>
-<span class="normal">259</span>
-<span class="normal">260</span>
-<span class="normal">261</span>
-<span class="normal">262</span>
-<span class="normal">263</span>
-<span class="normal">264</span>
-<span class="normal">265</span>
-<span class="normal">266</span>
-<span class="normal">267</span>
-<span class="normal">268</span>
-<span class="normal">269</span>
-<span class="normal">270</span>
-<span class="normal">271</span>
-<span class="normal">272</span>
-<span class="normal">273</span>
-<span class="normal">274</span>
-<span class="normal">275</span>
-<span class="normal">276</span>
-<span class="normal">277</span>
-<span class="normal">278</span>
-<span class="normal">279</span>
-<span class="normal">280</span>
-<span class="normal">281</span>
-<span class="normal">282</span>
-<span class="normal">283</span>
-<span class="normal">284</span>
-<span class="normal">285</span>
-<span class="normal">286</span>
-<span class="normal">287</span>
-<span class="normal">288</span>
-<span class="normal">289</span>
-<span class="normal">290</span>
-<span class="normal">291</span>
-<span class="normal">292</span>
-<span class="normal">293</span>
-<span class="normal">294</span>
-<span class="normal">295</span>
-<span class="normal">296</span>
-<span class="normal">297</span>
-<span class="normal">298</span>
-<span class="normal">299</span>
-<span class="normal">300</span>
-<span class="normal">301</span>
-<span class="normal">302</span>
-<span class="normal">303</span>
-<span class="normal">304</span>
-<span class="normal">305</span>
-<span class="normal">306</span>
-<span class="normal">307</span>
-<span class="normal">308</span>
-<span class="normal">309</span>
-<span class="normal">310</span>
-<span class="normal">311</span>
-<span class="normal">312</span>
-<span class="normal">313</span>
-<span class="normal">314</span>
-<span class="normal">315</span>
-<span class="normal">316</span>
-<span class="normal">317</span>
-<span class="normal">318</span>
-<span class="normal">319</span>
-<span class="normal">320</span>
-<span class="normal">321</span>
-<span class="normal">322</span>
-<span class="normal">323</span>
-<span class="normal">324</span>
-<span class="normal">325</span>
-<span class="normal">326</span>
-<span class="normal">327</span>
-<span class="normal">328</span>
-<span class="normal">329</span>
-<span class="normal">330</span>
-<span class="normal">331</span>
-<span class="normal">332</span>
-<span class="normal">333</span>
-<span class="normal">334</span>
-<span class="normal">335</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@classmethod</span>
-<span class="k">def</span> <span class="nf">clump_with_locus</span><span class="p">(</span>
-    <span class="bp">cls</span><span class="p">:</span> <span class="nb">type</span><span class="p">[</span><span class="n">WindowBasedClumping</span><span class="p">],</span>
-    <span class="n">summary_stats</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">,</span>
-    <span class="n">window_length</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
-    <span class="n">p_value_significance</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">5e-8</span><span class="p">,</span>
-    <span class="n">p_value_baseline</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.05</span><span class="p">,</span>
-    <span class="n">locus_window_length</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StudyLocus</span><span class="p">:</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;Clump significant associations while collecting locus around them.</span>
-
-<span class="sd">    Args:</span>
-<span class="sd">        summary_stats (SummaryStatistics): Input summary statistics dataset</span>
-<span class="sd">        window_length (int): Window size in  bp, used for distance based clumping.</span>
-<span class="sd">        p_value_significance (float): GWAS significance threshold used to filter peaks. Defaults to 5e-8.</span>
-<span class="sd">        p_value_baseline (float): Least significant threshold. Below this, all snps are dropped. Defaults to 0.05.</span>
-<span class="sd">        locus_window_length (int | None): The distance for collecting locus around the semi indices. Defaults to None.</span>
-
-<span class="sd">    Returns:</span>
-<span class="sd">        StudyLocus: StudyLocus after clumping with information about the `locus`</span>
-<span class="sd">    &quot;&quot;&quot;</span>
-    <span class="c1"># If no locus window provided, using the same value:</span>
-    <span class="k">if</span> <span class="n">locus_window_length</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="n">locus_window_length</span> <span class="o">=</span> <span class="n">window_length</span>
-
-    <span class="c1"># Run distance based clumping on the summary stats:</span>
-    <span class="n">clumped_dataframe</span> <span class="o">=</span> <span class="n">WindowBasedClumping</span><span class="o">.</span><span class="n">clump</span><span class="p">(</span>
-        <span class="n">summary_stats</span><span class="p">,</span>
-        <span class="n">window_length</span><span class="o">=</span><span class="n">window_length</span><span class="p">,</span>
-        <span class="n">p_value_significance</span><span class="o">=</span><span class="n">p_value_significance</span><span class="p">,</span>
-    <span class="p">)</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;clumped&quot;</span><span class="p">)</span>
-
-    <span class="c1"># Get list of columns from clumped dataset for further propagation:</span>
-    <span class="n">clumped_columns</span> <span class="o">=</span> <span class="n">clumped_dataframe</span><span class="o">.</span><span class="n">columns</span>
-
-    <span class="c1"># Dropping variants not meeting the baseline criteria:</span>
-    <span class="n">sumstats_baseline</span> <span class="o">=</span> <span class="n">summary_stats</span><span class="o">.</span><span class="n">pvalue_filter</span><span class="p">(</span><span class="n">p_value_baseline</span><span class="p">)</span><span class="o">.</span><span class="n">df</span>
-
-    <span class="c1"># Renaming columns:</span>
-    <span class="n">sumstats_baseline_renamed</span> <span class="o">=</span> <span class="n">sumstats_baseline</span><span class="o">.</span><span class="n">selectExpr</span><span class="p">(</span>
-        <span class="o">*</span><span class="p">[</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">col</span><span class="si">}</span><span class="s2"> as tag_</span><span class="si">{</span><span class="n">col</span><span class="si">}</span><span class="s2">&quot;</span> <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">sumstats_baseline</span><span class="o">.</span><span class="n">columns</span><span class="p">]</span>
-    <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;sumstat&quot;</span><span class="p">)</span>
-
-    <span class="n">study_locus_df</span> <span class="o">=</span> <span class="p">(</span>
-        <span class="n">sumstats_baseline_renamed</span>
-        <span class="c1"># Joining the two datasets together:</span>
-        <span class="o">.</span><span class="n">join</span><span class="p">(</span>
-            <span class="n">f</span><span class="o">.</span><span class="n">broadcast</span><span class="p">(</span><span class="n">clumped_dataframe</span><span class="p">),</span>
-            <span class="n">on</span><span class="o">=</span><span class="p">[</span>
-                <span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;sumstat.tag_studyId&quot;</span><span class="p">)</span> <span class="o">==</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;clumped.studyId&quot;</span><span class="p">))</span>
-                <span class="o">&amp;</span> <span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;sumstat.tag_chromosome&quot;</span><span class="p">)</span> <span class="o">==</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;clumped.chromosome&quot;</span><span class="p">))</span>
-                <span class="o">&amp;</span> <span class="p">(</span>
-                    <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;sumstat.tag_position&quot;</span><span class="p">)</span>
-                    <span class="o">&gt;=</span> <span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;clumped.position&quot;</span><span class="p">)</span> <span class="o">-</span> <span class="n">locus_window_length</span><span class="p">)</span>
-                <span class="p">)</span>
-                <span class="o">&amp;</span> <span class="p">(</span>
-                    <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;sumstat.tag_position&quot;</span><span class="p">)</span>
-                    <span class="o">&lt;=</span> <span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;clumped.position&quot;</span><span class="p">)</span> <span class="o">+</span> <span class="n">locus_window_length</span><span class="p">)</span>
-                <span class="p">)</span>
-            <span class="p">],</span>
-            <span class="n">how</span><span class="o">=</span><span class="s2">&quot;right&quot;</span><span class="p">,</span>
-        <span class="p">)</span>
-        <span class="o">.</span><span class="n">withColumn</span><span class="p">(</span>
-            <span class="s2">&quot;locus&quot;</span><span class="p">,</span>
-            <span class="n">f</span><span class="o">.</span><span class="n">struct</span><span class="p">(</span>
-                <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;tag_variantId&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;variantId&quot;</span><span class="p">),</span>
-                <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;tag_beta&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;beta&quot;</span><span class="p">),</span>
-                <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;tag_pValueMantissa&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;pValueMantissa&quot;</span><span class="p">),</span>
-                <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;tag_pValueExponent&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;pValueExponent&quot;</span><span class="p">),</span>
-                <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;tag_standardError&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;standardError&quot;</span><span class="p">),</span>
-            <span class="p">),</span>
-        <span class="p">)</span>
-        <span class="o">.</span><span class="n">groupby</span><span class="p">(</span><span class="s2">&quot;studyLocusId&quot;</span><span class="p">)</span>
-        <span class="o">.</span><span class="n">agg</span><span class="p">(</span>
-            <span class="o">*</span><span class="p">[</span>
-                <span class="n">f</span><span class="o">.</span><span class="n">first</span><span class="p">(</span><span class="n">col</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="n">col</span><span class="p">)</span>
-                <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">clumped_columns</span>
-                <span class="k">if</span> <span class="n">col</span> <span class="o">!=</span> <span class="s2">&quot;studyLocusId&quot;</span>
-            <span class="p">],</span>
-            <span class="n">f</span><span class="o">.</span><span class="n">collect_list</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;locus&quot;</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;locus&quot;</span><span class="p">),</span>
-        <span class="p">)</span>
-    <span class="p">)</span>
-
-    <span class="k">return</span> <span class="n">StudyLocus</span><span class="p">(</span>
-        <span class="n">_df</span><span class="o">=</span><span class="n">study_locus_df</span><span class="p">,</span>
-        <span class="n">_schema</span><span class="o">=</span><span class="n">StudyLocus</span><span class="o">.</span><span class="n">get_schema</span><span class="p">(),</span>
-    <span class="p">)</span>
-</code></pre></div></td></tr></table></div>
-          </details>
-  </div>
-
-</div>
-
-
 
   </div>
 
@@ -4276,7 +3828,7 @@ <h3 id="gentropy.method.clump.LDclumping.clump" class="doc doc-heading">
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -4287,7 +3839,7 @@ <h3 id="gentropy.method.clump.LDclumping.clump" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>associations</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_locus.StudyLocus" href="../../datasets/study_locus/#gentropy.dataset.study_locus.StudyLocus">StudyLocus</a></code>
@@ -4306,7 +3858,7 @@ <h3 id="gentropy.method.clump.LDclumping.clump" class="doc doc-heading">
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -4315,7 +3867,7 @@ <h3 id="gentropy.method.clump.LDclumping.clump" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StudyLocus</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_locus.StudyLocus" href="../../datasets/study_locus/#gentropy.dataset.study_locus.StudyLocus">StudyLocus</a></code>
           </td>
diff --git a/python_api/methods/coloc/index.html b/python_api/methods/coloc/index.html
index 4803a8a7b..23e47c64f 100644
--- a/python_api/methods/coloc/index.html
+++ b/python_api/methods/coloc/index.html
@@ -2257,6 +2257,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../susie_inf/" class="md-nav__link">
         
@@ -2288,7 +2330,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2304,14 +2346,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -2955,7 +2997,7 @@ <h2 id="gentropy.method.colocalisation.Coloc" class="doc doc-heading">
 
 
 
-  <p><strong>Attributes:</strong></p>
+  <p><span class="doc-section-title">Attributes:</span></p>
   <table>
     <thead>
       <tr>
@@ -2965,7 +3007,7 @@ <h2 id="gentropy.method.colocalisation.Coloc" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code><span title="gentropy.method.colocalisation.Coloc.PSEUDOCOUNT">PSEUDOCOUNT</span></code></td>
           <td>
                 <code>float</code>
@@ -2981,10 +3023,7 @@ <h2 id="gentropy.method.colocalisation.Coloc" class="doc doc-heading">
 
             <details class="quote">
               <summary>Source code in <code>src/gentropy/method/colocalisation.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 90</span>
-<span class="normal"> 91</span>
-<span class="normal"> 92</span>
-<span class="normal"> 93</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 93</span>
 <span class="normal"> 94</span>
 <span class="normal"> 95</span>
 <span class="normal"> 96</span>
@@ -3150,7 +3189,12 @@ <h2 id="gentropy.method.colocalisation.Coloc" class="doc doc-heading">
 <span class="normal">256</span>
 <span class="normal">257</span>
 <span class="normal">258</span>
-<span class="normal">259</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">Coloc</span><span class="p">:</span>
+<span class="normal">259</span>
+<span class="normal">260</span>
+<span class="normal">261</span>
+<span class="normal">262</span>
+<span class="normal">263</span>
+<span class="normal">264</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">Coloc</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Calculate bayesian colocalisation based on overlapping signals from credible sets.</span>
 
 <span class="sd">    Based on the [R COLOC package](https://github.com/chr1swallace/coloc/blob/main/R/claudia.R), which uses the Bayes factors from the credible set to estimate the posterior probability of colocalisation. This method makes the simplifying assumption that **only one single causal variant** exists for any given trait in any genomic region.</span>
@@ -3171,6 +3215,8 @@ <h2 id="gentropy.method.colocalisation.Coloc" class="doc doc-heading">
 <span class="sd">        PSEUDOCOUNT (float): Pseudocount to avoid log(0). Defaults to 1e-10.</span>
 <span class="sd">    &quot;&quot;&quot;</span>
 
+    <span class="n">METHOD_NAME</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;COLOC&quot;</span>
+    <span class="n">METHOD_METRIC</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;llr&quot;</span>
     <span class="n">PSEUDOCOUNT</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1e-10</span>
 
     <span class="nd">@staticmethod</span>
@@ -3217,24 +3263,24 @@ <h2 id="gentropy.method.colocalisation.Coloc" class="doc doc-heading">
         <span class="n">posteriors</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">udf</span><span class="p">(</span><span class="n">Coloc</span><span class="o">.</span><span class="n">_get_posteriors</span><span class="p">,</span> <span class="n">VectorUDT</span><span class="p">())</span>
         <span class="k">return</span> <span class="n">Colocalisation</span><span class="p">(</span>
             <span class="n">_df</span><span class="o">=</span><span class="p">(</span>
-                <span class="n">overlapping_signals</span><span class="o">.</span><span class="n">df</span>
+                <span class="n">overlapping_signals</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">&quot;*&quot;</span><span class="p">,</span> <span class="s2">&quot;statistics.*&quot;</span><span class="p">)</span>
                 <span class="c1"># Before summing log_BF columns nulls need to be filled with 0:</span>
-                <span class="o">.</span><span class="n">fillna</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">subset</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;statistics.left_logBF&quot;</span><span class="p">,</span> <span class="s2">&quot;statistics.right_logBF&quot;</span><span class="p">])</span>
+                <span class="o">.</span><span class="n">fillna</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">subset</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;left_logBF&quot;</span><span class="p">,</span> <span class="s2">&quot;right_logBF&quot;</span><span class="p">])</span>
                 <span class="c1"># Sum of log_BFs for each pair of signals</span>
                 <span class="o">.</span><span class="n">withColumn</span><span class="p">(</span>
                     <span class="s2">&quot;sum_log_bf&quot;</span><span class="p">,</span>
-                    <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;statistics.left_logBF&quot;</span><span class="p">)</span> <span class="o">+</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;statistics.right_logBF&quot;</span><span class="p">),</span>
+                    <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;left_logBF&quot;</span><span class="p">)</span> <span class="o">+</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;right_logBF&quot;</span><span class="p">),</span>
                 <span class="p">)</span>
                 <span class="c1"># Group by overlapping peak and generating dense vectors of log_BF:</span>
                 <span class="o">.</span><span class="n">groupBy</span><span class="p">(</span><span class="s2">&quot;chromosome&quot;</span><span class="p">,</span> <span class="s2">&quot;leftStudyLocusId&quot;</span><span class="p">,</span> <span class="s2">&quot;rightStudyLocusId&quot;</span><span class="p">)</span>
                 <span class="o">.</span><span class="n">agg</span><span class="p">(</span>
                     <span class="n">f</span><span class="o">.</span><span class="n">count</span><span class="p">(</span><span class="s2">&quot;*&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;numberColocalisingVariants&quot;</span><span class="p">),</span>
-                    <span class="n">fml</span><span class="o">.</span><span class="n">array_to_vector</span><span class="p">(</span>
-                        <span class="n">f</span><span class="o">.</span><span class="n">collect_list</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;statistics.left_logBF&quot;</span><span class="p">))</span>
-                    <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;left_logBF&quot;</span><span class="p">),</span>
-                    <span class="n">fml</span><span class="o">.</span><span class="n">array_to_vector</span><span class="p">(</span>
-                        <span class="n">f</span><span class="o">.</span><span class="n">collect_list</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;statistics.right_logBF&quot;</span><span class="p">))</span>
-                    <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;right_logBF&quot;</span><span class="p">),</span>
+                    <span class="n">fml</span><span class="o">.</span><span class="n">array_to_vector</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">collect_list</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;left_logBF&quot;</span><span class="p">)))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span>
+                        <span class="s2">&quot;left_logBF&quot;</span>
+                    <span class="p">),</span>
+                    <span class="n">fml</span><span class="o">.</span><span class="n">array_to_vector</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">collect_list</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;right_logBF&quot;</span><span class="p">)))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span>
+                        <span class="s2">&quot;right_logBF&quot;</span>
+                    <span class="p">),</span>
                     <span class="n">fml</span><span class="o">.</span><span class="n">array_to_vector</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">collect_list</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;sum_log_bf&quot;</span><span class="p">)))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span>
                         <span class="s2">&quot;sum_log_bf&quot;</span>
                     <span class="p">),</span>
@@ -3316,7 +3362,7 @@ <h2 id="gentropy.method.colocalisation.Coloc" class="doc doc-heading">
                     <span class="s2">&quot;lH3bf&quot;</span><span class="p">,</span>
                     <span class="s2">&quot;lH4bf&quot;</span><span class="p">,</span>
                 <span class="p">)</span>
-                <span class="o">.</span><span class="n">withColumn</span><span class="p">(</span><span class="s2">&quot;colocalisationMethod&quot;</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">&quot;COLOC&quot;</span><span class="p">))</span>
+                <span class="o">.</span><span class="n">withColumn</span><span class="p">(</span><span class="s2">&quot;colocalisationMethod&quot;</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="bp">cls</span><span class="o">.</span><span class="n">METHOD_NAME</span><span class="p">))</span>
             <span class="p">),</span>
             <span class="n">_schema</span><span class="o">=</span><span class="n">Colocalisation</span><span class="o">.</span><span class="n">get_schema</span><span class="p">(),</span>
         <span class="p">)</span>
@@ -3356,7 +3402,7 @@ <h3 id="gentropy.method.colocalisation.Coloc.colocalise" class="doc doc-heading"
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3367,7 +3413,7 @@ <h3 id="gentropy.method.colocalisation.Coloc.colocalise" class="doc doc-heading"
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>overlapping_signals</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_locus_overlap.StudyLocusOverlap" href="../../datasets/study_locus_overlap/#gentropy.dataset.study_locus_overlap.StudyLocusOverlap">StudyLocusOverlap</a></code>
@@ -3381,7 +3427,7 @@ <h3 id="gentropy.method.colocalisation.Coloc.colocalise" class="doc doc-heading"
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>priorc1</code></td>
           <td>
                 <code>float</code>
@@ -3395,7 +3441,7 @@ <h3 id="gentropy.method.colocalisation.Coloc.colocalise" class="doc doc-heading"
                 <code>0.0001</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>priorc2</code></td>
           <td>
                 <code>float</code>
@@ -3409,7 +3455,7 @@ <h3 id="gentropy.method.colocalisation.Coloc.colocalise" class="doc doc-heading"
                 <code>0.0001</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>priorc12</code></td>
           <td>
                 <code>float</code>
@@ -3428,7 +3474,7 @@ <h3 id="gentropy.method.colocalisation.Coloc.colocalise" class="doc doc-heading"
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3437,7 +3483,7 @@ <h3 id="gentropy.method.colocalisation.Coloc.colocalise" class="doc doc-heading"
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>Colocalisation</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.colocalisation.Colocalisation" href="../../datasets/colocalisation/#gentropy.dataset.colocalisation.Colocalisation">Colocalisation</a></code>
           </td>
@@ -3452,12 +3498,7 @@ <h3 id="gentropy.method.colocalisation.Coloc.colocalise" class="doc doc-heading"
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/method/colocalisation.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">132</span>
-<span class="normal">133</span>
-<span class="normal">134</span>
-<span class="normal">135</span>
-<span class="normal">136</span>
-<span class="normal">137</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">137</span>
 <span class="normal">138</span>
 <span class="normal">139</span>
 <span class="normal">140</span>
@@ -3579,7 +3620,12 @@ <h3 id="gentropy.method.colocalisation.Coloc.colocalise" class="doc doc-heading"
 <span class="normal">256</span>
 <span class="normal">257</span>
 <span class="normal">258</span>
-<span class="normal">259</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@classmethod</span>
+<span class="normal">259</span>
+<span class="normal">260</span>
+<span class="normal">261</span>
+<span class="normal">262</span>
+<span class="normal">263</span>
+<span class="normal">264</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@classmethod</span>
 <span class="k">def</span> <span class="nf">colocalise</span><span class="p">(</span>
     <span class="bp">cls</span><span class="p">:</span> <span class="nb">type</span><span class="p">[</span><span class="n">Coloc</span><span class="p">],</span>
     <span class="n">overlapping_signals</span><span class="p">:</span> <span class="n">StudyLocusOverlap</span><span class="p">,</span>
@@ -3604,24 +3650,24 @@ <h3 id="gentropy.method.colocalisation.Coloc.colocalise" class="doc doc-heading"
     <span class="n">posteriors</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">udf</span><span class="p">(</span><span class="n">Coloc</span><span class="o">.</span><span class="n">_get_posteriors</span><span class="p">,</span> <span class="n">VectorUDT</span><span class="p">())</span>
     <span class="k">return</span> <span class="n">Colocalisation</span><span class="p">(</span>
         <span class="n">_df</span><span class="o">=</span><span class="p">(</span>
-            <span class="n">overlapping_signals</span><span class="o">.</span><span class="n">df</span>
+            <span class="n">overlapping_signals</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">&quot;*&quot;</span><span class="p">,</span> <span class="s2">&quot;statistics.*&quot;</span><span class="p">)</span>
             <span class="c1"># Before summing log_BF columns nulls need to be filled with 0:</span>
-            <span class="o">.</span><span class="n">fillna</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">subset</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;statistics.left_logBF&quot;</span><span class="p">,</span> <span class="s2">&quot;statistics.right_logBF&quot;</span><span class="p">])</span>
+            <span class="o">.</span><span class="n">fillna</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">subset</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;left_logBF&quot;</span><span class="p">,</span> <span class="s2">&quot;right_logBF&quot;</span><span class="p">])</span>
             <span class="c1"># Sum of log_BFs for each pair of signals</span>
             <span class="o">.</span><span class="n">withColumn</span><span class="p">(</span>
                 <span class="s2">&quot;sum_log_bf&quot;</span><span class="p">,</span>
-                <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;statistics.left_logBF&quot;</span><span class="p">)</span> <span class="o">+</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;statistics.right_logBF&quot;</span><span class="p">),</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;left_logBF&quot;</span><span class="p">)</span> <span class="o">+</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;right_logBF&quot;</span><span class="p">),</span>
             <span class="p">)</span>
             <span class="c1"># Group by overlapping peak and generating dense vectors of log_BF:</span>
             <span class="o">.</span><span class="n">groupBy</span><span class="p">(</span><span class="s2">&quot;chromosome&quot;</span><span class="p">,</span> <span class="s2">&quot;leftStudyLocusId&quot;</span><span class="p">,</span> <span class="s2">&quot;rightStudyLocusId&quot;</span><span class="p">)</span>
             <span class="o">.</span><span class="n">agg</span><span class="p">(</span>
                 <span class="n">f</span><span class="o">.</span><span class="n">count</span><span class="p">(</span><span class="s2">&quot;*&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;numberColocalisingVariants&quot;</span><span class="p">),</span>
-                <span class="n">fml</span><span class="o">.</span><span class="n">array_to_vector</span><span class="p">(</span>
-                    <span class="n">f</span><span class="o">.</span><span class="n">collect_list</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;statistics.left_logBF&quot;</span><span class="p">))</span>
-                <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;left_logBF&quot;</span><span class="p">),</span>
-                <span class="n">fml</span><span class="o">.</span><span class="n">array_to_vector</span><span class="p">(</span>
-                    <span class="n">f</span><span class="o">.</span><span class="n">collect_list</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;statistics.right_logBF&quot;</span><span class="p">))</span>
-                <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;right_logBF&quot;</span><span class="p">),</span>
+                <span class="n">fml</span><span class="o">.</span><span class="n">array_to_vector</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">collect_list</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;left_logBF&quot;</span><span class="p">)))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span>
+                    <span class="s2">&quot;left_logBF&quot;</span>
+                <span class="p">),</span>
+                <span class="n">fml</span><span class="o">.</span><span class="n">array_to_vector</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">collect_list</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;right_logBF&quot;</span><span class="p">)))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span>
+                    <span class="s2">&quot;right_logBF&quot;</span>
+                <span class="p">),</span>
                 <span class="n">fml</span><span class="o">.</span><span class="n">array_to_vector</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">collect_list</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;sum_log_bf&quot;</span><span class="p">)))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span>
                     <span class="s2">&quot;sum_log_bf&quot;</span>
                 <span class="p">),</span>
@@ -3703,7 +3749,7 @@ <h3 id="gentropy.method.colocalisation.Coloc.colocalise" class="doc doc-heading"
                 <span class="s2">&quot;lH3bf&quot;</span><span class="p">,</span>
                 <span class="s2">&quot;lH4bf&quot;</span><span class="p">,</span>
             <span class="p">)</span>
-            <span class="o">.</span><span class="n">withColumn</span><span class="p">(</span><span class="s2">&quot;colocalisationMethod&quot;</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">&quot;COLOC&quot;</span><span class="p">))</span>
+            <span class="o">.</span><span class="n">withColumn</span><span class="p">(</span><span class="s2">&quot;colocalisationMethod&quot;</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="bp">cls</span><span class="o">.</span><span class="n">METHOD_NAME</span><span class="p">))</span>
         <span class="p">),</span>
         <span class="n">_schema</span><span class="o">=</span><span class="n">Colocalisation</span><span class="o">.</span><span class="n">get_schema</span><span class="p">(),</span>
     <span class="p">)</span>
diff --git a/python_api/methods/ecaviar/index.html b/python_api/methods/ecaviar/index.html
index fe11db025..7b07e450f 100644
--- a/python_api/methods/ecaviar/index.html
+++ b/python_api/methods/ecaviar/index.html
@@ -2257,6 +2257,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../susie_inf/" class="md-nav__link">
         
@@ -2288,7 +2330,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2304,14 +2346,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -2985,12 +3027,18 @@ <h2 id="gentropy.method.colocalisation.ECaviar" class="doc doc-heading">
 <span class="normal">84</span>
 <span class="normal">85</span>
 <span class="normal">86</span>
-<span class="normal">87</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">ECaviar</span><span class="p">:</span>
+<span class="normal">87</span>
+<span class="normal">88</span>
+<span class="normal">89</span>
+<span class="normal">90</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">ECaviar</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;ECaviar-based colocalisation analysis.</span>
 
 <span class="sd">    It extends [CAVIAR](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5142122/#bib18) framework to explicitly estimate the posterior probability that the same variant is causal in 2 studies while accounting for the uncertainty of LD. eCAVIAR computes the colocalization posterior probability (**CLPP**) by utilizing the marginal posterior probabilities. This framework allows for **multiple variants to be causal** in a single locus.</span>
 <span class="sd">    &quot;&quot;&quot;</span>
 
+    <span class="n">METHOD_NAME</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;eCAVIAR&quot;</span>
+    <span class="n">METHOD_METRIC</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;clpp&quot;</span>
+
     <span class="nd">@staticmethod</span>
     <span class="k">def</span> <span class="nf">_get_clpp</span><span class="p">(</span><span class="n">left_pp</span><span class="p">:</span> <span class="n">Column</span><span class="p">,</span> <span class="n">right_pp</span><span class="p">:</span> <span class="n">Column</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Column</span><span class="p">:</span>
 <span class="w">        </span><span class="sd">&quot;&quot;&quot;Calculate the colocalisation posterior probability (CLPP).</span>
@@ -3046,7 +3094,7 @@ <h2 id="gentropy.method.colocalisation.ECaviar" class="doc doc-heading">
                     <span class="n">f</span><span class="o">.</span><span class="n">count</span><span class="p">(</span><span class="s2">&quot;*&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;numberColocalisingVariants&quot;</span><span class="p">),</span>
                     <span class="n">f</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;clpp&quot;</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;clpp&quot;</span><span class="p">),</span>
                 <span class="p">)</span>
-                <span class="o">.</span><span class="n">withColumn</span><span class="p">(</span><span class="s2">&quot;colocalisationMethod&quot;</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">&quot;eCAVIAR&quot;</span><span class="p">))</span>
+                <span class="o">.</span><span class="n">withColumn</span><span class="p">(</span><span class="s2">&quot;colocalisationMethod&quot;</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="bp">cls</span><span class="o">.</span><span class="n">METHOD_NAME</span><span class="p">))</span>
             <span class="p">),</span>
             <span class="n">_schema</span><span class="o">=</span><span class="n">Colocalisation</span><span class="o">.</span><span class="n">get_schema</span><span class="p">(),</span>
         <span class="p">)</span>
@@ -3086,7 +3134,7 @@ <h3 id="gentropy.method.colocalisation.ECaviar.colocalise" class="doc doc-headin
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3097,7 +3145,7 @@ <h3 id="gentropy.method.colocalisation.ECaviar.colocalise" class="doc doc-headin
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>overlapping_signals</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_locus_overlap.StudyLocusOverlap" href="../../datasets/study_locus_overlap/#gentropy.dataset.study_locus_overlap.StudyLocusOverlap">StudyLocusOverlap</a></code>
@@ -3116,7 +3164,7 @@ <h3 id="gentropy.method.colocalisation.ECaviar.colocalise" class="doc doc-headin
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3125,7 +3173,7 @@ <h3 id="gentropy.method.colocalisation.ECaviar.colocalise" class="doc doc-headin
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>Colocalisation</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.colocalisation.Colocalisation" href="../../datasets/colocalisation/#gentropy.dataset.colocalisation.Colocalisation">Colocalisation</a></code>
           </td>
@@ -3140,10 +3188,7 @@ <h3 id="gentropy.method.colocalisation.ECaviar.colocalise" class="doc doc-headin
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/method/colocalisation.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">58</span>
-<span class="normal">59</span>
-<span class="normal">60</span>
-<span class="normal">61</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">61</span>
 <span class="normal">62</span>
 <span class="normal">63</span>
 <span class="normal">64</span>
@@ -3169,7 +3214,10 @@ <h3 id="gentropy.method.colocalisation.ECaviar.colocalise" class="doc doc-headin
 <span class="normal">84</span>
 <span class="normal">85</span>
 <span class="normal">86</span>
-<span class="normal">87</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@classmethod</span>
+<span class="normal">87</span>
+<span class="normal">88</span>
+<span class="normal">89</span>
+<span class="normal">90</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@classmethod</span>
 <span class="k">def</span> <span class="nf">colocalise</span><span class="p">(</span>
     <span class="bp">cls</span><span class="p">:</span> <span class="nb">type</span><span class="p">[</span><span class="n">ECaviar</span><span class="p">],</span> <span class="n">overlapping_signals</span><span class="p">:</span> <span class="n">StudyLocusOverlap</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Colocalisation</span><span class="p">:</span>
@@ -3195,7 +3243,7 @@ <h3 id="gentropy.method.colocalisation.ECaviar.colocalise" class="doc doc-headin
                 <span class="n">f</span><span class="o">.</span><span class="n">count</span><span class="p">(</span><span class="s2">&quot;*&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;numberColocalisingVariants&quot;</span><span class="p">),</span>
                 <span class="n">f</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;clpp&quot;</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;clpp&quot;</span><span class="p">),</span>
             <span class="p">)</span>
-            <span class="o">.</span><span class="n">withColumn</span><span class="p">(</span><span class="s2">&quot;colocalisationMethod&quot;</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">&quot;eCAVIAR&quot;</span><span class="p">))</span>
+            <span class="o">.</span><span class="n">withColumn</span><span class="p">(</span><span class="s2">&quot;colocalisationMethod&quot;</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="bp">cls</span><span class="o">.</span><span class="n">METHOD_NAME</span><span class="p">))</span>
         <span class="p">),</span>
         <span class="n">_schema</span><span class="o">=</span><span class="n">Colocalisation</span><span class="o">.</span><span class="n">get_schema</span><span class="p">(),</span>
     <span class="p">)</span>
diff --git a/python_api/methods/l2g/_l2g/index.html b/python_api/methods/l2g/_l2g/index.html
index 28c97b132..e24f2af4d 100644
--- a/python_api/methods/l2g/_l2g/index.html
+++ b/python_api/methods/l2g/_l2g/index.html
@@ -2192,6 +2192,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../susie_inf/" class="md-nav__link">
         
@@ -2225,7 +2267,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" checked>
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" checked>
         
           
           
@@ -2241,14 +2283,14 @@
             </a>
             
               
-              <label class="md-nav__link md-nav__link--active" for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link md-nav__link--active" for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="true">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="true">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
diff --git a/python_api/methods/l2g/evaluator/index.html b/python_api/methods/l2g/evaluator/index.html
index 3f3ace29b..c252b76cc 100644
--- a/python_api/methods/l2g/evaluator/index.html
+++ b/python_api/methods/l2g/evaluator/index.html
@@ -2197,6 +2197,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../susie_inf/" class="md-nav__link">
         
@@ -2230,7 +2272,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" checked>
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" checked>
         
           
           
@@ -2246,14 +2288,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="true">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="true">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3433,7 +3475,7 @@ <h3 id="gentropy.method.l2g.evaluator.WandbEvaluator.__init__" class="doc doc-he
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3444,7 +3486,7 @@ <h3 id="gentropy.method.l2g.evaluator.WandbEvaluator.__init__" class="doc doc-he
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>label_values</code></td>
           <td>
                 <code>list[str] | None</code>
@@ -3458,7 +3500,7 @@ <h3 id="gentropy.method.l2g.evaluator.WandbEvaluator.__init__" class="doc doc-he
                 <code>None</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>**kwargs</code></td>
           <td>
                 <code><span title="pyspark.ml.evaluation.BinaryClassificationEvaluator">BinaryClassificationEvaluator</span> | <span title="pyspark.ml.evaluation.MulticlassClassificationEvaluator">MulticlassClassificationEvaluator</span> | <span title="wandb.sdk.wandb_run.Run">Run</span></code>
@@ -3581,7 +3623,7 @@ <h3 id="gentropy.method.l2g.evaluator.WandbEvaluator.getlabel_values" class="doc
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3590,7 +3632,7 @@ <h3 id="gentropy.method.l2g.evaluator.WandbEvaluator.getlabel_values" class="doc
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td>
                 <code>list[str]</code>
           </td>
@@ -3641,7 +3683,7 @@ <h3 id="gentropy.method.l2g.evaluator.WandbEvaluator.getspark_ml_evaluator" clas
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3650,7 +3692,7 @@ <h3 id="gentropy.method.l2g.evaluator.WandbEvaluator.getspark_ml_evaluator" clas
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>Evaluator</code></td>          <td>
                 <code><span title="pyspark.ml.evaluation.Evaluator">Evaluator</span></code>
           </td>
@@ -3701,7 +3743,7 @@ <h3 id="gentropy.method.l2g.evaluator.WandbEvaluator.getwandb_project_name" clas
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3710,7 +3752,7 @@ <h3 id="gentropy.method.l2g.evaluator.WandbEvaluator.getwandb_project_name" clas
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>Any</code></td>          <td>
                 <code><span title="typing.Any">Any</span></code>
           </td>
@@ -3761,7 +3803,7 @@ <h3 id="gentropy.method.l2g.evaluator.WandbEvaluator.getwandb_run" class="doc do
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3770,7 +3812,7 @@ <h3 id="gentropy.method.l2g.evaluator.WandbEvaluator.getwandb_run" class="doc do
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>Run</code></td>          <td>
                 <code><span title="wandb.sdk.wandb_run.Run">Run</span></code>
           </td>
@@ -3821,7 +3863,7 @@ <h3 id="gentropy.method.l2g.evaluator.WandbEvaluator.setlabel_values" class="doc
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3832,7 +3874,7 @@ <h3 id="gentropy.method.l2g.evaluator.WandbEvaluator.setlabel_values" class="doc
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>value</code></td>
           <td>
                 <code>list[str]</code>
@@ -3887,7 +3929,7 @@ <h3 id="gentropy.method.l2g.evaluator.WandbEvaluator.setspark_ml_evaluator" clas
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3898,7 +3940,7 @@ <h3 id="gentropy.method.l2g.evaluator.WandbEvaluator.setspark_ml_evaluator" clas
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>value</code></td>
           <td>
                 <code><span title="pyspark.ml.evaluation.Evaluator">Evaluator</span></code>
diff --git a/python_api/methods/l2g/feature_factory/index.html b/python_api/methods/l2g/feature_factory/index.html
index 709d01745..24236bddb 100644
--- a/python_api/methods/l2g/feature_factory/index.html
+++ b/python_api/methods/l2g/feature_factory/index.html
@@ -2197,6 +2197,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../susie_inf/" class="md-nav__link">
         
@@ -2230,7 +2272,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" checked>
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" checked>
         
           
           
@@ -2246,14 +2288,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="true">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="true">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -2910,9 +2952,7 @@ <h2 id="gentropy.method.l2g.feature_factory.ColocalisationFactory" class="doc do
 
             <details class="quote">
               <summary>Source code in <code>src/gentropy/method/l2g/feature_factory.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 24</span>
-<span class="normal"> 25</span>
-<span class="normal"> 26</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 26</span>
 <span class="normal"> 27</span>
 <span class="normal"> 28</span>
 <span class="normal"> 29</span>
@@ -3046,259 +3086,139 @@ <h2 id="gentropy.method.l2g.feature_factory.ColocalisationFactory" class="doc do
 <span class="normal">157</span>
 <span class="normal">158</span>
 <span class="normal">159</span>
-<span class="normal">160</span>
-<span class="normal">161</span>
-<span class="normal">162</span>
-<span class="normal">163</span>
-<span class="normal">164</span>
-<span class="normal">165</span>
-<span class="normal">166</span>
-<span class="normal">167</span>
-<span class="normal">168</span>
-<span class="normal">169</span>
-<span class="normal">170</span>
-<span class="normal">171</span>
-<span class="normal">172</span>
-<span class="normal">173</span>
-<span class="normal">174</span>
-<span class="normal">175</span>
-<span class="normal">176</span>
-<span class="normal">177</span>
-<span class="normal">178</span>
-<span class="normal">179</span>
-<span class="normal">180</span>
-<span class="normal">181</span>
-<span class="normal">182</span>
-<span class="normal">183</span>
-<span class="normal">184</span>
-<span class="normal">185</span>
-<span class="normal">186</span>
-<span class="normal">187</span>
-<span class="normal">188</span>
-<span class="normal">189</span>
-<span class="normal">190</span>
-<span class="normal">191</span>
-<span class="normal">192</span>
-<span class="normal">193</span>
-<span class="normal">194</span>
-<span class="normal">195</span>
-<span class="normal">196</span>
-<span class="normal">197</span>
-<span class="normal">198</span>
-<span class="normal">199</span>
-<span class="normal">200</span>
-<span class="normal">201</span>
-<span class="normal">202</span>
-<span class="normal">203</span>
-<span class="normal">204</span>
-<span class="normal">205</span>
-<span class="normal">206</span>
-<span class="normal">207</span>
-<span class="normal">208</span>
-<span class="normal">209</span>
-<span class="normal">210</span>
-<span class="normal">211</span>
-<span class="normal">212</span>
-<span class="normal">213</span>
-<span class="normal">214</span>
-<span class="normal">215</span>
-<span class="normal">216</span>
-<span class="normal">217</span>
-<span class="normal">218</span>
-<span class="normal">219</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">ColocalisationFactory</span><span class="p">:</span>
+<span class="normal">160</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">ColocalisationFactory</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Feature extraction in colocalisation.&quot;&quot;&quot;</span>
 
+    <span class="nd">@classmethod</span>
+    <span class="k">def</span> <span class="nf">_add_colocalisation_metric</span><span class="p">(</span><span class="bp">cls</span><span class="p">:</span> <span class="nb">type</span><span class="p">[</span><span class="n">ColocalisationFactory</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">Column</span><span class="p">:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Expression that adds a `colocalisationMetric` column to the colocalisation dataframe in preparation for feature extraction.</span>
+
+<span class="sd">        Returns:</span>
+<span class="sd">            Column: The expression that adds a `colocalisationMetric` column with the derived metric</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">method_metric_map</span> <span class="o">=</span> <span class="p">{</span>
+            <span class="n">ECaviar</span><span class="o">.</span><span class="n">METHOD_NAME</span><span class="p">:</span> <span class="n">ECaviar</span><span class="o">.</span><span class="n">METHOD_METRIC</span><span class="p">,</span>
+            <span class="n">Coloc</span><span class="o">.</span><span class="n">METHOD_NAME</span><span class="p">:</span> <span class="n">Coloc</span><span class="o">.</span><span class="n">METHOD_METRIC</span><span class="p">,</span>
+        <span class="p">}</span>
+        <span class="n">map_expr</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">create_map</span><span class="p">(</span><span class="o">*</span><span class="p">[</span><span class="n">f</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">chain</span><span class="p">(</span><span class="o">*</span><span class="n">method_metric_map</span><span class="o">.</span><span class="n">items</span><span class="p">())])</span>
+        <span class="k">return</span> <span class="n">map_expr</span><span class="p">[</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;colocalisationMethod&quot;</span><span class="p">)]</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;colocalisationMetric&quot;</span><span class="p">)</span>
+
     <span class="nd">@staticmethod</span>
-    <span class="k">def</span> <span class="nf">_get_max_coloc_per_study_locus</span><span class="p">(</span>
-        <span class="n">study_locus</span><span class="p">:</span> <span class="n">StudyLocus</span><span class="p">,</span>
-        <span class="n">studies</span><span class="p">:</span> <span class="n">StudyIndex</span><span class="p">,</span>
+    <span class="k">def</span> <span class="nf">_get_max_coloc_per_credible_set</span><span class="p">(</span>
         <span class="n">colocalisation</span><span class="p">:</span> <span class="n">Colocalisation</span><span class="p">,</span>
-        <span class="n">colocalisation_method</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
+        <span class="n">credible_set</span><span class="p">:</span> <span class="n">StudyLocus</span><span class="p">,</span>
+        <span class="n">studies</span><span class="p">:</span> <span class="n">StudyIndex</span><span class="p">,</span>
     <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">L2GFeature</span><span class="p">:</span>
 <span class="w">        </span><span class="sd">&quot;&quot;&quot;Get the maximum colocalisation posterior probability for each pair of overlapping study-locus per type of colocalisation method and QTL type.</span>
 
 <span class="sd">        Args:</span>
-<span class="sd">            study_locus (StudyLocus): Study locus dataset</span>
-<span class="sd">            studies (StudyIndex): Study index dataset</span>
 <span class="sd">            colocalisation (Colocalisation): Colocalisation dataset</span>
-<span class="sd">            colocalisation_method (str): Colocalisation method to extract the max from</span>
+<span class="sd">            credible_set (StudyLocus): Study locus dataset</span>
+<span class="sd">            studies (StudyIndex): Study index dataset</span>
 
 <span class="sd">        Returns:</span>
 <span class="sd">            L2GFeature: Stores the features with the max coloc probabilities for each pair of study-locus</span>
-
-<span class="sd">        Raises:</span>
-<span class="sd">            ValueError: If the colocalisation method is not supported</span>
 <span class="sd">        &quot;&quot;&quot;</span>
-        <span class="k">if</span> <span class="n">colocalisation_method</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;COLOC&quot;</span><span class="p">,</span> <span class="s2">&quot;eCAVIAR&quot;</span><span class="p">]:</span>
-            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
-                <span class="sa">f</span><span class="s2">&quot;Colocalisation method </span><span class="si">{</span><span class="n">colocalisation_method</span><span class="si">}</span><span class="s2"> not supported&quot;</span>
-            <span class="p">)</span>
-        <span class="k">if</span> <span class="n">colocalisation_method</span> <span class="o">==</span> <span class="s2">&quot;COLOC&quot;</span><span class="p">:</span>
-            <span class="n">coloc_score_col_name</span> <span class="o">=</span> <span class="s2">&quot;log2h4h3&quot;</span>
-            <span class="n">coloc_feature_col_template</span> <span class="o">=</span> <span class="s2">&quot;ColocLlrMaximum&quot;</span>
-
-        <span class="k">elif</span> <span class="n">colocalisation_method</span> <span class="o">==</span> <span class="s2">&quot;eCAVIAR&quot;</span><span class="p">:</span>
-            <span class="n">coloc_score_col_name</span> <span class="o">=</span> <span class="s2">&quot;clpp&quot;</span>
-            <span class="n">coloc_feature_col_template</span> <span class="o">=</span> <span class="s2">&quot;ColocClppMaximum&quot;</span>
+        <span class="n">colocalisation_df</span> <span class="o">=</span> <span class="n">colocalisation</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+            <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;leftStudyLocusId&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;studyLocusId&quot;</span><span class="p">),</span>
+            <span class="s2">&quot;rightStudyLocusId&quot;</span><span class="p">,</span>
+            <span class="n">f</span><span class="o">.</span><span class="n">coalesce</span><span class="p">(</span><span class="s2">&quot;log2h4h3&quot;</span><span class="p">,</span> <span class="s2">&quot;clpp&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;score&quot;</span><span class="p">),</span>
+            <span class="n">ColocalisationFactory</span><span class="o">.</span><span class="n">_add_colocalisation_metric</span><span class="p">(),</span>
+        <span class="p">)</span>
 
-        <span class="n">colocalising_study_locus</span> <span class="o">=</span> <span class="p">(</span>
-            <span class="n">study_locus</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">&quot;studyLocusId&quot;</span><span class="p">,</span> <span class="s2">&quot;studyId&quot;</span><span class="p">)</span>
+        <span class="n">colocalising_credible_sets</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="n">credible_set</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">&quot;studyLocusId&quot;</span><span class="p">,</span> <span class="s2">&quot;studyId&quot;</span><span class="p">)</span>
             <span class="c1"># annotate studyLoci with overlapping IDs on the left - to just keep GWAS associations</span>
             <span class="o">.</span><span class="n">join</span><span class="p">(</span>
-                <span class="n">colocalisation</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">selectExpr</span><span class="p">(</span>
-                    <span class="s2">&quot;leftStudyLocusId as studyLocusId&quot;</span><span class="p">,</span>
-                    <span class="s2">&quot;rightStudyLocusId&quot;</span><span class="p">,</span>
-                    <span class="s2">&quot;colocalisationMethod&quot;</span><span class="p">,</span>
-                    <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">coloc_score_col_name</span><span class="si">}</span><span class="s2"> as coloc_score&quot;</span><span class="p">,</span>
-                <span class="p">),</span>
+                <span class="n">colocalisation_df</span><span class="p">,</span>
                 <span class="n">on</span><span class="o">=</span><span class="s2">&quot;studyLocusId&quot;</span><span class="p">,</span>
                 <span class="n">how</span><span class="o">=</span><span class="s2">&quot;inner&quot;</span><span class="p">,</span>
             <span class="p">)</span>
             <span class="c1"># bring study metadata to just keep QTL studies on the right</span>
             <span class="o">.</span><span class="n">join</span><span class="p">(</span>
-                <span class="n">study_locus</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">selectExpr</span><span class="p">(</span>
-                    <span class="s2">&quot;studyLocusId as rightStudyLocusId&quot;</span><span class="p">,</span> <span class="s2">&quot;studyId as right_studyId&quot;</span>
+                <span class="n">credible_set</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
+                    <span class="n">studies</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">,</span> <span class="s2">&quot;studyType&quot;</span><span class="p">,</span> <span class="s2">&quot;geneId&quot;</span><span class="p">),</span> <span class="s2">&quot;studyId&quot;</span>
+                <span class="p">)</span><span class="o">.</span><span class="n">selectExpr</span><span class="p">(</span>
+                    <span class="s2">&quot;studyLocusId as rightStudyLocusId&quot;</span><span class="p">,</span>
+                    <span class="s2">&quot;studyType as right_studyType&quot;</span><span class="p">,</span>
+                    <span class="s2">&quot;geneId&quot;</span><span class="p">,</span>
                 <span class="p">),</span>
                 <span class="n">on</span><span class="o">=</span><span class="s2">&quot;rightStudyLocusId&quot;</span><span class="p">,</span>
                 <span class="n">how</span><span class="o">=</span><span class="s2">&quot;inner&quot;</span><span class="p">,</span>
             <span class="p">)</span>
-            <span class="o">.</span><span class="n">join</span><span class="p">(</span>
-                <span class="n">f</span><span class="o">.</span><span class="n">broadcast</span><span class="p">(</span>
-                    <span class="n">studies</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">selectExpr</span><span class="p">(</span>
-                        <span class="s2">&quot;studyId as right_studyId&quot;</span><span class="p">,</span>
-                        <span class="s2">&quot;studyType as right_studyType&quot;</span><span class="p">,</span>
-                        <span class="s2">&quot;geneId&quot;</span><span class="p">,</span>
-                    <span class="p">)</span>
-                <span class="p">),</span>
-                <span class="n">on</span><span class="o">=</span><span class="s2">&quot;right_studyId&quot;</span><span class="p">,</span>
-                <span class="n">how</span><span class="o">=</span><span class="s2">&quot;inner&quot;</span><span class="p">,</span>
-            <span class="p">)</span>
-            <span class="o">.</span><span class="n">filter</span><span class="p">(</span>
-                <span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;colocalisationMethod&quot;</span><span class="p">)</span> <span class="o">==</span> <span class="n">colocalisation_method</span><span class="p">)</span>
-                <span class="o">&amp;</span> <span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;right_studyType&quot;</span><span class="p">)</span> <span class="o">!=</span> <span class="s2">&quot;gwas&quot;</span><span class="p">)</span>
+            <span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;right_studyType&quot;</span><span class="p">)</span> <span class="o">!=</span> <span class="s2">&quot;gwas&quot;</span><span class="p">)</span>
+            <span class="o">.</span><span class="n">select</span><span class="p">(</span>
+                <span class="s2">&quot;studyLocusId&quot;</span><span class="p">,</span>
+                <span class="s2">&quot;right_studyType&quot;</span><span class="p">,</span>
+                <span class="s2">&quot;geneId&quot;</span><span class="p">,</span>
+                <span class="s2">&quot;score&quot;</span><span class="p">,</span>
+                <span class="s2">&quot;colocalisationMetric&quot;</span><span class="p">,</span>
             <span class="p">)</span>
-            <span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">&quot;studyLocusId&quot;</span><span class="p">,</span> <span class="s2">&quot;right_studyType&quot;</span><span class="p">,</span> <span class="s2">&quot;geneId&quot;</span><span class="p">,</span> <span class="s2">&quot;coloc_score&quot;</span><span class="p">)</span>
         <span class="p">)</span>
 
-        <span class="c1"># Max PP calculation per studyLocus AND type of QTL</span>
-        <span class="n">local_max</span> <span class="o">=</span> <span class="n">get_record_with_maximum_value</span><span class="p">(</span>
-            <span class="n">colocalising_study_locus</span><span class="p">,</span>
-            <span class="p">[</span><span class="s2">&quot;studyLocusId&quot;</span><span class="p">,</span> <span class="s2">&quot;right_studyType&quot;</span><span class="p">,</span> <span class="s2">&quot;geneId&quot;</span><span class="p">],</span>
-            <span class="s2">&quot;coloc_score&quot;</span><span class="p">,</span>
-        <span class="p">)</span><span class="o">.</span><span class="n">persist</span><span class="p">()</span>
-
-        <span class="n">intercept</span> <span class="o">=</span> <span class="mf">0.0001</span>
-        <span class="n">neighbourhood_max</span> <span class="o">=</span> <span class="p">(</span>
-            <span class="p">(</span>
-                <span class="n">local_max</span><span class="o">.</span><span class="n">selectExpr</span><span class="p">(</span>
-                    <span class="s2">&quot;studyLocusId&quot;</span><span class="p">,</span> <span class="s2">&quot;coloc_score as coloc_local_max&quot;</span><span class="p">,</span> <span class="s2">&quot;geneId&quot;</span>
-                <span class="p">)</span>
-                <span class="o">.</span><span class="n">join</span><span class="p">(</span>
-                    <span class="c1"># Add maximum in the neighborhood</span>
-                    <span class="n">get_record_with_maximum_value</span><span class="p">(</span>
-                        <span class="n">colocalising_study_locus</span><span class="o">.</span><span class="n">withColumnRenamed</span><span class="p">(</span>
-                            <span class="s2">&quot;coloc_score&quot;</span><span class="p">,</span> <span class="s2">&quot;coloc_neighborhood_max&quot;</span>
-                        <span class="p">),</span>
-                        <span class="p">[</span><span class="s2">&quot;studyLocusId&quot;</span><span class="p">,</span> <span class="s2">&quot;right_studyType&quot;</span><span class="p">],</span>
-                        <span class="s2">&quot;coloc_neighborhood_max&quot;</span><span class="p">,</span>
-                    <span class="p">)</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="s2">&quot;geneId&quot;</span><span class="p">),</span>
-                    <span class="n">on</span><span class="o">=</span><span class="s2">&quot;studyLocusId&quot;</span><span class="p">,</span>
-                <span class="p">)</span>
-                <span class="o">.</span><span class="n">withColumn</span><span class="p">(</span>
-                    <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">coloc_feature_col_template</span><span class="si">}</span><span class="s2">Neighborhood&quot;</span><span class="p">,</span>
-                    <span class="n">f</span><span class="o">.</span><span class="n">log10</span><span class="p">(</span>
-                        <span class="n">f</span><span class="o">.</span><span class="n">abs</span><span class="p">(</span>
-                            <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;coloc_local_max&quot;</span><span class="p">)</span>
-                            <span class="o">-</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;coloc_neighborhood_max&quot;</span><span class="p">)</span>
-                            <span class="o">+</span> <span class="n">f</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="n">intercept</span><span class="p">)</span>
-                        <span class="p">)</span>
-                    <span class="p">),</span>
-                <span class="p">)</span>
+        <span class="c1"># Max PP calculation per credible set AND type of QTL AND colocalisation method</span>
+        <span class="n">local_max</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="n">get_record_with_maximum_value</span><span class="p">(</span>
+                <span class="n">colocalising_credible_sets</span><span class="p">,</span>
+                <span class="p">[</span><span class="s2">&quot;studyLocusId&quot;</span><span class="p">,</span> <span class="s2">&quot;right_studyType&quot;</span><span class="p">,</span> <span class="s2">&quot;geneId&quot;</span><span class="p">,</span> <span class="s2">&quot;colocalisationMetric&quot;</span><span class="p">],</span>
+                <span class="s2">&quot;score&quot;</span><span class="p">,</span>
             <span class="p">)</span>
-            <span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="s2">&quot;coloc_neighborhood_max&quot;</span><span class="p">,</span> <span class="s2">&quot;coloc_local_max&quot;</span><span class="p">)</span>
-            <span class="o">.</span><span class="n">persist</span><span class="p">()</span>
+            <span class="o">.</span><span class="n">select</span><span class="p">(</span>
+                <span class="s2">&quot;*&quot;</span><span class="p">,</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;score&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;max_score&quot;</span><span class="p">),</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">&quot;Local&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;score_type&quot;</span><span class="p">),</span>
+            <span class="p">)</span>
+            <span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="s2">&quot;score&quot;</span><span class="p">)</span>
         <span class="p">)</span>
 
-        <span class="c1"># Split feature per molQTL</span>
-        <span class="n">local_dfs</span> <span class="o">=</span> <span class="p">[]</span>
-        <span class="n">nbh_dfs</span> <span class="o">=</span> <span class="p">[]</span>
-        <span class="n">qtl_types</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span>
-            <span class="n">colocalising_study_locus</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">&quot;right_studyType&quot;</span><span class="p">)</span>
-            <span class="o">.</span><span class="n">distinct</span><span class="p">()</span>
-            <span class="o">.</span><span class="n">toPandas</span><span class="p">()[</span><span class="s2">&quot;right_studyType&quot;</span><span class="p">]</span>
-            <span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
-        <span class="p">)</span> <span class="ow">or</span> <span class="p">[</span><span class="s2">&quot;eqtl&quot;</span><span class="p">,</span> <span class="s2">&quot;pqtl&quot;</span><span class="p">,</span> <span class="s2">&quot;sqtl&quot;</span><span class="p">]</span>
-        <span class="k">for</span> <span class="n">qtl_type</span> <span class="ow">in</span> <span class="n">qtl_types</span><span class="p">:</span>
-            <span class="n">filtered_local_max</span> <span class="o">=</span> <span class="p">(</span>
-                <span class="n">local_max</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;right_studyType&quot;</span><span class="p">)</span> <span class="o">==</span> <span class="n">qtl_type</span><span class="p">)</span>
-                <span class="o">.</span><span class="n">withColumnRenamed</span><span class="p">(</span>
-                    <span class="s2">&quot;coloc_score&quot;</span><span class="p">,</span>
-                    <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">qtl_type</span><span class="si">}{</span><span class="n">coloc_feature_col_template</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">,</span>
-                <span class="p">)</span>
-                <span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="s2">&quot;right_studyType&quot;</span><span class="p">)</span>
+        <span class="n">neighbourhood_max</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="n">local_max</span><span class="o">.</span><span class="n">selectExpr</span><span class="p">(</span>
+                <span class="s2">&quot;studyLocusId&quot;</span><span class="p">,</span> <span class="s2">&quot;max_score as local_max_score&quot;</span><span class="p">,</span> <span class="s2">&quot;geneId&quot;</span>
             <span class="p">)</span>
-            <span class="n">local_dfs</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">filtered_local_max</span><span class="p">)</span>
-
-            <span class="n">filtered_neighbourhood_max</span> <span class="o">=</span> <span class="p">(</span>
-                <span class="n">neighbourhood_max</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;right_studyType&quot;</span><span class="p">)</span> <span class="o">==</span> <span class="n">qtl_type</span><span class="p">)</span>
-                <span class="o">.</span><span class="n">withColumnRenamed</span><span class="p">(</span>
-                    <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">coloc_feature_col_template</span><span class="si">}</span><span class="s2">Neighborhood&quot;</span><span class="p">,</span>
-                    <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">qtl_type</span><span class="si">}{</span><span class="n">coloc_feature_col_template</span><span class="si">}</span><span class="s2">Neighborhood&quot;</span><span class="p">,</span>
-                <span class="p">)</span>
-                <span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="s2">&quot;right_studyType&quot;</span><span class="p">)</span>
+            <span class="o">.</span><span class="n">join</span><span class="p">(</span>
+                <span class="c1"># Add maximum in the neighborhood</span>
+                <span class="n">get_record_with_maximum_value</span><span class="p">(</span>
+                    <span class="n">colocalising_credible_sets</span><span class="o">.</span><span class="n">withColumnRenamed</span><span class="p">(</span>
+                        <span class="s2">&quot;score&quot;</span><span class="p">,</span> <span class="s2">&quot;tmp_nbh_max_score&quot;</span>
+                    <span class="p">),</span>
+                    <span class="p">[</span><span class="s2">&quot;studyLocusId&quot;</span><span class="p">,</span> <span class="s2">&quot;right_studyType&quot;</span><span class="p">,</span> <span class="s2">&quot;colocalisationMetric&quot;</span><span class="p">],</span>
+                    <span class="s2">&quot;tmp_nbh_max_score&quot;</span><span class="p">,</span>
+                <span class="p">)</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="s2">&quot;geneId&quot;</span><span class="p">),</span>
+                <span class="n">on</span><span class="o">=</span><span class="s2">&quot;studyLocusId&quot;</span><span class="p">,</span>
             <span class="p">)</span>
-            <span class="n">nbh_dfs</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">filtered_neighbourhood_max</span><span class="p">)</span>
-
-        <span class="n">wide_dfs</span> <span class="o">=</span> <span class="n">reduce</span><span class="p">(</span>
-            <span class="k">lambda</span> <span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">:</span> <span class="n">x</span><span class="o">.</span><span class="n">unionByName</span><span class="p">(</span><span class="n">y</span><span class="p">,</span> <span class="n">allowMissingColumns</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
-            <span class="n">local_dfs</span> <span class="o">+</span> <span class="n">nbh_dfs</span><span class="p">,</span>
-        <span class="p">)</span>
-
-        <span class="k">return</span> <span class="n">L2GFeature</span><span class="p">(</span>
-            <span class="n">_df</span><span class="o">=</span><span class="n">convert_from_wide_to_long</span><span class="p">(</span>
-                <span class="n">wide_dfs</span><span class="o">.</span><span class="n">groupBy</span><span class="p">(</span><span class="s2">&quot;studyLocusId&quot;</span><span class="p">,</span> <span class="s2">&quot;geneId&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">agg</span><span class="p">(</span>
-                    <span class="o">*</span><span class="p">(</span>
-                        <span class="n">f</span><span class="o">.</span><span class="n">first</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="n">c</span><span class="p">),</span> <span class="n">ignorenulls</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="n">c</span><span class="p">)</span>
-                        <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">wide_dfs</span><span class="o">.</span><span class="n">columns</span>
-                        <span class="k">if</span> <span class="n">c</span>
-                        <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span>
-                            <span class="s2">&quot;studyLocusId&quot;</span><span class="p">,</span>
-                            <span class="s2">&quot;geneId&quot;</span><span class="p">,</span>
-                        <span class="p">]</span>
+            <span class="o">.</span><span class="n">withColumn</span><span class="p">(</span><span class="s2">&quot;score_type&quot;</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">&quot;Neighborhood&quot;</span><span class="p">))</span>
+            <span class="o">.</span><span class="n">withColumn</span><span class="p">(</span>
+                <span class="s2">&quot;max_score&quot;</span><span class="p">,</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">log10</span><span class="p">(</span>
+                    <span class="n">f</span><span class="o">.</span><span class="n">abs</span><span class="p">(</span>
+                        <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;local_max_score&quot;</span><span class="p">)</span>
+                        <span class="o">-</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;tmp_nbh_max_score&quot;</span><span class="p">)</span>
+                        <span class="o">+</span> <span class="n">f</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mf">0.0001</span><span class="p">)</span>  <span class="c1"># intercept</span>
                     <span class="p">)</span>
                 <span class="p">),</span>
-                <span class="n">id_vars</span><span class="o">=</span><span class="p">(</span><span class="s2">&quot;studyLocusId&quot;</span><span class="p">,</span> <span class="s2">&quot;geneId&quot;</span><span class="p">),</span>
-                <span class="n">var_name</span><span class="o">=</span><span class="s2">&quot;featureName&quot;</span><span class="p">,</span>
-                <span class="n">value_name</span><span class="o">=</span><span class="s2">&quot;featureValue&quot;</span><span class="p">,</span>
-            <span class="p">),</span>
-            <span class="n">_schema</span><span class="o">=</span><span class="n">L2GFeature</span><span class="o">.</span><span class="n">get_schema</span><span class="p">(),</span>
-        <span class="p">)</span>
-
-    <span class="nd">@staticmethod</span>
-    <span class="k">def</span> <span class="nf">_get_coloc_features</span><span class="p">(</span>
-        <span class="n">study_locus</span><span class="p">:</span> <span class="n">StudyLocus</span><span class="p">,</span> <span class="n">studies</span><span class="p">:</span> <span class="n">StudyIndex</span><span class="p">,</span> <span class="n">colocalisation</span><span class="p">:</span> <span class="n">Colocalisation</span>
-    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">L2GFeature</span><span class="p">:</span>
-<span class="w">        </span><span class="sd">&quot;&quot;&quot;Calls _get_max_coloc_per_study_locus for both methods and concatenates the results.</span>
-
-<span class="sd">        !!! note &quot;Colocalisation features are only available for the eCAVIAR results for now.&quot;</span>
-
-<span class="sd">        Args:</span>
-<span class="sd">            study_locus (StudyLocus): Study locus dataset</span>
-<span class="sd">            studies (StudyIndex): Study index dataset</span>
-<span class="sd">            colocalisation (Colocalisation): Colocalisation dataset</span>
-
-<span class="sd">        Returns:</span>
-<span class="sd">            L2GFeature: Stores the features with the max coloc probabilities for each pair of study-locus</span>
-<span class="sd">        &quot;&quot;&quot;</span>
-        <span class="n">coloc_clpp</span> <span class="o">=</span> <span class="n">ColocalisationFactory</span><span class="o">.</span><span class="n">_get_max_coloc_per_study_locus</span><span class="p">(</span>
-            <span class="n">study_locus</span><span class="p">,</span>
-            <span class="n">studies</span><span class="p">,</span>
-            <span class="n">colocalisation</span><span class="p">,</span>
-            <span class="s2">&quot;eCAVIAR&quot;</span><span class="p">,</span>
-        <span class="p">)</span>
+            <span class="p">)</span>
+        <span class="p">)</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="s2">&quot;tmp_nbh_max_score&quot;</span><span class="p">,</span> <span class="s2">&quot;local_max_score&quot;</span><span class="p">)</span>
 
         <span class="k">return</span> <span class="n">L2GFeature</span><span class="p">(</span>
-            <span class="n">_df</span><span class="o">=</span><span class="n">coloc_clpp</span><span class="o">.</span><span class="n">df</span><span class="p">,</span>
+            <span class="n">_df</span><span class="o">=</span><span class="p">(</span>
+                <span class="c1"># Combine local and neighborhood metrics</span>
+                <span class="n">local_max</span><span class="o">.</span><span class="n">unionByName</span><span class="p">(</span>
+                    <span class="n">neighbourhood_max</span><span class="p">,</span> <span class="n">allowMissingColumns</span><span class="o">=</span><span class="kc">True</span>
+                <span class="p">)</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+                    <span class="s2">&quot;studyLocusId&quot;</span><span class="p">,</span>
+                    <span class="s2">&quot;geneId&quot;</span><span class="p">,</span>
+                    <span class="c1"># Feature name is a concatenation of the QTL type, colocalisation metric and if it&#39;s local or in the vicinity</span>
+                    <span class="n">f</span><span class="o">.</span><span class="n">concat_ws</span><span class="p">(</span>
+                        <span class="s2">&quot;&quot;</span><span class="p">,</span>
+                        <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;right_studyType&quot;</span><span class="p">),</span>
+                        <span class="n">f</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">&quot;Coloc&quot;</span><span class="p">),</span>
+                        <span class="n">f</span><span class="o">.</span><span class="n">initcap</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;colocalisationMetric&quot;</span><span class="p">)),</span>
+                        <span class="n">f</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">&quot;Maximum&quot;</span><span class="p">),</span>
+                        <span class="n">f</span><span class="o">.</span><span class="n">regexp_replace</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;score_type&quot;</span><span class="p">),</span> <span class="s2">&quot;Local&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">),</span>
+                    <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;featureName&quot;</span><span class="p">),</span>
+                    <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;max_score&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">cast</span><span class="p">(</span><span class="s2">&quot;float&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;featureValue&quot;</span><span class="p">),</span>
+                <span class="p">)</span>
+            <span class="p">),</span>
             <span class="n">_schema</span><span class="o">=</span><span class="n">L2GFeature</span><span class="o">.</span><span class="n">get_schema</span><span class="p">(),</span>
         <span class="p">)</span>
 </code></pre></div></td></tr></table></div>
@@ -3332,6 +3252,10 @@ <h2 id="gentropy.method.l2g.feature_factory.ColocalisationFactory" class="doc do
 <h2 id="gentropy.method.l2g.feature_factory.StudyLocusFactory" class="doc doc-heading">
           <code>gentropy.method.l2g.feature_factory.StudyLocusFactory</code>
 
+  
+  <span class="doc doc-labels">
+      <small class="doc doc-label doc-label-dataclass"><code>dataclass</code></small>
+  </span>
 
 <a href="#gentropy.method.l2g.feature_factory.StudyLocusFactory" class="headerlink" title="Permanent link">&para;</a></h2>
 
@@ -3345,7 +3269,66 @@ <h2 id="gentropy.method.l2g.feature_factory.StudyLocusFactory" class="doc doc-he
 
             <details class="quote">
               <summary>Source code in <code>src/gentropy/method/l2g/feature_factory.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">222</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">163</span>
+<span class="normal">164</span>
+<span class="normal">165</span>
+<span class="normal">166</span>
+<span class="normal">167</span>
+<span class="normal">168</span>
+<span class="normal">169</span>
+<span class="normal">170</span>
+<span class="normal">171</span>
+<span class="normal">172</span>
+<span class="normal">173</span>
+<span class="normal">174</span>
+<span class="normal">175</span>
+<span class="normal">176</span>
+<span class="normal">177</span>
+<span class="normal">178</span>
+<span class="normal">179</span>
+<span class="normal">180</span>
+<span class="normal">181</span>
+<span class="normal">182</span>
+<span class="normal">183</span>
+<span class="normal">184</span>
+<span class="normal">185</span>
+<span class="normal">186</span>
+<span class="normal">187</span>
+<span class="normal">188</span>
+<span class="normal">189</span>
+<span class="normal">190</span>
+<span class="normal">191</span>
+<span class="normal">192</span>
+<span class="normal">193</span>
+<span class="normal">194</span>
+<span class="normal">195</span>
+<span class="normal">196</span>
+<span class="normal">197</span>
+<span class="normal">198</span>
+<span class="normal">199</span>
+<span class="normal">200</span>
+<span class="normal">201</span>
+<span class="normal">202</span>
+<span class="normal">203</span>
+<span class="normal">204</span>
+<span class="normal">205</span>
+<span class="normal">206</span>
+<span class="normal">207</span>
+<span class="normal">208</span>
+<span class="normal">209</span>
+<span class="normal">210</span>
+<span class="normal">211</span>
+<span class="normal">212</span>
+<span class="normal">213</span>
+<span class="normal">214</span>
+<span class="normal">215</span>
+<span class="normal">216</span>
+<span class="normal">217</span>
+<span class="normal">218</span>
+<span class="normal">219</span>
+<span class="normal">220</span>
+<span class="normal">221</span>
+<span class="normal">222</span>
 <span class="normal">223</span>
 <span class="normal">224</span>
 <span class="normal">225</span>
@@ -3463,93 +3446,47 @@ <h2 id="gentropy.method.l2g.feature_factory.StudyLocusFactory" class="doc doc-he
 <span class="normal">337</span>
 <span class="normal">338</span>
 <span class="normal">339</span>
-<span class="normal">340</span>
-<span class="normal">341</span>
-<span class="normal">342</span>
-<span class="normal">343</span>
-<span class="normal">344</span>
-<span class="normal">345</span>
-<span class="normal">346</span>
-<span class="normal">347</span>
-<span class="normal">348</span>
-<span class="normal">349</span>
-<span class="normal">350</span>
-<span class="normal">351</span>
-<span class="normal">352</span>
-<span class="normal">353</span>
-<span class="normal">354</span>
-<span class="normal">355</span>
-<span class="normal">356</span>
-<span class="normal">357</span>
-<span class="normal">358</span>
-<span class="normal">359</span>
-<span class="normal">360</span>
-<span class="normal">361</span>
-<span class="normal">362</span>
-<span class="normal">363</span>
-<span class="normal">364</span>
-<span class="normal">365</span>
-<span class="normal">366</span>
-<span class="normal">367</span>
-<span class="normal">368</span>
-<span class="normal">369</span>
-<span class="normal">370</span>
-<span class="normal">371</span>
-<span class="normal">372</span>
-<span class="normal">373</span>
-<span class="normal">374</span>
-<span class="normal">375</span>
-<span class="normal">376</span>
-<span class="normal">377</span>
-<span class="normal">378</span>
-<span class="normal">379</span>
-<span class="normal">380</span>
-<span class="normal">381</span>
-<span class="normal">382</span>
-<span class="normal">383</span>
-<span class="normal">384</span>
-<span class="normal">385</span>
-<span class="normal">386</span>
-<span class="normal">387</span>
-<span class="normal">388</span>
-<span class="normal">389</span>
-<span class="normal">390</span>
-<span class="normal">391</span>
-<span class="normal">392</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">StudyLocusFactory</span><span class="p">(</span><span class="n">StudyLocus</span><span class="p">):</span>
+<span class="normal">340</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">StudyLocusFactory</span><span class="p">(</span><span class="n">StudyLocus</span><span class="p">):</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Feature extraction in study locus.&quot;&quot;&quot;</span>
 
     <span class="nd">@staticmethod</span>
-    <span class="k">def</span> <span class="nf">_get_tss_distance_features</span><span class="p">(</span>
-        <span class="n">study_locus</span><span class="p">:</span> <span class="n">StudyLocus</span><span class="p">,</span> <span class="n">distances</span><span class="p">:</span> <span class="n">V2G</span>
-    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">L2GFeature</span><span class="p">:</span>
-<span class="w">        </span><span class="sd">&quot;&quot;&quot;Joins StudyLocus with the V2G to extract the minimum distance to a gene TSS of all variants in a StudyLocus credible set.</span>
+    <span class="k">def</span> <span class="nf">_get_tss_distance_features</span><span class="p">(</span><span class="n">credible_set</span><span class="p">:</span> <span class="n">StudyLocus</span><span class="p">,</span> <span class="n">v2g</span><span class="p">:</span> <span class="n">V2G</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">L2GFeature</span><span class="p">:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Joins StudyLocus with the V2G to extract a score that is based on the distance to a gene TSS of any variant weighted by its posterior probability in a credible set.</span>
 
 <span class="sd">        Args:</span>
-<span class="sd">            study_locus (StudyLocus): Study locus dataset</span>
-<span class="sd">            distances (V2G): Dataframe containing the distances of all variants to all genes TSS within a region</span>
+<span class="sd">            credible_set (StudyLocus): Credible set dataset</span>
+<span class="sd">            v2g (V2G): Dataframe containing the distances of all variants to all genes TSS within a region</span>
 
 <span class="sd">        Returns:</span>
-<span class="sd">            L2GFeature: Stores the features with the minimum distance among all variants in the credible set and a gene TSS.</span>
+<span class="sd">            L2GFeature: Stores the features with the score of weighting the distance to the TSS by the posterior probability of the variant</span>
 
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="n">wide_df</span> <span class="o">=</span> <span class="p">(</span>
-            <span class="n">study_locus</span><span class="o">.</span><span class="n">filter_credible_set</span><span class="p">(</span><span class="n">CredibleInterval</span><span class="o">.</span><span class="n">IS95</span><span class="p">)</span>
-            <span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+            <span class="n">credible_set</span><span class="o">.</span><span class="n">filter_credible_set</span><span class="p">(</span><span class="n">CredibleInterval</span><span class="o">.</span><span class="n">IS95</span><span class="p">)</span>
+            <span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">withColumn</span><span class="p">(</span><span class="s2">&quot;variantInLocus&quot;</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">explode_outer</span><span class="p">(</span><span class="s2">&quot;locus&quot;</span><span class="p">))</span>
+            <span class="o">.</span><span class="n">select</span><span class="p">(</span>
                 <span class="s2">&quot;studyLocusId&quot;</span><span class="p">,</span>
                 <span class="s2">&quot;variantId&quot;</span><span class="p">,</span>
-                <span class="n">f</span><span class="o">.</span><span class="n">explode</span><span class="p">(</span><span class="s2">&quot;locus.variantId&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;tagVariantId&quot;</span><span class="p">),</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;variantInLocus.variantId&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;variantInLocusId&quot;</span><span class="p">),</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;variantInLocus.posteriorProbability&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span>
+                    <span class="s2">&quot;variantInLocusPosteriorProbability&quot;</span>
+                <span class="p">),</span>
             <span class="p">)</span>
             <span class="o">.</span><span class="n">join</span><span class="p">(</span>
-                <span class="n">distances</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">selectExpr</span><span class="p">(</span>
-                    <span class="s2">&quot;variantId as tagVariantId&quot;</span><span class="p">,</span> <span class="s2">&quot;geneId&quot;</span><span class="p">,</span> <span class="s2">&quot;distance&quot;</span>
+                <span class="n">v2g</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;datasourceId&quot;</span><span class="p">)</span> <span class="o">==</span> <span class="s2">&quot;canonical_tss&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">selectExpr</span><span class="p">(</span>
+                    <span class="s2">&quot;variantId as variantInLocusId&quot;</span><span class="p">,</span> <span class="s2">&quot;geneId&quot;</span><span class="p">,</span> <span class="s2">&quot;score&quot;</span>
                 <span class="p">),</span>
-                <span class="n">on</span><span class="o">=</span><span class="s2">&quot;tagVariantId&quot;</span><span class="p">,</span>
+                <span class="n">on</span><span class="o">=</span><span class="s2">&quot;variantInLocusId&quot;</span><span class="p">,</span>
                 <span class="n">how</span><span class="o">=</span><span class="s2">&quot;inner&quot;</span><span class="p">,</span>
             <span class="p">)</span>
+            <span class="o">.</span><span class="n">withColumn</span><span class="p">(</span>
+                <span class="s2">&quot;weightedScore&quot;</span><span class="p">,</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;score&quot;</span><span class="p">)</span> <span class="o">*</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;variantInLocusPosteriorProbability&quot;</span><span class="p">),</span>
+            <span class="p">)</span>
             <span class="o">.</span><span class="n">groupBy</span><span class="p">(</span><span class="s2">&quot;studyLocusId&quot;</span><span class="p">,</span> <span class="s2">&quot;geneId&quot;</span><span class="p">)</span>
             <span class="o">.</span><span class="n">agg</span><span class="p">(</span>
-                <span class="n">f</span><span class="o">.</span><span class="n">min</span><span class="p">(</span><span class="s2">&quot;distance&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;distanceTssMinimum&quot;</span><span class="p">),</span>
-                <span class="n">f</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="s2">&quot;distance&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;distanceTssMean&quot;</span><span class="p">),</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">min</span><span class="p">(</span><span class="s2">&quot;weightedScore&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;distanceTssMinimum&quot;</span><span class="p">),</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="s2">&quot;weightedScore&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;distanceTssMean&quot;</span><span class="p">),</span>
             <span class="p">)</span>
         <span class="p">)</span>
 
@@ -3617,32 +3554,33 @@ <h2 id="gentropy.method.l2g.feature_factory.StudyLocusFactory" class="doc doc-he
 
         <span class="n">credible_set_w_variant_consequences</span> <span class="o">=</span> <span class="p">(</span>
             <span class="n">credible_set</span><span class="o">.</span><span class="n">filter_credible_set</span><span class="p">(</span><span class="n">CredibleInterval</span><span class="o">.</span><span class="n">IS95</span><span class="p">)</span>
-            <span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">withColumn</span><span class="p">(</span><span class="s2">&quot;variantInLocusId&quot;</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">explode</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;locus.variantId&quot;</span><span class="p">)))</span>
-            <span class="o">.</span><span class="n">withColumn</span><span class="p">(</span>
-                <span class="s2">&quot;variantInLocusPosteriorProbability&quot;</span><span class="p">,</span>
-                <span class="n">f</span><span class="o">.</span><span class="n">explode</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;locus.posteriorProbability&quot;</span><span class="p">)),</span>
+            <span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">withColumn</span><span class="p">(</span><span class="s2">&quot;variantInLocus&quot;</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">explode_outer</span><span class="p">(</span><span class="s2">&quot;locus&quot;</span><span class="p">))</span>
+            <span class="o">.</span><span class="n">select</span><span class="p">(</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;studyLocusId&quot;</span><span class="p">),</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;variantId&quot;</span><span class="p">),</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">),</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;variantInLocus.variantId&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;variantInLocusId&quot;</span><span class="p">),</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;variantInLocus.posteriorProbability&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span>
+                    <span class="s2">&quot;variantInLocusPosteriorProbability&quot;</span>
+                <span class="p">),</span>
             <span class="p">)</span>
             <span class="o">.</span><span class="n">join</span><span class="p">(</span>
                 <span class="c1"># Join with V2G to get variant consequences</span>
-                <span class="n">v2g</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span>
-                    <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;datasourceId&quot;</span><span class="p">)</span> <span class="o">==</span> <span class="s2">&quot;variantConsequence&quot;</span>
-                <span class="p">)</span><span class="o">.</span><span class="n">withColumnRenamed</span><span class="p">(</span><span class="s2">&quot;variantId&quot;</span><span class="p">,</span> <span class="s2">&quot;variantInLocusId&quot;</span><span class="p">),</span>
+                <span class="n">v2g</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;datasourceId&quot;</span><span class="p">)</span> <span class="o">==</span> <span class="s2">&quot;variantConsequence&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">selectExpr</span><span class="p">(</span>
+                    <span class="s2">&quot;variantId as variantInLocusId&quot;</span><span class="p">,</span> <span class="s2">&quot;geneId&quot;</span><span class="p">,</span> <span class="s2">&quot;score&quot;</span>
+                <span class="p">),</span>
                 <span class="n">on</span><span class="o">=</span><span class="s2">&quot;variantInLocusId&quot;</span><span class="p">,</span>
             <span class="p">)</span>
-            <span class="o">.</span><span class="n">withColumn</span><span class="p">(</span>
-                <span class="s2">&quot;weightedScore&quot;</span><span class="p">,</span>
-                <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;score&quot;</span><span class="p">)</span> <span class="o">*</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;variantInLocusPosteriorProbability&quot;</span><span class="p">),</span>
-            <span class="p">)</span>
             <span class="o">.</span><span class="n">select</span><span class="p">(</span>
                 <span class="s2">&quot;studyLocusId&quot;</span><span class="p">,</span>
                 <span class="s2">&quot;variantId&quot;</span><span class="p">,</span>
                 <span class="s2">&quot;studyId&quot;</span><span class="p">,</span>
                 <span class="s2">&quot;geneId&quot;</span><span class="p">,</span>
-                <span class="s2">&quot;score&quot;</span><span class="p">,</span>
-                <span class="s2">&quot;weightedScore&quot;</span><span class="p">,</span>
+                <span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;score&quot;</span><span class="p">)</span> <span class="o">*</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;variantInLocusPosteriorProbability&quot;</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span>
+                    <span class="s2">&quot;weightedScore&quot;</span>
+                <span class="p">),</span>
             <span class="p">)</span>
             <span class="o">.</span><span class="n">distinct</span><span class="p">()</span>
-            <span class="o">.</span><span class="n">persist</span><span class="p">()</span>
         <span class="p">)</span>
 
         <span class="k">return</span> <span class="n">L2GFeature</span><span class="p">(</span>
@@ -3653,14 +3591,14 @@ <h2 id="gentropy.method.l2g.feature_factory.StudyLocusFactory" class="doc doc-he
                         <span class="c1"># Calculate overall max VEP score for all genes in the vicinity</span>
                         <span class="n">credible_set_w_variant_consequences</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span>
                             <span class="n">_aggregate_vep_feature</span><span class="p">,</span>
-                            <span class="n">f</span><span class="o">.</span><span class="n">max</span><span class="p">(</span><span class="s2">&quot;score&quot;</span><span class="p">),</span>
+                            <span class="n">f</span><span class="o">.</span><span class="n">max</span><span class="p">(</span><span class="s2">&quot;weightedScore&quot;</span><span class="p">),</span>
                             <span class="p">[</span><span class="s2">&quot;studyLocusId&quot;</span><span class="p">],</span>
                             <span class="s2">&quot;vepMaximumNeighborhood&quot;</span><span class="p">,</span>
                         <span class="p">),</span>
                         <span class="c1"># Calculate overall max VEP score per gene</span>
                         <span class="n">credible_set_w_variant_consequences</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span>
                             <span class="n">_aggregate_vep_feature</span><span class="p">,</span>
-                            <span class="n">f</span><span class="o">.</span><span class="n">max</span><span class="p">(</span><span class="s2">&quot;score&quot;</span><span class="p">),</span>
+                            <span class="n">f</span><span class="o">.</span><span class="n">max</span><span class="p">(</span><span class="s2">&quot;weightedScore&quot;</span><span class="p">),</span>
                             <span class="p">[</span><span class="s2">&quot;studyLocusId&quot;</span><span class="p">,</span> <span class="s2">&quot;geneId&quot;</span><span class="p">],</span>
                             <span class="s2">&quot;vepMaximum&quot;</span><span class="p">,</span>
                         <span class="p">),</span>
diff --git a/python_api/methods/l2g/model/index.html b/python_api/methods/l2g/model/index.html
index 2a21c6d03..6b3ef9826 100644
--- a/python_api/methods/l2g/model/index.html
+++ b/python_api/methods/l2g/model/index.html
@@ -2197,6 +2197,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../susie_inf/" class="md-nav__link">
         
@@ -2230,7 +2272,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" checked>
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" checked>
         
           
           
@@ -2246,14 +2288,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="true">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="true">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3401,7 +3443,10 @@ <h2 id="gentropy.method.l2g.model.LocusToGeneModel" class="doc doc-heading">
 <span class="normal">302</span>
 <span class="normal">303</span>
 <span class="normal">304</span>
-<span class="normal">305</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@dataclass</span>
+<span class="normal">305</span>
+<span class="normal">306</span>
+<span class="normal">307</span>
+<span class="normal">308</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@dataclass</span>
 <span class="k">class</span> <span class="nc">LocusToGeneModel</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Wrapper for the Locus to Gene classifier.&quot;&quot;&quot;</span>
 
@@ -3583,7 +3628,7 @@ <h2 id="gentropy.method.l2g.model.LocusToGeneModel" class="doc doc-heading">
         <span class="n">results</span><span class="p">:</span> <span class="n">DataFrame</span><span class="p">,</span>
         <span class="n">hyperparameters</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">],</span>
         <span class="n">wandb_run_name</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span>
-        <span class="n">training_data</span><span class="p">:</span> <span class="n">L2GFeatureMatrix</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">gold_standard_data</span><span class="p">:</span> <span class="n">L2GFeatureMatrix</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
     <span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
 <span class="w">        </span><span class="sd">&quot;&quot;&quot;Perform evaluation of the model predictions for the test set and track the results with W&amp;B.</span>
 
@@ -3591,7 +3636,7 @@ <h2 id="gentropy.method.l2g.model.LocusToGeneModel" class="doc doc-heading">
 <span class="sd">            results (DataFrame): Dataframe containing the predictions</span>
 <span class="sd">            hyperparameters (dict[str, Any]): Hyperparameters used for the model</span>
 <span class="sd">            wandb_run_name (str | None): Descriptive name for the run to be tracked with W&amp;B</span>
-<span class="sd">            training_data (L2GFeatureMatrix | None): Training data used for the model. If provided, the ratio of positive to negative labels will be logged to W&amp;B</span>
+<span class="sd">            gold_standard_data (L2GFeatureMatrix | None): Feature matrix for the associations in the gold standard. If provided, the ratio of positive to negative labels will be logged to W&amp;B</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="n">binary_evaluator</span> <span class="o">=</span> <span class="n">BinaryClassificationEvaluator</span><span class="p">(</span>
             <span class="n">rawPredictionCol</span><span class="o">=</span><span class="s2">&quot;rawPrediction&quot;</span><span class="p">,</span> <span class="n">labelCol</span><span class="o">=</span><span class="s2">&quot;label&quot;</span>
@@ -3600,7 +3645,7 @@ <h2 id="gentropy.method.l2g.model.LocusToGeneModel" class="doc doc-heading">
             <span class="n">labelCol</span><span class="o">=</span><span class="s2">&quot;label&quot;</span><span class="p">,</span> <span class="n">predictionCol</span><span class="o">=</span><span class="s2">&quot;prediction&quot;</span>
         <span class="p">)</span>
 
-        <span class="k">if</span> <span class="n">wandb_run_name</span> <span class="ow">and</span> <span class="n">training_data</span><span class="p">:</span>
+        <span class="k">if</span> <span class="n">wandb_run_name</span> <span class="ow">and</span> <span class="n">gold_standard_data</span><span class="p">:</span>
             <span class="n">run</span> <span class="o">=</span> <span class="n">wandb_init</span><span class="p">(</span>
                 <span class="n">project</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">wandb_l2g_project_name</span><span class="p">,</span>
                 <span class="n">config</span><span class="o">=</span><span class="n">hyperparameters</span><span class="p">,</span>
@@ -3608,7 +3653,10 @@ <h2 id="gentropy.method.l2g.model.LocusToGeneModel" class="doc doc-heading">
             <span class="p">)</span>
             <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">run</span><span class="p">,</span> <span class="n">Run</span><span class="p">):</span>
                 <span class="bp">self</span><span class="o">.</span><span class="n">log_to_wandb</span><span class="p">(</span>
-                    <span class="n">results</span><span class="p">,</span> <span class="n">training_data</span><span class="p">,</span> <span class="p">[</span><span class="n">binary_evaluator</span><span class="p">,</span> <span class="n">multi_evaluator</span><span class="p">],</span> <span class="n">run</span>
+                    <span class="n">results</span><span class="p">,</span>
+                    <span class="n">gold_standard_data</span><span class="p">,</span>
+                    <span class="p">[</span><span class="n">binary_evaluator</span><span class="p">,</span> <span class="n">multi_evaluator</span><span class="p">],</span>
+                    <span class="n">run</span><span class="p">,</span>
                 <span class="p">)</span>
                 <span class="n">run</span><span class="o">.</span><span class="n">finish</span><span class="p">()</span>
 
@@ -3713,7 +3761,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.classifier" class="doc doc-he
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3722,7 +3770,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.classifier" class="doc doc-he
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>Any</code></td>          <td>
                 <code><span title="typing.Any">Any</span></code>
           </td>
@@ -3758,7 +3806,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.feature_name_map" class="doc
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3767,7 +3815,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.feature_name_map" class="doc
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td>
                 <code>dict[str, str]</code>
           </td>
@@ -3782,7 +3830,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.feature_name_map" class="doc
 
 
 
-  <p><strong>Raises:</strong></p>
+  <p><span class="doc-section-title">Raises:</span></p>
   <table>
     <thead>
       <tr>
@@ -3791,7 +3839,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.feature_name_map" class="doc
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td>
                 <code>ValueError</code>
           </td>
@@ -3826,7 +3874,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.add_pipeline_stage" class="do
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3837,7 +3885,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.add_pipeline_stage" class="do
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>transformer</code></td>
           <td>
                 <code><span title="pyspark.ml.Transformer">Transformer</span></code>
@@ -3856,7 +3904,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.add_pipeline_stage" class="do
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3865,7 +3913,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.add_pipeline_stage" class="do
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>LocusToGeneModel</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.method.l2g.model.LocusToGeneModel" href="#gentropy.method.l2g.model.LocusToGeneModel">LocusToGeneModel</a></code>
           </td>
@@ -3880,7 +3928,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.add_pipeline_stage" class="do
 
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pyspark.ml.regression</span> <span class="kn">import</span> <span class="n">LinearRegression</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">estimator</span> <span class="o">=</span> <span class="n">LinearRegression</span><span class="p">()</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">test_model</span> <span class="o">=</span> <span class="n">LocusToGeneModel</span><span class="p">(</span><span class="n">features_list</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;a&quot;</span><span class="p">,</span> <span class="s2">&quot;b&quot;</span><span class="p">])</span>
@@ -3951,7 +3999,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.add_pipeline_stage" class="do
 
 
 <h3 id="gentropy.method.l2g.model.LocusToGeneModel.evaluate" class="doc doc-heading">
-          <code class="highlight language-python"><span class="n">evaluate</span><span class="p">(</span><span class="n">results</span><span class="p">:</span> <span class="n">DataFrame</span><span class="p">,</span> <span class="n">hyperparameters</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">],</span> <span class="n">wandb_run_name</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span> <span class="n">training_data</span><span class="p">:</span> <span class="n">L2GFeatureMatrix</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span></code>
+          <code class="highlight language-python"><span class="n">evaluate</span><span class="p">(</span><span class="n">results</span><span class="p">:</span> <span class="n">DataFrame</span><span class="p">,</span> <span class="n">hyperparameters</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">],</span> <span class="n">wandb_run_name</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span> <span class="n">gold_standard_data</span><span class="p">:</span> <span class="n">L2GFeatureMatrix</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span></code>
 
 <a href="#gentropy.method.l2g.model.LocusToGeneModel.evaluate" class="headerlink" title="Permanent link">&para;</a></h3>
 
@@ -3962,7 +4010,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.evaluate" class="doc doc-head
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3973,7 +4021,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.evaluate" class="doc doc-head
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>results</code></td>
           <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
@@ -3987,7 +4035,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.evaluate" class="doc doc-head
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>hyperparameters</code></td>
           <td>
                 <code>dict[str, <span title="typing.Any">Any</span>]</code>
@@ -4001,7 +4049,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.evaluate" class="doc doc-head
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>wandb_run_name</code></td>
           <td>
                 <code>str | None</code>
@@ -4015,14 +4063,14 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.evaluate" class="doc doc-head
               <em>required</em>
           </td>
         </tr>
-        <tr>
-          <td><code>training_data</code></td>
+        <tr class="doc-section-item">
+          <td><code>gold_standard_data</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix" href="../../../datasets/l2g_feature_matrix/#gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix">L2GFeatureMatrix</a> | None</code>
           </td>
           <td>
             <div class="doc-md-description">
-              <p>Training data used for the model. If provided, the ratio of positive to negative labels will be logged to W&amp;B</p>
+              <p>Feature matrix for the associations in the gold standard. If provided, the ratio of positive to negative labels will be logged to W&amp;B</p>
             </div>
           </td>
           <td>
@@ -4066,12 +4114,15 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.evaluate" class="doc doc-head
 <span class="normal">234</span>
 <span class="normal">235</span>
 <span class="normal">236</span>
-<span class="normal">237</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">evaluate</span><span class="p">(</span>
+<span class="normal">237</span>
+<span class="normal">238</span>
+<span class="normal">239</span>
+<span class="normal">240</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">evaluate</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">:</span> <span class="n">LocusToGeneModel</span><span class="p">,</span>
     <span class="n">results</span><span class="p">:</span> <span class="n">DataFrame</span><span class="p">,</span>
     <span class="n">hyperparameters</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">],</span>
     <span class="n">wandb_run_name</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">training_data</span><span class="p">:</span> <span class="n">L2GFeatureMatrix</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">gold_standard_data</span><span class="p">:</span> <span class="n">L2GFeatureMatrix</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Perform evaluation of the model predictions for the test set and track the results with W&amp;B.</span>
 
@@ -4079,7 +4130,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.evaluate" class="doc doc-head
 <span class="sd">        results (DataFrame): Dataframe containing the predictions</span>
 <span class="sd">        hyperparameters (dict[str, Any]): Hyperparameters used for the model</span>
 <span class="sd">        wandb_run_name (str | None): Descriptive name for the run to be tracked with W&amp;B</span>
-<span class="sd">        training_data (L2GFeatureMatrix | None): Training data used for the model. If provided, the ratio of positive to negative labels will be logged to W&amp;B</span>
+<span class="sd">        gold_standard_data (L2GFeatureMatrix | None): Feature matrix for the associations in the gold standard. If provided, the ratio of positive to negative labels will be logged to W&amp;B</span>
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="n">binary_evaluator</span> <span class="o">=</span> <span class="n">BinaryClassificationEvaluator</span><span class="p">(</span>
         <span class="n">rawPredictionCol</span><span class="o">=</span><span class="s2">&quot;rawPrediction&quot;</span><span class="p">,</span> <span class="n">labelCol</span><span class="o">=</span><span class="s2">&quot;label&quot;</span>
@@ -4088,7 +4139,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.evaluate" class="doc doc-head
         <span class="n">labelCol</span><span class="o">=</span><span class="s2">&quot;label&quot;</span><span class="p">,</span> <span class="n">predictionCol</span><span class="o">=</span><span class="s2">&quot;prediction&quot;</span>
     <span class="p">)</span>
 
-    <span class="k">if</span> <span class="n">wandb_run_name</span> <span class="ow">and</span> <span class="n">training_data</span><span class="p">:</span>
+    <span class="k">if</span> <span class="n">wandb_run_name</span> <span class="ow">and</span> <span class="n">gold_standard_data</span><span class="p">:</span>
         <span class="n">run</span> <span class="o">=</span> <span class="n">wandb_init</span><span class="p">(</span>
             <span class="n">project</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">wandb_l2g_project_name</span><span class="p">,</span>
             <span class="n">config</span><span class="o">=</span><span class="n">hyperparameters</span><span class="p">,</span>
@@ -4096,7 +4147,10 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.evaluate" class="doc doc-head
         <span class="p">)</span>
         <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">run</span><span class="p">,</span> <span class="n">Run</span><span class="p">):</span>
             <span class="bp">self</span><span class="o">.</span><span class="n">log_to_wandb</span><span class="p">(</span>
-                <span class="n">results</span><span class="p">,</span> <span class="n">training_data</span><span class="p">,</span> <span class="p">[</span><span class="n">binary_evaluator</span><span class="p">,</span> <span class="n">multi_evaluator</span><span class="p">],</span> <span class="n">run</span>
+                <span class="n">results</span><span class="p">,</span>
+                <span class="n">gold_standard_data</span><span class="p">,</span>
+                <span class="p">[</span><span class="n">binary_evaluator</span><span class="p">,</span> <span class="n">multi_evaluator</span><span class="p">],</span>
+                <span class="n">run</span><span class="p">,</span>
             <span class="p">)</span>
             <span class="n">run</span><span class="o">.</span><span class="n">finish</span><span class="p">()</span>
 </code></pre></div></td></tr></table></div>
@@ -4126,7 +4180,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.features_vector_assembler" cl
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -4137,7 +4191,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.features_vector_assembler" cl
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>features_cols</code></td>
           <td>
                 <code>list[str]</code>
@@ -4156,7 +4210,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.features_vector_assembler" cl
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -4165,7 +4219,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.features_vector_assembler" cl
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>VectorAssembler</code></td>          <td>
                 <code><span title="pyspark.ml.feature.VectorAssembler">VectorAssembler</span></code>
           </td>
@@ -4180,7 +4234,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.features_vector_assembler" cl
 
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pyspark.ml.feature</span> <span class="kn">import</span> <span class="n">VectorAssembler</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">df</span> <span class="o">=</span> <span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">([(</span><span class="mf">5.2</span><span class="p">,</span> <span class="mf">3.5</span><span class="p">)],</span> <span class="n">schema</span><span class="o">=</span><span class="s2">&quot;feature_1 FLOAT, feature_2 FLOAT&quot;</span><span class="p">)</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">assembler</span> <span class="o">=</span> <span class="n">LocusToGeneModel</span><span class="o">.</span><span class="n">features_vector_assembler</span><span class="p">([</span><span class="s2">&quot;feature_1&quot;</span><span class="p">,</span> <span class="s2">&quot;feature_2&quot;</span><span class="p">])</span>
@@ -4270,7 +4324,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.fit" class="doc doc-heading">
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -4281,7 +4335,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.fit" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>feature_matrix</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix" href="../../../datasets/l2g_feature_matrix/#gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix">L2GFeatureMatrix</a></code>
@@ -4300,7 +4354,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.fit" class="doc doc-heading">
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -4309,7 +4363,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.fit" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>LocusToGeneModel</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.method.l2g.model.LocusToGeneModel" href="#gentropy.method.l2g.model.LocusToGeneModel">LocusToGeneModel</a></code>
           </td>
@@ -4324,10 +4378,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.fit" class="doc doc-heading">
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/method/l2g/model.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">273</span>
-<span class="normal">274</span>
-<span class="normal">275</span>
-<span class="normal">276</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">276</span>
 <span class="normal">277</span>
 <span class="normal">278</span>
 <span class="normal">279</span>
@@ -4337,7 +4388,10 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.fit" class="doc doc-heading">
 <span class="normal">283</span>
 <span class="normal">284</span>
 <span class="normal">285</span>
-<span class="normal">286</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit</span><span class="p">(</span>
+<span class="normal">286</span>
+<span class="normal">287</span>
+<span class="normal">288</span>
+<span class="normal">289</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">:</span> <span class="n">LocusToGeneModel</span><span class="p">,</span>
     <span class="n">feature_matrix</span><span class="p">:</span> <span class="n">L2GFeatureMatrix</span><span class="p">,</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">LocusToGeneModel</span><span class="p">:</span>
@@ -4374,7 +4428,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.get_feature_importance" class
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -4383,7 +4437,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.get_feature_importance" class
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td>
                 <code>dict[str, float]</code>
           </td>
@@ -4398,7 +4452,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.get_feature_importance" class
 
 
 
-  <p><strong>Raises:</strong></p>
+  <p><span class="doc-section-title">Raises:</span></p>
   <table>
     <thead>
       <tr>
@@ -4407,7 +4461,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.get_feature_importance" class
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td>
                 <code>ValueError</code>
           </td>
@@ -4422,10 +4476,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.get_feature_importance" class
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/method/l2g/model.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">255</span>
-<span class="normal">256</span>
-<span class="normal">257</span>
-<span class="normal">258</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">258</span>
 <span class="normal">259</span>
 <span class="normal">260</span>
 <span class="normal">261</span>
@@ -4438,7 +4489,10 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.get_feature_importance" class
 <span class="normal">268</span>
 <span class="normal">269</span>
 <span class="normal">270</span>
-<span class="normal">271</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">get_feature_importance</span><span class="p">(</span><span class="bp">self</span><span class="p">:</span> <span class="n">LocusToGeneModel</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">float</span><span class="p">]:</span>
+<span class="normal">271</span>
+<span class="normal">272</span>
+<span class="normal">273</span>
+<span class="normal">274</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">get_feature_importance</span><span class="p">(</span><span class="bp">self</span><span class="p">:</span> <span class="n">LocusToGeneModel</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">float</span><span class="p">]:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Return dictionary with relative importances of every feature in the model. Feature names are encoded and have to be mapped back to their original names.</span>
 
 <span class="sd">    Returns:</span>
@@ -4478,7 +4532,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.get_param_grid" class="doc do
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -4487,7 +4541,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.get_param_grid" class="doc do
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td>
                 <code>list[<span title="typing.Any">Any</span>]</code>
           </td>
@@ -4552,7 +4606,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.load_from_disk" class="doc do
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -4563,7 +4617,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.load_from_disk" class="doc do
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>path</code></td>
           <td>
                 <code>str</code>
@@ -4577,7 +4631,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.load_from_disk" class="doc do
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>features_list</code></td>
           <td>
                 <code>list[str]</code>
@@ -4596,7 +4650,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.load_from_disk" class="doc do
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -4605,7 +4659,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.load_from_disk" class="doc do
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>LocusToGeneModel</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.method.l2g.model.LocusToGeneModel" href="#gentropy.method.l2g.model.LocusToGeneModel">LocusToGeneModel</a></code>
           </td>
@@ -4670,7 +4724,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.log_to_wandb" class="doc doc-
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -4681,7 +4735,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.log_to_wandb" class="doc doc-
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>results</code></td>
           <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
@@ -4695,7 +4749,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.log_to_wandb" class="doc doc-
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>training_data</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix" href="../../../datasets/l2g_feature_matrix/#gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix">L2GFeatureMatrix</a></code>
@@ -4709,7 +4763,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.log_to_wandb" class="doc doc-
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>evaluators</code></td>
           <td>
                 <code>list[<span title="pyspark.ml.evaluation.BinaryClassificationEvaluator">BinaryClassificationEvaluator</span> | <span title="pyspark.ml.evaluation.MulticlassClassificationEvaluator">MulticlassClassificationEvaluator</span>]</code>
@@ -4723,7 +4777,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.log_to_wandb" class="doc doc-
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>wandb_run</code></td>
           <td>
                 <code><span title="wandb.wandb_run.Run">Run</span></code>
@@ -4840,7 +4894,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.predict" class="doc doc-headi
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -4851,7 +4905,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.predict" class="doc doc-headi
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>feature_matrix</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix" href="../../../datasets/l2g_feature_matrix/#gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix">L2GFeatureMatrix</a></code>
@@ -4870,7 +4924,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.predict" class="doc doc-headi
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -4879,7 +4933,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.predict" class="doc doc-headi
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>DataFrame</code></td>          <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
           </td>
@@ -4894,7 +4948,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.predict" class="doc doc-headi
 
 
 
-  <p><strong>Raises:</strong></p>
+  <p><span class="doc-section-title">Raises:</span></p>
   <table>
     <thead>
       <tr>
@@ -4903,7 +4957,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.predict" class="doc doc-headi
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td>
                 <code>ValueError</code>
           </td>
@@ -4918,10 +4972,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.predict" class="doc doc-headi
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/method/l2g/model.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">288</span>
-<span class="normal">289</span>
-<span class="normal">290</span>
-<span class="normal">291</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">291</span>
 <span class="normal">292</span>
 <span class="normal">293</span>
 <span class="normal">294</span>
@@ -4935,7 +4986,10 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.predict" class="doc doc-headi
 <span class="normal">302</span>
 <span class="normal">303</span>
 <span class="normal">304</span>
-<span class="normal">305</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">predict</span><span class="p">(</span>
+<span class="normal">305</span>
+<span class="normal">306</span>
+<span class="normal">307</span>
+<span class="normal">308</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">predict</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">:</span> <span class="n">LocusToGeneModel</span><span class="p">,</span>
     <span class="n">feature_matrix</span><span class="p">:</span> <span class="n">L2GFeatureMatrix</span><span class="p">,</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">DataFrame</span><span class="p">:</span>
@@ -4976,7 +5030,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.save" class="doc doc-heading"
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -4987,7 +5041,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.save" class="doc doc-heading"
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>path</code></td>
           <td>
                 <code>str</code>
@@ -5006,7 +5060,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.save" class="doc doc-heading"
 
 
 
-  <p><strong>Raises:</strong></p>
+  <p><span class="doc-section-title">Raises:</span></p>
   <table>
     <thead>
       <tr>
@@ -5015,7 +5069,7 @@ <h3 id="gentropy.method.l2g.model.LocusToGeneModel.save" class="doc doc-heading"
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td>
                 <code>ValueError</code>
           </td>
diff --git a/python_api/methods/l2g/trainer/index.html b/python_api/methods/l2g/trainer/index.html
index c1e101c57..3adead230 100644
--- a/python_api/methods/l2g/trainer/index.html
+++ b/python_api/methods/l2g/trainer/index.html
@@ -2197,6 +2197,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../susie_inf/" class="md-nav__link">
         
@@ -2230,7 +2272,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" checked>
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" checked>
         
           
           
@@ -2246,14 +2288,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="true">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="true">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3039,9 +3081,7 @@ <h2 id="gentropy.method.l2g.trainer.LocusToGeneTrainer" class="doc doc-heading">
 <span class="normal">107</span>
 <span class="normal">108</span>
 <span class="normal">109</span>
-<span class="normal">110</span>
-<span class="normal">111</span>
-<span class="normal">112</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@dataclass</span>
+<span class="normal">110</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@dataclass</span>
 <span class="k">class</span> <span class="nc">LocusToGeneTrainer</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Modelling of what is the most likely causal gene associated with a given locus.&quot;&quot;&quot;</span>
 
@@ -3051,9 +3091,8 @@ <h2 id="gentropy.method.l2g.trainer.LocusToGeneTrainer" class="doc doc-heading">
     <span class="nd">@classmethod</span>
     <span class="k">def</span> <span class="nf">train</span><span class="p">(</span>
         <span class="bp">cls</span><span class="p">:</span> <span class="nb">type</span><span class="p">[</span><span class="n">LocusToGeneTrainer</span><span class="p">],</span>
-        <span class="n">data</span><span class="p">:</span> <span class="n">L2GFeatureMatrix</span><span class="p">,</span>
+        <span class="n">gold_standard_data</span><span class="p">:</span> <span class="n">L2GFeatureMatrix</span><span class="p">,</span>
         <span class="n">l2g_model</span><span class="p">:</span> <span class="n">LocusToGeneModel</span><span class="p">,</span>
-        <span class="n">features_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span>
         <span class="n">evaluate</span><span class="p">:</span> <span class="nb">bool</span><span class="p">,</span>
         <span class="n">wandb_run_name</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
         <span class="n">model_path</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
@@ -3062,9 +3101,8 @@ <h2 id="gentropy.method.l2g.trainer.LocusToGeneTrainer" class="doc doc-heading">
 <span class="w">        </span><span class="sd">&quot;&quot;&quot;Train the Locus to Gene model.</span>
 
 <span class="sd">        Args:</span>
-<span class="sd">            data (L2GFeatureMatrix): Feature matrix containing the data</span>
+<span class="sd">            gold_standard_data (L2GFeatureMatrix): Feature matrix for the associations in the gold standard</span>
 <span class="sd">            l2g_model (LocusToGeneModel): Model to fit to the data on</span>
-<span class="sd">            features_list (list[str]): List of features to use for the model</span>
 <span class="sd">            evaluate (bool): Whether to evaluate the model on a test set</span>
 <span class="sd">            wandb_run_name (str | None): Descriptive name for the run to be tracked with W&amp;B</span>
 <span class="sd">            model_path (str | None): Path to save the model to</span>
@@ -3073,7 +3111,7 @@ <h2 id="gentropy.method.l2g.trainer.LocusToGeneTrainer" class="doc doc-heading">
 <span class="sd">        Returns:</span>
 <span class="sd">            LocusToGeneModel: Trained model</span>
 <span class="sd">        &quot;&quot;&quot;</span>
-        <span class="n">train</span><span class="p">,</span> <span class="n">test</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">select_features</span><span class="p">(</span><span class="n">features_list</span><span class="p">)</span><span class="o">.</span><span class="n">train_test_split</span><span class="p">(</span><span class="n">fraction</span><span class="o">=</span><span class="mf">0.8</span><span class="p">)</span>
+        <span class="n">train</span><span class="p">,</span> <span class="n">test</span> <span class="o">=</span> <span class="n">gold_standard_data</span><span class="o">.</span><span class="n">train_test_split</span><span class="p">(</span><span class="n">fraction</span><span class="o">=</span><span class="mf">0.8</span><span class="p">)</span>
 
         <span class="n">model</span> <span class="o">=</span> <span class="n">l2g_model</span><span class="o">.</span><span class="n">add_pipeline_stage</span><span class="p">(</span><span class="n">l2g_model</span><span class="o">.</span><span class="n">estimator</span><span class="p">)</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">train</span><span class="p">)</span>
 
@@ -3082,7 +3120,7 @@ <h2 id="gentropy.method.l2g.trainer.LocusToGeneTrainer" class="doc doc-heading">
                 <span class="n">results</span><span class="o">=</span><span class="n">model</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">test</span><span class="p">),</span>
                 <span class="n">hyperparameters</span><span class="o">=</span><span class="n">hyperparams</span><span class="p">,</span>
                 <span class="n">wandb_run_name</span><span class="o">=</span><span class="n">wandb_run_name</span><span class="p">,</span>
-                <span class="n">training_data</span><span class="o">=</span><span class="n">train</span><span class="p">,</span>
+                <span class="n">gold_standard_data</span><span class="o">=</span><span class="n">gold_standard_data</span><span class="p">,</span>
             <span class="p">)</span>
         <span class="k">if</span> <span class="n">model_path</span><span class="p">:</span>
             <span class="n">l2g_model</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="n">model_path</span><span class="p">)</span>
@@ -3177,7 +3215,7 @@ <h3 id="gentropy.method.l2g.trainer.LocusToGeneTrainer.cross_validate" class="do
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3188,7 +3226,7 @@ <h3 id="gentropy.method.l2g.trainer.LocusToGeneTrainer.cross_validate" class="do
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>l2g_model</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.method.l2g.model.LocusToGeneModel" href="../model/#gentropy.method.l2g.model.LocusToGeneModel">LocusToGeneModel</a></code>
@@ -3202,7 +3240,7 @@ <h3 id="gentropy.method.l2g.trainer.LocusToGeneTrainer.cross_validate" class="do
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>data</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix" href="../../../datasets/l2g_feature_matrix/#gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix">L2GFeatureMatrix</a></code>
@@ -3216,7 +3254,7 @@ <h3 id="gentropy.method.l2g.trainer.LocusToGeneTrainer.cross_validate" class="do
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>num_folds</code></td>
           <td>
                 <code>int</code>
@@ -3230,7 +3268,7 @@ <h3 id="gentropy.method.l2g.trainer.LocusToGeneTrainer.cross_validate" class="do
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>param_grid</code></td>
           <td>
                 <code><span title="typing.Optional">Optional</span>[list]</code>
@@ -3249,7 +3287,7 @@ <h3 id="gentropy.method.l2g.trainer.LocusToGeneTrainer.cross_validate" class="do
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3258,7 +3296,7 @@ <h3 id="gentropy.method.l2g.trainer.LocusToGeneTrainer.cross_validate" class="do
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>LocusToGeneModel</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.method.l2g.model.LocusToGeneModel" href="../model/#gentropy.method.l2g.model.LocusToGeneModel">LocusToGeneModel</a></code>
           </td>
@@ -3273,7 +3311,7 @@ <h3 id="gentropy.method.l2g.trainer.LocusToGeneTrainer.cross_validate" class="do
 
 
 
-  <p><strong>Raises:</strong></p>
+  <p><span class="doc-section-title">Raises:</span></p>
   <table>
     <thead>
       <tr>
@@ -3282,7 +3320,7 @@ <h3 id="gentropy.method.l2g.trainer.LocusToGeneTrainer.cross_validate" class="do
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td>
                 <code>ValueError</code>
           </td>
@@ -3292,7 +3330,7 @@ <h3 id="gentropy.method.l2g.trainer.LocusToGeneTrainer.cross_validate" class="do
             </div>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td>
                 <code>ValueError</code>
           </td>
@@ -3307,7 +3345,9 @@ <h3 id="gentropy.method.l2g.trainer.LocusToGeneTrainer.cross_validate" class="do
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/method/l2g/trainer.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 62</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 60</span>
+<span class="normal"> 61</span>
+<span class="normal"> 62</span>
 <span class="normal"> 63</span>
 <span class="normal"> 64</span>
 <span class="normal"> 65</span>
@@ -3355,9 +3395,7 @@ <h3 id="gentropy.method.l2g.trainer.LocusToGeneTrainer.cross_validate" class="do
 <span class="normal">107</span>
 <span class="normal">108</span>
 <span class="normal">109</span>
-<span class="normal">110</span>
-<span class="normal">111</span>
-<span class="normal">112</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@classmethod</span>
+<span class="normal">110</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@classmethod</span>
 <span class="k">def</span> <span class="nf">cross_validate</span><span class="p">(</span>
     <span class="bp">cls</span><span class="p">:</span> <span class="nb">type</span><span class="p">[</span><span class="n">LocusToGeneTrainer</span><span class="p">],</span>
     <span class="n">l2g_model</span><span class="p">:</span> <span class="n">LocusToGeneModel</span><span class="p">,</span>
@@ -3420,7 +3458,7 @@ <h3 id="gentropy.method.l2g.trainer.LocusToGeneTrainer.cross_validate" class="do
 
 
 <h3 id="gentropy.method.l2g.trainer.LocusToGeneTrainer.train" class="doc doc-heading">
-          <code class="highlight language-python"><span class="n">train</span><span class="p">(</span><span class="n">data</span><span class="p">:</span> <span class="n">L2GFeatureMatrix</span><span class="p">,</span> <span class="n">l2g_model</span><span class="p">:</span> <span class="n">LocusToGeneModel</span><span class="p">,</span> <span class="n">features_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="n">evaluate</span><span class="p">:</span> <span class="nb">bool</span><span class="p">,</span> <span class="n">wandb_run_name</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">model_path</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="o">**</span><span class="n">hyperparams</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">LocusToGeneModel</span></code>
+          <code class="highlight language-python"><span class="n">train</span><span class="p">(</span><span class="n">gold_standard_data</span><span class="p">:</span> <span class="n">L2GFeatureMatrix</span><span class="p">,</span> <span class="n">l2g_model</span><span class="p">:</span> <span class="n">LocusToGeneModel</span><span class="p">,</span> <span class="n">evaluate</span><span class="p">:</span> <span class="nb">bool</span><span class="p">,</span> <span class="n">wandb_run_name</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">model_path</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="o">**</span><span class="n">hyperparams</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">LocusToGeneModel</span></code>
   
   <span class="doc doc-labels">
       <small class="doc doc-label doc-label-classmethod"><code>classmethod</code></small>
@@ -3435,7 +3473,7 @@ <h3 id="gentropy.method.l2g.trainer.LocusToGeneTrainer.train" class="doc doc-hea
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3446,21 +3484,21 @@ <h3 id="gentropy.method.l2g.trainer.LocusToGeneTrainer.train" class="doc doc-hea
       </tr>
     </thead>
     <tbody>
-        <tr>
-          <td><code>data</code></td>
+        <tr class="doc-section-item">
+          <td><code>gold_standard_data</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix" href="../../../datasets/l2g_feature_matrix/#gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix">L2GFeatureMatrix</a></code>
           </td>
           <td>
             <div class="doc-md-description">
-              <p>Feature matrix containing the data</p>
+              <p>Feature matrix for the associations in the gold standard</p>
             </div>
           </td>
           <td>
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>l2g_model</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.method.l2g.model.LocusToGeneModel" href="../model/#gentropy.method.l2g.model.LocusToGeneModel">LocusToGeneModel</a></code>
@@ -3474,21 +3512,7 @@ <h3 id="gentropy.method.l2g.trainer.LocusToGeneTrainer.train" class="doc doc-hea
               <em>required</em>
           </td>
         </tr>
-        <tr>
-          <td><code>features_list</code></td>
-          <td>
-                <code>list[str]</code>
-          </td>
-          <td>
-            <div class="doc-md-description">
-              <p>List of features to use for the model</p>
-            </div>
-          </td>
-          <td>
-              <em>required</em>
-          </td>
-        </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>evaluate</code></td>
           <td>
                 <code>bool</code>
@@ -3502,7 +3526,7 @@ <h3 id="gentropy.method.l2g.trainer.LocusToGeneTrainer.train" class="doc doc-hea
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>wandb_run_name</code></td>
           <td>
                 <code>str | None</code>
@@ -3516,7 +3540,7 @@ <h3 id="gentropy.method.l2g.trainer.LocusToGeneTrainer.train" class="doc doc-hea
                 <code>None</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>model_path</code></td>
           <td>
                 <code>str | None</code>
@@ -3530,7 +3554,7 @@ <h3 id="gentropy.method.l2g.trainer.LocusToGeneTrainer.train" class="doc doc-hea
                 <code>None</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>**hyperparams</code></td>
           <td>
                 <code>dict[str, <span title="typing.Any">Any</span>]</code>
@@ -3549,7 +3573,7 @@ <h3 id="gentropy.method.l2g.trainer.LocusToGeneTrainer.train" class="doc doc-hea
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3558,7 +3582,7 @@ <h3 id="gentropy.method.l2g.trainer.LocusToGeneTrainer.train" class="doc doc-hea
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>LocusToGeneModel</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.method.l2g.model.LocusToGeneModel" href="../model/#gentropy.method.l2g.model.LocusToGeneModel">LocusToGeneModel</a></code>
           </td>
@@ -3609,14 +3633,11 @@ <h3 id="gentropy.method.l2g.trainer.LocusToGeneTrainer.train" class="doc doc-hea
 <span class="normal">55</span>
 <span class="normal">56</span>
 <span class="normal">57</span>
-<span class="normal">58</span>
-<span class="normal">59</span>
-<span class="normal">60</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@classmethod</span>
+<span class="normal">58</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@classmethod</span>
 <span class="k">def</span> <span class="nf">train</span><span class="p">(</span>
     <span class="bp">cls</span><span class="p">:</span> <span class="nb">type</span><span class="p">[</span><span class="n">LocusToGeneTrainer</span><span class="p">],</span>
-    <span class="n">data</span><span class="p">:</span> <span class="n">L2GFeatureMatrix</span><span class="p">,</span>
+    <span class="n">gold_standard_data</span><span class="p">:</span> <span class="n">L2GFeatureMatrix</span><span class="p">,</span>
     <span class="n">l2g_model</span><span class="p">:</span> <span class="n">LocusToGeneModel</span><span class="p">,</span>
-    <span class="n">features_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span>
     <span class="n">evaluate</span><span class="p">:</span> <span class="nb">bool</span><span class="p">,</span>
     <span class="n">wandb_run_name</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
     <span class="n">model_path</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
@@ -3625,9 +3646,8 @@ <h3 id="gentropy.method.l2g.trainer.LocusToGeneTrainer.train" class="doc doc-hea
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Train the Locus to Gene model.</span>
 
 <span class="sd">    Args:</span>
-<span class="sd">        data (L2GFeatureMatrix): Feature matrix containing the data</span>
+<span class="sd">        gold_standard_data (L2GFeatureMatrix): Feature matrix for the associations in the gold standard</span>
 <span class="sd">        l2g_model (LocusToGeneModel): Model to fit to the data on</span>
-<span class="sd">        features_list (list[str]): List of features to use for the model</span>
 <span class="sd">        evaluate (bool): Whether to evaluate the model on a test set</span>
 <span class="sd">        wandb_run_name (str | None): Descriptive name for the run to be tracked with W&amp;B</span>
 <span class="sd">        model_path (str | None): Path to save the model to</span>
@@ -3636,7 +3656,7 @@ <h3 id="gentropy.method.l2g.trainer.LocusToGeneTrainer.train" class="doc doc-hea
 <span class="sd">    Returns:</span>
 <span class="sd">        LocusToGeneModel: Trained model</span>
 <span class="sd">    &quot;&quot;&quot;</span>
-    <span class="n">train</span><span class="p">,</span> <span class="n">test</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">select_features</span><span class="p">(</span><span class="n">features_list</span><span class="p">)</span><span class="o">.</span><span class="n">train_test_split</span><span class="p">(</span><span class="n">fraction</span><span class="o">=</span><span class="mf">0.8</span><span class="p">)</span>
+    <span class="n">train</span><span class="p">,</span> <span class="n">test</span> <span class="o">=</span> <span class="n">gold_standard_data</span><span class="o">.</span><span class="n">train_test_split</span><span class="p">(</span><span class="n">fraction</span><span class="o">=</span><span class="mf">0.8</span><span class="p">)</span>
 
     <span class="n">model</span> <span class="o">=</span> <span class="n">l2g_model</span><span class="o">.</span><span class="n">add_pipeline_stage</span><span class="p">(</span><span class="n">l2g_model</span><span class="o">.</span><span class="n">estimator</span><span class="p">)</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">train</span><span class="p">)</span>
 
@@ -3645,7 +3665,7 @@ <h3 id="gentropy.method.l2g.trainer.LocusToGeneTrainer.train" class="doc doc-hea
             <span class="n">results</span><span class="o">=</span><span class="n">model</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">test</span><span class="p">),</span>
             <span class="n">hyperparameters</span><span class="o">=</span><span class="n">hyperparams</span><span class="p">,</span>
             <span class="n">wandb_run_name</span><span class="o">=</span><span class="n">wandb_run_name</span><span class="p">,</span>
-            <span class="n">training_data</span><span class="o">=</span><span class="n">train</span><span class="p">,</span>
+            <span class="n">gold_standard_data</span><span class="o">=</span><span class="n">gold_standard_data</span><span class="p">,</span>
         <span class="p">)</span>
     <span class="k">if</span> <span class="n">model_path</span><span class="p">:</span>
         <span class="n">l2g_model</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="n">model_path</span><span class="p">)</span>
diff --git a/python_api/methods/ld_annotator/index.html b/python_api/methods/ld_annotator/index.html
index 2c2d20e85..e3974cc6a 100644
--- a/python_api/methods/ld_annotator/index.html
+++ b/python_api/methods/ld_annotator/index.html
@@ -2257,6 +2257,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../susie_inf/" class="md-nav__link">
         
@@ -2288,7 +2330,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2304,14 +2346,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3314,7 +3356,7 @@ <h3 id="gentropy.method.ld.LDAnnotator.ld_annotate" class="doc doc-heading">
 </details>
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3325,7 +3367,7 @@ <h3 id="gentropy.method.ld.LDAnnotator.ld_annotate" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>associations</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_locus.StudyLocus" href="../../datasets/study_locus/#gentropy.dataset.study_locus.StudyLocus">StudyLocus</a></code>
@@ -3339,7 +3381,7 @@ <h3 id="gentropy.method.ld.LDAnnotator.ld_annotate" class="doc doc-heading">
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>studies</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_index.StudyIndex" href="../../datasets/study_index/#gentropy.dataset.study_index.StudyIndex">StudyIndex</a></code>
@@ -3353,7 +3395,7 @@ <h3 id="gentropy.method.ld.LDAnnotator.ld_annotate" class="doc doc-heading">
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>ld_index</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.ld_index.LDIndex" href="../../datasets/ld_index/#gentropy.dataset.ld_index.LDIndex">LDIndex</a></code>
@@ -3372,7 +3414,7 @@ <h3 id="gentropy.method.ld.LDAnnotator.ld_annotate" class="doc doc-heading">
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3381,7 +3423,7 @@ <h3 id="gentropy.method.ld.LDAnnotator.ld_annotate" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StudyLocus</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_locus.StudyLocus" href="../../datasets/study_locus/#gentropy.dataset.study_locus.StudyLocus">StudyLocus</a></code>
           </td>
diff --git a/python_api/methods/pics/index.html b/python_api/methods/pics/index.html
index 2ce18c68e..e0dfe1210 100644
--- a/python_api/methods/pics/index.html
+++ b/python_api/methods/pics/index.html
@@ -12,7 +12,7 @@
         <link rel="prev" href="../ld_annotator/">
       
       
-        <link rel="next" href="../susie_inf/">
+        <link rel="next" href="../sumstat_imputation/">
       
       
       <link rel="icon" href="../../../assets/imgs/favicon.png">
@@ -2257,6 +2257,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../susie_inf/" class="md-nav__link">
         
@@ -2288,7 +2330,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2304,14 +2346,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3453,7 +3495,7 @@ <h3 id="gentropy.method.pics.PICS.finemap" class="doc doc-heading">
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3464,7 +3506,7 @@ <h3 id="gentropy.method.pics.PICS.finemap" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>associations</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_locus.StudyLocus" href="../../datasets/study_locus/#gentropy.dataset.study_locus.StudyLocus">StudyLocus</a></code>
@@ -3478,7 +3520,7 @@ <h3 id="gentropy.method.pics.PICS.finemap" class="doc doc-heading">
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>k</code></td>
           <td>
                 <code>float</code>
@@ -3497,7 +3539,7 @@ <h3 id="gentropy.method.pics.PICS.finemap" class="doc doc-heading">
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3506,7 +3548,7 @@ <h3 id="gentropy.method.pics.PICS.finemap" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StudyLocus</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.dataset.study_locus.StudyLocus" href="../../datasets/study_locus/#gentropy.dataset.study_locus.StudyLocus">StudyLocus</a></code>
           </td>
diff --git a/python_api/methods/sumstat_imputation/index.html b/python_api/methods/sumstat_imputation/index.html
new file mode 100644
index 000000000..43bc38737
--- /dev/null
+++ b/python_api/methods/sumstat_imputation/index.html
@@ -0,0 +1,3729 @@
+
+<!doctype html>
+<html lang="en" class="no-js">
+  <head>
+    
+      <meta charset="utf-8">
+      <meta name="viewport" content="width=device-width,initial-scale=1">
+      
+      
+      
+      
+        <link rel="prev" href="../pics/">
+      
+      
+        <link rel="next" href="../sumstat_quality_controls/">
+      
+      
+      <link rel="icon" href="../../../assets/imgs/favicon.png">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.5.3">
+    
+    
+      
+        <title>Summary Statistics Imputation - Open Targets Gentropy</title>
+      
+    
+    
+      <link rel="stylesheet" href="../../../assets/stylesheets/main.50c56a3b.min.css">
+      
+        
+        <link rel="stylesheet" href="../../../assets/stylesheets/palette.06af60db.min.css">
+      
+      
+
+
+    
+    
+      
+    
+    
+      
+        
+        
+        <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+        <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
+        <style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
+      
+    
+    
+      <link rel="stylesheet" href="../../../css/timeago.css">
+    
+      <link rel="stylesheet" href="../../../assets/_mkdocstrings.css">
+    
+      <link rel="stylesheet" href="../../../assets/stylesheets/extra.css">
+    
+    <script>__md_scope=new URL("../../..",location),__md_hash=e=>[...e].reduce((e,_)=>(e<<5)-e+_.charCodeAt(0),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
+    
+      
+
+    
+    
+    
+  </head>
+  
+  
+    
+    
+      
+    
+    
+    
+    
+    <body dir="ltr" data-md-color-scheme="default" data-md-color-primary="blue-grey" data-md-color-accent="light-blue">
+  
+    
+    <input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
+    <input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
+    <label class="md-overlay" for="__drawer"></label>
+    <div data-md-component="skip">
+      
+        
+        <a href="#gentropy.method.sumstat_imputation.SummaryStatisticsImputation" class="md-skip">
+          Skip to content
+        </a>
+      
+    </div>
+    <div data-md-component="announce">
+      
+    </div>
+    
+    
+      
+
+  
+
+<header class="md-header md-header--shadow md-header--lifted" data-md-component="header">
+  <nav class="md-header__inner md-grid" aria-label="Header">
+    <a href="../../.." title="Open Targets Gentropy" class="md-header__button md-logo" aria-label="Open Targets Gentropy" data-md-component="logo">
+      
+  <img src="../../../assets/imgs/otlogo.png" alt="logo">
+
+    </a>
+    <label class="md-header__button md-icon" for="__drawer">
+      
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3V6m0 5h18v2H3v-2m0 5h18v2H3v-2Z"/></svg>
+    </label>
+    <div class="md-header__title" data-md-component="header-title">
+      <div class="md-header__ellipsis">
+        <div class="md-header__topic">
+          <span class="md-ellipsis">
+            Open Targets Gentropy
+          </span>
+        </div>
+        <div class="md-header__topic" data-md-component="header-topic">
+          <span class="md-ellipsis">
+            
+              Summary Statistics Imputation
+            
+          </span>
+        </div>
+      </div>
+    </div>
+    
+      
+        <form class="md-header__option" data-md-component="palette">
+  
+    
+    
+    
+    <input class="md-option" data-md-color-media="(prefers-color-scheme: light)" data-md-color-scheme="default" data-md-color-primary="blue-grey" data-md-color-accent="light-blue"  aria-label="Switch to dark mode"  type="radio" name="__palette" id="__palette_0">
+    
+      <label class="md-header__button md-icon" title="Switch to dark mode" for="__palette_1" hidden>
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M17 6H7c-3.31 0-6 2.69-6 6s2.69 6 6 6h10c3.31 0 6-2.69 6-6s-2.69-6-6-6zm0 10H7c-2.21 0-4-1.79-4-4s1.79-4 4-4h10c2.21 0 4 1.79 4 4s-1.79 4-4 4zM7 9c-1.66 0-3 1.34-3 3s1.34 3 3 3 3-1.34 3-3-1.34-3-3-3z"/></svg>
+      </label>
+    
+  
+    
+    
+    
+    <input class="md-option" data-md-color-media="(prefers-color-scheme: dark)" data-md-color-scheme="slate" data-md-color-primary="blue-grey" data-md-color-accent="light-blue"  aria-label="Switch to light mode"  type="radio" name="__palette" id="__palette_1">
+    
+      <label class="md-header__button md-icon" title="Switch to light mode" for="__palette_0" hidden>
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M17 7H7a5 5 0 0 0-5 5 5 5 0 0 0 5 5h10a5 5 0 0 0 5-5 5 5 0 0 0-5-5m0 8a3 3 0 0 1-3-3 3 3 0 0 1 3-3 3 3 0 0 1 3 3 3 3 0 0 1-3 3Z"/></svg>
+      </label>
+    
+  
+</form>
+      
+    
+    
+      <script>var media,input,key,value,palette=__md_get("__palette");if(palette&&palette.color){"(prefers-color-scheme)"===palette.color.media&&(media=matchMedia("(prefers-color-scheme: light)"),input=document.querySelector(media.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']"),palette.color.media=input.getAttribute("data-md-color-media"),palette.color.scheme=input.getAttribute("data-md-color-scheme"),palette.color.primary=input.getAttribute("data-md-color-primary"),palette.color.accent=input.getAttribute("data-md-color-accent"));for([key,value]of Object.entries(palette.color))document.body.setAttribute("data-md-color-"+key,value)}</script>
+    
+    
+    
+      <label class="md-header__button md-icon" for="__search">
+        
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
+      </label>
+      <div class="md-search" data-md-component="search" role="dialog">
+  <label class="md-search__overlay" for="__search"></label>
+  <div class="md-search__inner" role="search">
+    <form class="md-search__form" name="search">
+      <input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
+      <label class="md-search__icon md-icon" for="__search">
+        
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
+        
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12Z"/></svg>
+      </label>
+      <nav class="md-search__options" aria-label="Search">
+        
+        <button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
+          
+          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12 19 6.41Z"/></svg>
+        </button>
+      </nav>
+      
+        <div class="md-search__suggest" data-md-component="search-suggest"></div>
+      
+    </form>
+    <div class="md-search__output">
+      <div class="md-search__scrollwrap" data-md-scrollfix>
+        <div class="md-search-result" data-md-component="search-result">
+          <div class="md-search-result__meta">
+            Initializing search
+          </div>
+          <ol class="md-search-result__list" role="presentation"></ol>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+    
+    
+      <div class="md-header__source">
+        <a href="https://github.com/opentargets/gentropy" title="Go to repository" class="md-source" data-md-component="source">
+  <div class="md-source__icon md-icon">
+    
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+  </div>
+  <div class="md-source__repository">
+    opentargets/gentropy
+  </div>
+</a>
+      </div>
+    
+  </nav>
+  
+    
+      
+<nav class="md-tabs" aria-label="Tabs" data-md-component="tabs">
+  <div class="md-grid">
+    <ul class="md-tabs__list">
+      
+        
+  
+  
+  
+    <li class="md-tabs__item">
+      <a href="../../.." class="md-tabs__link">
+        
+  
+    
+  
+  Home
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
+    <li class="md-tabs__item">
+      <a href="../../../installation/" class="md-tabs__link">
+        
+  
+    
+  
+  Installation
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
+    <li class="md-tabs__item">
+      <a href="../../../howto/_howto/" class="md-tabs__link">
+        
+  
+    
+  
+  How-to
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
+    <li class="md-tabs__item">
+      <a href="../../../roadmap/" class="md-tabs__link">
+        
+  
+    
+  
+  Roadmap
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
+    <li class="md-tabs__item">
+      <a href="../../../development/_development/" class="md-tabs__link">
+        
+  
+    
+  
+  Development
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+    
+  
+  
+    <li class="md-tabs__item md-tabs__item--active">
+      <a href="../../_python_api/" class="md-tabs__link">
+        
+  
+    
+  
+  Python API
+
+      </a>
+    </li>
+  
+
+      
+    </ul>
+  </div>
+</nav>
+    
+  
+</header>
+    
+    <div class="md-container" data-md-component="container">
+      
+      
+        
+      
+      <main class="md-main" data-md-component="main">
+        <div class="md-main__inner md-grid">
+          
+            
+              
+              <div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
+                <div class="md-sidebar__scrollwrap">
+                  <div class="md-sidebar__inner">
+                    
+
+
+  
+
+
+<nav class="md-nav md-nav--primary md-nav--lifted" aria-label="Navigation" data-md-level="0">
+  <label class="md-nav__title" for="__drawer">
+    <a href="../../.." title="Open Targets Gentropy" class="md-nav__button md-logo" aria-label="Open Targets Gentropy" data-md-component="logo">
+      
+  <img src="../../../assets/imgs/otlogo.png" alt="logo">
+
+    </a>
+    Open Targets Gentropy
+  </label>
+  
+    <div class="md-nav__source">
+      <a href="https://github.com/opentargets/gentropy" title="Go to repository" class="md-source" data-md-component="source">
+  <div class="md-source__icon md-icon">
+    
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+  </div>
+  <div class="md-source__repository">
+    opentargets/gentropy
+  </div>
+</a>
+    </div>
+  
+  <ul class="md-nav__list" data-md-scrollfix>
+    
+      
+      
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../.." class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Home
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../installation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Installation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
+    
+    
+    
+    
+    
+      
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+        
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../../../howto/_howto/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    How-to
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_3" id="__nav_3_label" tabindex="">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3">
+            <span class="md-nav__icon md-icon"></span>
+            How-to
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    
+    
+    
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../../../howto/command_line/_command_line/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    Command line
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_3_1" id="__nav_3_1_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_1_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1">
+            <span class="md-nav__icon md-icon"></span>
+            Command line
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../howto/command_line/run_step_in_cli/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Run step in CLI
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../howto/command_line/run_step_using_config/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Run step using YAML config
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+    
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_2" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../../../howto/python_api/_python_api/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    Python api
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_3_2" id="__nav_3_2_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_2_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_2">
+            <span class="md-nav__icon md-icon"></span>
+            Python api
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../howto/python_api/a_creating_spark_session/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Creating a Spark Session
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../howto/python_api/b_create_dataset/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Create a dataset
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../howto/python_api/c_applying_methods/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Applying methods
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../howto/python_api/d_inspect_dataset/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Inspect a dataset
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../roadmap/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Roadmap
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
+    
+    
+    
+    
+    
+      
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+        
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_5" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../../../development/_development/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    Development
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_5" id="__nav_5_label" tabindex="">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_5">
+            <span class="md-nav__icon md-icon"></span>
+            Development
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../development/airflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Airflow configuration
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../development/contributing/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Contributing guidelines
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../development/troubleshooting/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Troubleshooting
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../development/workflows/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Pipeline workflows
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+    
+      
+      
+  
+  
+    
+  
+  
+  
+    
+    
+    
+    
+    
+      
+      
+    
+    <li class="md-nav__item md-nav__item--active md-nav__item--section md-nav__item--nested">
+      
+        
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6" checked>
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../../_python_api/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    Python API
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_6" id="__nav_6_label" tabindex="">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_6_label" aria-expanded="true">
+          <label class="md-nav__title" for="__nav_6">
+            <span class="md-nav__icon md-icon"></span>
+            Python API
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    
+    
+    
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_1" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../../datasets/_datasets/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    Datasets
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_6_1" id="__nav_6_1_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_6_1_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_1">
+            <span class="md-nav__icon md-icon"></span>
+            Datasets
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasets/colocalisation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Colocalisation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasets/gene_index/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Gene Index
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasets/intervals/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Intervals
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasets/l2g_feature/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    L2G Feature
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasets/l2g_feature_matrix/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    L2G Feature Matrix
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasets/l2g_gold_standard/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    L2G Gold Standard
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasets/l2g_prediction/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    L2G Prediction
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasets/ld_index/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    LD Index
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasets/study_index/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Study Index
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasets/study_locus/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Study Locus
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasets/study_locus_overlap/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Study Locus Overlap
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasets/summary_statistics/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasets/variant_annotation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Variant annotation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasets/variant_index/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Variant index
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasets/variant_to_gene/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Variant-to-gene
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+    
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_2" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../../datasources/_datasources/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    Datasources
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_6_2" id="__nav_6_2_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_6_2_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_2">
+            <span class="md-nav__icon md-icon"></span>
+            Datasources
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    
+    
+    
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_2_1" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../../datasources/eqtl_catalogue/_eqtl_catalogue/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    Eqtl catalogue
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_6_2_1" id="__nav_6_2_1_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_2_1_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_2_1">
+            <span class="md-nav__icon md-icon"></span>
+            Eqtl catalogue
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasources/eqtl_catalogue/finemapping/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Fine mapping results
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasources/eqtl_catalogue/study_index/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Study Index
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasources/eqtl_catalogue/summary_stats/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Stats
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+    
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_2_2" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../../datasources/finngen/_finngen/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    Finngen
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_6_2_2" id="__nav_6_2_2_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_2_2_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_2_2">
+            <span class="md-nav__icon md-icon"></span>
+            Finngen
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasources/finngen/finemapping/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Finemapping
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasources/finngen/study_index/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Study Index
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasources/finngen/summary_stats/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Study Index
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+    
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_2_3" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../../datasources/gnomad/_gnomad/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    Gnomad
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_6_2_3" id="__nav_6_2_3_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_2_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_2_3">
+            <span class="md-nav__icon md-icon"></span>
+            Gnomad
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasources/gnomad/gnomad_ld/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    LD Matrix
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasources/gnomad/gnomad_variants/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Variants
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+    
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_2_4" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../../datasources/gwas_catalog/_gwas_catalog/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    Gwas catalog
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_6_2_4" id="__nav_6_2_4_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_2_4_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_2_4">
+            <span class="md-nav__icon md-icon"></span>
+            Gwas catalog
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasources/gwas_catalog/associations/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Associations
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasources/gwas_catalog/study_index/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Study Index
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasources/gwas_catalog/study_splitter/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Study Splitter
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasources/gwas_catalog/summary_statistics/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+    
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_2_5" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../../datasources/intervals/_intervals/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    Intervals
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_6_2_5" id="__nav_6_2_5_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_2_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_2_5">
+            <span class="md-nav__icon md-icon"></span>
+            Intervals
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasources/intervals/andersson/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Andersson et al.
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasources/intervals/javierre/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Javierre et al.
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasources/intervals/jung/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Jung et al.
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasources/intervals/thurman/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Thurman et al.
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+    
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_2_6" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../../datasources/open_targets/_open_targets/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    Open targets
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_6_2_6" id="__nav_6_2_6_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_2_6_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_2_6">
+            <span class="md-nav__icon md-icon"></span>
+            Open targets
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasources/open_targets/l2g_gold_standard/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    L2G Gold Standard
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasources/open_targets/target/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Target
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+    
+  
+  
+  
+    
+    
+    
+    
+    
+    <li class="md-nav__item md-nav__item--active md-nav__item--nested">
+      
+        
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3" checked>
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../_methods/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    Methods
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_6_3" id="__nav_6_3_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_6_3_label" aria-expanded="true">
+          <label class="md-nav__title" for="__nav_6_3">
+            <span class="md-nav__icon md-icon"></span>
+            Methods
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../carma/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    CARMA
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../clumping/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Clumping
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../coloc/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Coloc
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../ecaviar/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    eCAVIAR
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../ld_annotator/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    LDAnnotator
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../pics/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    PICS
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+    
+  
+  
+  
+    <li class="md-nav__item md-nav__item--active">
+      
+      <input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
+      
+      
+      
+        <label class="md-nav__link md-nav__link--active" for="__toc">
+          
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+          <span class="md-nav__icon md-icon"></span>
+        </label>
+      
+      <a href="./" class="md-nav__link md-nav__link--active">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+      
+        
+
+<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
+  
+  
+  
+  
+    <label class="md-nav__title" for="__toc">
+      <span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
+      
+        <li class="md-nav__item">
+  <a href="#gentropy.method.sumstat_imputation.SummaryStatisticsImputation" class="md-nav__link">
+    <span class="md-ellipsis">
+      SummaryStatisticsImputation
+    </span>
+  </a>
+  
+    <nav class="md-nav" aria-label="SummaryStatisticsImputation">
+      <ul class="md-nav__list">
+        
+          <li class="md-nav__item">
+  <a href="#gentropy.method.sumstat_imputation.SummaryStatisticsImputation.raiss_model" class="md-nav__link">
+    <span class="md-ellipsis">
+      raiss_model
+    </span>
+  </a>
+  
+</li>
+        
+      </ul>
+    </nav>
+  
+</li>
+      
+    </ul>
+  
+</nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../susie_inf/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    SuSiE-inf
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+    
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../l2g/_l2g/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    L2g
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
+            <span class="md-nav__icon md-icon"></span>
+            L2g
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../l2g/evaluator/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    W&B evaluator
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../l2g/feature_factory/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    L2G Feature Factory
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../l2g/model/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    L2G Model
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../l2g/trainer/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    L2G Trainer
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+    
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_4" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../../steps/_steps/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    Steps
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_6_4" id="__nav_6_4_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_6_4_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_4">
+            <span class="md-nav__icon md-icon"></span>
+            Steps
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../steps/colocalisation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    colocalisation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../steps/eqtl_catalogue/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    eQTL Catalogue
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../steps/finngen_studies/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    finngen_studies
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../steps/finngen_sumstat_preprocess/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    finngen_sumstat_preprocess
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../steps/gene_index/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    gene_index
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../steps/gwas_catalog_curation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    gwas_catalog_study_curation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../steps/gwas_catalog_inclusion/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    gwas_catalog_study_inclusion
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../steps/gwas_catalog_ingestion/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    gwas_catalog_ingestion
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../steps/gwas_catalog_sumstat_preprocess/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    gwas_catalog_sumstat_preprocess
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../steps/l2g/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    locus_to_gene
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../steps/ld_clump/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    ld_based_clumping
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../steps/ld_index/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    ld_index
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../steps/pics/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    pics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../steps/variant_annotation_step/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    variant_annotation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../steps/variant_index_step/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    variant_index
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../steps/variant_to_gene_step/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    variant_to_gene
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../steps/window_based_clumping/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    window_based_clumping
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+    
+  </ul>
+</nav>
+                  </div>
+                </div>
+              </div>
+            
+            
+              
+              <div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
+                <div class="md-sidebar__scrollwrap">
+                  <div class="md-sidebar__inner">
+                    
+
+<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
+  
+  
+  
+  
+    <label class="md-nav__title" for="__toc">
+      <span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
+      
+        <li class="md-nav__item">
+  <a href="#gentropy.method.sumstat_imputation.SummaryStatisticsImputation" class="md-nav__link">
+    <span class="md-ellipsis">
+      SummaryStatisticsImputation
+    </span>
+  </a>
+  
+    <nav class="md-nav" aria-label="SummaryStatisticsImputation">
+      <ul class="md-nav__list">
+        
+          <li class="md-nav__item">
+  <a href="#gentropy.method.sumstat_imputation.SummaryStatisticsImputation.raiss_model" class="md-nav__link">
+    <span class="md-ellipsis">
+      raiss_model
+    </span>
+  </a>
+  
+</li>
+        
+      </ul>
+    </nav>
+  
+</li>
+      
+    </ul>
+  
+</nav>
+                  </div>
+                </div>
+              </div>
+            
+          
+          
+            <div class="md-content" data-md-component="content">
+              <article class="md-content__inner md-typeset">
+                
+                  
+
+  
+  
+
+
+  <h1>Summary Statistics Imputation</h1>
+
+<p>Summary statistics imputation leverages linkage disequilibrium (LD) information to compute Z-scores of missing SNPs from neighbouring observed SNPs
+SNPs by taking advantage of the Linkage Disequilibrium.</p>
+<p>We implemented the basic model from RAISS (Robust and Accurate Imputation from Summary Statistics) package (see the original <a href="https://academic.oup.com/bioinformatics/article/35/22/4837/5512360">paper</a>).</p>
+<p>The full repository for the RAISS package can be found <a href="https://gitlab.pasteur.fr/statistical-genetics/raiss">here</a>.</p>
+<p>The original model was suggested in 2014 by Bogdan Pasaniuc et al. <a href="https://pubmed.ncbi.nlm.nih.gov/24990607/">here</a>.</p>
+<p>It represents the following formula:</p>
+<p>E(z<em>i|z_t) = M</em>{i,t} \cdot (M_{t,t})^{-1} \cdot z_t</p>
+<p>Where:</p>
+<ul>
+<li>
+<p>E(z_i|z_t) represents the expected z-score of SNP 'i' given the observed z-scores at known SNP indexes 't'.</p>
+</li>
+<li>
+<p>M_{i,t} represents the LD (Linkage Disequilibrium) matrix between SNP 'i' and the known SNPs at indexes 't'.</p>
+</li>
+<li>
+<p>(M_{t,t})^{-1} represents the inverse of the LD matrix of the known SNPs at indexes 't'.</p>
+</li>
+<li>
+<p>z_t represents the vector of observed z-scores at the known SNP indexes 't'.</p>
+</li>
+</ul>
+
+
+<div class="doc doc-object doc-class">
+
+
+
+<h2 id="gentropy.method.sumstat_imputation.SummaryStatisticsImputation" class="doc doc-heading">
+          <code>gentropy.method.sumstat_imputation.SummaryStatisticsImputation</code>
+
+
+<a href="#gentropy.method.sumstat_imputation.SummaryStatisticsImputation" class="headerlink" title="Permanent link">&para;</a></h2>
+
+
+  <div class="doc doc-contents first">
+
+  
+      <p>Implementation of RAISS summary statstics imputation model.</p>
+
+            <details class="quote">
+              <summary>Source code in <code>src/gentropy/method/sumstat_imputation.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 11</span>
+<span class="normal"> 12</span>
+<span class="normal"> 13</span>
+<span class="normal"> 14</span>
+<span class="normal"> 15</span>
+<span class="normal"> 16</span>
+<span class="normal"> 17</span>
+<span class="normal"> 18</span>
+<span class="normal"> 19</span>
+<span class="normal"> 20</span>
+<span class="normal"> 21</span>
+<span class="normal"> 22</span>
+<span class="normal"> 23</span>
+<span class="normal"> 24</span>
+<span class="normal"> 25</span>
+<span class="normal"> 26</span>
+<span class="normal"> 27</span>
+<span class="normal"> 28</span>
+<span class="normal"> 29</span>
+<span class="normal"> 30</span>
+<span class="normal"> 31</span>
+<span class="normal"> 32</span>
+<span class="normal"> 33</span>
+<span class="normal"> 34</span>
+<span class="normal"> 35</span>
+<span class="normal"> 36</span>
+<span class="normal"> 37</span>
+<span class="normal"> 38</span>
+<span class="normal"> 39</span>
+<span class="normal"> 40</span>
+<span class="normal"> 41</span>
+<span class="normal"> 42</span>
+<span class="normal"> 43</span>
+<span class="normal"> 44</span>
+<span class="normal"> 45</span>
+<span class="normal"> 46</span>
+<span class="normal"> 47</span>
+<span class="normal"> 48</span>
+<span class="normal"> 49</span>
+<span class="normal"> 50</span>
+<span class="normal"> 51</span>
+<span class="normal"> 52</span>
+<span class="normal"> 53</span>
+<span class="normal"> 54</span>
+<span class="normal"> 55</span>
+<span class="normal"> 56</span>
+<span class="normal"> 57</span>
+<span class="normal"> 58</span>
+<span class="normal"> 59</span>
+<span class="normal"> 60</span>
+<span class="normal"> 61</span>
+<span class="normal"> 62</span>
+<span class="normal"> 63</span>
+<span class="normal"> 64</span>
+<span class="normal"> 65</span>
+<span class="normal"> 66</span>
+<span class="normal"> 67</span>
+<span class="normal"> 68</span>
+<span class="normal"> 69</span>
+<span class="normal"> 70</span>
+<span class="normal"> 71</span>
+<span class="normal"> 72</span>
+<span class="normal"> 73</span>
+<span class="normal"> 74</span>
+<span class="normal"> 75</span>
+<span class="normal"> 76</span>
+<span class="normal"> 77</span>
+<span class="normal"> 78</span>
+<span class="normal"> 79</span>
+<span class="normal"> 80</span>
+<span class="normal"> 81</span>
+<span class="normal"> 82</span>
+<span class="normal"> 83</span>
+<span class="normal"> 84</span>
+<span class="normal"> 85</span>
+<span class="normal"> 86</span>
+<span class="normal"> 87</span>
+<span class="normal"> 88</span>
+<span class="normal"> 89</span>
+<span class="normal"> 90</span>
+<span class="normal"> 91</span>
+<span class="normal"> 92</span>
+<span class="normal"> 93</span>
+<span class="normal"> 94</span>
+<span class="normal"> 95</span>
+<span class="normal"> 96</span>
+<span class="normal"> 97</span>
+<span class="normal"> 98</span>
+<span class="normal"> 99</span>
+<span class="normal">100</span>
+<span class="normal">101</span>
+<span class="normal">102</span>
+<span class="normal">103</span>
+<span class="normal">104</span>
+<span class="normal">105</span>
+<span class="normal">106</span>
+<span class="normal">107</span>
+<span class="normal">108</span>
+<span class="normal">109</span>
+<span class="normal">110</span>
+<span class="normal">111</span>
+<span class="normal">112</span>
+<span class="normal">113</span>
+<span class="normal">114</span>
+<span class="normal">115</span>
+<span class="normal">116</span>
+<span class="normal">117</span>
+<span class="normal">118</span>
+<span class="normal">119</span>
+<span class="normal">120</span>
+<span class="normal">121</span>
+<span class="normal">122</span>
+<span class="normal">123</span>
+<span class="normal">124</span>
+<span class="normal">125</span>
+<span class="normal">126</span>
+<span class="normal">127</span>
+<span class="normal">128</span>
+<span class="normal">129</span>
+<span class="normal">130</span>
+<span class="normal">131</span>
+<span class="normal">132</span>
+<span class="normal">133</span>
+<span class="normal">134</span>
+<span class="normal">135</span>
+<span class="normal">136</span>
+<span class="normal">137</span>
+<span class="normal">138</span>
+<span class="normal">139</span>
+<span class="normal">140</span>
+<span class="normal">141</span>
+<span class="normal">142</span>
+<span class="normal">143</span>
+<span class="normal">144</span>
+<span class="normal">145</span>
+<span class="normal">146</span>
+<span class="normal">147</span>
+<span class="normal">148</span>
+<span class="normal">149</span>
+<span class="normal">150</span>
+<span class="normal">151</span>
+<span class="normal">152</span>
+<span class="normal">153</span>
+<span class="normal">154</span>
+<span class="normal">155</span>
+<span class="normal">156</span>
+<span class="normal">157</span>
+<span class="normal">158</span>
+<span class="normal">159</span>
+<span class="normal">160</span>
+<span class="normal">161</span>
+<span class="normal">162</span>
+<span class="normal">163</span>
+<span class="normal">164</span>
+<span class="normal">165</span>
+<span class="normal">166</span>
+<span class="normal">167</span>
+<span class="normal">168</span>
+<span class="normal">169</span>
+<span class="normal">170</span>
+<span class="normal">171</span>
+<span class="normal">172</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">SummaryStatisticsImputation</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;Implementation of RAISS summary statstics imputation model.&quot;&quot;&quot;</span>
+
+    <span class="nd">@staticmethod</span>
+    <span class="k">def</span> <span class="nf">raiss_model</span><span class="p">(</span>
+        <span class="n">z_scores_known</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span>
+        <span class="n">ld_matrix_known</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span>
+        <span class="n">ld_matrix_known_missing</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span>
+        <span class="n">lamb</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.01</span><span class="p">,</span>
+        <span class="n">rtol</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.01</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Compute the imputation of the z-score using the RAISS model.</span>
+
+<span class="sd">        Args:</span>
+<span class="sd">            z_scores_known (np.ndarray): the vector of known Z scores</span>
+<span class="sd">            ld_matrix_known (np.ndarray) : the matrix of known LD correlations</span>
+<span class="sd">            ld_matrix_known_missing (np.ndarray): LD matrix of known SNPs with other unknown SNPs in large matrix (similar to ld[unknowns, :][:,known])</span>
+<span class="sd">            lamb (float): size of the small value added to the diagonal of the covariance matrix before inversion. Defaults to 0.01.</span>
+<span class="sd">            rtol (float): threshold to filter eigenvectos by its eigenvalue. It makes an inversion biased but much more numerically robust. Default to 0.01.</span>
+
+<span class="sd">        Returns:</span>
+<span class="sd">            dict[str, Any]:</span>
+<span class="sd">                - var (np.ndarray): variance of the imputed SNPs</span>
+<span class="sd">                - mu (np.ndarray): the estimation of the zscore of the imputed SNPs</span>
+<span class="sd">                - ld_score (np.ndarray): the linkage disequilibrium score of the imputed SNPs</span>
+<span class="sd">                - condition_number (np.ndarray): the condition number of the correlation matrix</span>
+<span class="sd">                - correct_inversion (np.ndarray): a boolean array indicating if the inversion was successful</span>
+<span class="sd">                - imputation_r2 (np.ndarray): the R2 of the imputation</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">sig_t_inv</span> <span class="o">=</span> <span class="n">SummaryStatisticsImputation</span><span class="o">.</span><span class="n">_invert_sig_t</span><span class="p">(</span>
+            <span class="n">ld_matrix_known</span><span class="p">,</span> <span class="n">lamb</span><span class="p">,</span> <span class="n">rtol</span>
+        <span class="p">)</span>
+        <span class="k">if</span> <span class="n">sig_t_inv</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">return</span> <span class="p">{</span>
+                <span class="s2">&quot;var&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
+                <span class="s2">&quot;mu&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
+                <span class="s2">&quot;ld_score&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
+                <span class="s2">&quot;condition_number&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
+                <span class="s2">&quot;correct_inversion&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
+                <span class="s2">&quot;imputation_r2&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
+            <span class="p">}</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">condition_number</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span>
+                <span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">cond</span><span class="p">(</span><span class="n">ld_matrix_known</span><span class="p">)]</span> <span class="o">*</span> <span class="n">ld_matrix_known_missing</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
+            <span class="p">)</span>
+            <span class="n">correct_inversion</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span>
+                <span class="p">[</span>
+                    <span class="n">SummaryStatisticsImputation</span><span class="o">.</span><span class="n">_check_inversion</span><span class="p">(</span>
+                        <span class="n">ld_matrix_known</span><span class="p">,</span> <span class="n">sig_t_inv</span>
+                    <span class="p">)</span>
+                <span class="p">]</span>
+                <span class="o">*</span> <span class="n">ld_matrix_known_missing</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
+            <span class="p">)</span>
+
+            <span class="n">var</span><span class="p">,</span> <span class="n">ld_score</span> <span class="o">=</span> <span class="n">SummaryStatisticsImputation</span><span class="o">.</span><span class="n">_compute_var</span><span class="p">(</span>
+                <span class="n">ld_matrix_known_missing</span><span class="p">,</span> <span class="n">sig_t_inv</span><span class="p">,</span> <span class="n">lamb</span>
+            <span class="p">)</span>
+
+            <span class="n">mu</span> <span class="o">=</span> <span class="n">SummaryStatisticsImputation</span><span class="o">.</span><span class="n">_compute_mu</span><span class="p">(</span>
+                <span class="n">ld_matrix_known_missing</span><span class="p">,</span> <span class="n">sig_t_inv</span><span class="p">,</span> <span class="n">z_scores_known</span>
+            <span class="p">)</span>
+            <span class="n">var_norm</span> <span class="o">=</span> <span class="n">SummaryStatisticsImputation</span><span class="o">.</span><span class="n">_var_in_boundaries</span><span class="p">(</span><span class="n">var</span><span class="p">,</span> <span class="n">lamb</span><span class="p">)</span>
+
+            <span class="n">R2</span> <span class="o">=</span> <span class="p">(</span><span class="mi">1</span> <span class="o">+</span> <span class="n">lamb</span><span class="p">)</span> <span class="o">-</span> <span class="n">var_norm</span>
+
+            <span class="n">mu</span> <span class="o">=</span> <span class="n">mu</span> <span class="o">/</span> <span class="n">np</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">R2</span><span class="p">)</span>
+            <span class="k">return</span> <span class="p">{</span>
+                <span class="s2">&quot;var&quot;</span><span class="p">:</span> <span class="n">var</span><span class="p">,</span>
+                <span class="s2">&quot;mu&quot;</span><span class="p">:</span> <span class="n">mu</span><span class="p">,</span>
+                <span class="s2">&quot;ld_score&quot;</span><span class="p">:</span> <span class="n">ld_score</span><span class="p">,</span>
+                <span class="s2">&quot;condition_number&quot;</span><span class="p">:</span> <span class="n">condition_number</span><span class="p">,</span>
+                <span class="s2">&quot;correct_inversion&quot;</span><span class="p">:</span> <span class="n">correct_inversion</span><span class="p">,</span>
+                <span class="s2">&quot;imputation_r2&quot;</span><span class="p">:</span> <span class="mi">1</span> <span class="o">-</span> <span class="n">var</span><span class="p">,</span>
+            <span class="p">}</span>
+
+    <span class="nd">@staticmethod</span>
+    <span class="k">def</span> <span class="nf">_compute_mu</span><span class="p">(</span>
+        <span class="n">sig_i_t</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">sig_t_inv</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">zt</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Compute the estimation of z-score from neighborring snp.</span>
+
+<span class="sd">        Args:</span>
+<span class="sd">            sig_i_t (np.ndarray) : correlation matrix with line corresponding to unknown Snp (snp to impute) and column to known SNPs</span>
+<span class="sd">            sig_t_inv (np.ndarray): inverse of the correlation matrix of known matrix</span>
+<span class="sd">            zt (np.ndarray): Zscores of known snp</span>
+<span class="sd">        Returns:</span>
+<span class="sd">            np.ndarray: a vector of length i containing the estimate of zscore</span>
+
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">dot</span><span class="p">(</span><span class="n">sig_i_t</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">dot</span><span class="p">(</span><span class="n">sig_t_inv</span><span class="p">,</span> <span class="n">zt</span><span class="p">))</span>
+
+    <span class="nd">@staticmethod</span>
+    <span class="k">def</span> <span class="nf">_compute_var</span><span class="p">(</span>
+        <span class="n">sig_i_t</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">sig_t_inv</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">lamb</span><span class="p">:</span> <span class="nb">float</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">tuple</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Compute the expected variance of the imputed SNPs.</span>
+
+<span class="sd">        Args:</span>
+<span class="sd">            sig_i_t (np.ndarray) : correlation matrix with line corresponding to unknown Snp (snp to impute) and column to known SNPs</span>
+<span class="sd">            sig_t_inv (np.ndarray): inverse of the correlation matrix of known matrix</span>
+<span class="sd">            lamb (float): regularization term added to matrix</span>
+
+<span class="sd">        Returns:</span>
+<span class="sd">            tuple[np.ndarray, np.ndarray]: a tuple containing the variance and the ld score</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">var</span> <span class="o">=</span> <span class="p">(</span><span class="mi">1</span> <span class="o">+</span> <span class="n">lamb</span><span class="p">)</span> <span class="o">-</span> <span class="n">np</span><span class="o">.</span><span class="n">einsum</span><span class="p">(</span>
+            <span class="s2">&quot;ij,jk,ki-&gt;i&quot;</span><span class="p">,</span> <span class="n">sig_i_t</span><span class="p">,</span> <span class="n">sig_t_inv</span><span class="p">,</span> <span class="n">sig_i_t</span><span class="o">.</span><span class="n">transpose</span><span class="p">()</span>
+        <span class="p">)</span>
+        <span class="n">ld_score</span> <span class="o">=</span> <span class="p">(</span><span class="n">sig_i_t</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">var</span><span class="p">,</span> <span class="n">ld_score</span>
+
+    <span class="nd">@staticmethod</span>
+    <span class="k">def</span> <span class="nf">_check_inversion</span><span class="p">(</span><span class="n">sig_t</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">sig_t_inv</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Check if the inversion is correct.</span>
+
+<span class="sd">        Args:</span>
+<span class="sd">            sig_t (np.ndarray): the correlation matrix</span>
+<span class="sd">            sig_t_inv (np.ndarray): the inverse of the correlation matrix</span>
+<span class="sd">        Returns:</span>
+<span class="sd">            bool: True if the inversion is correct, False otherwise</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">allclose</span><span class="p">(</span><span class="n">sig_t</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">dot</span><span class="p">(</span><span class="n">sig_t</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">dot</span><span class="p">(</span><span class="n">sig_t_inv</span><span class="p">,</span> <span class="n">sig_t</span><span class="p">)))</span>
+
+    <span class="nd">@staticmethod</span>
+    <span class="k">def</span> <span class="nf">_var_in_boundaries</span><span class="p">(</span><span class="n">var</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">lamb</span><span class="p">:</span> <span class="nb">float</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Forces the variance to be in the 0 to 1+lambda boundary. Theoritically we shouldn&#39;t have to do that.</span>
+
+<span class="sd">        Args:</span>
+<span class="sd">            var (np.ndarray): the variance of the imputed SNPs</span>
+<span class="sd">            lamb (float): regularization term added to the diagonal of the sig_t matrix</span>
+
+<span class="sd">        Returns:</span>
+<span class="sd">            np.ndarray: the variance of the imputed SNPs</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">id_neg</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">where</span><span class="p">(</span><span class="n">var</span> <span class="o">&lt;</span> <span class="mi">0</span><span class="p">)</span>
+        <span class="n">var</span><span class="p">[</span><span class="n">id_neg</span><span class="p">]</span> <span class="o">=</span> <span class="mi">0</span>
+        <span class="n">id_inf</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">where</span><span class="p">(</span><span class="n">var</span> <span class="o">&gt;</span> <span class="p">(</span><span class="mf">0.99999</span> <span class="o">+</span> <span class="n">lamb</span><span class="p">))</span>
+        <span class="n">var</span><span class="p">[</span><span class="n">id_inf</span><span class="p">]</span> <span class="o">=</span> <span class="mi">1</span>
+
+        <span class="k">return</span> <span class="n">var</span>
+
+    <span class="nd">@staticmethod</span>
+    <span class="k">def</span> <span class="nf">_invert_sig_t</span><span class="p">(</span><span class="n">sig_t</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">lamb</span><span class="p">:</span> <span class="nb">float</span><span class="p">,</span> <span class="n">rtol</span><span class="p">:</span> <span class="nb">float</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Invert the correlation matrix. If the provided regularization values are not enough to stabilize the inversion process for the given matrix, the function calls itself recursively, increasing lamb and rtol by 10%.</span>
+
+<span class="sd">        Args:</span>
+<span class="sd">            sig_t (np.ndarray): the correlation matrix</span>
+<span class="sd">            lamb (float): regularization term added to the diagonal of the sig_t matrix</span>
+<span class="sd">            rtol (float): threshold to filter eigenvector with a eigenvalue under rtol make inversion biased but much more numerically robust</span>
+
+<span class="sd">        Returns:</span>
+<span class="sd">            np.ndarray: the inverse of the correlation matrix</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">np</span><span class="o">.</span><span class="n">fill_diagonal</span><span class="p">(</span><span class="n">sig_t</span><span class="p">,</span> <span class="p">(</span><span class="mi">1</span> <span class="o">+</span> <span class="n">lamb</span><span class="p">))</span>
+            <span class="n">sig_t_inv</span> <span class="o">=</span> <span class="n">scipy</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">pinv</span><span class="p">(</span><span class="n">sig_t</span><span class="p">,</span> <span class="n">rtol</span><span class="o">=</span><span class="n">rtol</span><span class="p">,</span> <span class="n">atol</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
+            <span class="k">return</span> <span class="n">sig_t_inv</span>
+        <span class="k">except</span> <span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">LinAlgError</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">SummaryStatisticsImputation</span><span class="o">.</span><span class="n">_invert_sig_t</span><span class="p">(</span>
+                <span class="n">sig_t</span><span class="p">,</span> <span class="n">lamb</span> <span class="o">*</span> <span class="mf">1.1</span><span class="p">,</span> <span class="n">rtol</span> <span class="o">*</span> <span class="mf">1.1</span>
+            <span class="p">)</span>
+</code></pre></div></td></tr></table></div>
+            </details>
+
+  
+
+  <div class="doc doc-children">
+
+
+
+
+
+
+
+
+
+
+<div class="doc doc-object doc-function">
+
+
+
+<h3 id="gentropy.method.sumstat_imputation.SummaryStatisticsImputation.raiss_model" class="doc doc-heading">
+          <code class="highlight language-python"><span class="n">raiss_model</span><span class="p">(</span><span class="n">z_scores_known</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">ld_matrix_known</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">ld_matrix_known_missing</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">lamb</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.01</span><span class="p">,</span> <span class="n">rtol</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.01</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span></code>
+  
+  <span class="doc doc-labels">
+      <small class="doc doc-label doc-label-staticmethod"><code>staticmethod</code></small>
+  </span>
+
+<a href="#gentropy.method.sumstat_imputation.SummaryStatisticsImputation.raiss_model" class="headerlink" title="Permanent link">&para;</a></h3>
+
+
+  <div class="doc doc-contents ">
+  
+      <p>Compute the imputation of the z-score using the RAISS model.</p>
+
+
+
+  <p><span class="doc-section-title">Parameters:</span></p>
+  <table>
+    <thead>
+      <tr>
+        <th>Name</th>
+        <th>Type</th>
+        <th>Description</th>
+        <th>Default</th>
+      </tr>
+    </thead>
+    <tbody>
+        <tr class="doc-section-item">
+          <td><code>z_scores_known</code></td>
+          <td>
+                <code><span title="numpy.ndarray">ndarray</span></code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>the vector of known Z scores</p>
+            </div>
+          </td>
+          <td>
+              <em>required</em>
+          </td>
+        </tr>
+        <tr class="doc-section-item">
+          <td><code>ld_matrix_known</code></td>
+          <td>
+                <code>np.ndarray) </code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>the matrix of known LD correlations</p>
+            </div>
+          </td>
+          <td>
+              <em>required</em>
+          </td>
+        </tr>
+        <tr class="doc-section-item">
+          <td><code>ld_matrix_known_missing</code></td>
+          <td>
+                <code><span title="numpy.ndarray">ndarray</span></code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>LD matrix of known SNPs with other unknown SNPs in large matrix (similar to ld[unknowns, :][:,known])</p>
+            </div>
+          </td>
+          <td>
+              <em>required</em>
+          </td>
+        </tr>
+        <tr class="doc-section-item">
+          <td><code>lamb</code></td>
+          <td>
+                <code>float</code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>size of the small value added to the diagonal of the covariance matrix before inversion. Defaults to 0.01.</p>
+            </div>
+          </td>
+          <td>
+                <code>0.01</code>
+          </td>
+        </tr>
+        <tr class="doc-section-item">
+          <td><code>rtol</code></td>
+          <td>
+                <code>float</code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>threshold to filter eigenvectos by its eigenvalue. It makes an inversion biased but much more numerically robust. Default to 0.01.</p>
+            </div>
+          </td>
+          <td>
+                <code>0.01</code>
+          </td>
+        </tr>
+    </tbody>
+  </table>
+
+
+
+  <p><span class="doc-section-title">Returns:</span></p>
+  <table>
+    <thead>
+      <tr>
+        <th>Type</th>
+        <th>Description</th>
+      </tr>
+    </thead>
+    <tbody>
+        <tr class="doc-section-item">
+          <td>
+                <code>dict[str, <span title="typing.Any">Any</span>]</code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>dict[str, Any]:
+- var (np.ndarray): variance of the imputed SNPs
+- mu (np.ndarray): the estimation of the zscore of the imputed SNPs
+- ld_score (np.ndarray): the linkage disequilibrium score of the imputed SNPs
+- condition_number (np.ndarray): the condition number of the correlation matrix
+- correct_inversion (np.ndarray): a boolean array indicating if the inversion was successful
+- imputation_r2 (np.ndarray): the R2 of the imputation</p>
+            </div>
+          </td>
+        </tr>
+    </tbody>
+  </table>
+
+          <details class="quote">
+            <summary>Source code in <code>src/gentropy/method/sumstat_imputation.py</code></summary>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">14</span>
+<span class="normal">15</span>
+<span class="normal">16</span>
+<span class="normal">17</span>
+<span class="normal">18</span>
+<span class="normal">19</span>
+<span class="normal">20</span>
+<span class="normal">21</span>
+<span class="normal">22</span>
+<span class="normal">23</span>
+<span class="normal">24</span>
+<span class="normal">25</span>
+<span class="normal">26</span>
+<span class="normal">27</span>
+<span class="normal">28</span>
+<span class="normal">29</span>
+<span class="normal">30</span>
+<span class="normal">31</span>
+<span class="normal">32</span>
+<span class="normal">33</span>
+<span class="normal">34</span>
+<span class="normal">35</span>
+<span class="normal">36</span>
+<span class="normal">37</span>
+<span class="normal">38</span>
+<span class="normal">39</span>
+<span class="normal">40</span>
+<span class="normal">41</span>
+<span class="normal">42</span>
+<span class="normal">43</span>
+<span class="normal">44</span>
+<span class="normal">45</span>
+<span class="normal">46</span>
+<span class="normal">47</span>
+<span class="normal">48</span>
+<span class="normal">49</span>
+<span class="normal">50</span>
+<span class="normal">51</span>
+<span class="normal">52</span>
+<span class="normal">53</span>
+<span class="normal">54</span>
+<span class="normal">55</span>
+<span class="normal">56</span>
+<span class="normal">57</span>
+<span class="normal">58</span>
+<span class="normal">59</span>
+<span class="normal">60</span>
+<span class="normal">61</span>
+<span class="normal">62</span>
+<span class="normal">63</span>
+<span class="normal">64</span>
+<span class="normal">65</span>
+<span class="normal">66</span>
+<span class="normal">67</span>
+<span class="normal">68</span>
+<span class="normal">69</span>
+<span class="normal">70</span>
+<span class="normal">71</span>
+<span class="normal">72</span>
+<span class="normal">73</span>
+<span class="normal">74</span>
+<span class="normal">75</span>
+<span class="normal">76</span>
+<span class="normal">77</span>
+<span class="normal">78</span>
+<span class="normal">79</span>
+<span class="normal">80</span>
+<span class="normal">81</span>
+<span class="normal">82</span>
+<span class="normal">83</span>
+<span class="normal">84</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@staticmethod</span>
+<span class="k">def</span> <span class="nf">raiss_model</span><span class="p">(</span>
+    <span class="n">z_scores_known</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span>
+    <span class="n">ld_matrix_known</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span>
+    <span class="n">ld_matrix_known_missing</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span>
+    <span class="n">lamb</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.01</span><span class="p">,</span>
+    <span class="n">rtol</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.01</span><span class="p">,</span>
+<span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;Compute the imputation of the z-score using the RAISS model.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        z_scores_known (np.ndarray): the vector of known Z scores</span>
+<span class="sd">        ld_matrix_known (np.ndarray) : the matrix of known LD correlations</span>
+<span class="sd">        ld_matrix_known_missing (np.ndarray): LD matrix of known SNPs with other unknown SNPs in large matrix (similar to ld[unknowns, :][:,known])</span>
+<span class="sd">        lamb (float): size of the small value added to the diagonal of the covariance matrix before inversion. Defaults to 0.01.</span>
+<span class="sd">        rtol (float): threshold to filter eigenvectos by its eigenvalue. It makes an inversion biased but much more numerically robust. Default to 0.01.</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">        dict[str, Any]:</span>
+<span class="sd">            - var (np.ndarray): variance of the imputed SNPs</span>
+<span class="sd">            - mu (np.ndarray): the estimation of the zscore of the imputed SNPs</span>
+<span class="sd">            - ld_score (np.ndarray): the linkage disequilibrium score of the imputed SNPs</span>
+<span class="sd">            - condition_number (np.ndarray): the condition number of the correlation matrix</span>
+<span class="sd">            - correct_inversion (np.ndarray): a boolean array indicating if the inversion was successful</span>
+<span class="sd">            - imputation_r2 (np.ndarray): the R2 of the imputation</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">sig_t_inv</span> <span class="o">=</span> <span class="n">SummaryStatisticsImputation</span><span class="o">.</span><span class="n">_invert_sig_t</span><span class="p">(</span>
+        <span class="n">ld_matrix_known</span><span class="p">,</span> <span class="n">lamb</span><span class="p">,</span> <span class="n">rtol</span>
+    <span class="p">)</span>
+    <span class="k">if</span> <span class="n">sig_t_inv</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="k">return</span> <span class="p">{</span>
+            <span class="s2">&quot;var&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
+            <span class="s2">&quot;mu&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
+            <span class="s2">&quot;ld_score&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
+            <span class="s2">&quot;condition_number&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
+            <span class="s2">&quot;correct_inversion&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
+            <span class="s2">&quot;imputation_r2&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="p">}</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="n">condition_number</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span>
+            <span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">cond</span><span class="p">(</span><span class="n">ld_matrix_known</span><span class="p">)]</span> <span class="o">*</span> <span class="n">ld_matrix_known_missing</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
+        <span class="p">)</span>
+        <span class="n">correct_inversion</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span>
+            <span class="p">[</span>
+                <span class="n">SummaryStatisticsImputation</span><span class="o">.</span><span class="n">_check_inversion</span><span class="p">(</span>
+                    <span class="n">ld_matrix_known</span><span class="p">,</span> <span class="n">sig_t_inv</span>
+                <span class="p">)</span>
+            <span class="p">]</span>
+            <span class="o">*</span> <span class="n">ld_matrix_known_missing</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
+        <span class="p">)</span>
+
+        <span class="n">var</span><span class="p">,</span> <span class="n">ld_score</span> <span class="o">=</span> <span class="n">SummaryStatisticsImputation</span><span class="o">.</span><span class="n">_compute_var</span><span class="p">(</span>
+            <span class="n">ld_matrix_known_missing</span><span class="p">,</span> <span class="n">sig_t_inv</span><span class="p">,</span> <span class="n">lamb</span>
+        <span class="p">)</span>
+
+        <span class="n">mu</span> <span class="o">=</span> <span class="n">SummaryStatisticsImputation</span><span class="o">.</span><span class="n">_compute_mu</span><span class="p">(</span>
+            <span class="n">ld_matrix_known_missing</span><span class="p">,</span> <span class="n">sig_t_inv</span><span class="p">,</span> <span class="n">z_scores_known</span>
+        <span class="p">)</span>
+        <span class="n">var_norm</span> <span class="o">=</span> <span class="n">SummaryStatisticsImputation</span><span class="o">.</span><span class="n">_var_in_boundaries</span><span class="p">(</span><span class="n">var</span><span class="p">,</span> <span class="n">lamb</span><span class="p">)</span>
+
+        <span class="n">R2</span> <span class="o">=</span> <span class="p">(</span><span class="mi">1</span> <span class="o">+</span> <span class="n">lamb</span><span class="p">)</span> <span class="o">-</span> <span class="n">var_norm</span>
+
+        <span class="n">mu</span> <span class="o">=</span> <span class="n">mu</span> <span class="o">/</span> <span class="n">np</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">R2</span><span class="p">)</span>
+        <span class="k">return</span> <span class="p">{</span>
+            <span class="s2">&quot;var&quot;</span><span class="p">:</span> <span class="n">var</span><span class="p">,</span>
+            <span class="s2">&quot;mu&quot;</span><span class="p">:</span> <span class="n">mu</span><span class="p">,</span>
+            <span class="s2">&quot;ld_score&quot;</span><span class="p">:</span> <span class="n">ld_score</span><span class="p">,</span>
+            <span class="s2">&quot;condition_number&quot;</span><span class="p">:</span> <span class="n">condition_number</span><span class="p">,</span>
+            <span class="s2">&quot;correct_inversion&quot;</span><span class="p">:</span> <span class="n">correct_inversion</span><span class="p">,</span>
+            <span class="s2">&quot;imputation_r2&quot;</span><span class="p">:</span> <span class="mi">1</span> <span class="o">-</span> <span class="n">var</span><span class="p">,</span>
+        <span class="p">}</span>
+</code></pre></div></td></tr></table></div>
+          </details>
+  </div>
+
+</div>
+
+
+
+  </div>
+
+  </div>
+
+
+</div>
+<hr />
+<div class="md-source-file">
+  <small>
+    <ul
+      class="metadata page-metadata"
+      data-bi-name="page info"
+      lang="en-us"
+      dir="ltr"
+    >
+      <!-- mkdocs-git-revision-date-plugin -->
+      
+      <li>
+        <span class="icon twemoji">
+          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
+        </span>
+        <span class="label"> <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-timeago"><span class="timeago" datetime="2024-03-12T15:35:22+00:00" locale="en"></span></span><span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-03-12</span> </span>
+      </li>
+       
+      <li>
+        <span class="icon twemoji">
+          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
+        </span>
+        <span class="label"> <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-timeago"><span class="timeago" datetime="2024-03-12T15:35:22+00:00" locale="en"></span></span><span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-03-12</span> </span>
+      </li>
+      
+
+      <li>
+        <span class="icon twemoji">
+          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 2A10 10 0 0 0 2 12c0 4.42 2.87 8.17 6.84 9.5.5.08.66-.23.66-.5v-1.69c-2.77.6-3.36-1.34-3.36-1.34-.46-1.16-1.11-1.47-1.11-1.47-.91-.62.07-.6.07-.6 1 .07 1.53 1.03 1.53 1.03.87 1.52 2.34 1.07 2.91.83.09-.65.35-1.09.63-1.34-2.22-.25-4.55-1.11-4.55-4.92 0-1.11.38-2 1.03-2.71-.1-.25-.45-1.29.1-2.64 0 0 .84-.27 2.75 1.02.79-.22 1.65-.33 2.5-.33.85 0 1.71.11 2.5.33 1.91-1.29 2.75-1.02 2.75-1.02.55 1.35.2 2.39.1 2.64.65.71 1.03 1.6 1.03 2.71 0 3.82-2.34 4.66-4.57 4.91.36.31.69.92.69 1.85V21c0 .27.16.59.67.5C19.14 20.16 22 16.42 22 12A10 10 0 0 0 12 2Z"/></svg>
+        </span>
+        <span class="label"> Contributors </span>
+      </li>
+      <li class="contributors-holder">
+        <ul class="contributors" data-bi-name="contributors"></ul>
+      </li>
+    </ul>
+  </small>
+</div>
+
+
+
+
+                
+              </article>
+            </div>
+          
+          
+<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
+        </div>
+        
+          <button type="button" class="md-top md-icon" data-md-component="top" hidden>
+  
+  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8v12Z"/></svg>
+  Back to top
+</button>
+        
+      </main>
+      
+        <footer class="md-footer">
+  
+  <div class="md-footer-meta md-typeset">
+    <div class="md-footer-meta__inner md-grid">
+      <div class="md-copyright">
+  
+  
+    Made with
+    <a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
+      Material for MkDocs
+    </a>
+  
+</div>
+      
+        <div class="md-social">
+  
+    
+    
+    
+    
+      
+      
+    
+    <a href="https://github.com/gentropy" target="_blank" rel="noopener" title="github.com" class="md-social__link">
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 496 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6zm-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3zm44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9zM244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8zM97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1zm-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7zm32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1zm-11.4-14.7c-1.6 1-1.6 3.6 0 5.9 1.6 2.3 4.3 3.3 5.6 2.3 1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2z"/></svg>
+    </a>
+  
+</div>
+      
+    </div>
+  </div>
+</footer>
+      
+    </div>
+    <div class="md-dialog" data-md-component="dialog">
+      <div class="md-dialog__inner md-typeset"></div>
+    </div>
+    
+    
+    <script id="__config" type="application/json">{"base": "../../..", "features": ["navigation.instant", "navigation.top", "navigation.tabs", "navigation.tabs.sticky", "content.code.annotate", "search.suggest", "search.highlight"], "search": "../../../assets/javascripts/workers/search.f886a092.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
+    
+    
+      <script src="../../../assets/javascripts/bundle.d7c377c4.min.js"></script>
+      
+        <script src="../../../js/timeago.min.js"></script>
+      
+        <script src="../../../js/timeago_mkdocs_material.js"></script>
+      
+        <script src="../../../assets/javascripts/extra.js"></script>
+      
+    
+  </body>
+</html>
\ No newline at end of file
diff --git a/python_api/methods/sumstat_quality_controls/index.html b/python_api/methods/sumstat_quality_controls/index.html
new file mode 100644
index 000000000..ced3087b3
--- /dev/null
+++ b/python_api/methods/sumstat_quality_controls/index.html
@@ -0,0 +1,4831 @@
+
+<!doctype html>
+<html lang="en" class="no-js">
+  <head>
+    
+      <meta charset="utf-8">
+      <meta name="viewport" content="width=device-width,initial-scale=1">
+      
+      
+      
+      
+        <link rel="prev" href="../sumstat_imputation/">
+      
+      
+        <link rel="next" href="../susie_inf/">
+      
+      
+      <link rel="icon" href="../../../assets/imgs/favicon.png">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.5.3">
+    
+    
+      
+        <title>QC of GWAS Summary Statistics - Open Targets Gentropy</title>
+      
+    
+    
+      <link rel="stylesheet" href="../../../assets/stylesheets/main.50c56a3b.min.css">
+      
+        
+        <link rel="stylesheet" href="../../../assets/stylesheets/palette.06af60db.min.css">
+      
+      
+
+
+    
+    
+      
+    
+    
+      
+        
+        
+        <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+        <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
+        <style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
+      
+    
+    
+      <link rel="stylesheet" href="../../../css/timeago.css">
+    
+      <link rel="stylesheet" href="../../../assets/_mkdocstrings.css">
+    
+      <link rel="stylesheet" href="../../../assets/stylesheets/extra.css">
+    
+    <script>__md_scope=new URL("../../..",location),__md_hash=e=>[...e].reduce((e,_)=>(e<<5)-e+_.charCodeAt(0),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
+    
+      
+
+    
+    
+    
+  </head>
+  
+  
+    
+    
+      
+    
+    
+    
+    
+    <body dir="ltr" data-md-color-scheme="default" data-md-color-primary="blue-grey" data-md-color-accent="light-blue">
+  
+    
+    <input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
+    <input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
+    <label class="md-overlay" for="__drawer"></label>
+    <div data-md-component="skip">
+      
+        
+        <a href="#gentropy.method.sumstat_quality_controls.SummaryStatisticsQC" class="md-skip">
+          Skip to content
+        </a>
+      
+    </div>
+    <div data-md-component="announce">
+      
+    </div>
+    
+    
+      
+
+  
+
+<header class="md-header md-header--shadow md-header--lifted" data-md-component="header">
+  <nav class="md-header__inner md-grid" aria-label="Header">
+    <a href="../../.." title="Open Targets Gentropy" class="md-header__button md-logo" aria-label="Open Targets Gentropy" data-md-component="logo">
+      
+  <img src="../../../assets/imgs/otlogo.png" alt="logo">
+
+    </a>
+    <label class="md-header__button md-icon" for="__drawer">
+      
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3V6m0 5h18v2H3v-2m0 5h18v2H3v-2Z"/></svg>
+    </label>
+    <div class="md-header__title" data-md-component="header-title">
+      <div class="md-header__ellipsis">
+        <div class="md-header__topic">
+          <span class="md-ellipsis">
+            Open Targets Gentropy
+          </span>
+        </div>
+        <div class="md-header__topic" data-md-component="header-topic">
+          <span class="md-ellipsis">
+            
+              QC of GWAS Summary Statistics
+            
+          </span>
+        </div>
+      </div>
+    </div>
+    
+      
+        <form class="md-header__option" data-md-component="palette">
+  
+    
+    
+    
+    <input class="md-option" data-md-color-media="(prefers-color-scheme: light)" data-md-color-scheme="default" data-md-color-primary="blue-grey" data-md-color-accent="light-blue"  aria-label="Switch to dark mode"  type="radio" name="__palette" id="__palette_0">
+    
+      <label class="md-header__button md-icon" title="Switch to dark mode" for="__palette_1" hidden>
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M17 6H7c-3.31 0-6 2.69-6 6s2.69 6 6 6h10c3.31 0 6-2.69 6-6s-2.69-6-6-6zm0 10H7c-2.21 0-4-1.79-4-4s1.79-4 4-4h10c2.21 0 4 1.79 4 4s-1.79 4-4 4zM7 9c-1.66 0-3 1.34-3 3s1.34 3 3 3 3-1.34 3-3-1.34-3-3-3z"/></svg>
+      </label>
+    
+  
+    
+    
+    
+    <input class="md-option" data-md-color-media="(prefers-color-scheme: dark)" data-md-color-scheme="slate" data-md-color-primary="blue-grey" data-md-color-accent="light-blue"  aria-label="Switch to light mode"  type="radio" name="__palette" id="__palette_1">
+    
+      <label class="md-header__button md-icon" title="Switch to light mode" for="__palette_0" hidden>
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M17 7H7a5 5 0 0 0-5 5 5 5 0 0 0 5 5h10a5 5 0 0 0 5-5 5 5 0 0 0-5-5m0 8a3 3 0 0 1-3-3 3 3 0 0 1 3-3 3 3 0 0 1 3 3 3 3 0 0 1-3 3Z"/></svg>
+      </label>
+    
+  
+</form>
+      
+    
+    
+      <script>var media,input,key,value,palette=__md_get("__palette");if(palette&&palette.color){"(prefers-color-scheme)"===palette.color.media&&(media=matchMedia("(prefers-color-scheme: light)"),input=document.querySelector(media.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']"),palette.color.media=input.getAttribute("data-md-color-media"),palette.color.scheme=input.getAttribute("data-md-color-scheme"),palette.color.primary=input.getAttribute("data-md-color-primary"),palette.color.accent=input.getAttribute("data-md-color-accent"));for([key,value]of Object.entries(palette.color))document.body.setAttribute("data-md-color-"+key,value)}</script>
+    
+    
+    
+      <label class="md-header__button md-icon" for="__search">
+        
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
+      </label>
+      <div class="md-search" data-md-component="search" role="dialog">
+  <label class="md-search__overlay" for="__search"></label>
+  <div class="md-search__inner" role="search">
+    <form class="md-search__form" name="search">
+      <input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
+      <label class="md-search__icon md-icon" for="__search">
+        
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
+        
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12Z"/></svg>
+      </label>
+      <nav class="md-search__options" aria-label="Search">
+        
+        <button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
+          
+          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12 19 6.41Z"/></svg>
+        </button>
+      </nav>
+      
+        <div class="md-search__suggest" data-md-component="search-suggest"></div>
+      
+    </form>
+    <div class="md-search__output">
+      <div class="md-search__scrollwrap" data-md-scrollfix>
+        <div class="md-search-result" data-md-component="search-result">
+          <div class="md-search-result__meta">
+            Initializing search
+          </div>
+          <ol class="md-search-result__list" role="presentation"></ol>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+    
+    
+      <div class="md-header__source">
+        <a href="https://github.com/opentargets/gentropy" title="Go to repository" class="md-source" data-md-component="source">
+  <div class="md-source__icon md-icon">
+    
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+  </div>
+  <div class="md-source__repository">
+    opentargets/gentropy
+  </div>
+</a>
+      </div>
+    
+  </nav>
+  
+    
+      
+<nav class="md-tabs" aria-label="Tabs" data-md-component="tabs">
+  <div class="md-grid">
+    <ul class="md-tabs__list">
+      
+        
+  
+  
+  
+    <li class="md-tabs__item">
+      <a href="../../.." class="md-tabs__link">
+        
+  
+    
+  
+  Home
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
+    <li class="md-tabs__item">
+      <a href="../../../installation/" class="md-tabs__link">
+        
+  
+    
+  
+  Installation
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
+    <li class="md-tabs__item">
+      <a href="../../../howto/_howto/" class="md-tabs__link">
+        
+  
+    
+  
+  How-to
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
+    <li class="md-tabs__item">
+      <a href="../../../roadmap/" class="md-tabs__link">
+        
+  
+    
+  
+  Roadmap
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
+    <li class="md-tabs__item">
+      <a href="../../../development/_development/" class="md-tabs__link">
+        
+  
+    
+  
+  Development
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+    
+  
+  
+    <li class="md-tabs__item md-tabs__item--active">
+      <a href="../../_python_api/" class="md-tabs__link">
+        
+  
+    
+  
+  Python API
+
+      </a>
+    </li>
+  
+
+      
+    </ul>
+  </div>
+</nav>
+    
+  
+</header>
+    
+    <div class="md-container" data-md-component="container">
+      
+      
+        
+      
+      <main class="md-main" data-md-component="main">
+        <div class="md-main__inner md-grid">
+          
+            
+              
+              <div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
+                <div class="md-sidebar__scrollwrap">
+                  <div class="md-sidebar__inner">
+                    
+
+
+  
+
+
+<nav class="md-nav md-nav--primary md-nav--lifted" aria-label="Navigation" data-md-level="0">
+  <label class="md-nav__title" for="__drawer">
+    <a href="../../.." title="Open Targets Gentropy" class="md-nav__button md-logo" aria-label="Open Targets Gentropy" data-md-component="logo">
+      
+  <img src="../../../assets/imgs/otlogo.png" alt="logo">
+
+    </a>
+    Open Targets Gentropy
+  </label>
+  
+    <div class="md-nav__source">
+      <a href="https://github.com/opentargets/gentropy" title="Go to repository" class="md-source" data-md-component="source">
+  <div class="md-source__icon md-icon">
+    
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+  </div>
+  <div class="md-source__repository">
+    opentargets/gentropy
+  </div>
+</a>
+    </div>
+  
+  <ul class="md-nav__list" data-md-scrollfix>
+    
+      
+      
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../.." class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Home
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../installation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Installation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
+    
+    
+    
+    
+    
+      
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+        
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../../../howto/_howto/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    How-to
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_3" id="__nav_3_label" tabindex="">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3">
+            <span class="md-nav__icon md-icon"></span>
+            How-to
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    
+    
+    
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../../../howto/command_line/_command_line/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    Command line
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_3_1" id="__nav_3_1_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_1_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1">
+            <span class="md-nav__icon md-icon"></span>
+            Command line
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../howto/command_line/run_step_in_cli/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Run step in CLI
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../howto/command_line/run_step_using_config/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Run step using YAML config
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+    
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_2" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../../../howto/python_api/_python_api/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    Python api
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_3_2" id="__nav_3_2_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_2_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_2">
+            <span class="md-nav__icon md-icon"></span>
+            Python api
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../howto/python_api/a_creating_spark_session/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Creating a Spark Session
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../howto/python_api/b_create_dataset/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Create a dataset
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../howto/python_api/c_applying_methods/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Applying methods
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../howto/python_api/d_inspect_dataset/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Inspect a dataset
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../roadmap/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Roadmap
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
+    
+    
+    
+    
+    
+      
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+        
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_5" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../../../development/_development/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    Development
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_5" id="__nav_5_label" tabindex="">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_5">
+            <span class="md-nav__icon md-icon"></span>
+            Development
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../development/airflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Airflow configuration
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../development/contributing/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Contributing guidelines
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../development/troubleshooting/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Troubleshooting
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../development/workflows/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Pipeline workflows
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+    
+      
+      
+  
+  
+    
+  
+  
+  
+    
+    
+    
+    
+    
+      
+      
+    
+    <li class="md-nav__item md-nav__item--active md-nav__item--section md-nav__item--nested">
+      
+        
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6" checked>
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../../_python_api/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    Python API
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_6" id="__nav_6_label" tabindex="">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_6_label" aria-expanded="true">
+          <label class="md-nav__title" for="__nav_6">
+            <span class="md-nav__icon md-icon"></span>
+            Python API
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    
+    
+    
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_1" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../../datasets/_datasets/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    Datasets
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_6_1" id="__nav_6_1_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_6_1_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_1">
+            <span class="md-nav__icon md-icon"></span>
+            Datasets
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasets/colocalisation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Colocalisation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasets/gene_index/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Gene Index
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasets/intervals/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Intervals
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasets/l2g_feature/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    L2G Feature
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasets/l2g_feature_matrix/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    L2G Feature Matrix
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasets/l2g_gold_standard/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    L2G Gold Standard
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasets/l2g_prediction/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    L2G Prediction
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasets/ld_index/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    LD Index
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasets/study_index/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Study Index
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasets/study_locus/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Study Locus
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasets/study_locus_overlap/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Study Locus Overlap
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasets/summary_statistics/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasets/variant_annotation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Variant annotation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasets/variant_index/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Variant index
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasets/variant_to_gene/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Variant-to-gene
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+    
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_2" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../../datasources/_datasources/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    Datasources
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_6_2" id="__nav_6_2_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_6_2_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_2">
+            <span class="md-nav__icon md-icon"></span>
+            Datasources
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    
+    
+    
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_2_1" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../../datasources/eqtl_catalogue/_eqtl_catalogue/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    Eqtl catalogue
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_6_2_1" id="__nav_6_2_1_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_2_1_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_2_1">
+            <span class="md-nav__icon md-icon"></span>
+            Eqtl catalogue
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasources/eqtl_catalogue/finemapping/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Fine mapping results
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasources/eqtl_catalogue/study_index/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Study Index
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasources/eqtl_catalogue/summary_stats/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Stats
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+    
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_2_2" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../../datasources/finngen/_finngen/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    Finngen
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_6_2_2" id="__nav_6_2_2_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_2_2_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_2_2">
+            <span class="md-nav__icon md-icon"></span>
+            Finngen
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasources/finngen/finemapping/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Finemapping
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasources/finngen/study_index/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Study Index
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasources/finngen/summary_stats/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Study Index
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+    
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_2_3" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../../datasources/gnomad/_gnomad/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    Gnomad
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_6_2_3" id="__nav_6_2_3_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_2_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_2_3">
+            <span class="md-nav__icon md-icon"></span>
+            Gnomad
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasources/gnomad/gnomad_ld/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    LD Matrix
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasources/gnomad/gnomad_variants/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Variants
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+    
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_2_4" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../../datasources/gwas_catalog/_gwas_catalog/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    Gwas catalog
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_6_2_4" id="__nav_6_2_4_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_2_4_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_2_4">
+            <span class="md-nav__icon md-icon"></span>
+            Gwas catalog
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasources/gwas_catalog/associations/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Associations
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasources/gwas_catalog/study_index/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Study Index
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasources/gwas_catalog/study_splitter/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Study Splitter
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasources/gwas_catalog/summary_statistics/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+    
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_2_5" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../../datasources/intervals/_intervals/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    Intervals
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_6_2_5" id="__nav_6_2_5_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_2_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_2_5">
+            <span class="md-nav__icon md-icon"></span>
+            Intervals
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasources/intervals/andersson/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Andersson et al.
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasources/intervals/javierre/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Javierre et al.
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasources/intervals/jung/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Jung et al.
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasources/intervals/thurman/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Thurman et al.
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+    
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_2_6" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../../datasources/open_targets/_open_targets/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    Open targets
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_6_2_6" id="__nav_6_2_6_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_2_6_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_2_6">
+            <span class="md-nav__icon md-icon"></span>
+            Open targets
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasources/open_targets/l2g_gold_standard/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    L2G Gold Standard
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../datasources/open_targets/target/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Target
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+    
+  
+  
+  
+    
+    
+    
+    
+    
+    <li class="md-nav__item md-nav__item--active md-nav__item--nested">
+      
+        
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3" checked>
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../_methods/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    Methods
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_6_3" id="__nav_6_3_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_6_3_label" aria-expanded="true">
+          <label class="md-nav__title" for="__nav_6_3">
+            <span class="md-nav__icon md-icon"></span>
+            Methods
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../carma/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    CARMA
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../clumping/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Clumping
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../coloc/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Coloc
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../ecaviar/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    eCAVIAR
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../ld_annotator/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    LDAnnotator
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../pics/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    PICS
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+    
+  
+  
+  
+    <li class="md-nav__item md-nav__item--active">
+      
+      <input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
+      
+      
+      
+        <label class="md-nav__link md-nav__link--active" for="__toc">
+          
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+          <span class="md-nav__icon md-icon"></span>
+        </label>
+      
+      <a href="./" class="md-nav__link md-nav__link--active">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+      
+        
+
+<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
+  
+  
+  
+  
+    <label class="md-nav__title" for="__toc">
+      <span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
+      
+        <li class="md-nav__item">
+  <a href="#gentropy.method.sumstat_quality_controls.SummaryStatisticsQC" class="md-nav__link">
+    <span class="md-ellipsis">
+      SummaryStatisticsQC
+    </span>
+  </a>
+  
+    <nav class="md-nav" aria-label="SummaryStatisticsQC">
+      <ul class="md-nav__list">
+        
+          <li class="md-nav__item">
+  <a href="#gentropy.method.sumstat_quality_controls.SummaryStatisticsQC.gc_lambda_check" class="md-nav__link">
+    <span class="md-ellipsis">
+      gc_lambda_check
+    </span>
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#gentropy.method.sumstat_quality_controls.SummaryStatisticsQC.get_quality_control_metrics" class="md-nav__link">
+    <span class="md-ellipsis">
+      get_quality_control_metrics
+    </span>
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#gentropy.method.sumstat_quality_controls.SummaryStatisticsQC.number_of_snps" class="md-nav__link">
+    <span class="md-ellipsis">
+      number_of_snps
+    </span>
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#gentropy.method.sumstat_quality_controls.SummaryStatisticsQC.sumstat_n_eff_check" class="md-nav__link">
+    <span class="md-ellipsis">
+      sumstat_n_eff_check
+    </span>
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#gentropy.method.sumstat_quality_controls.SummaryStatisticsQC.sumstat_qc_beta_check" class="md-nav__link">
+    <span class="md-ellipsis">
+      sumstat_qc_beta_check
+    </span>
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#gentropy.method.sumstat_quality_controls.SummaryStatisticsQC.sumstat_qc_pz_check" class="md-nav__link">
+    <span class="md-ellipsis">
+      sumstat_qc_pz_check
+    </span>
+  </a>
+  
+</li>
+        
+      </ul>
+    </nav>
+  
+</li>
+      
+    </ul>
+  
+</nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../susie_inf/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    SuSiE-inf
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+    
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../l2g/_l2g/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    L2g
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
+            <span class="md-nav__icon md-icon"></span>
+            L2g
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../l2g/evaluator/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    W&B evaluator
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../l2g/feature_factory/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    L2G Feature Factory
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../l2g/model/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    L2G Model
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../l2g/trainer/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    L2G Trainer
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+    
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_4" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../../steps/_steps/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    Steps
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_6_4" id="__nav_6_4_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_6_4_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_4">
+            <span class="md-nav__icon md-icon"></span>
+            Steps
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../steps/colocalisation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    colocalisation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../steps/eqtl_catalogue/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    eQTL Catalogue
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../steps/finngen_studies/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    finngen_studies
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../steps/finngen_sumstat_preprocess/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    finngen_sumstat_preprocess
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../steps/gene_index/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    gene_index
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../steps/gwas_catalog_curation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    gwas_catalog_study_curation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../steps/gwas_catalog_inclusion/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    gwas_catalog_study_inclusion
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../steps/gwas_catalog_ingestion/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    gwas_catalog_ingestion
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../steps/gwas_catalog_sumstat_preprocess/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    gwas_catalog_sumstat_preprocess
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../steps/l2g/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    locus_to_gene
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../steps/ld_clump/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    ld_based_clumping
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../steps/ld_index/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    ld_index
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../steps/pics/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    pics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../steps/variant_annotation_step/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    variant_annotation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../steps/variant_index_step/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    variant_index
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../steps/variant_to_gene_step/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    variant_to_gene
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../steps/window_based_clumping/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    window_based_clumping
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+    
+  </ul>
+</nav>
+                  </div>
+                </div>
+              </div>
+            
+            
+              
+              <div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
+                <div class="md-sidebar__scrollwrap">
+                  <div class="md-sidebar__inner">
+                    
+
+<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
+  
+  
+  
+  
+    <label class="md-nav__title" for="__toc">
+      <span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
+      
+        <li class="md-nav__item">
+  <a href="#gentropy.method.sumstat_quality_controls.SummaryStatisticsQC" class="md-nav__link">
+    <span class="md-ellipsis">
+      SummaryStatisticsQC
+    </span>
+  </a>
+  
+    <nav class="md-nav" aria-label="SummaryStatisticsQC">
+      <ul class="md-nav__list">
+        
+          <li class="md-nav__item">
+  <a href="#gentropy.method.sumstat_quality_controls.SummaryStatisticsQC.gc_lambda_check" class="md-nav__link">
+    <span class="md-ellipsis">
+      gc_lambda_check
+    </span>
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#gentropy.method.sumstat_quality_controls.SummaryStatisticsQC.get_quality_control_metrics" class="md-nav__link">
+    <span class="md-ellipsis">
+      get_quality_control_metrics
+    </span>
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#gentropy.method.sumstat_quality_controls.SummaryStatisticsQC.number_of_snps" class="md-nav__link">
+    <span class="md-ellipsis">
+      number_of_snps
+    </span>
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#gentropy.method.sumstat_quality_controls.SummaryStatisticsQC.sumstat_n_eff_check" class="md-nav__link">
+    <span class="md-ellipsis">
+      sumstat_n_eff_check
+    </span>
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#gentropy.method.sumstat_quality_controls.SummaryStatisticsQC.sumstat_qc_beta_check" class="md-nav__link">
+    <span class="md-ellipsis">
+      sumstat_qc_beta_check
+    </span>
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#gentropy.method.sumstat_quality_controls.SummaryStatisticsQC.sumstat_qc_pz_check" class="md-nav__link">
+    <span class="md-ellipsis">
+      sumstat_qc_pz_check
+    </span>
+  </a>
+  
+</li>
+        
+      </ul>
+    </nav>
+  
+</li>
+      
+    </ul>
+  
+</nav>
+                  </div>
+                </div>
+              </div>
+            
+          
+          
+            <div class="md-content" data-md-component="content">
+              <article class="md-content__inner md-typeset">
+                
+                  
+
+  
+  
+
+
+  <h1>QC of GWAS Summary Statistics</h1>
+
+<p>This class consists of several general quality control checks for GWAS with full summary statistics.
+There are several checks included:</p>
+<ol>
+<li>
+<p>Genomic control lambda (median of the distribution of Chi2 statistics divided by expected for Chi2 with df=1). Lambda should be reasonably close to 1. Ideally not bigger than 2.</p>
+</li>
+<li>
+<p>P-Z check: the linear regression between log10 of reported p-values and log10 of p-values inferred from betas and standard errors. Intercept of the regression should be close to 0, slope close to 1.</p>
+</li>
+<li>
+<p>Mean beta check: mean of beta. Should be close to 0.</p>
+</li>
+<li>
+<p>The N_eff check: It estimates the ratio between effective sample size and the expected one and checks its distribution. It is possible to conduct only if the effective allele frequency is provided in the study. The median ratio is always close to 1, standard error should be close to 0.</p>
+</li>
+<li>
+<p>Number of SNPs and number of significant SNPs.</p>
+</li>
+</ol>
+
+
+<div class="doc doc-object doc-class">
+
+
+
+<h2 id="gentropy.method.sumstat_quality_controls.SummaryStatisticsQC" class="doc doc-heading">
+          <code>gentropy.method.sumstat_quality_controls.SummaryStatisticsQC</code>
+
+
+<a href="#gentropy.method.sumstat_quality_controls.SummaryStatisticsQC" class="headerlink" title="Permanent link">&para;</a></h2>
+
+
+  <div class="doc doc-contents first">
+
+  
+      <p>Summary statistics QC methods.</p>
+<p>This module contains methods for quality control of GWAS summary statistics.
+The list of methods includes:</p>
+<div class="highlight"><pre><span></span><code>- sumstat_qc_beta_check: This is the mean beta check. The mean beta should be close to 0.
+
+- sumstat_qc_pz_check: This is the PZ check. It runs a linear regression between reported p-values and p-values inferred from z-scores.
+
+- sumstat_n_eff_check: This is the effective sample size check. It estimates the ratio between the effective sample size and the expected one and checks its distribution.
+
+- gc_lambda_check: This is the genomic control lambda check.
+
+- number_of_snps: This function calculates the number of SNPs and the number of SNPs with a p-value less than 5e-8.
+</code></pre></div>
+
+            <details class="quote">
+              <summary>Source code in <code>src/gentropy/method/sumstat_quality_controls.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 16</span>
+<span class="normal"> 17</span>
+<span class="normal"> 18</span>
+<span class="normal"> 19</span>
+<span class="normal"> 20</span>
+<span class="normal"> 21</span>
+<span class="normal"> 22</span>
+<span class="normal"> 23</span>
+<span class="normal"> 24</span>
+<span class="normal"> 25</span>
+<span class="normal"> 26</span>
+<span class="normal"> 27</span>
+<span class="normal"> 28</span>
+<span class="normal"> 29</span>
+<span class="normal"> 30</span>
+<span class="normal"> 31</span>
+<span class="normal"> 32</span>
+<span class="normal"> 33</span>
+<span class="normal"> 34</span>
+<span class="normal"> 35</span>
+<span class="normal"> 36</span>
+<span class="normal"> 37</span>
+<span class="normal"> 38</span>
+<span class="normal"> 39</span>
+<span class="normal"> 40</span>
+<span class="normal"> 41</span>
+<span class="normal"> 42</span>
+<span class="normal"> 43</span>
+<span class="normal"> 44</span>
+<span class="normal"> 45</span>
+<span class="normal"> 46</span>
+<span class="normal"> 47</span>
+<span class="normal"> 48</span>
+<span class="normal"> 49</span>
+<span class="normal"> 50</span>
+<span class="normal"> 51</span>
+<span class="normal"> 52</span>
+<span class="normal"> 53</span>
+<span class="normal"> 54</span>
+<span class="normal"> 55</span>
+<span class="normal"> 56</span>
+<span class="normal"> 57</span>
+<span class="normal"> 58</span>
+<span class="normal"> 59</span>
+<span class="normal"> 60</span>
+<span class="normal"> 61</span>
+<span class="normal"> 62</span>
+<span class="normal"> 63</span>
+<span class="normal"> 64</span>
+<span class="normal"> 65</span>
+<span class="normal"> 66</span>
+<span class="normal"> 67</span>
+<span class="normal"> 68</span>
+<span class="normal"> 69</span>
+<span class="normal"> 70</span>
+<span class="normal"> 71</span>
+<span class="normal"> 72</span>
+<span class="normal"> 73</span>
+<span class="normal"> 74</span>
+<span class="normal"> 75</span>
+<span class="normal"> 76</span>
+<span class="normal"> 77</span>
+<span class="normal"> 78</span>
+<span class="normal"> 79</span>
+<span class="normal"> 80</span>
+<span class="normal"> 81</span>
+<span class="normal"> 82</span>
+<span class="normal"> 83</span>
+<span class="normal"> 84</span>
+<span class="normal"> 85</span>
+<span class="normal"> 86</span>
+<span class="normal"> 87</span>
+<span class="normal"> 88</span>
+<span class="normal"> 89</span>
+<span class="normal"> 90</span>
+<span class="normal"> 91</span>
+<span class="normal"> 92</span>
+<span class="normal"> 93</span>
+<span class="normal"> 94</span>
+<span class="normal"> 95</span>
+<span class="normal"> 96</span>
+<span class="normal"> 97</span>
+<span class="normal"> 98</span>
+<span class="normal"> 99</span>
+<span class="normal">100</span>
+<span class="normal">101</span>
+<span class="normal">102</span>
+<span class="normal">103</span>
+<span class="normal">104</span>
+<span class="normal">105</span>
+<span class="normal">106</span>
+<span class="normal">107</span>
+<span class="normal">108</span>
+<span class="normal">109</span>
+<span class="normal">110</span>
+<span class="normal">111</span>
+<span class="normal">112</span>
+<span class="normal">113</span>
+<span class="normal">114</span>
+<span class="normal">115</span>
+<span class="normal">116</span>
+<span class="normal">117</span>
+<span class="normal">118</span>
+<span class="normal">119</span>
+<span class="normal">120</span>
+<span class="normal">121</span>
+<span class="normal">122</span>
+<span class="normal">123</span>
+<span class="normal">124</span>
+<span class="normal">125</span>
+<span class="normal">126</span>
+<span class="normal">127</span>
+<span class="normal">128</span>
+<span class="normal">129</span>
+<span class="normal">130</span>
+<span class="normal">131</span>
+<span class="normal">132</span>
+<span class="normal">133</span>
+<span class="normal">134</span>
+<span class="normal">135</span>
+<span class="normal">136</span>
+<span class="normal">137</span>
+<span class="normal">138</span>
+<span class="normal">139</span>
+<span class="normal">140</span>
+<span class="normal">141</span>
+<span class="normal">142</span>
+<span class="normal">143</span>
+<span class="normal">144</span>
+<span class="normal">145</span>
+<span class="normal">146</span>
+<span class="normal">147</span>
+<span class="normal">148</span>
+<span class="normal">149</span>
+<span class="normal">150</span>
+<span class="normal">151</span>
+<span class="normal">152</span>
+<span class="normal">153</span>
+<span class="normal">154</span>
+<span class="normal">155</span>
+<span class="normal">156</span>
+<span class="normal">157</span>
+<span class="normal">158</span>
+<span class="normal">159</span>
+<span class="normal">160</span>
+<span class="normal">161</span>
+<span class="normal">162</span>
+<span class="normal">163</span>
+<span class="normal">164</span>
+<span class="normal">165</span>
+<span class="normal">166</span>
+<span class="normal">167</span>
+<span class="normal">168</span>
+<span class="normal">169</span>
+<span class="normal">170</span>
+<span class="normal">171</span>
+<span class="normal">172</span>
+<span class="normal">173</span>
+<span class="normal">174</span>
+<span class="normal">175</span>
+<span class="normal">176</span>
+<span class="normal">177</span>
+<span class="normal">178</span>
+<span class="normal">179</span>
+<span class="normal">180</span>
+<span class="normal">181</span>
+<span class="normal">182</span>
+<span class="normal">183</span>
+<span class="normal">184</span>
+<span class="normal">185</span>
+<span class="normal">186</span>
+<span class="normal">187</span>
+<span class="normal">188</span>
+<span class="normal">189</span>
+<span class="normal">190</span>
+<span class="normal">191</span>
+<span class="normal">192</span>
+<span class="normal">193</span>
+<span class="normal">194</span>
+<span class="normal">195</span>
+<span class="normal">196</span>
+<span class="normal">197</span>
+<span class="normal">198</span>
+<span class="normal">199</span>
+<span class="normal">200</span>
+<span class="normal">201</span>
+<span class="normal">202</span>
+<span class="normal">203</span>
+<span class="normal">204</span>
+<span class="normal">205</span>
+<span class="normal">206</span>
+<span class="normal">207</span>
+<span class="normal">208</span>
+<span class="normal">209</span>
+<span class="normal">210</span>
+<span class="normal">211</span>
+<span class="normal">212</span>
+<span class="normal">213</span>
+<span class="normal">214</span>
+<span class="normal">215</span>
+<span class="normal">216</span>
+<span class="normal">217</span>
+<span class="normal">218</span>
+<span class="normal">219</span>
+<span class="normal">220</span>
+<span class="normal">221</span>
+<span class="normal">222</span>
+<span class="normal">223</span>
+<span class="normal">224</span>
+<span class="normal">225</span>
+<span class="normal">226</span>
+<span class="normal">227</span>
+<span class="normal">228</span>
+<span class="normal">229</span>
+<span class="normal">230</span>
+<span class="normal">231</span>
+<span class="normal">232</span>
+<span class="normal">233</span>
+<span class="normal">234</span>
+<span class="normal">235</span>
+<span class="normal">236</span>
+<span class="normal">237</span>
+<span class="normal">238</span>
+<span class="normal">239</span>
+<span class="normal">240</span>
+<span class="normal">241</span>
+<span class="normal">242</span>
+<span class="normal">243</span>
+<span class="normal">244</span>
+<span class="normal">245</span>
+<span class="normal">246</span>
+<span class="normal">247</span>
+<span class="normal">248</span>
+<span class="normal">249</span>
+<span class="normal">250</span>
+<span class="normal">251</span>
+<span class="normal">252</span>
+<span class="normal">253</span>
+<span class="normal">254</span>
+<span class="normal">255</span>
+<span class="normal">256</span>
+<span class="normal">257</span>
+<span class="normal">258</span>
+<span class="normal">259</span>
+<span class="normal">260</span>
+<span class="normal">261</span>
+<span class="normal">262</span>
+<span class="normal">263</span>
+<span class="normal">264</span>
+<span class="normal">265</span>
+<span class="normal">266</span>
+<span class="normal">267</span>
+<span class="normal">268</span>
+<span class="normal">269</span>
+<span class="normal">270</span>
+<span class="normal">271</span>
+<span class="normal">272</span>
+<span class="normal">273</span>
+<span class="normal">274</span>
+<span class="normal">275</span>
+<span class="normal">276</span>
+<span class="normal">277</span>
+<span class="normal">278</span>
+<span class="normal">279</span>
+<span class="normal">280</span>
+<span class="normal">281</span>
+<span class="normal">282</span>
+<span class="normal">283</span>
+<span class="normal">284</span>
+<span class="normal">285</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">SummaryStatisticsQC</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;Summary statistics QC methods.</span>
+
+<span class="sd">    This module contains methods for quality control of GWAS summary statistics.</span>
+<span class="sd">    The list of methods includes:</span>
+
+<span class="sd">        - sumstat_qc_beta_check: This is the mean beta check. The mean beta should be close to 0.</span>
+
+<span class="sd">        - sumstat_qc_pz_check: This is the PZ check. It runs a linear regression between reported p-values and p-values inferred from z-scores.</span>
+
+<span class="sd">        - sumstat_n_eff_check: This is the effective sample size check. It estimates the ratio between the effective sample size and the expected one and checks its distribution.</span>
+
+<span class="sd">        - gc_lambda_check: This is the genomic control lambda check.</span>
+
+<span class="sd">        - number_of_snps: This function calculates the number of SNPs and the number of SNPs with a p-value less than 5e-8.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="nd">@staticmethod</span>
+    <span class="k">def</span> <span class="nf">sumstat_qc_beta_check</span><span class="p">(</span>
+        <span class="n">gwas_for_qc</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">DataFrame</span><span class="p">:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;The mean beta check for QC of GWAS summary statstics.</span>
+
+<span class="sd">        Args:</span>
+<span class="sd">            gwas_for_qc (SummaryStatistics): The instance of the SummaryStatistics class.</span>
+
+<span class="sd">        Returns:</span>
+<span class="sd">            DataFrame: PySpark DataFrame with the mean beta for each study.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">gwas_df</span> <span class="o">=</span> <span class="n">gwas_for_qc</span><span class="o">.</span><span class="n">_df</span>
+        <span class="n">qc_c</span> <span class="o">=</span> <span class="n">gwas_df</span><span class="o">.</span><span class="n">groupBy</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">agg</span><span class="p">(</span>
+            <span class="n">f</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="s2">&quot;beta&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;mean_beta&quot;</span><span class="p">),</span>
+        <span class="p">)</span>
+        <span class="k">return</span> <span class="n">qc_c</span>
+
+    <span class="nd">@staticmethod</span>
+    <span class="k">def</span> <span class="nf">_calculate_logpval</span><span class="p">(</span><span class="n">z2</span><span class="p">:</span> <span class="nb">float</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">float</span><span class="p">:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Calculate negative log10-pval from Z-score.</span>
+
+<span class="sd">        Args:</span>
+<span class="sd">            z2 (float): Z-score squared.</span>
+
+<span class="sd">        Returns:</span>
+<span class="sd">            float: log10-pval.</span>
+
+<span class="sd">        Examples:</span>
+<span class="sd">            &gt;&gt;&gt; SummaryStatisticsQC._calculate_logpval(1.0)</span>
+<span class="sd">            0.49851554582799334</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">logpval</span> <span class="o">=</span> <span class="o">-</span><span class="n">np</span><span class="o">.</span><span class="n">log10</span><span class="p">(</span><span class="n">sc</span><span class="o">.</span><span class="n">stats</span><span class="o">.</span><span class="n">chi2</span><span class="o">.</span><span class="n">sf</span><span class="p">((</span><span class="n">z2</span><span class="p">),</span> <span class="mi">1</span><span class="p">))</span>
+        <span class="k">return</span> <span class="nb">float</span><span class="p">(</span><span class="n">logpval</span><span class="p">)</span>
+
+    <span class="nd">@staticmethod</span>
+    <span class="k">def</span> <span class="nf">sumstat_qc_pz_check</span><span class="p">(</span>
+        <span class="n">gwas_for_qc</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">,</span>
+        <span class="n">limit</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">10_000_000</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">DataFrame</span><span class="p">:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;The PZ check for QC of GWAS summary statstics. It runs linear regression between reported p-values and p-values infered from z-scores.</span>
+
+<span class="sd">        Args:</span>
+<span class="sd">            gwas_for_qc (SummaryStatistics): The instance of the SummaryStatistics class.</span>
+<span class="sd">            limit (int): The limit for the number of variants to be used for the estimation.</span>
+
+<span class="sd">        Returns:</span>
+<span class="sd">            DataFrame: PySpark DataFrame with the results of the linear regression for each study.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">gwas_df</span> <span class="o">=</span> <span class="n">gwas_for_qc</span><span class="o">.</span><span class="n">_df</span>
+
+        <span class="n">calculate_logpval_udf</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">udf</span><span class="p">(</span>
+            <span class="n">SummaryStatisticsQC</span><span class="o">.</span><span class="n">_calculate_logpval</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">DoubleType</span><span class="p">()</span>
+        <span class="p">)</span>
+
+        <span class="n">window</span> <span class="o">=</span> <span class="n">Window</span><span class="o">.</span><span class="n">partitionBy</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">orderBy</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">)</span>
+
+        <span class="n">gwas_df</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="n">gwas_df</span><span class="o">.</span><span class="n">withColumn</span><span class="p">(</span><span class="s2">&quot;row_num&quot;</span><span class="p">,</span> <span class="n">row_number</span><span class="p">()</span><span class="o">.</span><span class="n">over</span><span class="p">(</span><span class="n">window</span><span class="p">))</span>
+            <span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;row_num&quot;</span><span class="p">)</span> <span class="o">&lt;=</span> <span class="n">limit</span><span class="p">)</span>
+            <span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="s2">&quot;row_num&quot;</span><span class="p">)</span>
+        <span class="p">)</span>
+
+        <span class="n">qc_c</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="n">gwas_df</span><span class="o">.</span><span class="n">withColumn</span><span class="p">(</span><span class="s2">&quot;zscore&quot;</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;beta&quot;</span><span class="p">)</span> <span class="o">/</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;standardError&quot;</span><span class="p">))</span>
+            <span class="o">.</span><span class="n">withColumn</span><span class="p">(</span><span class="s2">&quot;new_logpval&quot;</span><span class="p">,</span> <span class="n">calculate_logpval_udf</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;zscore&quot;</span><span class="p">)</span> <span class="o">**</span> <span class="mi">2</span><span class="p">))</span>
+            <span class="o">.</span><span class="n">withColumn</span><span class="p">(</span><span class="s2">&quot;log_mantissa&quot;</span><span class="p">,</span> <span class="n">log10</span><span class="p">(</span><span class="s2">&quot;pValueMantissa&quot;</span><span class="p">))</span>
+            <span class="o">.</span><span class="n">withColumn</span><span class="p">(</span>
+                <span class="s2">&quot;diffpval&quot;</span><span class="p">,</span>
+                <span class="o">-</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;log_mantissa&quot;</span><span class="p">)</span> <span class="o">-</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;pValueExponent&quot;</span><span class="p">)</span> <span class="o">-</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;new_logpval&quot;</span><span class="p">),</span>
+            <span class="p">)</span>
+            <span class="o">.</span><span class="n">groupBy</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">)</span>
+            <span class="o">.</span><span class="n">agg</span><span class="p">(</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="s2">&quot;diffpval&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;mean_diff_pz&quot;</span><span class="p">),</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">stddev</span><span class="p">(</span><span class="s2">&quot;diffpval&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;se_diff_pz&quot;</span><span class="p">),</span>
+            <span class="p">)</span>
+            <span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">,</span> <span class="s2">&quot;mean_diff_pz&quot;</span><span class="p">,</span> <span class="s2">&quot;se_diff_pz&quot;</span><span class="p">)</span>
+        <span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">qc_c</span>
+
+    <span class="nd">@staticmethod</span>
+    <span class="k">def</span> <span class="nf">sumstat_n_eff_check</span><span class="p">(</span>
+        <span class="n">gwas_for_qc</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">,</span>
+        <span class="n">n_total</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">100_000</span><span class="p">,</span>
+        <span class="n">limit</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">10_000_000</span><span class="p">,</span>
+        <span class="n">min_count</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">100</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">DataFrame</span><span class="p">:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;The effective sample size check for QC of GWAS summary statstics.</span>
+
+<span class="sd">        It estiamtes the ratio between effective sample size and the expected one and checks it&#39;s distribution.</span>
+<span class="sd">        It is possible to conduct only if the effective allele frequency is provided in the study.</span>
+<span class="sd">        The median rartio is always close to 1, but standard error could be inflated.</span>
+
+<span class="sd">        Args:</span>
+<span class="sd">            gwas_for_qc (SummaryStatistics): The instance of the SummaryStatistics class.</span>
+<span class="sd">            n_total (int): The reported sample size of the study. The QC metrics is robust toward the sample size.</span>
+<span class="sd">            limit (int): The limit for the number of variants to be used for the estimation.</span>
+<span class="sd">            min_count (int): The minimum number of variants to be used for the estimation.</span>
+
+<span class="sd">        Returns:</span>
+<span class="sd">            DataFrame: PySpark DataFrame with the effective sample size ratio for each study.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">gwas_df</span> <span class="o">=</span> <span class="n">gwas_for_qc</span><span class="o">.</span><span class="n">_df</span>
+
+        <span class="n">gwas_df</span> <span class="o">=</span> <span class="n">gwas_df</span><span class="o">.</span><span class="n">dropna</span><span class="p">(</span><span class="n">subset</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;effectAlleleFrequencyFromSource&quot;</span><span class="p">])</span>
+
+        <span class="n">counts_df</span> <span class="o">=</span> <span class="n">gwas_df</span><span class="o">.</span><span class="n">groupBy</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">count</span><span class="p">()</span>
+
+        <span class="c1"># Join the original DataFrame with the counts DataFrame</span>
+        <span class="n">df_with_counts</span> <span class="o">=</span> <span class="n">gwas_df</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">counts_df</span><span class="p">,</span> <span class="n">on</span><span class="o">=</span><span class="s2">&quot;studyId&quot;</span><span class="p">)</span>
+
+        <span class="c1"># Filter the DataFrame to keep only the groups with count greater than or equal to min_count</span>
+        <span class="n">filtered_df</span> <span class="o">=</span> <span class="n">df_with_counts</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;count&quot;</span><span class="p">)</span> <span class="o">&gt;=</span> <span class="n">min_count</span><span class="p">)</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="s2">&quot;count&quot;</span><span class="p">)</span>
+
+        <span class="n">window</span> <span class="o">=</span> <span class="n">Window</span><span class="o">.</span><span class="n">partitionBy</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">orderBy</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">)</span>
+        <span class="n">gwas_df</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="n">filtered_df</span><span class="o">.</span><span class="n">withColumn</span><span class="p">(</span><span class="s2">&quot;row_num&quot;</span><span class="p">,</span> <span class="n">row_number</span><span class="p">()</span><span class="o">.</span><span class="n">over</span><span class="p">(</span><span class="n">window</span><span class="p">))</span>
+            <span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;row_num&quot;</span><span class="p">)</span> <span class="o">&lt;=</span> <span class="n">limit</span><span class="p">)</span>
+            <span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="s2">&quot;row_num&quot;</span><span class="p">)</span>
+        <span class="p">)</span>
+
+        <span class="n">gwas_df</span> <span class="o">=</span> <span class="n">gwas_df</span><span class="o">.</span><span class="n">withColumn</span><span class="p">(</span>
+            <span class="s2">&quot;var_af&quot;</span><span class="p">,</span>
+            <span class="mi">2</span>
+            <span class="o">*</span> <span class="p">(</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;effectAlleleFrequencyFromSource&quot;</span><span class="p">)</span>
+                <span class="o">*</span> <span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;effectAlleleFrequencyFromSource&quot;</span><span class="p">))</span>
+            <span class="p">),</span>
+        <span class="p">)</span><span class="o">.</span><span class="n">withColumn</span><span class="p">(</span>
+            <span class="s2">&quot;pheno_var&quot;</span><span class="p">,</span>
+            <span class="p">((</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;standardError&quot;</span><span class="p">)</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span> <span class="o">*</span> <span class="n">n_total</span> <span class="o">*</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;var_af&quot;</span><span class="p">))</span>
+            <span class="o">+</span> <span class="p">((</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;beta&quot;</span><span class="p">)</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span> <span class="o">*</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;var_af&quot;</span><span class="p">)),</span>
+        <span class="p">)</span>
+
+        <span class="n">window</span> <span class="o">=</span> <span class="n">Window</span><span class="o">.</span><span class="n">partitionBy</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">orderBy</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">)</span>
+
+        <span class="c1"># Calculate the median of &#39;pheno_var&#39; for each &#39;studyId&#39; and add it as a new column</span>
+        <span class="n">gwas_df</span> <span class="o">=</span> <span class="n">gwas_df</span><span class="o">.</span><span class="n">withColumn</span><span class="p">(</span>
+            <span class="s2">&quot;pheno_median&quot;</span><span class="p">,</span> <span class="n">expr</span><span class="p">(</span><span class="s2">&quot;percentile_approx(pheno_var, 0.5)&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">over</span><span class="p">(</span><span class="n">window</span><span class="p">)</span>
+        <span class="p">)</span>
+
+        <span class="n">gwas_df</span> <span class="o">=</span> <span class="n">gwas_df</span><span class="o">.</span><span class="n">withColumn</span><span class="p">(</span>
+            <span class="s2">&quot;N_hat_ratio&quot;</span><span class="p">,</span>
+            <span class="p">(</span>
+                <span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;pheno_median&quot;</span><span class="p">)</span> <span class="o">-</span> <span class="p">((</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;beta&quot;</span><span class="p">)</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span> <span class="o">*</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;var_af&quot;</span><span class="p">)))</span>
+                <span class="o">/</span> <span class="p">((</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;standardError&quot;</span><span class="p">)</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span> <span class="o">*</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;var_af&quot;</span><span class="p">)</span> <span class="o">*</span> <span class="n">n_total</span><span class="p">)</span>
+            <span class="p">),</span>
+        <span class="p">)</span>
+
+        <span class="n">qc_c</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="n">gwas_df</span><span class="o">.</span><span class="n">groupBy</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">)</span>
+            <span class="o">.</span><span class="n">agg</span><span class="p">(</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">stddev</span><span class="p">(</span><span class="s2">&quot;N_hat_ratio&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;se_N&quot;</span><span class="p">),</span>
+            <span class="p">)</span>
+            <span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">,</span> <span class="s2">&quot;se_N&quot;</span><span class="p">)</span>
+        <span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">qc_c</span>
+
+    <span class="nd">@staticmethod</span>
+    <span class="k">def</span> <span class="nf">gc_lambda_check</span><span class="p">(</span>
+        <span class="n">gwas_for_qc</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">,</span>
+        <span class="n">limit</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">10_000_000</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">DataFrame</span><span class="p">:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;The genomic control lambda check for QC of GWAS summary statstics.</span>
+
+<span class="sd">        Args:</span>
+<span class="sd">            gwas_for_qc (SummaryStatistics): The instance of the SummaryStatistics class.</span>
+<span class="sd">            limit (int): The limit for the number of variants to be used for the estimation.</span>
+
+<span class="sd">        Returns:</span>
+<span class="sd">            DataFrame: PySpark DataFrame with the genomic control lambda for each study.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">gwas_df</span> <span class="o">=</span> <span class="n">gwas_for_qc</span><span class="o">.</span><span class="n">_df</span>
+        <span class="n">window</span> <span class="o">=</span> <span class="n">Window</span><span class="o">.</span><span class="n">partitionBy</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">orderBy</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">)</span>
+        <span class="n">gwas_df</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="n">gwas_df</span><span class="o">.</span><span class="n">withColumn</span><span class="p">(</span><span class="s2">&quot;row_num&quot;</span><span class="p">,</span> <span class="n">row_number</span><span class="p">()</span><span class="o">.</span><span class="n">over</span><span class="p">(</span><span class="n">window</span><span class="p">))</span>
+            <span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;row_num&quot;</span><span class="p">)</span> <span class="o">&lt;=</span> <span class="n">limit</span><span class="p">)</span>
+            <span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="s2">&quot;row_num&quot;</span><span class="p">)</span>
+        <span class="p">)</span>
+
+        <span class="n">qc_c</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="n">gwas_df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">,</span> <span class="s2">&quot;beta&quot;</span><span class="p">,</span> <span class="s2">&quot;standardError&quot;</span><span class="p">)</span>
+            <span class="o">.</span><span class="n">withColumn</span><span class="p">(</span><span class="s2">&quot;Z2&quot;</span><span class="p">,</span> <span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;beta&quot;</span><span class="p">)</span> <span class="o">/</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;standardError&quot;</span><span class="p">))</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span>
+            <span class="o">.</span><span class="n">groupBy</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">)</span>
+            <span class="o">.</span><span class="n">agg</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">expr</span><span class="p">(</span><span class="s2">&quot;percentile_approx(Z2, 0.5)&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;gc_lambda&quot;</span><span class="p">))</span>
+            <span class="o">.</span><span class="n">withColumn</span><span class="p">(</span><span class="s2">&quot;gc_lambda&quot;</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;gc_lambda&quot;</span><span class="p">)</span> <span class="o">/</span> <span class="n">chi2</span><span class="o">.</span><span class="n">ppf</span><span class="p">(</span><span class="mf">0.5</span><span class="p">,</span> <span class="n">df</span><span class="o">=</span><span class="mi">1</span><span class="p">))</span>
+            <span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">,</span> <span class="s2">&quot;gc_lambda&quot;</span><span class="p">)</span>
+        <span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">qc_c</span>
+
+    <span class="nd">@staticmethod</span>
+    <span class="k">def</span> <span class="nf">number_of_snps</span><span class="p">(</span>
+        <span class="n">gwas_for_qc</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">,</span> <span class="n">pval_threhod</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">5e-8</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">DataFrame</span><span class="p">:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;The function caluates number of SNPs and number of SNPs with p-value less than 5e-8.</span>
+
+<span class="sd">        Args:</span>
+<span class="sd">            gwas_for_qc (SummaryStatistics): The instance of the SummaryStatistics class.</span>
+<span class="sd">            pval_threhod (float): The threshold for the p-value.</span>
+
+<span class="sd">        Returns:</span>
+<span class="sd">            DataFrame: PySpark DataFrame with the number of SNPs and number of SNPs with p-value less than threshold.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">gwas_df</span> <span class="o">=</span> <span class="n">gwas_for_qc</span><span class="o">.</span><span class="n">_df</span>
+
+        <span class="n">snp_counts</span> <span class="o">=</span> <span class="n">gwas_df</span><span class="o">.</span><span class="n">groupBy</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">agg</span><span class="p">(</span>
+            <span class="n">f</span><span class="o">.</span><span class="n">count</span><span class="p">(</span><span class="s2">&quot;*&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;n_variants&quot;</span><span class="p">),</span>
+            <span class="n">f</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span>
+                <span class="p">(</span>
+                    <span class="n">f</span><span class="o">.</span><span class="n">log10</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;pValueMantissa&quot;</span><span class="p">))</span> <span class="o">+</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;pValueExponent&quot;</span><span class="p">)</span>
+                    <span class="o">&lt;=</span> <span class="n">np</span><span class="o">.</span><span class="n">log10</span><span class="p">(</span><span class="n">pval_threhod</span><span class="p">)</span>
+                <span class="p">)</span><span class="o">.</span><span class="n">cast</span><span class="p">(</span><span class="s2">&quot;int&quot;</span><span class="p">)</span>
+            <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;n_variants_sig&quot;</span><span class="p">),</span>
+        <span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">snp_counts</span>
+
+    <span class="nd">@staticmethod</span>
+    <span class="k">def</span> <span class="nf">get_quality_control_metrics</span><span class="p">(</span>
+        <span class="n">gwas</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">,</span>
+        <span class="n">limit</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">100_000_000</span><span class="p">,</span>
+        <span class="n">min_count</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">100_000</span><span class="p">,</span>
+        <span class="n">n_total</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">100_000</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">DataFrame</span><span class="p">:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;The function calculates the quality control metrics for the summary statistics.</span>
+
+<span class="sd">        Args:</span>
+<span class="sd">            gwas (SummaryStatistics): The instance of the SummaryStatistics class.</span>
+<span class="sd">            limit (int): The limit for the number of variants to be used for the estimation.</span>
+<span class="sd">            min_count (int): The minimum number of variants to be used for the estimation.</span>
+<span class="sd">            n_total (int): The total sample size.</span>
+
+<span class="sd">        Returns:</span>
+<span class="sd">            DataFrame: PySpark DataFrame with the quality control metrics for the summary statistics.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">qc1</span> <span class="o">=</span> <span class="n">SummaryStatisticsQC</span><span class="o">.</span><span class="n">sumstat_qc_beta_check</span><span class="p">(</span><span class="n">gwas_for_qc</span><span class="o">=</span><span class="n">gwas</span><span class="p">)</span>
+        <span class="n">qc2</span> <span class="o">=</span> <span class="n">SummaryStatisticsQC</span><span class="o">.</span><span class="n">sumstat_qc_pz_check</span><span class="p">(</span><span class="n">gwas_for_qc</span><span class="o">=</span><span class="n">gwas</span><span class="p">,</span> <span class="n">limit</span><span class="o">=</span><span class="n">limit</span><span class="p">)</span>
+        <span class="n">qc3</span> <span class="o">=</span> <span class="n">SummaryStatisticsQC</span><span class="o">.</span><span class="n">sumstat_n_eff_check</span><span class="p">(</span>
+            <span class="n">gwas_for_qc</span><span class="o">=</span><span class="n">gwas</span><span class="p">,</span> <span class="n">n_total</span><span class="o">=</span><span class="n">n_total</span><span class="p">,</span> <span class="n">limit</span><span class="o">=</span><span class="n">limit</span><span class="p">,</span> <span class="n">min_count</span><span class="o">=</span><span class="n">min_count</span>
+        <span class="p">)</span>
+        <span class="n">qc4</span> <span class="o">=</span> <span class="n">SummaryStatisticsQC</span><span class="o">.</span><span class="n">gc_lambda_check</span><span class="p">(</span><span class="n">gwas_for_qc</span><span class="o">=</span><span class="n">gwas</span><span class="p">,</span> <span class="n">limit</span><span class="o">=</span><span class="n">limit</span><span class="p">)</span>
+        <span class="n">qc5</span> <span class="o">=</span> <span class="n">SummaryStatisticsQC</span><span class="o">.</span><span class="n">number_of_snps</span><span class="p">(</span><span class="n">gwas_for_qc</span><span class="o">=</span><span class="n">gwas</span><span class="p">)</span>
+        <span class="n">df</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="n">qc1</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">qc2</span><span class="p">,</span> <span class="n">on</span><span class="o">=</span><span class="s2">&quot;studyId&quot;</span><span class="p">,</span> <span class="n">how</span><span class="o">=</span><span class="s2">&quot;outer&quot;</span><span class="p">)</span>
+            <span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">qc3</span><span class="p">,</span> <span class="n">on</span><span class="o">=</span><span class="s2">&quot;studyId&quot;</span><span class="p">,</span> <span class="n">how</span><span class="o">=</span><span class="s2">&quot;outer&quot;</span><span class="p">)</span>
+            <span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">qc4</span><span class="p">,</span> <span class="n">on</span><span class="o">=</span><span class="s2">&quot;studyId&quot;</span><span class="p">,</span> <span class="n">how</span><span class="o">=</span><span class="s2">&quot;outer&quot;</span><span class="p">)</span>
+            <span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">qc5</span><span class="p">,</span> <span class="n">on</span><span class="o">=</span><span class="s2">&quot;studyId&quot;</span><span class="p">,</span> <span class="n">how</span><span class="o">=</span><span class="s2">&quot;outer&quot;</span><span class="p">)</span>
+        <span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">df</span>
+</code></pre></div></td></tr></table></div>
+            </details>
+
+  
+
+  <div class="doc doc-children">
+
+
+
+
+
+
+
+
+
+
+<div class="doc doc-object doc-function">
+
+
+
+<h3 id="gentropy.method.sumstat_quality_controls.SummaryStatisticsQC.gc_lambda_check" class="doc doc-heading">
+          <code class="highlight language-python"><span class="n">gc_lambda_check</span><span class="p">(</span><span class="n">gwas_for_qc</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">,</span> <span class="n">limit</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">10000000</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">DataFrame</span></code>
+  
+  <span class="doc doc-labels">
+      <small class="doc doc-label doc-label-staticmethod"><code>staticmethod</code></small>
+  </span>
+
+<a href="#gentropy.method.sumstat_quality_controls.SummaryStatisticsQC.gc_lambda_check" class="headerlink" title="Permanent link">&para;</a></h3>
+
+
+  <div class="doc doc-contents ">
+  
+      <p>The genomic control lambda check for QC of GWAS summary statstics.</p>
+
+
+
+  <p><span class="doc-section-title">Parameters:</span></p>
+  <table>
+    <thead>
+      <tr>
+        <th>Name</th>
+        <th>Type</th>
+        <th>Description</th>
+        <th>Default</th>
+      </tr>
+    </thead>
+    <tbody>
+        <tr class="doc-section-item">
+          <td><code>gwas_for_qc</code></td>
+          <td>
+                <code><a class="autorefs autorefs-internal" title="gentropy.dataset.summary_statistics.SummaryStatistics" href="../../datasets/summary_statistics/#gentropy.dataset.summary_statistics.SummaryStatistics">SummaryStatistics</a></code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>The instance of the SummaryStatistics class.</p>
+            </div>
+          </td>
+          <td>
+              <em>required</em>
+          </td>
+        </tr>
+        <tr class="doc-section-item">
+          <td><code>limit</code></td>
+          <td>
+                <code>int</code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>The limit for the number of variants to be used for the estimation.</p>
+            </div>
+          </td>
+          <td>
+                <code>10000000</code>
+          </td>
+        </tr>
+    </tbody>
+  </table>
+
+
+
+  <p><span class="doc-section-title">Returns:</span></p>
+  <table>
+    <thead>
+      <tr>
+<th>Name</th>        <th>Type</th>
+        <th>Description</th>
+      </tr>
+    </thead>
+    <tbody>
+        <tr class="doc-section-item">
+<td><code>DataFrame</code></td>          <td>
+                <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>PySpark DataFrame with the genomic control lambda for each study.</p>
+            </div>
+          </td>
+        </tr>
+    </tbody>
+  </table>
+
+          <details class="quote">
+            <summary>Source code in <code>src/gentropy/method/sumstat_quality_controls.py</code></summary>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">193</span>
+<span class="normal">194</span>
+<span class="normal">195</span>
+<span class="normal">196</span>
+<span class="normal">197</span>
+<span class="normal">198</span>
+<span class="normal">199</span>
+<span class="normal">200</span>
+<span class="normal">201</span>
+<span class="normal">202</span>
+<span class="normal">203</span>
+<span class="normal">204</span>
+<span class="normal">205</span>
+<span class="normal">206</span>
+<span class="normal">207</span>
+<span class="normal">208</span>
+<span class="normal">209</span>
+<span class="normal">210</span>
+<span class="normal">211</span>
+<span class="normal">212</span>
+<span class="normal">213</span>
+<span class="normal">214</span>
+<span class="normal">215</span>
+<span class="normal">216</span>
+<span class="normal">217</span>
+<span class="normal">218</span>
+<span class="normal">219</span>
+<span class="normal">220</span>
+<span class="normal">221</span>
+<span class="normal">222</span>
+<span class="normal">223</span>
+<span class="normal">224</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@staticmethod</span>
+<span class="k">def</span> <span class="nf">gc_lambda_check</span><span class="p">(</span>
+    <span class="n">gwas_for_qc</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">,</span>
+    <span class="n">limit</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">10_000_000</span><span class="p">,</span>
+<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">DataFrame</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;The genomic control lambda check for QC of GWAS summary statstics.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        gwas_for_qc (SummaryStatistics): The instance of the SummaryStatistics class.</span>
+<span class="sd">        limit (int): The limit for the number of variants to be used for the estimation.</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">        DataFrame: PySpark DataFrame with the genomic control lambda for each study.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">gwas_df</span> <span class="o">=</span> <span class="n">gwas_for_qc</span><span class="o">.</span><span class="n">_df</span>
+    <span class="n">window</span> <span class="o">=</span> <span class="n">Window</span><span class="o">.</span><span class="n">partitionBy</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">orderBy</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">)</span>
+    <span class="n">gwas_df</span> <span class="o">=</span> <span class="p">(</span>
+        <span class="n">gwas_df</span><span class="o">.</span><span class="n">withColumn</span><span class="p">(</span><span class="s2">&quot;row_num&quot;</span><span class="p">,</span> <span class="n">row_number</span><span class="p">()</span><span class="o">.</span><span class="n">over</span><span class="p">(</span><span class="n">window</span><span class="p">))</span>
+        <span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;row_num&quot;</span><span class="p">)</span> <span class="o">&lt;=</span> <span class="n">limit</span><span class="p">)</span>
+        <span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="s2">&quot;row_num&quot;</span><span class="p">)</span>
+    <span class="p">)</span>
+
+    <span class="n">qc_c</span> <span class="o">=</span> <span class="p">(</span>
+        <span class="n">gwas_df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">,</span> <span class="s2">&quot;beta&quot;</span><span class="p">,</span> <span class="s2">&quot;standardError&quot;</span><span class="p">)</span>
+        <span class="o">.</span><span class="n">withColumn</span><span class="p">(</span><span class="s2">&quot;Z2&quot;</span><span class="p">,</span> <span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;beta&quot;</span><span class="p">)</span> <span class="o">/</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;standardError&quot;</span><span class="p">))</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span>
+        <span class="o">.</span><span class="n">groupBy</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">)</span>
+        <span class="o">.</span><span class="n">agg</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">expr</span><span class="p">(</span><span class="s2">&quot;percentile_approx(Z2, 0.5)&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;gc_lambda&quot;</span><span class="p">))</span>
+        <span class="o">.</span><span class="n">withColumn</span><span class="p">(</span><span class="s2">&quot;gc_lambda&quot;</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;gc_lambda&quot;</span><span class="p">)</span> <span class="o">/</span> <span class="n">chi2</span><span class="o">.</span><span class="n">ppf</span><span class="p">(</span><span class="mf">0.5</span><span class="p">,</span> <span class="n">df</span><span class="o">=</span><span class="mi">1</span><span class="p">))</span>
+        <span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">,</span> <span class="s2">&quot;gc_lambda&quot;</span><span class="p">)</span>
+    <span class="p">)</span>
+
+    <span class="k">return</span> <span class="n">qc_c</span>
+</code></pre></div></td></tr></table></div>
+          </details>
+  </div>
+
+</div>
+
+
+<div class="doc doc-object doc-function">
+
+
+
+<h3 id="gentropy.method.sumstat_quality_controls.SummaryStatisticsQC.get_quality_control_metrics" class="doc doc-heading">
+          <code class="highlight language-python"><span class="n">get_quality_control_metrics</span><span class="p">(</span><span class="n">gwas</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">,</span> <span class="n">limit</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">100000000</span><span class="p">,</span> <span class="n">min_count</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">100000</span><span class="p">,</span> <span class="n">n_total</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">100000</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">DataFrame</span></code>
+  
+  <span class="doc doc-labels">
+      <small class="doc doc-label doc-label-staticmethod"><code>staticmethod</code></small>
+  </span>
+
+<a href="#gentropy.method.sumstat_quality_controls.SummaryStatisticsQC.get_quality_control_metrics" class="headerlink" title="Permanent link">&para;</a></h3>
+
+
+  <div class="doc doc-contents ">
+  
+      <p>The function calculates the quality control metrics for the summary statistics.</p>
+
+
+
+  <p><span class="doc-section-title">Parameters:</span></p>
+  <table>
+    <thead>
+      <tr>
+        <th>Name</th>
+        <th>Type</th>
+        <th>Description</th>
+        <th>Default</th>
+      </tr>
+    </thead>
+    <tbody>
+        <tr class="doc-section-item">
+          <td><code>gwas</code></td>
+          <td>
+                <code><a class="autorefs autorefs-internal" title="gentropy.dataset.summary_statistics.SummaryStatistics" href="../../datasets/summary_statistics/#gentropy.dataset.summary_statistics.SummaryStatistics">SummaryStatistics</a></code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>The instance of the SummaryStatistics class.</p>
+            </div>
+          </td>
+          <td>
+              <em>required</em>
+          </td>
+        </tr>
+        <tr class="doc-section-item">
+          <td><code>limit</code></td>
+          <td>
+                <code>int</code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>The limit for the number of variants to be used for the estimation.</p>
+            </div>
+          </td>
+          <td>
+                <code>100000000</code>
+          </td>
+        </tr>
+        <tr class="doc-section-item">
+          <td><code>min_count</code></td>
+          <td>
+                <code>int</code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>The minimum number of variants to be used for the estimation.</p>
+            </div>
+          </td>
+          <td>
+                <code>100000</code>
+          </td>
+        </tr>
+        <tr class="doc-section-item">
+          <td><code>n_total</code></td>
+          <td>
+                <code>int</code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>The total sample size.</p>
+            </div>
+          </td>
+          <td>
+                <code>100000</code>
+          </td>
+        </tr>
+    </tbody>
+  </table>
+
+
+
+  <p><span class="doc-section-title">Returns:</span></p>
+  <table>
+    <thead>
+      <tr>
+<th>Name</th>        <th>Type</th>
+        <th>Description</th>
+      </tr>
+    </thead>
+    <tbody>
+        <tr class="doc-section-item">
+<td><code>DataFrame</code></td>          <td>
+                <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>PySpark DataFrame with the quality control metrics for the summary statistics.</p>
+            </div>
+          </td>
+        </tr>
+    </tbody>
+  </table>
+
+          <details class="quote">
+            <summary>Source code in <code>src/gentropy/method/sumstat_quality_controls.py</code></summary>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">253</span>
+<span class="normal">254</span>
+<span class="normal">255</span>
+<span class="normal">256</span>
+<span class="normal">257</span>
+<span class="normal">258</span>
+<span class="normal">259</span>
+<span class="normal">260</span>
+<span class="normal">261</span>
+<span class="normal">262</span>
+<span class="normal">263</span>
+<span class="normal">264</span>
+<span class="normal">265</span>
+<span class="normal">266</span>
+<span class="normal">267</span>
+<span class="normal">268</span>
+<span class="normal">269</span>
+<span class="normal">270</span>
+<span class="normal">271</span>
+<span class="normal">272</span>
+<span class="normal">273</span>
+<span class="normal">274</span>
+<span class="normal">275</span>
+<span class="normal">276</span>
+<span class="normal">277</span>
+<span class="normal">278</span>
+<span class="normal">279</span>
+<span class="normal">280</span>
+<span class="normal">281</span>
+<span class="normal">282</span>
+<span class="normal">283</span>
+<span class="normal">284</span>
+<span class="normal">285</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@staticmethod</span>
+<span class="k">def</span> <span class="nf">get_quality_control_metrics</span><span class="p">(</span>
+    <span class="n">gwas</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">,</span>
+    <span class="n">limit</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">100_000_000</span><span class="p">,</span>
+    <span class="n">min_count</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">100_000</span><span class="p">,</span>
+    <span class="n">n_total</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">100_000</span><span class="p">,</span>
+<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">DataFrame</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;The function calculates the quality control metrics for the summary statistics.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        gwas (SummaryStatistics): The instance of the SummaryStatistics class.</span>
+<span class="sd">        limit (int): The limit for the number of variants to be used for the estimation.</span>
+<span class="sd">        min_count (int): The minimum number of variants to be used for the estimation.</span>
+<span class="sd">        n_total (int): The total sample size.</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">        DataFrame: PySpark DataFrame with the quality control metrics for the summary statistics.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">qc1</span> <span class="o">=</span> <span class="n">SummaryStatisticsQC</span><span class="o">.</span><span class="n">sumstat_qc_beta_check</span><span class="p">(</span><span class="n">gwas_for_qc</span><span class="o">=</span><span class="n">gwas</span><span class="p">)</span>
+    <span class="n">qc2</span> <span class="o">=</span> <span class="n">SummaryStatisticsQC</span><span class="o">.</span><span class="n">sumstat_qc_pz_check</span><span class="p">(</span><span class="n">gwas_for_qc</span><span class="o">=</span><span class="n">gwas</span><span class="p">,</span> <span class="n">limit</span><span class="o">=</span><span class="n">limit</span><span class="p">)</span>
+    <span class="n">qc3</span> <span class="o">=</span> <span class="n">SummaryStatisticsQC</span><span class="o">.</span><span class="n">sumstat_n_eff_check</span><span class="p">(</span>
+        <span class="n">gwas_for_qc</span><span class="o">=</span><span class="n">gwas</span><span class="p">,</span> <span class="n">n_total</span><span class="o">=</span><span class="n">n_total</span><span class="p">,</span> <span class="n">limit</span><span class="o">=</span><span class="n">limit</span><span class="p">,</span> <span class="n">min_count</span><span class="o">=</span><span class="n">min_count</span>
+    <span class="p">)</span>
+    <span class="n">qc4</span> <span class="o">=</span> <span class="n">SummaryStatisticsQC</span><span class="o">.</span><span class="n">gc_lambda_check</span><span class="p">(</span><span class="n">gwas_for_qc</span><span class="o">=</span><span class="n">gwas</span><span class="p">,</span> <span class="n">limit</span><span class="o">=</span><span class="n">limit</span><span class="p">)</span>
+    <span class="n">qc5</span> <span class="o">=</span> <span class="n">SummaryStatisticsQC</span><span class="o">.</span><span class="n">number_of_snps</span><span class="p">(</span><span class="n">gwas_for_qc</span><span class="o">=</span><span class="n">gwas</span><span class="p">)</span>
+    <span class="n">df</span> <span class="o">=</span> <span class="p">(</span>
+        <span class="n">qc1</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">qc2</span><span class="p">,</span> <span class="n">on</span><span class="o">=</span><span class="s2">&quot;studyId&quot;</span><span class="p">,</span> <span class="n">how</span><span class="o">=</span><span class="s2">&quot;outer&quot;</span><span class="p">)</span>
+        <span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">qc3</span><span class="p">,</span> <span class="n">on</span><span class="o">=</span><span class="s2">&quot;studyId&quot;</span><span class="p">,</span> <span class="n">how</span><span class="o">=</span><span class="s2">&quot;outer&quot;</span><span class="p">)</span>
+        <span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">qc4</span><span class="p">,</span> <span class="n">on</span><span class="o">=</span><span class="s2">&quot;studyId&quot;</span><span class="p">,</span> <span class="n">how</span><span class="o">=</span><span class="s2">&quot;outer&quot;</span><span class="p">)</span>
+        <span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">qc5</span><span class="p">,</span> <span class="n">on</span><span class="o">=</span><span class="s2">&quot;studyId&quot;</span><span class="p">,</span> <span class="n">how</span><span class="o">=</span><span class="s2">&quot;outer&quot;</span><span class="p">)</span>
+    <span class="p">)</span>
+
+    <span class="k">return</span> <span class="n">df</span>
+</code></pre></div></td></tr></table></div>
+          </details>
+  </div>
+
+</div>
+
+
+<div class="doc doc-object doc-function">
+
+
+
+<h3 id="gentropy.method.sumstat_quality_controls.SummaryStatisticsQC.number_of_snps" class="doc doc-heading">
+          <code class="highlight language-python"><span class="n">number_of_snps</span><span class="p">(</span><span class="n">gwas_for_qc</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">,</span> <span class="n">pval_threhod</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">5e-08</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">DataFrame</span></code>
+  
+  <span class="doc doc-labels">
+      <small class="doc doc-label doc-label-staticmethod"><code>staticmethod</code></small>
+  </span>
+
+<a href="#gentropy.method.sumstat_quality_controls.SummaryStatisticsQC.number_of_snps" class="headerlink" title="Permanent link">&para;</a></h3>
+
+
+  <div class="doc doc-contents ">
+  
+      <p>The function caluates number of SNPs and number of SNPs with p-value less than 5e-8.</p>
+
+
+
+  <p><span class="doc-section-title">Parameters:</span></p>
+  <table>
+    <thead>
+      <tr>
+        <th>Name</th>
+        <th>Type</th>
+        <th>Description</th>
+        <th>Default</th>
+      </tr>
+    </thead>
+    <tbody>
+        <tr class="doc-section-item">
+          <td><code>gwas_for_qc</code></td>
+          <td>
+                <code><a class="autorefs autorefs-internal" title="gentropy.dataset.summary_statistics.SummaryStatistics" href="../../datasets/summary_statistics/#gentropy.dataset.summary_statistics.SummaryStatistics">SummaryStatistics</a></code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>The instance of the SummaryStatistics class.</p>
+            </div>
+          </td>
+          <td>
+              <em>required</em>
+          </td>
+        </tr>
+        <tr class="doc-section-item">
+          <td><code>pval_threhod</code></td>
+          <td>
+                <code>float</code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>The threshold for the p-value.</p>
+            </div>
+          </td>
+          <td>
+                <code>5e-08</code>
+          </td>
+        </tr>
+    </tbody>
+  </table>
+
+
+
+  <p><span class="doc-section-title">Returns:</span></p>
+  <table>
+    <thead>
+      <tr>
+<th>Name</th>        <th>Type</th>
+        <th>Description</th>
+      </tr>
+    </thead>
+    <tbody>
+        <tr class="doc-section-item">
+<td><code>DataFrame</code></td>          <td>
+                <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>PySpark DataFrame with the number of SNPs and number of SNPs with p-value less than threshold.</p>
+            </div>
+          </td>
+        </tr>
+    </tbody>
+  </table>
+
+          <details class="quote">
+            <summary>Source code in <code>src/gentropy/method/sumstat_quality_controls.py</code></summary>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">226</span>
+<span class="normal">227</span>
+<span class="normal">228</span>
+<span class="normal">229</span>
+<span class="normal">230</span>
+<span class="normal">231</span>
+<span class="normal">232</span>
+<span class="normal">233</span>
+<span class="normal">234</span>
+<span class="normal">235</span>
+<span class="normal">236</span>
+<span class="normal">237</span>
+<span class="normal">238</span>
+<span class="normal">239</span>
+<span class="normal">240</span>
+<span class="normal">241</span>
+<span class="normal">242</span>
+<span class="normal">243</span>
+<span class="normal">244</span>
+<span class="normal">245</span>
+<span class="normal">246</span>
+<span class="normal">247</span>
+<span class="normal">248</span>
+<span class="normal">249</span>
+<span class="normal">250</span>
+<span class="normal">251</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@staticmethod</span>
+<span class="k">def</span> <span class="nf">number_of_snps</span><span class="p">(</span>
+    <span class="n">gwas_for_qc</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">,</span> <span class="n">pval_threhod</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">5e-8</span>
+<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">DataFrame</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;The function caluates number of SNPs and number of SNPs with p-value less than 5e-8.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        gwas_for_qc (SummaryStatistics): The instance of the SummaryStatistics class.</span>
+<span class="sd">        pval_threhod (float): The threshold for the p-value.</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">        DataFrame: PySpark DataFrame with the number of SNPs and number of SNPs with p-value less than threshold.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">gwas_df</span> <span class="o">=</span> <span class="n">gwas_for_qc</span><span class="o">.</span><span class="n">_df</span>
+
+    <span class="n">snp_counts</span> <span class="o">=</span> <span class="n">gwas_df</span><span class="o">.</span><span class="n">groupBy</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">agg</span><span class="p">(</span>
+        <span class="n">f</span><span class="o">.</span><span class="n">count</span><span class="p">(</span><span class="s2">&quot;*&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;n_variants&quot;</span><span class="p">),</span>
+        <span class="n">f</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span>
+            <span class="p">(</span>
+                <span class="n">f</span><span class="o">.</span><span class="n">log10</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;pValueMantissa&quot;</span><span class="p">))</span> <span class="o">+</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;pValueExponent&quot;</span><span class="p">)</span>
+                <span class="o">&lt;=</span> <span class="n">np</span><span class="o">.</span><span class="n">log10</span><span class="p">(</span><span class="n">pval_threhod</span><span class="p">)</span>
+            <span class="p">)</span><span class="o">.</span><span class="n">cast</span><span class="p">(</span><span class="s2">&quot;int&quot;</span><span class="p">)</span>
+        <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;n_variants_sig&quot;</span><span class="p">),</span>
+    <span class="p">)</span>
+
+    <span class="k">return</span> <span class="n">snp_counts</span>
+</code></pre></div></td></tr></table></div>
+          </details>
+  </div>
+
+</div>
+
+
+<div class="doc doc-object doc-function">
+
+
+
+<h3 id="gentropy.method.sumstat_quality_controls.SummaryStatisticsQC.sumstat_n_eff_check" class="doc doc-heading">
+          <code class="highlight language-python"><span class="n">sumstat_n_eff_check</span><span class="p">(</span><span class="n">gwas_for_qc</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">,</span> <span class="n">n_total</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">100000</span><span class="p">,</span> <span class="n">limit</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">10000000</span><span class="p">,</span> <span class="n">min_count</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">100</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">DataFrame</span></code>
+  
+  <span class="doc doc-labels">
+      <small class="doc doc-label doc-label-staticmethod"><code>staticmethod</code></small>
+  </span>
+
+<a href="#gentropy.method.sumstat_quality_controls.SummaryStatisticsQC.sumstat_n_eff_check" class="headerlink" title="Permanent link">&para;</a></h3>
+
+
+  <div class="doc doc-contents ">
+  
+      <p>The effective sample size check for QC of GWAS summary statstics.</p>
+<p>It estiamtes the ratio between effective sample size and the expected one and checks it's distribution.
+It is possible to conduct only if the effective allele frequency is provided in the study.
+The median rartio is always close to 1, but standard error could be inflated.</p>
+
+
+
+  <p><span class="doc-section-title">Parameters:</span></p>
+  <table>
+    <thead>
+      <tr>
+        <th>Name</th>
+        <th>Type</th>
+        <th>Description</th>
+        <th>Default</th>
+      </tr>
+    </thead>
+    <tbody>
+        <tr class="doc-section-item">
+          <td><code>gwas_for_qc</code></td>
+          <td>
+                <code><a class="autorefs autorefs-internal" title="gentropy.dataset.summary_statistics.SummaryStatistics" href="../../datasets/summary_statistics/#gentropy.dataset.summary_statistics.SummaryStatistics">SummaryStatistics</a></code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>The instance of the SummaryStatistics class.</p>
+            </div>
+          </td>
+          <td>
+              <em>required</em>
+          </td>
+        </tr>
+        <tr class="doc-section-item">
+          <td><code>n_total</code></td>
+          <td>
+                <code>int</code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>The reported sample size of the study. The QC metrics is robust toward the sample size.</p>
+            </div>
+          </td>
+          <td>
+                <code>100000</code>
+          </td>
+        </tr>
+        <tr class="doc-section-item">
+          <td><code>limit</code></td>
+          <td>
+                <code>int</code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>The limit for the number of variants to be used for the estimation.</p>
+            </div>
+          </td>
+          <td>
+                <code>10000000</code>
+          </td>
+        </tr>
+        <tr class="doc-section-item">
+          <td><code>min_count</code></td>
+          <td>
+                <code>int</code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>The minimum number of variants to be used for the estimation.</p>
+            </div>
+          </td>
+          <td>
+                <code>100</code>
+          </td>
+        </tr>
+    </tbody>
+  </table>
+
+
+
+  <p><span class="doc-section-title">Returns:</span></p>
+  <table>
+    <thead>
+      <tr>
+<th>Name</th>        <th>Type</th>
+        <th>Description</th>
+      </tr>
+    </thead>
+    <tbody>
+        <tr class="doc-section-item">
+<td><code>DataFrame</code></td>          <td>
+                <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>PySpark DataFrame with the effective sample size ratio for each study.</p>
+            </div>
+          </td>
+        </tr>
+    </tbody>
+  </table>
+
+          <details class="quote">
+            <summary>Source code in <code>src/gentropy/method/sumstat_quality_controls.py</code></summary>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">114</span>
+<span class="normal">115</span>
+<span class="normal">116</span>
+<span class="normal">117</span>
+<span class="normal">118</span>
+<span class="normal">119</span>
+<span class="normal">120</span>
+<span class="normal">121</span>
+<span class="normal">122</span>
+<span class="normal">123</span>
+<span class="normal">124</span>
+<span class="normal">125</span>
+<span class="normal">126</span>
+<span class="normal">127</span>
+<span class="normal">128</span>
+<span class="normal">129</span>
+<span class="normal">130</span>
+<span class="normal">131</span>
+<span class="normal">132</span>
+<span class="normal">133</span>
+<span class="normal">134</span>
+<span class="normal">135</span>
+<span class="normal">136</span>
+<span class="normal">137</span>
+<span class="normal">138</span>
+<span class="normal">139</span>
+<span class="normal">140</span>
+<span class="normal">141</span>
+<span class="normal">142</span>
+<span class="normal">143</span>
+<span class="normal">144</span>
+<span class="normal">145</span>
+<span class="normal">146</span>
+<span class="normal">147</span>
+<span class="normal">148</span>
+<span class="normal">149</span>
+<span class="normal">150</span>
+<span class="normal">151</span>
+<span class="normal">152</span>
+<span class="normal">153</span>
+<span class="normal">154</span>
+<span class="normal">155</span>
+<span class="normal">156</span>
+<span class="normal">157</span>
+<span class="normal">158</span>
+<span class="normal">159</span>
+<span class="normal">160</span>
+<span class="normal">161</span>
+<span class="normal">162</span>
+<span class="normal">163</span>
+<span class="normal">164</span>
+<span class="normal">165</span>
+<span class="normal">166</span>
+<span class="normal">167</span>
+<span class="normal">168</span>
+<span class="normal">169</span>
+<span class="normal">170</span>
+<span class="normal">171</span>
+<span class="normal">172</span>
+<span class="normal">173</span>
+<span class="normal">174</span>
+<span class="normal">175</span>
+<span class="normal">176</span>
+<span class="normal">177</span>
+<span class="normal">178</span>
+<span class="normal">179</span>
+<span class="normal">180</span>
+<span class="normal">181</span>
+<span class="normal">182</span>
+<span class="normal">183</span>
+<span class="normal">184</span>
+<span class="normal">185</span>
+<span class="normal">186</span>
+<span class="normal">187</span>
+<span class="normal">188</span>
+<span class="normal">189</span>
+<span class="normal">190</span>
+<span class="normal">191</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@staticmethod</span>
+<span class="k">def</span> <span class="nf">sumstat_n_eff_check</span><span class="p">(</span>
+    <span class="n">gwas_for_qc</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">,</span>
+    <span class="n">n_total</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">100_000</span><span class="p">,</span>
+    <span class="n">limit</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">10_000_000</span><span class="p">,</span>
+    <span class="n">min_count</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">100</span><span class="p">,</span>
+<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">DataFrame</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;The effective sample size check for QC of GWAS summary statstics.</span>
+
+<span class="sd">    It estiamtes the ratio between effective sample size and the expected one and checks it&#39;s distribution.</span>
+<span class="sd">    It is possible to conduct only if the effective allele frequency is provided in the study.</span>
+<span class="sd">    The median rartio is always close to 1, but standard error could be inflated.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        gwas_for_qc (SummaryStatistics): The instance of the SummaryStatistics class.</span>
+<span class="sd">        n_total (int): The reported sample size of the study. The QC metrics is robust toward the sample size.</span>
+<span class="sd">        limit (int): The limit for the number of variants to be used for the estimation.</span>
+<span class="sd">        min_count (int): The minimum number of variants to be used for the estimation.</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">        DataFrame: PySpark DataFrame with the effective sample size ratio for each study.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">gwas_df</span> <span class="o">=</span> <span class="n">gwas_for_qc</span><span class="o">.</span><span class="n">_df</span>
+
+    <span class="n">gwas_df</span> <span class="o">=</span> <span class="n">gwas_df</span><span class="o">.</span><span class="n">dropna</span><span class="p">(</span><span class="n">subset</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;effectAlleleFrequencyFromSource&quot;</span><span class="p">])</span>
+
+    <span class="n">counts_df</span> <span class="o">=</span> <span class="n">gwas_df</span><span class="o">.</span><span class="n">groupBy</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">count</span><span class="p">()</span>
+
+    <span class="c1"># Join the original DataFrame with the counts DataFrame</span>
+    <span class="n">df_with_counts</span> <span class="o">=</span> <span class="n">gwas_df</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">counts_df</span><span class="p">,</span> <span class="n">on</span><span class="o">=</span><span class="s2">&quot;studyId&quot;</span><span class="p">)</span>
+
+    <span class="c1"># Filter the DataFrame to keep only the groups with count greater than or equal to min_count</span>
+    <span class="n">filtered_df</span> <span class="o">=</span> <span class="n">df_with_counts</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;count&quot;</span><span class="p">)</span> <span class="o">&gt;=</span> <span class="n">min_count</span><span class="p">)</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="s2">&quot;count&quot;</span><span class="p">)</span>
+
+    <span class="n">window</span> <span class="o">=</span> <span class="n">Window</span><span class="o">.</span><span class="n">partitionBy</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">orderBy</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">)</span>
+    <span class="n">gwas_df</span> <span class="o">=</span> <span class="p">(</span>
+        <span class="n">filtered_df</span><span class="o">.</span><span class="n">withColumn</span><span class="p">(</span><span class="s2">&quot;row_num&quot;</span><span class="p">,</span> <span class="n">row_number</span><span class="p">()</span><span class="o">.</span><span class="n">over</span><span class="p">(</span><span class="n">window</span><span class="p">))</span>
+        <span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;row_num&quot;</span><span class="p">)</span> <span class="o">&lt;=</span> <span class="n">limit</span><span class="p">)</span>
+        <span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="s2">&quot;row_num&quot;</span><span class="p">)</span>
+    <span class="p">)</span>
+
+    <span class="n">gwas_df</span> <span class="o">=</span> <span class="n">gwas_df</span><span class="o">.</span><span class="n">withColumn</span><span class="p">(</span>
+        <span class="s2">&quot;var_af&quot;</span><span class="p">,</span>
+        <span class="mi">2</span>
+        <span class="o">*</span> <span class="p">(</span>
+            <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;effectAlleleFrequencyFromSource&quot;</span><span class="p">)</span>
+            <span class="o">*</span> <span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;effectAlleleFrequencyFromSource&quot;</span><span class="p">))</span>
+        <span class="p">),</span>
+    <span class="p">)</span><span class="o">.</span><span class="n">withColumn</span><span class="p">(</span>
+        <span class="s2">&quot;pheno_var&quot;</span><span class="p">,</span>
+        <span class="p">((</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;standardError&quot;</span><span class="p">)</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span> <span class="o">*</span> <span class="n">n_total</span> <span class="o">*</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;var_af&quot;</span><span class="p">))</span>
+        <span class="o">+</span> <span class="p">((</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;beta&quot;</span><span class="p">)</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span> <span class="o">*</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;var_af&quot;</span><span class="p">)),</span>
+    <span class="p">)</span>
+
+    <span class="n">window</span> <span class="o">=</span> <span class="n">Window</span><span class="o">.</span><span class="n">partitionBy</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">orderBy</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">)</span>
+
+    <span class="c1"># Calculate the median of &#39;pheno_var&#39; for each &#39;studyId&#39; and add it as a new column</span>
+    <span class="n">gwas_df</span> <span class="o">=</span> <span class="n">gwas_df</span><span class="o">.</span><span class="n">withColumn</span><span class="p">(</span>
+        <span class="s2">&quot;pheno_median&quot;</span><span class="p">,</span> <span class="n">expr</span><span class="p">(</span><span class="s2">&quot;percentile_approx(pheno_var, 0.5)&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">over</span><span class="p">(</span><span class="n">window</span><span class="p">)</span>
+    <span class="p">)</span>
+
+    <span class="n">gwas_df</span> <span class="o">=</span> <span class="n">gwas_df</span><span class="o">.</span><span class="n">withColumn</span><span class="p">(</span>
+        <span class="s2">&quot;N_hat_ratio&quot;</span><span class="p">,</span>
+        <span class="p">(</span>
+            <span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;pheno_median&quot;</span><span class="p">)</span> <span class="o">-</span> <span class="p">((</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;beta&quot;</span><span class="p">)</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span> <span class="o">*</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;var_af&quot;</span><span class="p">)))</span>
+            <span class="o">/</span> <span class="p">((</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;standardError&quot;</span><span class="p">)</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span> <span class="o">*</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;var_af&quot;</span><span class="p">)</span> <span class="o">*</span> <span class="n">n_total</span><span class="p">)</span>
+        <span class="p">),</span>
+    <span class="p">)</span>
+
+    <span class="n">qc_c</span> <span class="o">=</span> <span class="p">(</span>
+        <span class="n">gwas_df</span><span class="o">.</span><span class="n">groupBy</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">)</span>
+        <span class="o">.</span><span class="n">agg</span><span class="p">(</span>
+            <span class="n">f</span><span class="o">.</span><span class="n">stddev</span><span class="p">(</span><span class="s2">&quot;N_hat_ratio&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;se_N&quot;</span><span class="p">),</span>
+        <span class="p">)</span>
+        <span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">,</span> <span class="s2">&quot;se_N&quot;</span><span class="p">)</span>
+    <span class="p">)</span>
+
+    <span class="k">return</span> <span class="n">qc_c</span>
+</code></pre></div></td></tr></table></div>
+          </details>
+  </div>
+
+</div>
+
+
+<div class="doc doc-object doc-function">
+
+
+
+<h3 id="gentropy.method.sumstat_quality_controls.SummaryStatisticsQC.sumstat_qc_beta_check" class="doc doc-heading">
+          <code class="highlight language-python"><span class="n">sumstat_qc_beta_check</span><span class="p">(</span><span class="n">gwas_for_qc</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">DataFrame</span></code>
+  
+  <span class="doc doc-labels">
+      <small class="doc doc-label doc-label-staticmethod"><code>staticmethod</code></small>
+  </span>
+
+<a href="#gentropy.method.sumstat_quality_controls.SummaryStatisticsQC.sumstat_qc_beta_check" class="headerlink" title="Permanent link">&para;</a></h3>
+
+
+  <div class="doc doc-contents ">
+  
+      <p>The mean beta check for QC of GWAS summary statstics.</p>
+
+
+
+  <p><span class="doc-section-title">Parameters:</span></p>
+  <table>
+    <thead>
+      <tr>
+        <th>Name</th>
+        <th>Type</th>
+        <th>Description</th>
+        <th>Default</th>
+      </tr>
+    </thead>
+    <tbody>
+        <tr class="doc-section-item">
+          <td><code>gwas_for_qc</code></td>
+          <td>
+                <code><a class="autorefs autorefs-internal" title="gentropy.dataset.summary_statistics.SummaryStatistics" href="../../datasets/summary_statistics/#gentropy.dataset.summary_statistics.SummaryStatistics">SummaryStatistics</a></code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>The instance of the SummaryStatistics class.</p>
+            </div>
+          </td>
+          <td>
+              <em>required</em>
+          </td>
+        </tr>
+    </tbody>
+  </table>
+
+
+
+  <p><span class="doc-section-title">Returns:</span></p>
+  <table>
+    <thead>
+      <tr>
+<th>Name</th>        <th>Type</th>
+        <th>Description</th>
+      </tr>
+    </thead>
+    <tbody>
+        <tr class="doc-section-item">
+<td><code>DataFrame</code></td>          <td>
+                <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>PySpark DataFrame with the mean beta for each study.</p>
+            </div>
+          </td>
+        </tr>
+    </tbody>
+  </table>
+
+          <details class="quote">
+            <summary>Source code in <code>src/gentropy/method/sumstat_quality_controls.py</code></summary>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">33</span>
+<span class="normal">34</span>
+<span class="normal">35</span>
+<span class="normal">36</span>
+<span class="normal">37</span>
+<span class="normal">38</span>
+<span class="normal">39</span>
+<span class="normal">40</span>
+<span class="normal">41</span>
+<span class="normal">42</span>
+<span class="normal">43</span>
+<span class="normal">44</span>
+<span class="normal">45</span>
+<span class="normal">46</span>
+<span class="normal">47</span>
+<span class="normal">48</span>
+<span class="normal">49</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@staticmethod</span>
+<span class="k">def</span> <span class="nf">sumstat_qc_beta_check</span><span class="p">(</span>
+    <span class="n">gwas_for_qc</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">,</span>
+<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">DataFrame</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;The mean beta check for QC of GWAS summary statstics.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        gwas_for_qc (SummaryStatistics): The instance of the SummaryStatistics class.</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">        DataFrame: PySpark DataFrame with the mean beta for each study.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">gwas_df</span> <span class="o">=</span> <span class="n">gwas_for_qc</span><span class="o">.</span><span class="n">_df</span>
+    <span class="n">qc_c</span> <span class="o">=</span> <span class="n">gwas_df</span><span class="o">.</span><span class="n">groupBy</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">agg</span><span class="p">(</span>
+        <span class="n">f</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="s2">&quot;beta&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;mean_beta&quot;</span><span class="p">),</span>
+    <span class="p">)</span>
+    <span class="k">return</span> <span class="n">qc_c</span>
+</code></pre></div></td></tr></table></div>
+          </details>
+  </div>
+
+</div>
+
+
+<div class="doc doc-object doc-function">
+
+
+
+<h3 id="gentropy.method.sumstat_quality_controls.SummaryStatisticsQC.sumstat_qc_pz_check" class="doc doc-heading">
+          <code class="highlight language-python"><span class="n">sumstat_qc_pz_check</span><span class="p">(</span><span class="n">gwas_for_qc</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">,</span> <span class="n">limit</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">10000000</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">DataFrame</span></code>
+  
+  <span class="doc doc-labels">
+      <small class="doc doc-label doc-label-staticmethod"><code>staticmethod</code></small>
+  </span>
+
+<a href="#gentropy.method.sumstat_quality_controls.SummaryStatisticsQC.sumstat_qc_pz_check" class="headerlink" title="Permanent link">&para;</a></h3>
+
+
+  <div class="doc doc-contents ">
+  
+      <p>The PZ check for QC of GWAS summary statstics. It runs linear regression between reported p-values and p-values infered from z-scores.</p>
+
+
+
+  <p><span class="doc-section-title">Parameters:</span></p>
+  <table>
+    <thead>
+      <tr>
+        <th>Name</th>
+        <th>Type</th>
+        <th>Description</th>
+        <th>Default</th>
+      </tr>
+    </thead>
+    <tbody>
+        <tr class="doc-section-item">
+          <td><code>gwas_for_qc</code></td>
+          <td>
+                <code><a class="autorefs autorefs-internal" title="gentropy.dataset.summary_statistics.SummaryStatistics" href="../../datasets/summary_statistics/#gentropy.dataset.summary_statistics.SummaryStatistics">SummaryStatistics</a></code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>The instance of the SummaryStatistics class.</p>
+            </div>
+          </td>
+          <td>
+              <em>required</em>
+          </td>
+        </tr>
+        <tr class="doc-section-item">
+          <td><code>limit</code></td>
+          <td>
+                <code>int</code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>The limit for the number of variants to be used for the estimation.</p>
+            </div>
+          </td>
+          <td>
+                <code>10000000</code>
+          </td>
+        </tr>
+    </tbody>
+  </table>
+
+
+
+  <p><span class="doc-section-title">Returns:</span></p>
+  <table>
+    <thead>
+      <tr>
+<th>Name</th>        <th>Type</th>
+        <th>Description</th>
+      </tr>
+    </thead>
+    <tbody>
+        <tr class="doc-section-item">
+<td><code>DataFrame</code></td>          <td>
+                <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>PySpark DataFrame with the results of the linear regression for each study.</p>
+            </div>
+          </td>
+        </tr>
+    </tbody>
+  </table>
+
+          <details class="quote">
+            <summary>Source code in <code>src/gentropy/method/sumstat_quality_controls.py</code></summary>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 68</span>
+<span class="normal"> 69</span>
+<span class="normal"> 70</span>
+<span class="normal"> 71</span>
+<span class="normal"> 72</span>
+<span class="normal"> 73</span>
+<span class="normal"> 74</span>
+<span class="normal"> 75</span>
+<span class="normal"> 76</span>
+<span class="normal"> 77</span>
+<span class="normal"> 78</span>
+<span class="normal"> 79</span>
+<span class="normal"> 80</span>
+<span class="normal"> 81</span>
+<span class="normal"> 82</span>
+<span class="normal"> 83</span>
+<span class="normal"> 84</span>
+<span class="normal"> 85</span>
+<span class="normal"> 86</span>
+<span class="normal"> 87</span>
+<span class="normal"> 88</span>
+<span class="normal"> 89</span>
+<span class="normal"> 90</span>
+<span class="normal"> 91</span>
+<span class="normal"> 92</span>
+<span class="normal"> 93</span>
+<span class="normal"> 94</span>
+<span class="normal"> 95</span>
+<span class="normal"> 96</span>
+<span class="normal"> 97</span>
+<span class="normal"> 98</span>
+<span class="normal"> 99</span>
+<span class="normal">100</span>
+<span class="normal">101</span>
+<span class="normal">102</span>
+<span class="normal">103</span>
+<span class="normal">104</span>
+<span class="normal">105</span>
+<span class="normal">106</span>
+<span class="normal">107</span>
+<span class="normal">108</span>
+<span class="normal">109</span>
+<span class="normal">110</span>
+<span class="normal">111</span>
+<span class="normal">112</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@staticmethod</span>
+<span class="k">def</span> <span class="nf">sumstat_qc_pz_check</span><span class="p">(</span>
+    <span class="n">gwas_for_qc</span><span class="p">:</span> <span class="n">SummaryStatistics</span><span class="p">,</span>
+    <span class="n">limit</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">10_000_000</span><span class="p">,</span>
+<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">DataFrame</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;The PZ check for QC of GWAS summary statstics. It runs linear regression between reported p-values and p-values infered from z-scores.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        gwas_for_qc (SummaryStatistics): The instance of the SummaryStatistics class.</span>
+<span class="sd">        limit (int): The limit for the number of variants to be used for the estimation.</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">        DataFrame: PySpark DataFrame with the results of the linear regression for each study.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">gwas_df</span> <span class="o">=</span> <span class="n">gwas_for_qc</span><span class="o">.</span><span class="n">_df</span>
+
+    <span class="n">calculate_logpval_udf</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">udf</span><span class="p">(</span>
+        <span class="n">SummaryStatisticsQC</span><span class="o">.</span><span class="n">_calculate_logpval</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">DoubleType</span><span class="p">()</span>
+    <span class="p">)</span>
+
+    <span class="n">window</span> <span class="o">=</span> <span class="n">Window</span><span class="o">.</span><span class="n">partitionBy</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">orderBy</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">)</span>
+
+    <span class="n">gwas_df</span> <span class="o">=</span> <span class="p">(</span>
+        <span class="n">gwas_df</span><span class="o">.</span><span class="n">withColumn</span><span class="p">(</span><span class="s2">&quot;row_num&quot;</span><span class="p">,</span> <span class="n">row_number</span><span class="p">()</span><span class="o">.</span><span class="n">over</span><span class="p">(</span><span class="n">window</span><span class="p">))</span>
+        <span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;row_num&quot;</span><span class="p">)</span> <span class="o">&lt;=</span> <span class="n">limit</span><span class="p">)</span>
+        <span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="s2">&quot;row_num&quot;</span><span class="p">)</span>
+    <span class="p">)</span>
+
+    <span class="n">qc_c</span> <span class="o">=</span> <span class="p">(</span>
+        <span class="n">gwas_df</span><span class="o">.</span><span class="n">withColumn</span><span class="p">(</span><span class="s2">&quot;zscore&quot;</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;beta&quot;</span><span class="p">)</span> <span class="o">/</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;standardError&quot;</span><span class="p">))</span>
+        <span class="o">.</span><span class="n">withColumn</span><span class="p">(</span><span class="s2">&quot;new_logpval&quot;</span><span class="p">,</span> <span class="n">calculate_logpval_udf</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;zscore&quot;</span><span class="p">)</span> <span class="o">**</span> <span class="mi">2</span><span class="p">))</span>
+        <span class="o">.</span><span class="n">withColumn</span><span class="p">(</span><span class="s2">&quot;log_mantissa&quot;</span><span class="p">,</span> <span class="n">log10</span><span class="p">(</span><span class="s2">&quot;pValueMantissa&quot;</span><span class="p">))</span>
+        <span class="o">.</span><span class="n">withColumn</span><span class="p">(</span>
+            <span class="s2">&quot;diffpval&quot;</span><span class="p">,</span>
+            <span class="o">-</span><span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;log_mantissa&quot;</span><span class="p">)</span> <span class="o">-</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;pValueExponent&quot;</span><span class="p">)</span> <span class="o">-</span> <span class="n">f</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;new_logpval&quot;</span><span class="p">),</span>
+        <span class="p">)</span>
+        <span class="o">.</span><span class="n">groupBy</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">)</span>
+        <span class="o">.</span><span class="n">agg</span><span class="p">(</span>
+            <span class="n">f</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="s2">&quot;diffpval&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;mean_diff_pz&quot;</span><span class="p">),</span>
+            <span class="n">f</span><span class="o">.</span><span class="n">stddev</span><span class="p">(</span><span class="s2">&quot;diffpval&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">&quot;se_diff_pz&quot;</span><span class="p">),</span>
+        <span class="p">)</span>
+        <span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">&quot;studyId&quot;</span><span class="p">,</span> <span class="s2">&quot;mean_diff_pz&quot;</span><span class="p">,</span> <span class="s2">&quot;se_diff_pz&quot;</span><span class="p">)</span>
+    <span class="p">)</span>
+
+    <span class="k">return</span> <span class="n">qc_c</span>
+</code></pre></div></td></tr></table></div>
+          </details>
+  </div>
+
+</div>
+
+
+
+  </div>
+
+  </div>
+
+
+</div>
+<hr />
+<div class="md-source-file">
+  <small>
+    <ul
+      class="metadata page-metadata"
+      data-bi-name="page info"
+      lang="en-us"
+      dir="ltr"
+    >
+      <!-- mkdocs-git-revision-date-plugin -->
+      
+      <li>
+        <span class="icon twemoji">
+          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
+        </span>
+        <span class="label"> <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-timeago"><span class="timeago" datetime="2024-04-24T08:55:51+00:00" locale="en"></span></span><span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-24</span> </span>
+      </li>
+       
+      <li>
+        <span class="icon twemoji">
+          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
+        </span>
+        <span class="label"> <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-timeago"><span class="timeago" datetime="2024-04-24T08:55:51+00:00" locale="en"></span></span><span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-24</span> </span>
+      </li>
+      
+
+      <li>
+        <span class="icon twemoji">
+          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 2A10 10 0 0 0 2 12c0 4.42 2.87 8.17 6.84 9.5.5.08.66-.23.66-.5v-1.69c-2.77.6-3.36-1.34-3.36-1.34-.46-1.16-1.11-1.47-1.11-1.47-.91-.62.07-.6.07-.6 1 .07 1.53 1.03 1.53 1.03.87 1.52 2.34 1.07 2.91.83.09-.65.35-1.09.63-1.34-2.22-.25-4.55-1.11-4.55-4.92 0-1.11.38-2 1.03-2.71-.1-.25-.45-1.29.1-2.64 0 0 .84-.27 2.75 1.02.79-.22 1.65-.33 2.5-.33.85 0 1.71.11 2.5.33 1.91-1.29 2.75-1.02 2.75-1.02.55 1.35.2 2.39.1 2.64.65.71 1.03 1.6 1.03 2.71 0 3.82-2.34 4.66-4.57 4.91.36.31.69.92.69 1.85V21c0 .27.16.59.67.5C19.14 20.16 22 16.42 22 12A10 10 0 0 0 12 2Z"/></svg>
+        </span>
+        <span class="label"> Contributors </span>
+      </li>
+      <li class="contributors-holder">
+        <ul class="contributors" data-bi-name="contributors"></ul>
+      </li>
+    </ul>
+  </small>
+</div>
+
+
+
+
+                
+              </article>
+            </div>
+          
+          
+<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
+        </div>
+        
+          <button type="button" class="md-top md-icon" data-md-component="top" hidden>
+  
+  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8v12Z"/></svg>
+  Back to top
+</button>
+        
+      </main>
+      
+        <footer class="md-footer">
+  
+  <div class="md-footer-meta md-typeset">
+    <div class="md-footer-meta__inner md-grid">
+      <div class="md-copyright">
+  
+  
+    Made with
+    <a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
+      Material for MkDocs
+    </a>
+  
+</div>
+      
+        <div class="md-social">
+  
+    
+    
+    
+    
+      
+      
+    
+    <a href="https://github.com/gentropy" target="_blank" rel="noopener" title="github.com" class="md-social__link">
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 496 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6zm-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3zm44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9zM244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8zM97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1zm-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7zm32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1zm-11.4-14.7c-1.6 1-1.6 3.6 0 5.9 1.6 2.3 4.3 3.3 5.6 2.3 1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2z"/></svg>
+    </a>
+  
+</div>
+      
+    </div>
+  </div>
+</footer>
+      
+    </div>
+    <div class="md-dialog" data-md-component="dialog">
+      <div class="md-dialog__inner md-typeset"></div>
+    </div>
+    
+    
+    <script id="__config" type="application/json">{"base": "../../..", "features": ["navigation.instant", "navigation.top", "navigation.tabs", "navigation.tabs.sticky", "content.code.annotate", "search.suggest", "search.highlight"], "search": "../../../assets/javascripts/workers/search.f886a092.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
+    
+    
+      <script src="../../../assets/javascripts/bundle.d7c377c4.min.js"></script>
+      
+        <script src="../../../js/timeago.min.js"></script>
+      
+        <script src="../../../js/timeago_mkdocs_material.js"></script>
+      
+        <script src="../../../assets/javascripts/extra.js"></script>
+      
+    
+  </body>
+</html>
\ No newline at end of file
diff --git a/python_api/methods/susie_inf/index.html b/python_api/methods/susie_inf/index.html
index 09731c13b..f927a4b0e 100644
--- a/python_api/methods/susie_inf/index.html
+++ b/python_api/methods/susie_inf/index.html
@@ -9,7 +9,7 @@
       
       
       
-        <link rel="prev" href="../pics/">
+        <link rel="prev" href="../sumstat_quality_controls/">
       
       
         <link rel="next" href="../l2g/_l2g/">
@@ -2195,6 +2195,48 @@
                 
   
   
+  
+  
+    <li class="md-nav__item">
+      <a href="../sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
     
   
   
@@ -2301,7 +2343,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2317,14 +2359,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3422,7 +3464,7 @@ <h2 id="gentropy.method.susie_inf.SUSIE_inf" class="doc doc-heading">
         <span class="n">ssq_range</span><span class="p">:</span> <span class="nb">tuple</span><span class="p">[</span><span class="nb">float</span><span class="p">,</span> <span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span>
         <span class="n">pi0</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
         <span class="n">est_sigmasq</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
-        <span class="n">est_tausq</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
+        <span class="n">est_tausq</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
         <span class="n">sigmasq</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
         <span class="n">tausq</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span>
         <span class="n">sigmasq_range</span><span class="p">:</span> <span class="nb">tuple</span><span class="p">[</span><span class="nb">float</span><span class="p">,</span> <span class="nb">float</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
@@ -3787,7 +3829,7 @@ <h2 id="gentropy.method.susie_inf.SUSIE_inf" class="doc doc-heading">
     <span class="k">def</span> <span class="nf">cred_inf</span><span class="p">(</span>
         <span class="n">PIP</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span>
         <span class="n">n</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">100_000</span><span class="p">,</span>
-        <span class="n">coverage</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.9</span><span class="p">,</span>
+        <span class="n">coverage</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.99</span><span class="p">,</span>
         <span class="n">purity</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.5</span><span class="p">,</span>
         <span class="n">LD</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
         <span class="n">V</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
@@ -3866,7 +3908,7 @@ <h2 id="gentropy.method.susie_inf.SUSIE_inf" class="doc doc-heading">
 
 
 <h3 id="gentropy.method.susie_inf.SUSIE_inf.cred_inf" class="doc doc-heading">
-          <code class="highlight language-python"><span class="n">cred_inf</span><span class="p">(</span><span class="n">PIP</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">n</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">100000</span><span class="p">,</span> <span class="n">coverage</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.9</span><span class="p">,</span> <span class="n">purity</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.5</span><span class="p">,</span> <span class="n">LD</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">V</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">Dsq</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">dedup</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">list</span><span class="p">[</span><span class="n">Any</span><span class="p">]</span></code>
+          <code class="highlight language-python"><span class="n">cred_inf</span><span class="p">(</span><span class="n">PIP</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">n</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">100000</span><span class="p">,</span> <span class="n">coverage</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.99</span><span class="p">,</span> <span class="n">purity</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.5</span><span class="p">,</span> <span class="n">LD</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">V</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">Dsq</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">dedup</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">list</span><span class="p">[</span><span class="n">Any</span><span class="p">]</span></code>
   
   <span class="doc doc-labels">
       <small class="doc doc-label doc-label-staticmethod"><code>staticmethod</code></small>
@@ -3881,7 +3923,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.cred_inf" class="doc doc-heading">
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3892,7 +3934,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.cred_inf" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>PIP</code></td>
           <td>
                 <code><span title="numpy.ndarray">ndarray</span></code>
@@ -3906,7 +3948,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.cred_inf" class="doc doc-heading">
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>n</code></td>
           <td>
                 <code>int</code>
@@ -3920,7 +3962,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.cred_inf" class="doc doc-heading">
                 <code>100000</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>coverage</code></td>
           <td>
                 <code>float</code>
@@ -3931,10 +3973,10 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.cred_inf" class="doc doc-heading">
             </div>
           </td>
           <td>
-                <code>0.9</code>
+                <code>0.99</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>purity</code></td>
           <td>
                 <code>float</code>
@@ -3948,7 +3990,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.cred_inf" class="doc doc-heading">
                 <code>0.5</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>LD</code></td>
           <td>
                 <code><span title="numpy.ndarray">ndarray</span> | None</code>
@@ -3962,7 +4004,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.cred_inf" class="doc doc-heading">
                 <code>None</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>V</code></td>
           <td>
                 <code><span title="numpy.ndarray">ndarray</span> | None</code>
@@ -3976,7 +4018,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.cred_inf" class="doc doc-heading">
                 <code>None</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>Dsq</code></td>
           <td>
                 <code><span title="numpy.ndarray">ndarray</span> | None</code>
@@ -3990,7 +4032,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.cred_inf" class="doc doc-heading">
                 <code>None</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>dedup</code></td>
           <td>
                 <code>bool</code>
@@ -4009,7 +4051,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.cred_inf" class="doc doc-heading">
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -4018,7 +4060,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.cred_inf" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td>
                 <code>list[<span title="typing.Any">Any</span>]</code>
           </td>
@@ -4033,7 +4075,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.cred_inf" class="doc doc-heading">
 
 
 
-  <p><strong>Raises:</strong></p>
+  <p><span class="doc-section-title">Raises:</span></p>
   <table>
     <thead>
       <tr>
@@ -4042,7 +4084,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.cred_inf" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td>
                 <code>RuntimeError</code>
           </td>
@@ -4122,7 +4164,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.cred_inf" class="doc doc-heading">
 <span class="k">def</span> <span class="nf">cred_inf</span><span class="p">(</span>
     <span class="n">PIP</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span>
     <span class="n">n</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">100_000</span><span class="p">,</span>
-    <span class="n">coverage</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.9</span><span class="p">,</span>
+    <span class="n">coverage</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.99</span><span class="p">,</span>
     <span class="n">purity</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.5</span><span class="p">,</span>
     <span class="n">LD</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
     <span class="n">V</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
@@ -4192,7 +4234,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.cred_inf" class="doc doc-heading">
 
 
 <h3 id="gentropy.method.susie_inf.SUSIE_inf.susie_inf" class="doc doc-heading">
-          <code class="highlight language-python"><span class="n">susie_inf</span><span class="p">(</span><span class="n">z</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">meansq</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span> <span class="n">n</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">100000</span><span class="p">,</span> <span class="n">L</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">10</span><span class="p">,</span> <span class="n">LD</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">V</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">Dsq</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">est_ssq</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> <span class="n">ssq</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">ssq_range</span><span class="p">:</span> <span class="nb">tuple</span><span class="p">[</span><span class="nb">float</span><span class="p">,</span> <span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="n">pi0</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">est_sigmasq</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> <span class="n">est_tausq</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> <span class="n">sigmasq</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span> <span class="n">tausq</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span> <span class="n">sigmasq_range</span><span class="p">:</span> <span class="nb">tuple</span><span class="p">[</span><span class="nb">float</span><span class="p">,</span> <span class="nb">float</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">tausq_range</span><span class="p">:</span> <span class="nb">tuple</span><span class="p">[</span><span class="nb">float</span><span class="p">,</span> <span class="nb">float</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">PIP</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">mu</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">method</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s1">&#39;moments&#39;</span><span class="p">,</span> <span class="n">maxiter</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">100</span><span class="p">,</span> <span class="n">PIP_tol</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.001</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span></code>
+          <code class="highlight language-python"><span class="n">susie_inf</span><span class="p">(</span><span class="n">z</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">meansq</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span> <span class="n">n</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">100000</span><span class="p">,</span> <span class="n">L</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">10</span><span class="p">,</span> <span class="n">LD</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">V</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">Dsq</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">est_ssq</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> <span class="n">ssq</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">ssq_range</span><span class="p">:</span> <span class="nb">tuple</span><span class="p">[</span><span class="nb">float</span><span class="p">,</span> <span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="n">pi0</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">est_sigmasq</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> <span class="n">est_tausq</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span> <span class="n">sigmasq</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span> <span class="n">tausq</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span> <span class="n">sigmasq_range</span><span class="p">:</span> <span class="nb">tuple</span><span class="p">[</span><span class="nb">float</span><span class="p">,</span> <span class="nb">float</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">tausq_range</span><span class="p">:</span> <span class="nb">tuple</span><span class="p">[</span><span class="nb">float</span><span class="p">,</span> <span class="nb">float</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">PIP</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">mu</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">method</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s1">&#39;moments&#39;</span><span class="p">,</span> <span class="n">maxiter</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">100</span><span class="p">,</span> <span class="n">PIP_tol</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.001</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span></code>
   
   <span class="doc doc-labels">
       <small class="doc doc-label doc-label-staticmethod"><code>staticmethod</code></small>
@@ -4207,7 +4249,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.susie_inf" class="doc doc-heading">
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -4218,7 +4260,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.susie_inf" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>z</code></td>
           <td>
                 <code><span title="numpy.ndarray">ndarray</span></code>
@@ -4232,7 +4274,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.susie_inf" class="doc doc-heading">
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>meansq</code></td>
           <td>
                 <code>float</code>
@@ -4246,7 +4288,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.susie_inf" class="doc doc-heading">
                 <code>1</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>n</code></td>
           <td>
                 <code>int</code>
@@ -4260,7 +4302,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.susie_inf" class="doc doc-heading">
                 <code>100000</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>L</code></td>
           <td>
                 <code>int</code>
@@ -4274,7 +4316,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.susie_inf" class="doc doc-heading">
                 <code>10</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>LD</code></td>
           <td>
                 <code><span title="numpy.ndarray">ndarray</span> | None</code>
@@ -4288,7 +4330,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.susie_inf" class="doc doc-heading">
                 <code>None</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>V</code></td>
           <td>
                 <code><span title="numpy.ndarray">ndarray</span> | None</code>
@@ -4302,7 +4344,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.susie_inf" class="doc doc-heading">
                 <code>None</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>Dsq</code></td>
           <td>
                 <code><span title="numpy.ndarray">ndarray</span> | None</code>
@@ -4316,7 +4358,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.susie_inf" class="doc doc-heading">
                 <code>None</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>est_ssq</code></td>
           <td>
                 <code>bool</code>
@@ -4330,7 +4372,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.susie_inf" class="doc doc-heading">
                 <code>True</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>ssq</code></td>
           <td>
                 <code><span title="numpy.ndarray">ndarray</span> | None</code>
@@ -4344,7 +4386,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.susie_inf" class="doc doc-heading">
                 <code>None</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>ssq_range</code></td>
           <td>
                 <code>tuple[float, float]</code>
@@ -4358,7 +4400,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.susie_inf" class="doc doc-heading">
                 <code>(0, 1)</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>pi0</code></td>
           <td>
                 <code><span title="numpy.ndarray">ndarray</span> | None</code>
@@ -4372,7 +4414,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.susie_inf" class="doc doc-heading">
                 <code>None</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>est_sigmasq</code></td>
           <td>
                 <code>bool</code>
@@ -4386,7 +4428,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.susie_inf" class="doc doc-heading">
                 <code>True</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>est_tausq</code></td>
           <td>
                 <code>bool</code>
@@ -4397,10 +4439,10 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.susie_inf" class="doc doc-heading">
             </div>
           </td>
           <td>
-                <code>True</code>
+                <code>False</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>sigmasq</code></td>
           <td>
                 <code>float</code>
@@ -4414,7 +4456,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.susie_inf" class="doc doc-heading">
                 <code>1</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>tausq</code></td>
           <td>
                 <code>float</code>
@@ -4428,7 +4470,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.susie_inf" class="doc doc-heading">
                 <code>0</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>sigmasq_range</code></td>
           <td>
                 <code>tuple[float, float] | None</code>
@@ -4442,7 +4484,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.susie_inf" class="doc doc-heading">
                 <code>None</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>tausq_range</code></td>
           <td>
                 <code>tuple[float, float] | None</code>
@@ -4456,7 +4498,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.susie_inf" class="doc doc-heading">
                 <code>None</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>PIP</code></td>
           <td>
                 <code><span title="numpy.ndarray">ndarray</span> | None</code>
@@ -4470,7 +4512,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.susie_inf" class="doc doc-heading">
                 <code>None</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>mu</code></td>
           <td>
                 <code><span title="numpy.ndarray">ndarray</span> | None</code>
@@ -4484,7 +4526,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.susie_inf" class="doc doc-heading">
                 <code>None</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>method</code></td>
           <td>
                 <code>str</code>
@@ -4498,7 +4540,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.susie_inf" class="doc doc-heading">
                 <code>&#39;moments&#39;</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>maxiter</code></td>
           <td>
                 <code>int</code>
@@ -4512,7 +4554,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.susie_inf" class="doc doc-heading">
                 <code>100</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>PIP_tol</code></td>
           <td>
                 <code>float</code>
@@ -4531,7 +4573,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.susie_inf" class="doc doc-heading">
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -4540,7 +4582,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.susie_inf" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td>
                 <code>dict[str, <span title="typing.Any">Any</span>]</code>
           </td>
@@ -4564,7 +4606,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.susie_inf" class="doc doc-heading">
 
 
 
-  <p><strong>Raises:</strong></p>
+  <p><span class="doc-section-title">Raises:</span></p>
   <table>
     <thead>
       <tr>
@@ -4573,7 +4615,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.susie_inf" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td>
                 <code>RuntimeError</code>
           </td>
@@ -4583,7 +4625,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.susie_inf" class="doc doc-heading">
             </div>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td>
                 <code>RuntimeError</code>
           </td>
@@ -4815,7 +4857,7 @@ <h3 id="gentropy.method.susie_inf.SUSIE_inf.susie_inf" class="doc doc-heading">
     <span class="n">ssq_range</span><span class="p">:</span> <span class="nb">tuple</span><span class="p">[</span><span class="nb">float</span><span class="p">,</span> <span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span>
     <span class="n">pi0</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
     <span class="n">est_sigmasq</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
-    <span class="n">est_tausq</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
+    <span class="n">est_tausq</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
     <span class="n">sigmasq</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
     <span class="n">tausq</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span>
     <span class="n">sigmasq_range</span><span class="p">:</span> <span class="nb">tuple</span><span class="p">[</span><span class="nb">float</span><span class="p">,</span> <span class="nb">float</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
diff --git a/python_api/steps/_steps/index.html b/python_api/steps/_steps/index.html
index f50381789..d91863925 100644
--- a/python_api/steps/_steps/index.html
+++ b/python_api/steps/_steps/index.html
@@ -2195,6 +2195,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../methods/susie_inf/" class="md-nav__link">
         
@@ -2226,7 +2268,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2242,14 +2284,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
diff --git a/python_api/steps/colocalisation/index.html b/python_api/steps/colocalisation/index.html
index a51cce52c..5336c4aff 100644
--- a/python_api/steps/colocalisation/index.html
+++ b/python_api/steps/colocalisation/index.html
@@ -2195,6 +2195,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../methods/susie_inf/" class="md-nav__link">
         
@@ -2226,7 +2268,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2242,14 +2284,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -2921,12 +2963,7 @@ <h2 id="gentropy.colocalisation.ColocalisationStep" class="doc doc-heading">
 
             <details class="quote">
               <summary>Source code in <code>src/gentropy/colocalisation.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">10</span>
-<span class="normal">11</span>
-<span class="normal">12</span>
-<span class="normal">13</span>
-<span class="normal">14</span>
-<span class="normal">15</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">15</span>
 <span class="normal">16</span>
 <span class="normal">17</span>
 <span class="normal">18</span>
@@ -2957,7 +2994,53 @@ <h2 id="gentropy.colocalisation.ColocalisationStep" class="doc doc-heading">
 <span class="normal">43</span>
 <span class="normal">44</span>
 <span class="normal">45</span>
-<span class="normal">46</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">ColocalisationStep</span><span class="p">:</span>
+<span class="normal">46</span>
+<span class="normal">47</span>
+<span class="normal">48</span>
+<span class="normal">49</span>
+<span class="normal">50</span>
+<span class="normal">51</span>
+<span class="normal">52</span>
+<span class="normal">53</span>
+<span class="normal">54</span>
+<span class="normal">55</span>
+<span class="normal">56</span>
+<span class="normal">57</span>
+<span class="normal">58</span>
+<span class="normal">59</span>
+<span class="normal">60</span>
+<span class="normal">61</span>
+<span class="normal">62</span>
+<span class="normal">63</span>
+<span class="normal">64</span>
+<span class="normal">65</span>
+<span class="normal">66</span>
+<span class="normal">67</span>
+<span class="normal">68</span>
+<span class="normal">69</span>
+<span class="normal">70</span>
+<span class="normal">71</span>
+<span class="normal">72</span>
+<span class="normal">73</span>
+<span class="normal">74</span>
+<span class="normal">75</span>
+<span class="normal">76</span>
+<span class="normal">77</span>
+<span class="normal">78</span>
+<span class="normal">79</span>
+<span class="normal">80</span>
+<span class="normal">81</span>
+<span class="normal">82</span>
+<span class="normal">83</span>
+<span class="normal">84</span>
+<span class="normal">85</span>
+<span class="normal">86</span>
+<span class="normal">87</span>
+<span class="normal">88</span>
+<span class="normal">89</span>
+<span class="normal">90</span>
+<span class="normal">91</span>
+<span class="normal">92</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">ColocalisationStep</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Colocalisation step.</span>
 
 <span class="sd">    This workflow runs colocalisation analyses that assess the degree to which independent signals of the association share the same causal variant in a region of the genome, typically limited by linkage disequilibrium (LD).</span>
@@ -2969,6 +3052,7 @@ <h2 id="gentropy.colocalisation.ColocalisationStep" class="doc doc-heading">
         <span class="n">credible_set_path</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
         <span class="n">study_index_path</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
         <span class="n">coloc_path</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
+        <span class="n">colocalisation_method</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
     <span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
 <span class="w">        </span><span class="sd">&quot;&quot;&quot;Run Colocalisation step.</span>
 
@@ -2977,10 +3061,18 @@ <h2 id="gentropy.colocalisation.ColocalisationStep" class="doc doc-heading">
 <span class="sd">            credible_set_path (str): Input credible sets path.</span>
 <span class="sd">            study_index_path (str): Input study index path.</span>
 <span class="sd">            coloc_path (str): Output Colocalisation path.</span>
+<span class="sd">            colocalisation_method (str): Colocalisation method.</span>
 <span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">colocalisation_class</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_colocalisation_class</span><span class="p">(</span><span class="n">colocalisation_method</span><span class="p">)</span>
         <span class="c1"># Extract</span>
-        <span class="n">credible_set</span> <span class="o">=</span> <span class="n">StudyLocus</span><span class="o">.</span><span class="n">from_parquet</span><span class="p">(</span>
-            <span class="n">session</span><span class="p">,</span> <span class="n">credible_set_path</span><span class="p">,</span> <span class="n">recursiveFileLookup</span><span class="o">=</span><span class="kc">True</span>
+        <span class="n">credible_set</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="n">StudyLocus</span><span class="o">.</span><span class="n">from_parquet</span><span class="p">(</span>
+                <span class="n">session</span><span class="p">,</span> <span class="n">credible_set_path</span><span class="p">,</span> <span class="n">recursiveFileLookup</span><span class="o">=</span><span class="kc">True</span>
+            <span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;finemappingMethod&quot;</span><span class="p">)</span> <span class="o">==</span> <span class="s2">&quot;SuSie&quot;</span><span class="p">)</span>
+            <span class="k">if</span> <span class="n">colocalisation_class</span> <span class="ow">is</span> <span class="n">Coloc</span>
+            <span class="k">else</span> <span class="n">StudyLocus</span><span class="o">.</span><span class="n">from_parquet</span><span class="p">(</span>
+                <span class="n">session</span><span class="p">,</span> <span class="n">credible_set_path</span><span class="p">,</span> <span class="n">recursiveFileLookup</span><span class="o">=</span><span class="kc">True</span>
+            <span class="p">)</span>
         <span class="p">)</span>
         <span class="n">si</span> <span class="o">=</span> <span class="n">StudyIndex</span><span class="o">.</span><span class="n">from_parquet</span><span class="p">(</span>
             <span class="n">session</span><span class="p">,</span> <span class="n">study_index_path</span><span class="p">,</span> <span class="n">recursiveFileLookup</span><span class="o">=</span><span class="kc">True</span>
@@ -2990,10 +3082,42 @@ <h2 id="gentropy.colocalisation.ColocalisationStep" class="doc doc-heading">
         <span class="n">overlaps</span> <span class="o">=</span> <span class="n">credible_set</span><span class="o">.</span><span class="n">filter_credible_set</span><span class="p">(</span>
             <span class="n">CredibleInterval</span><span class="o">.</span><span class="n">IS95</span>
         <span class="p">)</span><span class="o">.</span><span class="n">find_overlaps</span><span class="p">(</span><span class="n">si</span><span class="p">)</span>
-        <span class="n">ecaviar_results</span> <span class="o">=</span> <span class="n">ECaviar</span><span class="o">.</span><span class="n">colocalise</span><span class="p">(</span><span class="n">overlaps</span><span class="p">)</span>
+        <span class="n">colocalisation_results</span> <span class="o">=</span> <span class="n">colocalisation_class</span><span class="o">.</span><span class="n">colocalise</span><span class="p">(</span><span class="n">overlaps</span><span class="p">)</span>  <span class="c1"># type: ignore</span>
 
         <span class="c1"># Load</span>
-        <span class="n">ecaviar_results</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">write</span><span class="o">.</span><span class="n">mode</span><span class="p">(</span><span class="n">session</span><span class="o">.</span><span class="n">write_mode</span><span class="p">)</span><span class="o">.</span><span class="n">parquet</span><span class="p">(</span><span class="n">coloc_path</span><span class="p">)</span>
+        <span class="n">colocalisation_results</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">write</span><span class="o">.</span><span class="n">mode</span><span class="p">(</span><span class="n">session</span><span class="o">.</span><span class="n">write_mode</span><span class="p">)</span><span class="o">.</span><span class="n">parquet</span><span class="p">(</span>
+            <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">coloc_path</span><span class="si">}</span><span class="s2">/</span><span class="si">{</span><span class="n">colocalisation_method</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span><span class="si">}</span><span class="s2">&quot;</span>
+        <span class="p">)</span>
+
+    <span class="nd">@classmethod</span>
+    <span class="k">def</span> <span class="nf">_get_colocalisation_class</span><span class="p">(</span><span class="bp">cls</span><span class="p">:</span> <span class="nb">type</span><span class="p">[</span><span class="n">ColocalisationStep</span><span class="p">],</span> <span class="n">method</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">type</span><span class="p">:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Get colocalisation class.</span>
+
+<span class="sd">        Args:</span>
+<span class="sd">            method (str): Colocalisation method.</span>
+
+<span class="sd">        Returns:</span>
+<span class="sd">            type: Colocalisation class.</span>
+
+<span class="sd">        Raises:</span>
+<span class="sd">            ValueError: if method not available.</span>
+
+<span class="sd">        Examples:</span>
+<span class="sd">            &gt;&gt;&gt; ColocalisationStep._get_colocalisation_class(&quot;ECaviar&quot;)</span>
+<span class="sd">            &lt;class &#39;gentropy.method.colocalisation.ECaviar&#39;&gt;</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">module_name</span> <span class="o">=</span> <span class="s2">&quot;gentropy.method.colocalisation&quot;</span>
+        <span class="n">module</span> <span class="o">=</span> <span class="n">import_module</span><span class="p">(</span><span class="n">module_name</span><span class="p">)</span>
+
+        <span class="n">available_methods</span> <span class="o">=</span> <span class="p">[]</span>
+        <span class="k">for</span> <span class="n">class_name</span><span class="p">,</span> <span class="n">class_obj</span> <span class="ow">in</span> <span class="n">inspect</span><span class="o">.</span><span class="n">getmembers</span><span class="p">(</span><span class="n">module</span><span class="p">,</span> <span class="n">inspect</span><span class="o">.</span><span class="n">isclass</span><span class="p">):</span>
+            <span class="k">if</span> <span class="n">class_obj</span><span class="o">.</span><span class="vm">__module__</span> <span class="o">==</span> <span class="n">module_name</span><span class="p">:</span>
+                <span class="n">available_methods</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">class_name</span><span class="p">)</span>
+                <span class="k">if</span> <span class="n">class_name</span> <span class="o">==</span> <span class="n">method</span><span class="p">:</span>
+                    <span class="k">return</span> <span class="n">class_obj</span>
+        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+            <span class="sa">f</span><span class="s2">&quot;Method </span><span class="si">{</span><span class="n">method</span><span class="si">}</span><span class="s2"> is not supported. Available: </span><span class="si">{</span><span class="p">(</span><span class="s1">&#39;, &#39;</span><span class="p">)</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">available_methods</span><span class="p">)</span><span class="si">}</span><span class="s2">&quot;</span>
+        <span class="p">)</span>
 </code></pre></div></td></tr></table></div>
             </details>
 
@@ -3015,7 +3139,7 @@ <h2 id="gentropy.colocalisation.ColocalisationStep" class="doc doc-heading">
 
 
 <h3 id="gentropy.colocalisation.ColocalisationStep.__init__" class="doc doc-heading">
-          <code class="highlight language-python"><span class="fm">__init__</span><span class="p">(</span><span class="n">session</span><span class="p">:</span> <span class="n">Session</span><span class="p">,</span> <span class="n">credible_set_path</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">study_index_path</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">coloc_path</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span></code>
+          <code class="highlight language-python"><span class="fm">__init__</span><span class="p">(</span><span class="n">session</span><span class="p">:</span> <span class="n">Session</span><span class="p">,</span> <span class="n">credible_set_path</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">study_index_path</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">coloc_path</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">colocalisation_method</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span></code>
 
 <a href="#gentropy.colocalisation.ColocalisationStep.__init__" class="headerlink" title="Permanent link">&para;</a></h3>
 
@@ -3026,7 +3150,7 @@ <h3 id="gentropy.colocalisation.ColocalisationStep.__init__" class="doc doc-head
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3037,7 +3161,7 @@ <h3 id="gentropy.colocalisation.ColocalisationStep.__init__" class="doc doc-head
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>session</code></td>
           <td>
                 <code><span title="gentropy.common.session.Session">Session</span></code>
@@ -3051,7 +3175,7 @@ <h3 id="gentropy.colocalisation.ColocalisationStep.__init__" class="doc doc-head
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>credible_set_path</code></td>
           <td>
                 <code>str</code>
@@ -3065,7 +3189,7 @@ <h3 id="gentropy.colocalisation.ColocalisationStep.__init__" class="doc doc-head
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>study_index_path</code></td>
           <td>
                 <code>str</code>
@@ -3079,7 +3203,7 @@ <h3 id="gentropy.colocalisation.ColocalisationStep.__init__" class="doc doc-head
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>coloc_path</code></td>
           <td>
                 <code>str</code>
@@ -3093,17 +3217,26 @@ <h3 id="gentropy.colocalisation.ColocalisationStep.__init__" class="doc doc-head
               <em>required</em>
           </td>
         </tr>
+        <tr class="doc-section-item">
+          <td><code>colocalisation_method</code></td>
+          <td>
+                <code>str</code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>Colocalisation method.</p>
+            </div>
+          </td>
+          <td>
+              <em>required</em>
+          </td>
+        </tr>
     </tbody>
   </table>
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/colocalisation.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">16</span>
-<span class="normal">17</span>
-<span class="normal">18</span>
-<span class="normal">19</span>
-<span class="normal">20</span>
-<span class="normal">21</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">21</span>
 <span class="normal">22</span>
 <span class="normal">23</span>
 <span class="normal">24</span>
@@ -3128,12 +3261,29 @@ <h3 id="gentropy.colocalisation.ColocalisationStep.__init__" class="doc doc-head
 <span class="normal">43</span>
 <span class="normal">44</span>
 <span class="normal">45</span>
-<span class="normal">46</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+<span class="normal">46</span>
+<span class="normal">47</span>
+<span class="normal">48</span>
+<span class="normal">49</span>
+<span class="normal">50</span>
+<span class="normal">51</span>
+<span class="normal">52</span>
+<span class="normal">53</span>
+<span class="normal">54</span>
+<span class="normal">55</span>
+<span class="normal">56</span>
+<span class="normal">57</span>
+<span class="normal">58</span>
+<span class="normal">59</span>
+<span class="normal">60</span>
+<span class="normal">61</span>
+<span class="normal">62</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">session</span><span class="p">:</span> <span class="n">Session</span><span class="p">,</span>
     <span class="n">credible_set_path</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
     <span class="n">study_index_path</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
     <span class="n">coloc_path</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
+    <span class="n">colocalisation_method</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Run Colocalisation step.</span>
 
@@ -3142,10 +3292,18 @@ <h3 id="gentropy.colocalisation.ColocalisationStep.__init__" class="doc doc-head
 <span class="sd">        credible_set_path (str): Input credible sets path.</span>
 <span class="sd">        study_index_path (str): Input study index path.</span>
 <span class="sd">        coloc_path (str): Output Colocalisation path.</span>
+<span class="sd">        colocalisation_method (str): Colocalisation method.</span>
 <span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">colocalisation_class</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_colocalisation_class</span><span class="p">(</span><span class="n">colocalisation_method</span><span class="p">)</span>
     <span class="c1"># Extract</span>
-    <span class="n">credible_set</span> <span class="o">=</span> <span class="n">StudyLocus</span><span class="o">.</span><span class="n">from_parquet</span><span class="p">(</span>
-        <span class="n">session</span><span class="p">,</span> <span class="n">credible_set_path</span><span class="p">,</span> <span class="n">recursiveFileLookup</span><span class="o">=</span><span class="kc">True</span>
+    <span class="n">credible_set</span> <span class="o">=</span> <span class="p">(</span>
+        <span class="n">StudyLocus</span><span class="o">.</span><span class="n">from_parquet</span><span class="p">(</span>
+            <span class="n">session</span><span class="p">,</span> <span class="n">credible_set_path</span><span class="p">,</span> <span class="n">recursiveFileLookup</span><span class="o">=</span><span class="kc">True</span>
+        <span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;finemappingMethod&quot;</span><span class="p">)</span> <span class="o">==</span> <span class="s2">&quot;SuSie&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">colocalisation_class</span> <span class="ow">is</span> <span class="n">Coloc</span>
+        <span class="k">else</span> <span class="n">StudyLocus</span><span class="o">.</span><span class="n">from_parquet</span><span class="p">(</span>
+            <span class="n">session</span><span class="p">,</span> <span class="n">credible_set_path</span><span class="p">,</span> <span class="n">recursiveFileLookup</span><span class="o">=</span><span class="kc">True</span>
+        <span class="p">)</span>
     <span class="p">)</span>
     <span class="n">si</span> <span class="o">=</span> <span class="n">StudyIndex</span><span class="o">.</span><span class="n">from_parquet</span><span class="p">(</span>
         <span class="n">session</span><span class="p">,</span> <span class="n">study_index_path</span><span class="p">,</span> <span class="n">recursiveFileLookup</span><span class="o">=</span><span class="kc">True</span>
@@ -3155,10 +3313,12 @@ <h3 id="gentropy.colocalisation.ColocalisationStep.__init__" class="doc doc-head
     <span class="n">overlaps</span> <span class="o">=</span> <span class="n">credible_set</span><span class="o">.</span><span class="n">filter_credible_set</span><span class="p">(</span>
         <span class="n">CredibleInterval</span><span class="o">.</span><span class="n">IS95</span>
     <span class="p">)</span><span class="o">.</span><span class="n">find_overlaps</span><span class="p">(</span><span class="n">si</span><span class="p">)</span>
-    <span class="n">ecaviar_results</span> <span class="o">=</span> <span class="n">ECaviar</span><span class="o">.</span><span class="n">colocalise</span><span class="p">(</span><span class="n">overlaps</span><span class="p">)</span>
+    <span class="n">colocalisation_results</span> <span class="o">=</span> <span class="n">colocalisation_class</span><span class="o">.</span><span class="n">colocalise</span><span class="p">(</span><span class="n">overlaps</span><span class="p">)</span>  <span class="c1"># type: ignore</span>
 
     <span class="c1"># Load</span>
-    <span class="n">ecaviar_results</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">write</span><span class="o">.</span><span class="n">mode</span><span class="p">(</span><span class="n">session</span><span class="o">.</span><span class="n">write_mode</span><span class="p">)</span><span class="o">.</span><span class="n">parquet</span><span class="p">(</span><span class="n">coloc_path</span><span class="p">)</span>
+    <span class="n">colocalisation_results</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">write</span><span class="o">.</span><span class="n">mode</span><span class="p">(</span><span class="n">session</span><span class="o">.</span><span class="n">write_mode</span><span class="p">)</span><span class="o">.</span><span class="n">parquet</span><span class="p">(</span>
+        <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">coloc_path</span><span class="si">}</span><span class="s2">/</span><span class="si">{</span><span class="n">colocalisation_method</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span><span class="si">}</span><span class="s2">&quot;</span>
+    <span class="p">)</span>
 </code></pre></div></td></tr></table></div>
           </details>
   </div>
diff --git a/python_api/steps/eqtl_catalogue/index.html b/python_api/steps/eqtl_catalogue/index.html
index c045e7a52..e3accb9b5 100644
--- a/python_api/steps/eqtl_catalogue/index.html
+++ b/python_api/steps/eqtl_catalogue/index.html
@@ -2195,6 +2195,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../methods/susie_inf/" class="md-nav__link">
         
@@ -2226,7 +2268,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2242,14 +2284,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3074,7 +3116,7 @@ <h3 id="gentropy.eqtl_catalogue.EqtlCatalogueStep.__init__" class="doc doc-headi
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3085,7 +3127,7 @@ <h3 id="gentropy.eqtl_catalogue.EqtlCatalogueStep.__init__" class="doc doc-headi
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>session</code></td>
           <td>
                 <code><span title="gentropy.common.session.Session">Session</span></code>
@@ -3099,7 +3141,7 @@ <h3 id="gentropy.eqtl_catalogue.EqtlCatalogueStep.__init__" class="doc doc-headi
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>mqtl_quantification_methods_blacklist</code></td>
           <td>
                 <code>list[str]</code>
@@ -3113,7 +3155,7 @@ <h3 id="gentropy.eqtl_catalogue.EqtlCatalogueStep.__init__" class="doc doc-headi
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>eqtl_catalogue_paths_imported</code></td>
           <td>
                 <code>str</code>
@@ -3127,7 +3169,7 @@ <h3 id="gentropy.eqtl_catalogue.EqtlCatalogueStep.__init__" class="doc doc-headi
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>eqtl_catalogue_study_index_out</code></td>
           <td>
                 <code>str</code>
@@ -3141,7 +3183,7 @@ <h3 id="gentropy.eqtl_catalogue.EqtlCatalogueStep.__init__" class="doc doc-headi
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>eqtl_catalogue_credible_sets_out</code></td>
           <td>
                 <code>str</code>
diff --git a/python_api/steps/finngen_studies/index.html b/python_api/steps/finngen_studies/index.html
index 94f13f0ea..2f5a5dc86 100644
--- a/python_api/steps/finngen_studies/index.html
+++ b/python_api/steps/finngen_studies/index.html
@@ -2195,6 +2195,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../methods/susie_inf/" class="md-nav__link">
         
@@ -2226,7 +2268,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2242,14 +2284,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -2979,7 +3021,7 @@ <h3 id="gentropy.finngen_studies.FinnGenStudiesStep.__init__" class="doc doc-hea
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -2990,7 +3032,7 @@ <h3 id="gentropy.finngen_studies.FinnGenStudiesStep.__init__" class="doc doc-hea
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>session</code></td>
           <td>
                 <code><span title="gentropy.common.session.Session">Session</span></code>
@@ -3004,7 +3046,7 @@ <h3 id="gentropy.finngen_studies.FinnGenStudiesStep.__init__" class="doc doc-hea
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>finngen_study_index_out</code></td>
           <td>
                 <code>str</code>
diff --git a/python_api/steps/finngen_sumstat_preprocess/index.html b/python_api/steps/finngen_sumstat_preprocess/index.html
index 9dbfc9723..ffe647aad 100644
--- a/python_api/steps/finngen_sumstat_preprocess/index.html
+++ b/python_api/steps/finngen_sumstat_preprocess/index.html
@@ -2195,6 +2195,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../methods/susie_inf/" class="md-nav__link">
         
@@ -2226,7 +2268,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2242,14 +2284,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -2989,7 +3031,7 @@ <h3 id="gentropy.finngen_sumstat_preprocess.FinnGenSumstatPreprocessStep.__init_
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3000,7 +3042,7 @@ <h3 id="gentropy.finngen_sumstat_preprocess.FinnGenSumstatPreprocessStep.__init_
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>session</code></td>
           <td>
                 <code><span title="gentropy.common.session.Session">Session</span></code>
@@ -3014,7 +3056,7 @@ <h3 id="gentropy.finngen_sumstat_preprocess.FinnGenSumstatPreprocessStep.__init_
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>raw_sumstats_path</code></td>
           <td>
                 <code>str</code>
@@ -3028,7 +3070,7 @@ <h3 id="gentropy.finngen_sumstat_preprocess.FinnGenSumstatPreprocessStep.__init_
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>out_sumstats_path</code></td>
           <td>
                 <code>str</code>
diff --git a/python_api/steps/gene_index/index.html b/python_api/steps/gene_index/index.html
index 1626fe4ed..c8ae1306e 100644
--- a/python_api/steps/gene_index/index.html
+++ b/python_api/steps/gene_index/index.html
@@ -2195,6 +2195,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../methods/susie_inf/" class="md-nav__link">
         
@@ -2226,7 +2268,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2242,14 +2284,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3000,7 +3042,7 @@ <h3 id="gentropy.gene_index.GeneIndexStep.__init__" class="doc doc-heading">
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3011,7 +3053,7 @@ <h3 id="gentropy.gene_index.GeneIndexStep.__init__" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>session</code></td>
           <td>
                 <code><span title="gentropy.common.session.Session">Session</span></code>
@@ -3025,7 +3067,7 @@ <h3 id="gentropy.gene_index.GeneIndexStep.__init__" class="doc doc-heading">
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>target_path</code></td>
           <td>
                 <code>str</code>
@@ -3039,7 +3081,7 @@ <h3 id="gentropy.gene_index.GeneIndexStep.__init__" class="doc doc-heading">
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>gene_index_path</code></td>
           <td>
                 <code>str</code>
diff --git a/python_api/steps/gwas_catalog_curation/index.html b/python_api/steps/gwas_catalog_curation/index.html
index 1de893412..fd930bade 100644
--- a/python_api/steps/gwas_catalog_curation/index.html
+++ b/python_api/steps/gwas_catalog_curation/index.html
@@ -2195,6 +2195,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../methods/susie_inf/" class="md-nav__link">
         
@@ -2226,7 +2268,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2242,14 +2284,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3047,7 +3089,7 @@ <h3 id="gentropy.gwas_catalog_study_curation.GWASCatalogStudyCurationStep.__init
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3058,7 +3100,7 @@ <h3 id="gentropy.gwas_catalog_study_curation.GWASCatalogStudyCurationStep.__init
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>session</code></td>
           <td>
                 <code><span title="gentropy.common.session.Session">Session</span></code>
@@ -3072,7 +3114,7 @@ <h3 id="gentropy.gwas_catalog_study_curation.GWASCatalogStudyCurationStep.__init
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>catalog_study_files</code></td>
           <td>
                 <code>list[str]</code>
@@ -3086,7 +3128,7 @@ <h3 id="gentropy.gwas_catalog_study_curation.GWASCatalogStudyCurationStep.__init
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>catalog_ancestry_files</code></td>
           <td>
                 <code>list[str]</code>
@@ -3100,7 +3142,7 @@ <h3 id="gentropy.gwas_catalog_study_curation.GWASCatalogStudyCurationStep.__init
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>catalog_sumstats_lut</code></td>
           <td>
                 <code>str</code>
@@ -3114,7 +3156,7 @@ <h3 id="gentropy.gwas_catalog_study_curation.GWASCatalogStudyCurationStep.__init
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>gwas_catalog_study_curation_out</code></td>
           <td>
                 <code>str</code>
@@ -3128,7 +3170,7 @@ <h3 id="gentropy.gwas_catalog_study_curation.GWASCatalogStudyCurationStep.__init
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>gwas_catalog_study_curation_file</code></td>
           <td>
                 <code>str | None</code>
diff --git a/python_api/steps/gwas_catalog_inclusion/index.html b/python_api/steps/gwas_catalog_inclusion/index.html
index e56a638a6..5a98ce817 100644
--- a/python_api/steps/gwas_catalog_inclusion/index.html
+++ b/python_api/steps/gwas_catalog_inclusion/index.html
@@ -2195,6 +2195,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../methods/susie_inf/" class="md-nav__link">
         
@@ -2226,7 +2268,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2242,14 +2284,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3333,7 +3375,7 @@ <h3 id="gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3344,7 +3386,7 @@ <h3 id="gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>session</code></td>
           <td>
                 <code><span title="gentropy.common.session.Session">Session</span></code>
@@ -3358,7 +3400,7 @@ <h3 id="gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>catalog_study_files</code></td>
           <td>
                 <code>list[str]</code>
@@ -3372,7 +3414,7 @@ <h3 id="gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>catalog_ancestry_files</code></td>
           <td>
                 <code>list[str]</code>
@@ -3386,7 +3428,7 @@ <h3 id="gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>catalog_associations_file</code></td>
           <td>
                 <code>str</code>
@@ -3400,7 +3442,7 @@ <h3 id="gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>gwas_catalog_study_curation_file</code></td>
           <td>
                 <code>str</code>
@@ -3414,7 +3456,7 @@ <h3 id="gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>variant_annotation_path</code></td>
           <td>
                 <code>str</code>
@@ -3428,7 +3470,7 @@ <h3 id="gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>harmonised_study_file</code></td>
           <td>
                 <code>str</code>
@@ -3442,7 +3484,7 @@ <h3 id="gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>criteria</code></td>
           <td>
                 <code>str</code>
@@ -3456,7 +3498,7 @@ <h3 id="gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>inclusion_list_path</code></td>
           <td>
                 <code>str</code>
@@ -3470,7 +3512,7 @@ <h3 id="gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>exclusion_list_path</code></td>
           <td>
                 <code>str</code>
@@ -3615,7 +3657,7 @@ <h3 id="gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3626,7 +3668,7 @@ <h3 id="gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>study_index</code></td>
           <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog" href="../../datasources/gwas_catalog/study_index/#gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog">StudyIndexGWASCatalog</a></code>
@@ -3640,7 +3682,7 @@ <h3 id="gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>criteria</code></td>
           <td>
                 <code>str</code>
@@ -3659,7 +3701,7 @@ <h3 id="gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator
 
 
 
-  <p><strong>Raises:</strong></p>
+  <p><span class="doc-section-title">Raises:</span></p>
   <table>
     <thead>
       <tr>
@@ -3668,7 +3710,7 @@ <h3 id="gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td>
                 <code>ValueError</code>
           </td>
@@ -3683,7 +3725,7 @@ <h3 id="gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3692,7 +3734,7 @@ <h3 id="gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>DataFrame</code></td>          <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
           </td>
@@ -3819,7 +3861,7 @@ <h3 id="gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3830,7 +3872,7 @@ <h3 id="gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>session</code></td>
           <td>
                 <code><span title="gentropy.common.session.Session">Session</span></code>
@@ -3844,7 +3886,7 @@ <h3 id="gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>variant_annotation_path</code></td>
           <td>
                 <code>str</code>
@@ -3858,7 +3900,7 @@ <h3 id="gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>catalog_study_files</code></td>
           <td>
                 <code>list[str]</code>
@@ -3872,7 +3914,7 @@ <h3 id="gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>catalog_ancestry_files</code></td>
           <td>
                 <code>list[str]</code>
@@ -3886,7 +3928,7 @@ <h3 id="gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>harmonised_study_file</code></td>
           <td>
                 <code>str</code>
@@ -3900,7 +3942,7 @@ <h3 id="gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>catalog_associations_file</code></td>
           <td>
                 <code>str</code>
@@ -3914,7 +3956,7 @@ <h3 id="gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>gwas_catalog_study_curation_file</code></td>
           <td>
                 <code>str</code>
@@ -3933,7 +3975,7 @@ <h3 id="gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -3942,7 +3984,7 @@ <h3 id="gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>StudyIndexGWASCatalog</code></td>          <td>
                 <code><a class="autorefs autorefs-internal" title="gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog" href="../../datasources/gwas_catalog/study_index/#gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog">StudyIndexGWASCatalog</a></code>
           </td>
@@ -4089,7 +4131,7 @@ <h3 id="gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -4100,7 +4142,7 @@ <h3 id="gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>studies</code></td>
           <td>
                 <code>list[str]</code>
@@ -4114,7 +4156,7 @@ <h3 id="gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>session</code></td>
           <td>
                 <code><span title="gentropy.common.session.Session">Session</span></code>
@@ -4133,7 +4175,7 @@ <h3 id="gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator
 
 
 
-  <p><strong>Returns:</strong></p>
+  <p><span class="doc-section-title">Returns:</span></p>
   <table>
     <thead>
       <tr>
@@ -4142,7 +4184,7 @@ <h3 id="gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
 <td><code>DataFrame</code></td>          <td>
                 <code><span title="pyspark.sql.DataFrame">DataFrame</span></code>
           </td>
diff --git a/python_api/steps/gwas_catalog_ingestion/index.html b/python_api/steps/gwas_catalog_ingestion/index.html
index ca7e43f41..7f37ac3a3 100644
--- a/python_api/steps/gwas_catalog_ingestion/index.html
+++ b/python_api/steps/gwas_catalog_ingestion/index.html
@@ -2195,6 +2195,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../methods/susie_inf/" class="md-nav__link">
         
@@ -2226,7 +2268,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2242,14 +2284,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3094,7 +3136,7 @@ <h3 id="gentropy.gwas_catalog_ingestion.GWASCatalogIngestionStep.__init__" class
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3105,7 +3147,7 @@ <h3 id="gentropy.gwas_catalog_ingestion.GWASCatalogIngestionStep.__init__" class
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>session</code></td>
           <td>
                 <code><span title="gentropy.common.session.Session">Session</span></code>
@@ -3119,7 +3161,7 @@ <h3 id="gentropy.gwas_catalog_ingestion.GWASCatalogIngestionStep.__init__" class
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>catalog_study_files</code></td>
           <td>
                 <code>list[str]</code>
@@ -3133,7 +3175,7 @@ <h3 id="gentropy.gwas_catalog_ingestion.GWASCatalogIngestionStep.__init__" class
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>catalog_ancestry_files</code></td>
           <td>
                 <code>list[str]</code>
@@ -3147,7 +3189,7 @@ <h3 id="gentropy.gwas_catalog_ingestion.GWASCatalogIngestionStep.__init__" class
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>catalog_sumstats_lut</code></td>
           <td>
                 <code>str</code>
@@ -3161,7 +3203,7 @@ <h3 id="gentropy.gwas_catalog_ingestion.GWASCatalogIngestionStep.__init__" class
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>catalog_associations_file</code></td>
           <td>
                 <code>str</code>
@@ -3175,7 +3217,7 @@ <h3 id="gentropy.gwas_catalog_ingestion.GWASCatalogIngestionStep.__init__" class
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>variant_annotation_path</code></td>
           <td>
                 <code>str</code>
@@ -3189,7 +3231,7 @@ <h3 id="gentropy.gwas_catalog_ingestion.GWASCatalogIngestionStep.__init__" class
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>catalog_studies_out</code></td>
           <td>
                 <code>str</code>
@@ -3203,7 +3245,7 @@ <h3 id="gentropy.gwas_catalog_ingestion.GWASCatalogIngestionStep.__init__" class
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>catalog_associations_out</code></td>
           <td>
                 <code>str</code>
@@ -3217,7 +3259,7 @@ <h3 id="gentropy.gwas_catalog_ingestion.GWASCatalogIngestionStep.__init__" class
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>gwas_catalog_study_curation_file</code></td>
           <td>
                 <code>str | None</code>
@@ -3231,7 +3273,7 @@ <h3 id="gentropy.gwas_catalog_ingestion.GWASCatalogIngestionStep.__init__" class
                 <code>None</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>inclusion_list_path</code></td>
           <td>
                 <code>str | None</code>
diff --git a/python_api/steps/gwas_catalog_sumstat_preprocess/index.html b/python_api/steps/gwas_catalog_sumstat_preprocess/index.html
index 1619dfc26..ebe77c0e3 100644
--- a/python_api/steps/gwas_catalog_sumstat_preprocess/index.html
+++ b/python_api/steps/gwas_catalog_sumstat_preprocess/index.html
@@ -2195,6 +2195,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../methods/susie_inf/" class="md-nav__link">
         
@@ -2226,7 +2268,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2242,14 +2284,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -2987,7 +3029,7 @@ <h3 id="gentropy.gwas_catalog_sumstat_preprocess.GWASCatalogSumstatsPreprocessSt
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -2998,7 +3040,7 @@ <h3 id="gentropy.gwas_catalog_sumstat_preprocess.GWASCatalogSumstatsPreprocessSt
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>session</code></td>
           <td>
                 <code><span title="gentropy.common.session.Session">Session</span></code>
@@ -3012,7 +3054,7 @@ <h3 id="gentropy.gwas_catalog_sumstat_preprocess.GWASCatalogSumstatsPreprocessSt
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>raw_sumstats_path</code></td>
           <td>
                 <code>str</code>
@@ -3026,7 +3068,7 @@ <h3 id="gentropy.gwas_catalog_sumstat_preprocess.GWASCatalogSumstatsPreprocessSt
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>out_sumstats_path</code></td>
           <td>
                 <code>str</code>
diff --git a/python_api/steps/l2g/index.html b/python_api/steps/l2g/index.html
index 2d331650d..f5134802a 100644
--- a/python_api/steps/l2g/index.html
+++ b/python_api/steps/l2g/index.html
@@ -2195,6 +2195,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../methods/susie_inf/" class="md-nav__link">
         
@@ -2226,7 +2268,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2242,14 +2284,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3074,7 +3116,12 @@ <h2 id="gentropy.l2g.LocusToGeneStep" class="doc doc-heading">
 <span class="normal">173</span>
 <span class="normal">174</span>
 <span class="normal">175</span>
-<span class="normal">176</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">LocusToGeneStep</span><span class="p">:</span>
+<span class="normal">176</span>
+<span class="normal">177</span>
+<span class="normal">178</span>
+<span class="normal">179</span>
+<span class="normal">180</span>
+<span class="normal">181</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">LocusToGeneStep</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Locus to gene step.&quot;&quot;&quot;</span>
 
     <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
@@ -3128,7 +3175,9 @@ <h2 id="gentropy.l2g.LocusToGeneStep" class="doc doc-heading">
             <span class="n">session</span><span class="p">,</span> <span class="n">study_index_path</span><span class="p">,</span> <span class="n">recursiveFileLookup</span><span class="o">=</span><span class="kc">True</span>
         <span class="p">)</span>
         <span class="n">v2g</span> <span class="o">=</span> <span class="n">V2G</span><span class="o">.</span><span class="n">from_parquet</span><span class="p">(</span><span class="n">session</span><span class="p">,</span> <span class="n">variant_gene_path</span><span class="p">)</span>
-        <span class="n">coloc</span> <span class="o">=</span> <span class="n">Colocalisation</span><span class="o">.</span><span class="n">from_parquet</span><span class="p">(</span><span class="n">session</span><span class="p">,</span> <span class="n">colocalisation_path</span><span class="p">)</span>
+        <span class="n">coloc</span> <span class="o">=</span> <span class="n">Colocalisation</span><span class="o">.</span><span class="n">from_parquet</span><span class="p">(</span>
+            <span class="n">session</span><span class="p">,</span> <span class="n">colocalisation_path</span><span class="p">,</span> <span class="n">recursiveFileLookup</span><span class="o">=</span><span class="kc">True</span>
+        <span class="p">)</span>
 
         <span class="k">if</span> <span class="n">run_mode</span> <span class="o">==</span> <span class="s2">&quot;predict&quot;</span><span class="p">:</span>
             <span class="k">if</span> <span class="ow">not</span> <span class="n">model_path</span> <span class="ow">or</span> <span class="ow">not</span> <span class="n">predictions_path</span><span class="p">:</span>
@@ -3146,7 +3195,7 @@ <h2 id="gentropy.l2g.LocusToGeneStep" class="doc doc-heading">
             <span class="ow">and</span> <span class="n">gene_interactions_path</span>
         <span class="p">):</span>
             <span class="c1"># Process gold standard and L2G features</span>
-            <span class="n">gs_curation</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">read</span><span class="o">.</span><span class="n">json</span><span class="p">(</span><span class="n">gold_standard_curation_path</span><span class="p">)</span><span class="o">.</span><span class="n">persist</span><span class="p">()</span>
+            <span class="n">gs_curation</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">read</span><span class="o">.</span><span class="n">json</span><span class="p">(</span><span class="n">gold_standard_curation_path</span><span class="p">)</span>
             <span class="n">interactions</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">read</span><span class="o">.</span><span class="n">parquet</span><span class="p">(</span><span class="n">gene_interactions_path</span><span class="p">)</span>
             <span class="n">study_locus_overlap</span> <span class="o">=</span> <span class="n">StudyLocus</span><span class="p">(</span>
                 <span class="c1"># We just extract overlaps of associations in the gold standard. This parsing is a duplication of the one in the gold standard curation,</span>
@@ -3181,23 +3230,27 @@ <h2 id="gentropy.l2g.LocusToGeneStep" class="doc doc-heading">
 
             <span class="n">fm</span> <span class="o">=</span> <span class="n">L2GFeatureMatrix</span><span class="o">.</span><span class="n">generate_features</span><span class="p">(</span>
                 <span class="n">features_list</span><span class="o">=</span><span class="n">features_list</span><span class="p">,</span>
-                <span class="n">study_locus</span><span class="o">=</span><span class="n">credible_set</span><span class="p">,</span>
+                <span class="n">credible_set</span><span class="o">=</span><span class="n">credible_set</span><span class="p">,</span>
                 <span class="n">study_index</span><span class="o">=</span><span class="n">studies</span><span class="p">,</span>
                 <span class="n">variant_gene</span><span class="o">=</span><span class="n">v2g</span><span class="p">,</span>
                 <span class="n">colocalisation</span><span class="o">=</span><span class="n">coloc</span><span class="p">,</span>
             <span class="p">)</span>
 
-            <span class="c1"># Join and fill null values with 0</span>
-            <span class="n">data</span> <span class="o">=</span> <span class="n">L2GFeatureMatrix</span><span class="p">(</span>
-                <span class="n">_df</span><span class="o">=</span><span class="n">fm</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
-                    <span class="n">f</span><span class="o">.</span><span class="n">broadcast</span><span class="p">(</span>
-                        <span class="n">gold_standards</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="s2">&quot;variantId&quot;</span><span class="p">,</span> <span class="s2">&quot;studyId&quot;</span><span class="p">,</span> <span class="s2">&quot;sources&quot;</span><span class="p">)</span>
+            <span class="n">data</span> <span class="o">=</span> <span class="p">(</span>
+                <span class="c1"># Annotate gold standards with features</span>
+                <span class="n">L2GFeatureMatrix</span><span class="p">(</span>
+                    <span class="n">_df</span><span class="o">=</span><span class="n">fm</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
+                        <span class="n">f</span><span class="o">.</span><span class="n">broadcast</span><span class="p">(</span>
+                            <span class="n">gold_standards</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="s2">&quot;variantId&quot;</span><span class="p">,</span> <span class="s2">&quot;studyId&quot;</span><span class="p">,</span> <span class="s2">&quot;sources&quot;</span><span class="p">)</span>
+                        <span class="p">),</span>
+                        <span class="n">on</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;studyLocusId&quot;</span><span class="p">,</span> <span class="s2">&quot;geneId&quot;</span><span class="p">],</span>
+                        <span class="n">how</span><span class="o">=</span><span class="s2">&quot;inner&quot;</span><span class="p">,</span>
                     <span class="p">),</span>
-                    <span class="n">on</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;studyLocusId&quot;</span><span class="p">,</span> <span class="s2">&quot;geneId&quot;</span><span class="p">],</span>
-                    <span class="n">how</span><span class="o">=</span><span class="s2">&quot;inner&quot;</span><span class="p">,</span>
-                <span class="p">),</span>
-                <span class="n">_schema</span><span class="o">=</span><span class="n">L2GFeatureMatrix</span><span class="o">.</span><span class="n">get_schema</span><span class="p">(),</span>
-            <span class="p">)</span><span class="o">.</span><span class="n">fill_na</span><span class="p">()</span>
+                    <span class="n">_schema</span><span class="o">=</span><span class="n">L2GFeatureMatrix</span><span class="o">.</span><span class="n">get_schema</span><span class="p">(),</span>
+                <span class="p">)</span>
+                <span class="o">.</span><span class="n">fill_na</span><span class="p">()</span>
+                <span class="o">.</span><span class="n">select_features</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="n">features_list</span><span class="p">))</span>
+            <span class="p">)</span>
 
             <span class="c1"># Instantiate classifier</span>
             <span class="n">estimator</span> <span class="o">=</span> <span class="n">SparkXGBClassifier</span><span class="p">(</span>
@@ -3220,9 +3273,8 @@ <h2 id="gentropy.l2g.LocusToGeneStep" class="doc doc-heading">
             <span class="k">else</span><span class="p">:</span>
                 <span class="c1"># Train model</span>
                 <span class="n">LocusToGeneTrainer</span><span class="o">.</span><span class="n">train</span><span class="p">(</span>
-                    <span class="n">data</span><span class="o">=</span><span class="n">data</span><span class="p">,</span>
+                    <span class="n">gold_standard_data</span><span class="o">=</span><span class="n">data</span><span class="p">,</span>
                     <span class="n">l2g_model</span><span class="o">=</span><span class="n">l2g_model</span><span class="p">,</span>
-                    <span class="n">features_list</span><span class="o">=</span><span class="nb">list</span><span class="p">(</span><span class="n">features_list</span><span class="p">),</span>
                     <span class="n">model_path</span><span class="o">=</span><span class="n">model_path</span><span class="p">,</span>
                     <span class="n">evaluate</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
                     <span class="n">wandb_run_name</span><span class="o">=</span><span class="n">wandb_run_name</span><span class="p">,</span>
@@ -3261,7 +3313,7 @@ <h3 id="gentropy.l2g.LocusToGeneStep.__init__" class="doc doc-heading">
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3272,7 +3324,7 @@ <h3 id="gentropy.l2g.LocusToGeneStep.__init__" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>session</code></td>
           <td>
                 <code><span title="gentropy.common.session.Session">Session</span></code>
@@ -3286,7 +3338,7 @@ <h3 id="gentropy.l2g.LocusToGeneStep.__init__" class="doc doc-heading">
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>run_mode</code></td>
           <td>
                 <code>str</code>
@@ -3300,7 +3352,7 @@ <h3 id="gentropy.l2g.LocusToGeneStep.__init__" class="doc doc-heading">
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>model_path</code></td>
           <td>
                 <code>str</code>
@@ -3314,7 +3366,7 @@ <h3 id="gentropy.l2g.LocusToGeneStep.__init__" class="doc doc-heading">
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>predictions_path</code></td>
           <td>
                 <code>str</code>
@@ -3328,7 +3380,7 @@ <h3 id="gentropy.l2g.LocusToGeneStep.__init__" class="doc doc-heading">
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>credible_set_path</code></td>
           <td>
                 <code>str</code>
@@ -3342,7 +3394,7 @@ <h3 id="gentropy.l2g.LocusToGeneStep.__init__" class="doc doc-heading">
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>variant_gene_path</code></td>
           <td>
                 <code>str</code>
@@ -3356,7 +3408,7 @@ <h3 id="gentropy.l2g.LocusToGeneStep.__init__" class="doc doc-heading">
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>colocalisation_path</code></td>
           <td>
                 <code>str</code>
@@ -3370,7 +3422,7 @@ <h3 id="gentropy.l2g.LocusToGeneStep.__init__" class="doc doc-heading">
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>study_index_path</code></td>
           <td>
                 <code>str</code>
@@ -3384,7 +3436,7 @@ <h3 id="gentropy.l2g.LocusToGeneStep.__init__" class="doc doc-heading">
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>gold_standard_curation_path</code></td>
           <td>
                 <code>str</code>
@@ -3398,7 +3450,7 @@ <h3 id="gentropy.l2g.LocusToGeneStep.__init__" class="doc doc-heading">
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>gene_interactions_path</code></td>
           <td>
                 <code>str</code>
@@ -3412,7 +3464,7 @@ <h3 id="gentropy.l2g.LocusToGeneStep.__init__" class="doc doc-heading">
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>features_list</code></td>
           <td>
                 <code>list[str]</code>
@@ -3426,7 +3478,7 @@ <h3 id="gentropy.l2g.LocusToGeneStep.__init__" class="doc doc-heading">
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>hyperparameters</code></td>
           <td>
                 <code>dict[str, <span title="typing.Any">Any</span>]</code>
@@ -3440,7 +3492,7 @@ <h3 id="gentropy.l2g.LocusToGeneStep.__init__" class="doc doc-heading">
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>wandb_run_name</code></td>
           <td>
                 <code>str | None</code>
@@ -3454,7 +3506,7 @@ <h3 id="gentropy.l2g.LocusToGeneStep.__init__" class="doc doc-heading">
                 <code>None</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>perform_cross_validation</code></td>
           <td>
                 <code>bool</code>
@@ -3473,7 +3525,7 @@ <h3 id="gentropy.l2g.LocusToGeneStep.__init__" class="doc doc-heading">
 
 
 
-  <p><strong>Raises:</strong></p>
+  <p><span class="doc-section-title">Raises:</span></p>
   <table>
     <thead>
       <tr>
@@ -3482,7 +3534,7 @@ <h3 id="gentropy.l2g.LocusToGeneStep.__init__" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td>
                 <code>ValueError</code>
           </td>
@@ -3648,7 +3700,12 @@ <h3 id="gentropy.l2g.LocusToGeneStep.__init__" class="doc doc-heading">
 <span class="normal">173</span>
 <span class="normal">174</span>
 <span class="normal">175</span>
-<span class="normal">176</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+<span class="normal">176</span>
+<span class="normal">177</span>
+<span class="normal">178</span>
+<span class="normal">179</span>
+<span class="normal">180</span>
+<span class="normal">181</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">session</span><span class="p">:</span> <span class="n">Session</span><span class="p">,</span>
     <span class="n">run_mode</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
@@ -3699,7 +3756,9 @@ <h3 id="gentropy.l2g.LocusToGeneStep.__init__" class="doc doc-heading">
         <span class="n">session</span><span class="p">,</span> <span class="n">study_index_path</span><span class="p">,</span> <span class="n">recursiveFileLookup</span><span class="o">=</span><span class="kc">True</span>
     <span class="p">)</span>
     <span class="n">v2g</span> <span class="o">=</span> <span class="n">V2G</span><span class="o">.</span><span class="n">from_parquet</span><span class="p">(</span><span class="n">session</span><span class="p">,</span> <span class="n">variant_gene_path</span><span class="p">)</span>
-    <span class="n">coloc</span> <span class="o">=</span> <span class="n">Colocalisation</span><span class="o">.</span><span class="n">from_parquet</span><span class="p">(</span><span class="n">session</span><span class="p">,</span> <span class="n">colocalisation_path</span><span class="p">)</span>
+    <span class="n">coloc</span> <span class="o">=</span> <span class="n">Colocalisation</span><span class="o">.</span><span class="n">from_parquet</span><span class="p">(</span>
+        <span class="n">session</span><span class="p">,</span> <span class="n">colocalisation_path</span><span class="p">,</span> <span class="n">recursiveFileLookup</span><span class="o">=</span><span class="kc">True</span>
+    <span class="p">)</span>
 
     <span class="k">if</span> <span class="n">run_mode</span> <span class="o">==</span> <span class="s2">&quot;predict&quot;</span><span class="p">:</span>
         <span class="k">if</span> <span class="ow">not</span> <span class="n">model_path</span> <span class="ow">or</span> <span class="ow">not</span> <span class="n">predictions_path</span><span class="p">:</span>
@@ -3717,7 +3776,7 @@ <h3 id="gentropy.l2g.LocusToGeneStep.__init__" class="doc doc-heading">
         <span class="ow">and</span> <span class="n">gene_interactions_path</span>
     <span class="p">):</span>
         <span class="c1"># Process gold standard and L2G features</span>
-        <span class="n">gs_curation</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">read</span><span class="o">.</span><span class="n">json</span><span class="p">(</span><span class="n">gold_standard_curation_path</span><span class="p">)</span><span class="o">.</span><span class="n">persist</span><span class="p">()</span>
+        <span class="n">gs_curation</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">read</span><span class="o">.</span><span class="n">json</span><span class="p">(</span><span class="n">gold_standard_curation_path</span><span class="p">)</span>
         <span class="n">interactions</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">read</span><span class="o">.</span><span class="n">parquet</span><span class="p">(</span><span class="n">gene_interactions_path</span><span class="p">)</span>
         <span class="n">study_locus_overlap</span> <span class="o">=</span> <span class="n">StudyLocus</span><span class="p">(</span>
             <span class="c1"># We just extract overlaps of associations in the gold standard. This parsing is a duplication of the one in the gold standard curation,</span>
@@ -3752,23 +3811,27 @@ <h3 id="gentropy.l2g.LocusToGeneStep.__init__" class="doc doc-heading">
 
         <span class="n">fm</span> <span class="o">=</span> <span class="n">L2GFeatureMatrix</span><span class="o">.</span><span class="n">generate_features</span><span class="p">(</span>
             <span class="n">features_list</span><span class="o">=</span><span class="n">features_list</span><span class="p">,</span>
-            <span class="n">study_locus</span><span class="o">=</span><span class="n">credible_set</span><span class="p">,</span>
+            <span class="n">credible_set</span><span class="o">=</span><span class="n">credible_set</span><span class="p">,</span>
             <span class="n">study_index</span><span class="o">=</span><span class="n">studies</span><span class="p">,</span>
             <span class="n">variant_gene</span><span class="o">=</span><span class="n">v2g</span><span class="p">,</span>
             <span class="n">colocalisation</span><span class="o">=</span><span class="n">coloc</span><span class="p">,</span>
         <span class="p">)</span>
 
-        <span class="c1"># Join and fill null values with 0</span>
-        <span class="n">data</span> <span class="o">=</span> <span class="n">L2GFeatureMatrix</span><span class="p">(</span>
-            <span class="n">_df</span><span class="o">=</span><span class="n">fm</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
-                <span class="n">f</span><span class="o">.</span><span class="n">broadcast</span><span class="p">(</span>
-                    <span class="n">gold_standards</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="s2">&quot;variantId&quot;</span><span class="p">,</span> <span class="s2">&quot;studyId&quot;</span><span class="p">,</span> <span class="s2">&quot;sources&quot;</span><span class="p">)</span>
+        <span class="n">data</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="c1"># Annotate gold standards with features</span>
+            <span class="n">L2GFeatureMatrix</span><span class="p">(</span>
+                <span class="n">_df</span><span class="o">=</span><span class="n">fm</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
+                    <span class="n">f</span><span class="o">.</span><span class="n">broadcast</span><span class="p">(</span>
+                        <span class="n">gold_standards</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="s2">&quot;variantId&quot;</span><span class="p">,</span> <span class="s2">&quot;studyId&quot;</span><span class="p">,</span> <span class="s2">&quot;sources&quot;</span><span class="p">)</span>
+                    <span class="p">),</span>
+                    <span class="n">on</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;studyLocusId&quot;</span><span class="p">,</span> <span class="s2">&quot;geneId&quot;</span><span class="p">],</span>
+                    <span class="n">how</span><span class="o">=</span><span class="s2">&quot;inner&quot;</span><span class="p">,</span>
                 <span class="p">),</span>
-                <span class="n">on</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;studyLocusId&quot;</span><span class="p">,</span> <span class="s2">&quot;geneId&quot;</span><span class="p">],</span>
-                <span class="n">how</span><span class="o">=</span><span class="s2">&quot;inner&quot;</span><span class="p">,</span>
-            <span class="p">),</span>
-            <span class="n">_schema</span><span class="o">=</span><span class="n">L2GFeatureMatrix</span><span class="o">.</span><span class="n">get_schema</span><span class="p">(),</span>
-        <span class="p">)</span><span class="o">.</span><span class="n">fill_na</span><span class="p">()</span>
+                <span class="n">_schema</span><span class="o">=</span><span class="n">L2GFeatureMatrix</span><span class="o">.</span><span class="n">get_schema</span><span class="p">(),</span>
+            <span class="p">)</span>
+            <span class="o">.</span><span class="n">fill_na</span><span class="p">()</span>
+            <span class="o">.</span><span class="n">select_features</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="n">features_list</span><span class="p">))</span>
+        <span class="p">)</span>
 
         <span class="c1"># Instantiate classifier</span>
         <span class="n">estimator</span> <span class="o">=</span> <span class="n">SparkXGBClassifier</span><span class="p">(</span>
@@ -3791,9 +3854,8 @@ <h3 id="gentropy.l2g.LocusToGeneStep.__init__" class="doc doc-heading">
         <span class="k">else</span><span class="p">:</span>
             <span class="c1"># Train model</span>
             <span class="n">LocusToGeneTrainer</span><span class="o">.</span><span class="n">train</span><span class="p">(</span>
-                <span class="n">data</span><span class="o">=</span><span class="n">data</span><span class="p">,</span>
+                <span class="n">gold_standard_data</span><span class="o">=</span><span class="n">data</span><span class="p">,</span>
                 <span class="n">l2g_model</span><span class="o">=</span><span class="n">l2g_model</span><span class="p">,</span>
-                <span class="n">features_list</span><span class="o">=</span><span class="nb">list</span><span class="p">(</span><span class="n">features_list</span><span class="p">),</span>
                 <span class="n">model_path</span><span class="o">=</span><span class="n">model_path</span><span class="p">,</span>
                 <span class="n">evaluate</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
                 <span class="n">wandb_run_name</span><span class="o">=</span><span class="n">wandb_run_name</span><span class="p">,</span>
diff --git a/python_api/steps/ld_clump/index.html b/python_api/steps/ld_clump/index.html
index c53adf6d0..45ebef4ed 100644
--- a/python_api/steps/ld_clump/index.html
+++ b/python_api/steps/ld_clump/index.html
@@ -2195,6 +2195,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../methods/susie_inf/" class="md-nav__link">
         
@@ -2226,7 +2268,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2242,14 +2284,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3033,7 +3075,7 @@ <h3 id="gentropy.ld_based_clumping.LDBasedClumpingStep.__init__" class="doc doc-
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3044,7 +3086,7 @@ <h3 id="gentropy.ld_based_clumping.LDBasedClumpingStep.__init__" class="doc doc-
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>session</code></td>
           <td>
                 <code><span title="gentropy.common.session.Session">Session</span></code>
@@ -3058,7 +3100,7 @@ <h3 id="gentropy.ld_based_clumping.LDBasedClumpingStep.__init__" class="doc doc-
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>study_locus_input_path</code></td>
           <td>
                 <code>str</code>
@@ -3072,7 +3114,7 @@ <h3 id="gentropy.ld_based_clumping.LDBasedClumpingStep.__init__" class="doc doc-
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>study_index_path</code></td>
           <td>
                 <code>str</code>
@@ -3086,7 +3128,7 @@ <h3 id="gentropy.ld_based_clumping.LDBasedClumpingStep.__init__" class="doc doc-
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>ld_index_path</code></td>
           <td>
                 <code>str</code>
@@ -3100,7 +3142,7 @@ <h3 id="gentropy.ld_based_clumping.LDBasedClumpingStep.__init__" class="doc doc-
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>clumped_study_locus_output_path</code></td>
           <td>
                 <code>str</code>
diff --git a/python_api/steps/ld_index/index.html b/python_api/steps/ld_index/index.html
index 9ff0b0571..2d7625efb 100644
--- a/python_api/steps/ld_index/index.html
+++ b/python_api/steps/ld_index/index.html
@@ -2195,6 +2195,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../methods/susie_inf/" class="md-nav__link">
         
@@ -2226,7 +2268,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2242,14 +2284,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3005,7 +3047,7 @@ <h3 id="gentropy.ld_index.LDIndexStep.__init__" class="doc doc-heading">
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3016,7 +3058,7 @@ <h3 id="gentropy.ld_index.LDIndexStep.__init__" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>session</code></td>
           <td>
                 <code><span title="gentropy.common.session.Session">Session</span></code>
@@ -3030,7 +3072,7 @@ <h3 id="gentropy.ld_index.LDIndexStep.__init__" class="doc doc-heading">
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>min_r2</code></td>
           <td>
                 <code>float</code>
@@ -3044,7 +3086,7 @@ <h3 id="gentropy.ld_index.LDIndexStep.__init__" class="doc doc-heading">
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>ld_index_out</code></td>
           <td>
                 <code>str</code>
diff --git a/python_api/steps/pics/index.html b/python_api/steps/pics/index.html
index 5096315ab..a4dadfb39 100644
--- a/python_api/steps/pics/index.html
+++ b/python_api/steps/pics/index.html
@@ -2195,6 +2195,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../methods/susie_inf/" class="md-nav__link">
         
@@ -2226,7 +2268,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2242,14 +2284,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -2943,7 +2985,11 @@ <h2 id="gentropy.pics.PICSStep" class="doc doc-heading">
 <span class="normal">30</span>
 <span class="normal">31</span>
 <span class="normal">32</span>
-<span class="normal">33</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">PICSStep</span><span class="p">:</span>
+<span class="normal">33</span>
+<span class="normal">34</span>
+<span class="normal">35</span>
+<span class="normal">36</span>
+<span class="normal">37</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">PICSStep</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;PICS finemapping of LD-annotated StudyLocus.&quot;&quot;&quot;</span>
 
     <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
@@ -2964,7 +3010,11 @@ <h2 id="gentropy.pics.PICSStep" class="doc doc-heading">
             <span class="n">session</span><span class="p">,</span> <span class="n">study_locus_ld_annotated_in</span>
         <span class="p">)</span>
         <span class="c1"># PICS</span>
-        <span class="n">picsed_sl</span> <span class="o">=</span> <span class="n">PICS</span><span class="o">.</span><span class="n">finemap</span><span class="p">(</span><span class="n">study_locus_ld_annotated</span><span class="p">)</span><span class="o">.</span><span class="n">annotate_credible_sets</span><span class="p">()</span>
+        <span class="n">picsed_sl</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="n">PICS</span><span class="o">.</span><span class="n">finemap</span><span class="p">(</span><span class="n">study_locus_ld_annotated</span><span class="p">)</span>
+            <span class="o">.</span><span class="n">annotate_credible_sets</span><span class="p">()</span>
+            <span class="o">.</span><span class="n">filter_credible_set</span><span class="p">(</span><span class="n">credible_interval</span><span class="o">=</span><span class="n">CredibleInterval</span><span class="o">.</span><span class="n">IS99</span><span class="p">)</span>
+        <span class="p">)</span>
         <span class="c1"># Write</span>
         <span class="n">picsed_sl</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">write</span><span class="o">.</span><span class="n">mode</span><span class="p">(</span><span class="n">session</span><span class="o">.</span><span class="n">write_mode</span><span class="p">)</span><span class="o">.</span><span class="n">parquet</span><span class="p">(</span><span class="n">picsed_study_locus_out</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
@@ -2999,7 +3049,7 @@ <h3 id="gentropy.pics.PICSStep.__init__" class="doc doc-heading">
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3010,7 +3060,7 @@ <h3 id="gentropy.pics.PICSStep.__init__" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>session</code></td>
           <td>
                 <code><span title="gentropy.common.session.Session">Session</span></code>
@@ -3024,7 +3074,7 @@ <h3 id="gentropy.pics.PICSStep.__init__" class="doc doc-heading">
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>study_locus_ld_annotated_in</code></td>
           <td>
                 <code>str</code>
@@ -3038,7 +3088,7 @@ <h3 id="gentropy.pics.PICSStep.__init__" class="doc doc-heading">
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>picsed_study_locus_out</code></td>
           <td>
                 <code>str</code>
@@ -3077,7 +3127,11 @@ <h3 id="gentropy.pics.PICSStep.__init__" class="doc doc-heading">
 <span class="normal">30</span>
 <span class="normal">31</span>
 <span class="normal">32</span>
-<span class="normal">33</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+<span class="normal">33</span>
+<span class="normal">34</span>
+<span class="normal">35</span>
+<span class="normal">36</span>
+<span class="normal">37</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">session</span><span class="p">:</span> <span class="n">Session</span><span class="p">,</span>
     <span class="n">study_locus_ld_annotated_in</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
@@ -3095,7 +3149,11 @@ <h3 id="gentropy.pics.PICSStep.__init__" class="doc doc-heading">
         <span class="n">session</span><span class="p">,</span> <span class="n">study_locus_ld_annotated_in</span>
     <span class="p">)</span>
     <span class="c1"># PICS</span>
-    <span class="n">picsed_sl</span> <span class="o">=</span> <span class="n">PICS</span><span class="o">.</span><span class="n">finemap</span><span class="p">(</span><span class="n">study_locus_ld_annotated</span><span class="p">)</span><span class="o">.</span><span class="n">annotate_credible_sets</span><span class="p">()</span>
+    <span class="n">picsed_sl</span> <span class="o">=</span> <span class="p">(</span>
+        <span class="n">PICS</span><span class="o">.</span><span class="n">finemap</span><span class="p">(</span><span class="n">study_locus_ld_annotated</span><span class="p">)</span>
+        <span class="o">.</span><span class="n">annotate_credible_sets</span><span class="p">()</span>
+        <span class="o">.</span><span class="n">filter_credible_set</span><span class="p">(</span><span class="n">credible_interval</span><span class="o">=</span><span class="n">CredibleInterval</span><span class="o">.</span><span class="n">IS99</span><span class="p">)</span>
+    <span class="p">)</span>
     <span class="c1"># Write</span>
     <span class="n">picsed_sl</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">write</span><span class="o">.</span><span class="n">mode</span><span class="p">(</span><span class="n">session</span><span class="o">.</span><span class="n">write_mode</span><span class="p">)</span><span class="o">.</span><span class="n">parquet</span><span class="p">(</span><span class="n">picsed_study_locus_out</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
diff --git a/python_api/steps/variant_annotation_step/index.html b/python_api/steps/variant_annotation_step/index.html
index c3cdbfeee..b55ab9225 100644
--- a/python_api/steps/variant_annotation_step/index.html
+++ b/python_api/steps/variant_annotation_step/index.html
@@ -2195,6 +2195,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../methods/susie_inf/" class="md-nav__link">
         
@@ -2226,7 +2268,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2242,14 +2284,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -2998,7 +3040,7 @@ <h3 id="gentropy.variant_annotation.VariantAnnotationStep.__init__" class="doc d
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3009,7 +3051,7 @@ <h3 id="gentropy.variant_annotation.VariantAnnotationStep.__init__" class="doc d
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>session</code></td>
           <td>
                 <code><span title="gentropy.common.session.Session">Session</span></code>
@@ -3023,7 +3065,7 @@ <h3 id="gentropy.variant_annotation.VariantAnnotationStep.__init__" class="doc d
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>variant_annotation_path</code></td>
           <td>
                 <code>str</code>
diff --git a/python_api/steps/variant_index_step/index.html b/python_api/steps/variant_index_step/index.html
index c38ae79e3..1a78e5b31 100644
--- a/python_api/steps/variant_index_step/index.html
+++ b/python_api/steps/variant_index_step/index.html
@@ -2195,6 +2195,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../methods/susie_inf/" class="md-nav__link">
         
@@ -2226,7 +2268,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2242,14 +2284,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -3022,7 +3064,7 @@ <h3 id="gentropy.variant_index.VariantIndexStep.__init__" class="doc doc-heading
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3033,7 +3075,7 @@ <h3 id="gentropy.variant_index.VariantIndexStep.__init__" class="doc doc-heading
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>session</code></td>
           <td>
                 <code><span title="gentropy.common.session.Session">Session</span></code>
@@ -3047,7 +3089,7 @@ <h3 id="gentropy.variant_index.VariantIndexStep.__init__" class="doc doc-heading
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>variant_annotation_path</code></td>
           <td>
                 <code>str</code>
@@ -3061,7 +3103,7 @@ <h3 id="gentropy.variant_index.VariantIndexStep.__init__" class="doc doc-heading
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>credible_set_path</code></td>
           <td>
                 <code>str</code>
@@ -3075,7 +3117,7 @@ <h3 id="gentropy.variant_index.VariantIndexStep.__init__" class="doc doc-heading
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>variant_index_path</code></td>
           <td>
                 <code>str</code>
diff --git a/python_api/steps/variant_to_gene_step/index.html b/python_api/steps/variant_to_gene_step/index.html
index a25986bd9..2b94685d1 100644
--- a/python_api/steps/variant_to_gene_step/index.html
+++ b/python_api/steps/variant_to_gene_step/index.html
@@ -2195,6 +2195,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../methods/susie_inf/" class="md-nav__link">
         
@@ -2226,7 +2268,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2242,14 +2284,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -2926,7 +2968,7 @@ <h2 id="gentropy.v2g.V2GStep" class="doc doc-heading">
 
 
 
-  <p><strong>Attributes:</strong></p>
+  <p><span class="doc-section-title">Attributes:</span></p>
   <table>
     <thead>
       <tr>
@@ -2936,7 +2978,7 @@ <h2 id="gentropy.v2g.V2GStep" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code><span title="gentropy.v2g.V2GStep.session">session</span></code></td>
           <td>
                 <code><span title="gentropy.common.session.Session">Session</span></code>
@@ -2947,7 +2989,7 @@ <h2 id="gentropy.v2g.V2GStep" class="doc doc-heading">
             </div>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code><span title="gentropy.v2g.V2GStep.variant_index_path">variant_index_path</span></code></td>
           <td>
                 <code>str</code>
@@ -2958,7 +3000,7 @@ <h2 id="gentropy.v2g.V2GStep" class="doc doc-heading">
             </div>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code><span title="gentropy.v2g.V2GStep.variant_annotation_path">variant_annotation_path</span></code></td>
           <td>
                 <code>str</code>
@@ -2969,7 +3011,7 @@ <h2 id="gentropy.v2g.V2GStep" class="doc doc-heading">
             </div>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code><span title="gentropy.v2g.V2GStep.gene_index_path">gene_index_path</span></code></td>
           <td>
                 <code>str</code>
@@ -2980,7 +3022,7 @@ <h2 id="gentropy.v2g.V2GStep" class="doc doc-heading">
             </div>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code><span title="gentropy.v2g.V2GStep.vep_consequences_path">vep_consequences_path</span></code></td>
           <td>
                 <code>str</code>
@@ -2991,7 +3033,7 @@ <h2 id="gentropy.v2g.V2GStep" class="doc doc-heading">
             </div>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code><span title="gentropy.v2g.V2GStep.liftover_chain_file_path">liftover_chain_file_path</span></code></td>
           <td>
                 <code>str</code>
@@ -3002,7 +3044,7 @@ <h2 id="gentropy.v2g.V2GStep" class="doc doc-heading">
             </div>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code><span title="gentropy.v2g.V2GStep.liftover_max_length_difference">liftover_max_length_difference</span></code></td>
           <td>
                 <code>str</code>
@@ -3013,7 +3055,7 @@ <h2 id="gentropy.v2g.V2GStep" class="doc doc-heading">
             </div>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code><span title="gentropy.v2g.V2GStep.max_distance">max_distance</span></code></td>
           <td>
                 <code>int</code>
@@ -3024,7 +3066,7 @@ <h2 id="gentropy.v2g.V2GStep" class="doc doc-heading">
             </div>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code><span title="gentropy.v2g.V2GStep.approved_biotypes">approved_biotypes</span></code></td>
           <td>
                 <code>list[str]</code>
@@ -3035,7 +3077,7 @@ <h2 id="gentropy.v2g.V2GStep" class="doc doc-heading">
             </div>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code><span title="gentropy.v2g.V2GStep.intervals">intervals</span></code></td>
           <td>
                 <code>dict</code>
@@ -3046,7 +3088,7 @@ <h2 id="gentropy.v2g.V2GStep" class="doc doc-heading">
             </div>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code><span title="gentropy.v2g.V2GStep.v2g_path">v2g_path</span></code></td>
           <td>
                 <code>str</code>
@@ -3319,7 +3361,7 @@ <h3 id="gentropy.v2g.V2GStep.__init__" class="doc doc-heading">
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3330,7 +3372,7 @@ <h3 id="gentropy.v2g.V2GStep.__init__" class="doc doc-heading">
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>session</code></td>
           <td>
                 <code><span title="gentropy.common.session.Session">Session</span></code>
@@ -3344,7 +3386,7 @@ <h3 id="gentropy.v2g.V2GStep.__init__" class="doc doc-heading">
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>variant_index_path</code></td>
           <td>
                 <code>str</code>
@@ -3358,7 +3400,7 @@ <h3 id="gentropy.v2g.V2GStep.__init__" class="doc doc-heading">
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>variant_annotation_path</code></td>
           <td>
                 <code>str</code>
@@ -3372,7 +3414,7 @@ <h3 id="gentropy.v2g.V2GStep.__init__" class="doc doc-heading">
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>gene_index_path</code></td>
           <td>
                 <code>str</code>
@@ -3386,7 +3428,7 @@ <h3 id="gentropy.v2g.V2GStep.__init__" class="doc doc-heading">
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>vep_consequences_path</code></td>
           <td>
                 <code>str</code>
@@ -3400,7 +3442,7 @@ <h3 id="gentropy.v2g.V2GStep.__init__" class="doc doc-heading">
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>liftover_chain_file_path</code></td>
           <td>
                 <code>str</code>
@@ -3414,7 +3456,7 @@ <h3 id="gentropy.v2g.V2GStep.__init__" class="doc doc-heading">
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>approved_biotypes</code></td>
           <td>
                 <code>list[str]</code>
@@ -3428,7 +3470,7 @@ <h3 id="gentropy.v2g.V2GStep.__init__" class="doc doc-heading">
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>interval_sources</code></td>
           <td>
                 <code>dict[str, str]</code>
@@ -3442,7 +3484,7 @@ <h3 id="gentropy.v2g.V2GStep.__init__" class="doc doc-heading">
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>v2g_path</code></td>
           <td>
                 <code>str</code>
@@ -3456,7 +3498,7 @@ <h3 id="gentropy.v2g.V2GStep.__init__" class="doc doc-heading">
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>max_distance</code></td>
           <td>
                 <code>int</code>
@@ -3470,7 +3512,7 @@ <h3 id="gentropy.v2g.V2GStep.__init__" class="doc doc-heading">
                 <code>500000</code>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>liftover_max_length_difference</code></td>
           <td>
                 <code>int</code>
diff --git a/python_api/steps/window_based_clumping/index.html b/python_api/steps/window_based_clumping/index.html
index c88f29d86..6a51f9bd5 100644
--- a/python_api/steps/window_based_clumping/index.html
+++ b/python_api/steps/window_based_clumping/index.html
@@ -2193,6 +2193,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../methods/susie_inf/" class="md-nav__link">
         
@@ -2224,7 +2266,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2240,14 +2282,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
@@ -2918,8 +2960,7 @@ <h2 id="gentropy.window_based_clumping.WindowBasedClumpingStep" class="doc doc-h
 
             <details class="quote">
               <summary>Source code in <code>src/gentropy/window_based_clumping.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 8</span>
-<span class="normal"> 9</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 9</span>
 <span class="normal">10</span>
 <span class="normal">11</span>
 <span class="normal">12</span>
@@ -2960,7 +3001,18 @@ <h2 id="gentropy.window_based_clumping.WindowBasedClumpingStep" class="doc doc-h
 <span class="normal">47</span>
 <span class="normal">48</span>
 <span class="normal">49</span>
-<span class="normal">50</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">WindowBasedClumpingStep</span><span class="p">:</span>
+<span class="normal">50</span>
+<span class="normal">51</span>
+<span class="normal">52</span>
+<span class="normal">53</span>
+<span class="normal">54</span>
+<span class="normal">55</span>
+<span class="normal">56</span>
+<span class="normal">57</span>
+<span class="normal">58</span>
+<span class="normal">59</span>
+<span class="normal">60</span>
+<span class="normal">61</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">WindowBasedClumpingStep</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Apply window based clumping on summary statistics datasets.&quot;&quot;&quot;</span>
 
     <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
@@ -2968,8 +3020,10 @@ <h2 id="gentropy.window_based_clumping.WindowBasedClumpingStep" class="doc doc-h
         <span class="n">session</span><span class="p">:</span> <span class="n">Session</span><span class="p">,</span>
         <span class="n">summary_statistics_input_path</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
         <span class="n">study_locus_output_path</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
+        <span class="n">distance</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">500_000</span><span class="p">,</span>
+        <span class="n">collect_locus</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">collect_locus_distance</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">500_000</span><span class="p">,</span>
         <span class="n">inclusion_list_path</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-        <span class="n">locus_collect_distance</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
     <span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
 <span class="w">        </span><span class="sd">&quot;&quot;&quot;Run window-based clumping step.</span>
 
@@ -2977,8 +3031,10 @@ <h2 id="gentropy.window_based_clumping.WindowBasedClumpingStep" class="doc doc-h
 <span class="sd">            session (Session): Session object.</span>
 <span class="sd">            summary_statistics_input_path (str): Path to the harmonized summary statistics dataset.</span>
 <span class="sd">            study_locus_output_path (str): Output path for the resulting study locus dataset.</span>
+<span class="sd">            distance (int): Distance, within which tagging variants are collected around the semi-index. Optional.</span>
+<span class="sd">            collect_locus (bool): Whether to collect locus around semi-indices. Optional.</span>
+<span class="sd">            collect_locus_distance (int): Distance, within which tagging variants are collected around the semi-index. Optional.</span>
 <span class="sd">            inclusion_list_path (str | None): Path to the inclusion list (list of white-listed study identifier). Optional.</span>
-<span class="sd">            locus_collect_distance (int | None): Distance, within which tagging variants are collected around the semi-index. Optional.</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="c1"># If inclusion list path is provided, only these studies will be read:</span>
         <span class="k">if</span> <span class="n">inclusion_list_path</span><span class="p">:</span>
@@ -2990,19 +3046,25 @@ <h2 id="gentropy.window_based_clumping.WindowBasedClumpingStep" class="doc doc-h
             <span class="c1"># If no inclusion list is provided, read all summary stats in folder:</span>
             <span class="n">study_ids_to_ingest</span> <span class="o">=</span> <span class="p">[</span><span class="n">summary_statistics_input_path</span><span class="p">]</span>
 
-        <span class="p">(</span>
-            <span class="n">SummaryStatistics</span><span class="o">.</span><span class="n">from_parquet</span><span class="p">(</span>
-                <span class="n">session</span><span class="p">,</span>
-                <span class="n">study_ids_to_ingest</span><span class="p">,</span>
-                <span class="n">recursiveFileLookup</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
-            <span class="p">)</span>
-            <span class="o">.</span><span class="n">coalesce</span><span class="p">(</span><span class="mi">4000</span><span class="p">)</span>
-            <span class="c1"># Applying window based clumping:</span>
-            <span class="o">.</span><span class="n">window_based_clumping</span><span class="p">(</span><span class="n">locus_collect_distance</span><span class="o">=</span><span class="n">locus_collect_distance</span><span class="p">)</span>
-            <span class="c1"># Save resulting study locus dataset:</span>
-            <span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">write</span><span class="o">.</span><span class="n">mode</span><span class="p">(</span><span class="n">session</span><span class="o">.</span><span class="n">write_mode</span><span class="p">)</span>
-            <span class="o">.</span><span class="n">parquet</span><span class="p">(</span><span class="n">study_locus_output_path</span><span class="p">)</span>
+        <span class="n">ss</span> <span class="o">=</span> <span class="n">SummaryStatistics</span><span class="o">.</span><span class="n">from_parquet</span><span class="p">(</span>
+            <span class="n">session</span><span class="p">,</span>
+            <span class="n">study_ids_to_ingest</span><span class="p">,</span>
+            <span class="n">recursiveFileLookup</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
         <span class="p">)</span>
+
+        <span class="c1"># Clumping:</span>
+        <span class="n">study_locus</span> <span class="o">=</span> <span class="n">ss</span><span class="o">.</span><span class="n">window_based_clumping</span><span class="p">(</span>
+            <span class="n">distance</span><span class="o">=</span><span class="n">distance</span><span class="p">,</span>
+        <span class="p">)</span>
+
+        <span class="c1"># Optional locus collection:</span>
+        <span class="k">if</span> <span class="n">collect_locus</span><span class="p">:</span>
+            <span class="c1"># Collecting locus around semi-indices:</span>
+            <span class="n">study_locus</span> <span class="o">=</span> <span class="n">study_locus</span><span class="o">.</span><span class="n">annotate_locus_statistics</span><span class="p">(</span>
+                <span class="n">ss</span><span class="p">,</span> <span class="n">collect_locus_distance</span><span class="o">=</span><span class="n">collect_locus_distance</span>
+            <span class="p">)</span>
+
+        <span class="n">study_locus</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">write</span><span class="o">.</span><span class="n">mode</span><span class="p">(</span><span class="n">session</span><span class="o">.</span><span class="n">write_mode</span><span class="p">)</span><span class="o">.</span><span class="n">parquet</span><span class="p">(</span><span class="n">study_locus_output_path</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
             </details>
 
@@ -3024,7 +3086,7 @@ <h2 id="gentropy.window_based_clumping.WindowBasedClumpingStep" class="doc doc-h
 
 
 <h3 id="gentropy.window_based_clumping.WindowBasedClumpingStep.__init__" class="doc doc-heading">
-          <code class="highlight language-python"><span class="fm">__init__</span><span class="p">(</span><span class="n">session</span><span class="p">:</span> <span class="n">Session</span><span class="p">,</span> <span class="n">summary_statistics_input_path</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">study_locus_output_path</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">inclusion_list_path</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">locus_collect_distance</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span></code>
+          <code class="highlight language-python"><span class="fm">__init__</span><span class="p">(</span><span class="n">session</span><span class="p">:</span> <span class="n">Session</span><span class="p">,</span> <span class="n">summary_statistics_input_path</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">study_locus_output_path</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">distance</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">500000</span><span class="p">,</span> <span class="n">collect_locus</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span> <span class="n">collect_locus_distance</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">500000</span><span class="p">,</span> <span class="n">inclusion_list_path</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span></code>
 
 <a href="#gentropy.window_based_clumping.WindowBasedClumpingStep.__init__" class="headerlink" title="Permanent link">&para;</a></h3>
 
@@ -3035,7 +3097,7 @@ <h3 id="gentropy.window_based_clumping.WindowBasedClumpingStep.__init__" class="
 
 
 
-  <p><strong>Parameters:</strong></p>
+  <p><span class="doc-section-title">Parameters:</span></p>
   <table>
     <thead>
       <tr>
@@ -3046,7 +3108,7 @@ <h3 id="gentropy.window_based_clumping.WindowBasedClumpingStep.__init__" class="
       </tr>
     </thead>
     <tbody>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>session</code></td>
           <td>
                 <code><span title="gentropy.common.session.Session">Session</span></code>
@@ -3060,7 +3122,7 @@ <h3 id="gentropy.window_based_clumping.WindowBasedClumpingStep.__init__" class="
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>summary_statistics_input_path</code></td>
           <td>
                 <code>str</code>
@@ -3074,7 +3136,7 @@ <h3 id="gentropy.window_based_clumping.WindowBasedClumpingStep.__init__" class="
               <em>required</em>
           </td>
         </tr>
-        <tr>
+        <tr class="doc-section-item">
           <td><code>study_locus_output_path</code></td>
           <td>
                 <code>str</code>
@@ -3088,30 +3150,58 @@ <h3 id="gentropy.window_based_clumping.WindowBasedClumpingStep.__init__" class="
               <em>required</em>
           </td>
         </tr>
-        <tr>
-          <td><code>inclusion_list_path</code></td>
+        <tr class="doc-section-item">
+          <td><code>distance</code></td>
           <td>
-                <code>str | None</code>
+                <code>int</code>
           </td>
           <td>
             <div class="doc-md-description">
-              <p>Path to the inclusion list (list of white-listed study identifier). Optional.</p>
+              <p>Distance, within which tagging variants are collected around the semi-index. Optional.</p>
             </div>
           </td>
           <td>
-                <code>None</code>
+                <code>500000</code>
           </td>
         </tr>
-        <tr>
-          <td><code>locus_collect_distance</code></td>
+        <tr class="doc-section-item">
+          <td><code>collect_locus</code></td>
           <td>
-                <code>int | None</code>
+                <code>bool</code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>Whether to collect locus around semi-indices. Optional.</p>
+            </div>
+          </td>
+          <td>
+                <code>False</code>
+          </td>
+        </tr>
+        <tr class="doc-section-item">
+          <td><code>collect_locus_distance</code></td>
+          <td>
+                <code>int</code>
           </td>
           <td>
             <div class="doc-md-description">
               <p>Distance, within which tagging variants are collected around the semi-index. Optional.</p>
             </div>
           </td>
+          <td>
+                <code>500000</code>
+          </td>
+        </tr>
+        <tr class="doc-section-item">
+          <td><code>inclusion_list_path</code></td>
+          <td>
+                <code>str | None</code>
+          </td>
+          <td>
+            <div class="doc-md-description">
+              <p>Path to the inclusion list (list of white-listed study identifier). Optional.</p>
+            </div>
+          </td>
           <td>
                 <code>None</code>
           </td>
@@ -3121,8 +3211,7 @@ <h3 id="gentropy.window_based_clumping.WindowBasedClumpingStep.__init__" class="
 
           <details class="quote">
             <summary>Source code in <code>src/gentropy/window_based_clumping.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">11</span>
-<span class="normal">12</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">12</span>
 <span class="normal">13</span>
 <span class="normal">14</span>
 <span class="normal">15</span>
@@ -3160,13 +3249,26 @@ <h3 id="gentropy.window_based_clumping.WindowBasedClumpingStep.__init__" class="
 <span class="normal">47</span>
 <span class="normal">48</span>
 <span class="normal">49</span>
-<span class="normal">50</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+<span class="normal">50</span>
+<span class="normal">51</span>
+<span class="normal">52</span>
+<span class="normal">53</span>
+<span class="normal">54</span>
+<span class="normal">55</span>
+<span class="normal">56</span>
+<span class="normal">57</span>
+<span class="normal">58</span>
+<span class="normal">59</span>
+<span class="normal">60</span>
+<span class="normal">61</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">session</span><span class="p">:</span> <span class="n">Session</span><span class="p">,</span>
     <span class="n">summary_statistics_input_path</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
     <span class="n">study_locus_output_path</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
+    <span class="n">distance</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">500_000</span><span class="p">,</span>
+    <span class="n">collect_locus</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+    <span class="n">collect_locus_distance</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">500_000</span><span class="p">,</span>
     <span class="n">inclusion_list_path</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">locus_collect_distance</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Run window-based clumping step.</span>
 
@@ -3174,8 +3276,10 @@ <h3 id="gentropy.window_based_clumping.WindowBasedClumpingStep.__init__" class="
 <span class="sd">        session (Session): Session object.</span>
 <span class="sd">        summary_statistics_input_path (str): Path to the harmonized summary statistics dataset.</span>
 <span class="sd">        study_locus_output_path (str): Output path for the resulting study locus dataset.</span>
+<span class="sd">        distance (int): Distance, within which tagging variants are collected around the semi-index. Optional.</span>
+<span class="sd">        collect_locus (bool): Whether to collect locus around semi-indices. Optional.</span>
+<span class="sd">        collect_locus_distance (int): Distance, within which tagging variants are collected around the semi-index. Optional.</span>
 <span class="sd">        inclusion_list_path (str | None): Path to the inclusion list (list of white-listed study identifier). Optional.</span>
-<span class="sd">        locus_collect_distance (int | None): Distance, within which tagging variants are collected around the semi-index. Optional.</span>
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="c1"># If inclusion list path is provided, only these studies will be read:</span>
     <span class="k">if</span> <span class="n">inclusion_list_path</span><span class="p">:</span>
@@ -3187,19 +3291,25 @@ <h3 id="gentropy.window_based_clumping.WindowBasedClumpingStep.__init__" class="
         <span class="c1"># If no inclusion list is provided, read all summary stats in folder:</span>
         <span class="n">study_ids_to_ingest</span> <span class="o">=</span> <span class="p">[</span><span class="n">summary_statistics_input_path</span><span class="p">]</span>
 
-    <span class="p">(</span>
-        <span class="n">SummaryStatistics</span><span class="o">.</span><span class="n">from_parquet</span><span class="p">(</span>
-            <span class="n">session</span><span class="p">,</span>
-            <span class="n">study_ids_to_ingest</span><span class="p">,</span>
-            <span class="n">recursiveFileLookup</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
-        <span class="p">)</span>
-        <span class="o">.</span><span class="n">coalesce</span><span class="p">(</span><span class="mi">4000</span><span class="p">)</span>
-        <span class="c1"># Applying window based clumping:</span>
-        <span class="o">.</span><span class="n">window_based_clumping</span><span class="p">(</span><span class="n">locus_collect_distance</span><span class="o">=</span><span class="n">locus_collect_distance</span><span class="p">)</span>
-        <span class="c1"># Save resulting study locus dataset:</span>
-        <span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">write</span><span class="o">.</span><span class="n">mode</span><span class="p">(</span><span class="n">session</span><span class="o">.</span><span class="n">write_mode</span><span class="p">)</span>
-        <span class="o">.</span><span class="n">parquet</span><span class="p">(</span><span class="n">study_locus_output_path</span><span class="p">)</span>
+    <span class="n">ss</span> <span class="o">=</span> <span class="n">SummaryStatistics</span><span class="o">.</span><span class="n">from_parquet</span><span class="p">(</span>
+        <span class="n">session</span><span class="p">,</span>
+        <span class="n">study_ids_to_ingest</span><span class="p">,</span>
+        <span class="n">recursiveFileLookup</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
     <span class="p">)</span>
+
+    <span class="c1"># Clumping:</span>
+    <span class="n">study_locus</span> <span class="o">=</span> <span class="n">ss</span><span class="o">.</span><span class="n">window_based_clumping</span><span class="p">(</span>
+        <span class="n">distance</span><span class="o">=</span><span class="n">distance</span><span class="p">,</span>
+    <span class="p">)</span>
+
+    <span class="c1"># Optional locus collection:</span>
+    <span class="k">if</span> <span class="n">collect_locus</span><span class="p">:</span>
+        <span class="c1"># Collecting locus around semi-indices:</span>
+        <span class="n">study_locus</span> <span class="o">=</span> <span class="n">study_locus</span><span class="o">.</span><span class="n">annotate_locus_statistics</span><span class="p">(</span>
+            <span class="n">ss</span><span class="p">,</span> <span class="n">collect_locus_distance</span><span class="o">=</span><span class="n">collect_locus_distance</span>
+        <span class="p">)</span>
+
+    <span class="n">study_locus</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">write</span><span class="o">.</span><span class="n">mode</span><span class="p">(</span><span class="n">session</span><span class="o">.</span><span class="n">write_mode</span><span class="p">)</span><span class="o">.</span><span class="n">parquet</span><span class="p">(</span><span class="n">study_locus_output_path</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
           </details>
   </div>
diff --git a/roadmap/index.html b/roadmap/index.html
index 2cbeb00d9..61d0d7bd7 100644
--- a/roadmap/index.html
+++ b/roadmap/index.html
@@ -2205,6 +2205,48 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../python_api/methods/sumstat_imputation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Summary Statistics Imputation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../python_api/methods/sumstat_quality_controls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    QC of GWAS Summary Statistics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../python_api/methods/susie_inf/" class="md-nav__link">
         
@@ -2236,7 +2278,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_3_10" >
         
           
           
@@ -2252,14 +2294,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_6_3_8" id="__nav_6_3_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_6_3_10" id="__nav_6_3_10_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_6_3_8">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_3_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_3_10">
             <span class="md-nav__icon md-icon"></span>
             L2g
           </label>
diff --git a/search/search_index.json b/search/search_index.json
index 75531965e..fc8c16a1d 100644
--- a/search/search_index.json
+++ b/search/search_index.json
@@ -1 +1 @@
-{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Open Targets Gentropy","text":"<p>Open Targets Gentropy is a Python package to facilitate the interpretation and analysis of GWAS and functional genomic studies for target identification. This package contains a toolkit for the harmonisation, statistical analysis and prioritisation of genetic signals to assist drug discovery.</p>"},{"location":"#key-features","title":"Key Features:","text":"<ul> <li>Specialized Datatypes: Introduces essential genetics datatypes like StudyLocus, LocusToGene, and SummaryStatistics.</li> <li>Performance-Oriented: Optimized for large-scale genetic data analysis, including locus-to-gene scoring, fine mapping, and colocalization analysis.</li> <li>User-Friendly: The package is designed to be intuitive, allowing both beginners and experienced researchers to conduct complex genetic with ease.</li> </ul>"},{"location":"#about-open-targets","title":"About Open Targets","text":"<p>Open Targets is a pre-competitive, public-private partnership that uses human genetics and genomics data to systematically identify and prioritise drug targets. Through large-scale genomic experiments and the development of innovative computational techniques, the partnership aims to help researchers select the best targets for the development of new therapies. For more information, visit the Open Targets website.</p>"},{"location":"installation/","title":"Installation","text":"<p>Note</p> <p>In the early stages of development, we are using Python version 3.10. We recommend using pyenv or similar tools to manage your local Python version. We intend to support more Python versions in the future.</p>"},{"location":"installation/#pypi","title":"Pypi","text":"<p>We recommend installing Open Targets Gentropy using Pypi:</p> <pre><code>pip install gentropy\n</code></pre>"},{"location":"installation/#source","title":"Source","text":"<p>Alternatively, you can install Open Targets Gentropy from source. Check the contributing section for more information.</p>"},{"location":"installation/#uninstall","title":"Uninstall","text":"<pre><code>pip uninstall gentropy -y\n</code></pre> <p>For any issues with the installation, check the troubleshooting section.</p>"},{"location":"roadmap/","title":"Roadmap","text":"<p>The Open Targets core team is working on refactoring Open Targets Genetics, aiming to:</p> <ul> <li>Re-focus the product around Target ID</li> <li>Create a gold standard toolkit for post-GWAS analysis</li> <li>Faster/robust addition of new datasets and datatypes</li> <li>Reduce computational and financial cost</li> </ul> <p>See here for a list of open issues for this project.</p> <p>Schematic diagram representing the drafted process:</p> <p></p>"},{"location":"development/_development/","title":"Development","text":"<p>This section contains various technical information on how to develop and run the code.</p>"},{"location":"development/airflow/","title":"Airflow configuration","text":"<p>This section describes how to set up a local Airflow server which will orchestrate running workflows in Google Cloud Platform. This is useful for testing and debugging, but for production use, it is recommended to run Airflow on a dedicated server.</p>"},{"location":"development/airflow/#install-pre-requisites","title":"Install pre-requisites","text":"<ul> <li>Docker</li> <li>Google Cloud SDK</li> </ul> <p>Warning</p> <p>On macOS, the default amount of memory available for Docker might not be enough to get Airflow up and running. Allocate at least 4GB of memory for the Docker Engine (ideally 8GB). More info</p>"},{"location":"development/airflow/#configure-airflow-access-to-google-cloud-platform","title":"Configure Airflow access to Google Cloud Platform","text":"<p>Warning</p> <p>Run the next two command with the appropriate Google Cloud project ID and service account name to ensure the correct Google default application credentials are set up.</p> <p>Authenticate to Google Cloud:</p> <pre><code>gcloud auth application-default login --project=&lt;PROJECT&gt;\n</code></pre> <p>Create the service account key file that will be used by Airflow to access Google Cloud Platform resources:</p> <pre><code>gcloud iam service-accounts keys create ~/.config/gcloud/service_account_credentials.json --iam-account=&lt;PROJECT&gt;@appspot.gserviceaccount.com\n</code></pre>"},{"location":"development/airflow/#set-up-airflow","title":"Set up Airflow","text":"<p>Change the working directory so that all subsequent commands will work:</p> <pre><code>cd src/airflow\n</code></pre>"},{"location":"development/airflow/#build-docker-image","title":"Build Docker image","text":"<p>Note</p> <p>The custom Dockerfile built by the command below extends the official Airflow Docker Compose YAML. We add support for Google Cloud SDK, Google Dataproc operators, and access to GCP credentials.</p> <pre><code>docker build . --tag extending_airflow:latest\n</code></pre>"},{"location":"development/airflow/#set-airflow-user-id","title":"Set Airflow user ID","text":"<p>Note</p> <p>These commands allow Airflow running inside Docker to access the credentials file which was generated earlier.</p> <pre><code># If any user ID is already specified in .env, remove it.\ngrep -v \"AIRFLOW_UID\" .env &gt; .env.tmp\n# Add the correct user ID.\necho \"AIRFLOW_UID=$(id -u)\" &gt;&gt; .env.tmp\n# Move the file.\nmv .env.tmp .env\n</code></pre>"},{"location":"development/airflow/#initialise","title":"Initialise","text":"<p>Before starting Airflow, initialise the database:</p> <pre><code>docker compose up airflow-init\n</code></pre> <p>Now start all services:</p> <pre><code>docker compose up -d\n</code></pre> <p>Airflow UI will now be available at <code>http://localhost:8080/</code>. Default username and password are both <code>airflow</code>.</p> <p>For additional information on how to use Airflow visit the official documentation.</p>"},{"location":"development/airflow/#cleaning-up","title":"Cleaning up","text":"<p>At any time, you can check the status of your containers with:</p> <pre><code>docker ps\n</code></pre> <p>To stop Airflow, run:</p> <pre><code>docker compose down\n</code></pre> <p>To cleanup the Airflow database, run:</p> <pre><code>docker compose down --volumes --remove-orphans\n</code></pre>"},{"location":"development/airflow/#advanced-configuration","title":"Advanced configuration","text":"<p>More information on running Airflow with Docker Compose can be found in the official docs.</p> <ol> <li>Increase Airflow concurrency. Modify the <code>docker-compose.yaml</code> and add the following to the x-airflow-common \u2192 environment section:</li> </ol> <pre><code>AIRFLOW__CORE__PARALLELISM: 32\nAIRFLOW__CORE__MAX_ACTIVE_TASKS_PER_DAG: 32\nAIRFLOW__SCHEDULER__MAX_TIS_PER_QUERY: 16\nAIRFLOW__CORE__MAX_ACTIVE_RUNS_PER_DAG: 1\n# Also add the following line if you are using CeleryExecutor (by default, LocalExecutor is used).\nAIRFLOW__CELERY__WORKER_CONCURRENCY: 32\n</code></pre> <ol> <li>Additional pip packages. They can be added to the <code>requirements.txt</code> file.</li> </ol>"},{"location":"development/airflow/#troubleshooting","title":"Troubleshooting","text":"<p>Note that when you a a new workflow under <code>dags/</code>, Airflow will not pick that up immediately. By default the filesystem is only scanned for new DAGs every 300s. However, once the DAG is added, updates are applied nearly instantaneously.</p> <p>Also, if you edit the DAG while an instance of it is running, it might cause problems with the run, as Airflow will try to update the tasks and their properties in DAG according to the file changes.</p>"},{"location":"development/contributing/","title":"Contributing guidelines","text":""},{"location":"development/contributing/#one-time-configuration","title":"One-time configuration","text":"<p>The steps in this section only ever need to be done once on any particular system.</p> <p>For Google Cloud configuration:</p> <ol> <li> <p>Install Google Cloud SDK: https://cloud.google.com/sdk/docs/install.</p> </li> <li> <p>Log in to your work Google Account: run <code>gcloud auth login</code> and follow instructions.</p> </li> <li> <p>Obtain Google application credentials: run <code>gcloud auth application-default login</code> and follow instructions.</p> </li> </ol> <p>Check that you have the <code>make</code> utility installed, and if not (which is unlikely), install it using your system package manager.</p> <p>Check that you have <code>java</code> installed.</p>"},{"location":"development/contributing/#environment-configuration","title":"Environment configuration","text":"<p>Run <code>make setup-dev</code> to install/update the necessary packages and activate the development environment. You need to do this every time you open a new shell.</p> <p>It is recommended to use VS Code as an IDE for development.</p>"},{"location":"development/contributing/#how-to-run-the-code","title":"How to run the code","text":"<p>All pipelines in this repository are intended to be run in Google Dataproc. Running them locally is not currently supported.</p> <p>In order to run the code:</p> <ol> <li> <p>Manually edit your local <code>src/airflow/dags/*</code> file and comment out the steps you do not want to run.</p> </li> <li> <p>Manually edit your local <code>pyproject.toml</code> file and modify the version of the code.</p> </li> <li> <p>This must be different from the version used by any other people working on the repository to avoid any deployment conflicts, so it's a good idea to use your name, for example: <code>1.2.3+jdoe</code>.</p> </li> <li>You can also add a brief branch description, for example: <code>1.2.3+jdoe.myfeature</code>.</li> <li>Note that the version must comply with PEP440 conventions, otherwise Poetry will not allow it to be deployed.</li> <li> <p>Do not use underscores or hyphens in your version name. When building the WHL file, they will be automatically converted to dots, which means the file name will no longer match the version and the build will fail. Use dots instead.</p> </li> <li> <p>Manually edit your local <code>src/airflow/dags/common_airflow.py</code> and set <code>OTG_VERSION</code> to the same version as you did in the previous step.</p> </li> <li> <p>Run <code>make build</code>.</p> </li> <li> <p>This will create a bundle containing the neccessary code, configuration and dependencies to run the ETL pipeline, and then upload this bundle to Google Cloud.</p> </li> <li>A version specific subpath is used, so uploading the code will not affect any branches but your own.</li> <li> <p>If there was already a code bundle uploaded with the same version number, it will be replaced.</p> </li> <li> <p>Open Airflow UI and run the DAG.</p> </li> </ol>"},{"location":"development/contributing/#contributing-checklist","title":"Contributing checklist","text":"<p>When making changes, and especially when implementing a new module or feature, it's essential to ensure that all relevant sections of the code base are modified.</p> <ul> <li>[ ] Run <code>make check</code>. This will run the linter and formatter to ensure that the code is compliant with the project conventions.</li> <li>[ ] Develop unit tests for your code and run <code>make test</code>. This will run all unit tests in the repository, including the examples appended in the docstrings of some methods.</li> <li>[ ] Update the configuration if necessary.</li> <li>[ ] Update the documentation and check it with <code>make build-documentation</code>. This will start a local server to browse it (URL will be printed, usually <code>http://127.0.0.1:8000/</code>)</li> </ul> <p>For more details on each of these steps, see the sections below.</p>"},{"location":"development/contributing/#documentation","title":"Documentation","text":"<ul> <li>If during development you had a question which wasn't covered in the documentation, and someone explained it to you, add it to the documentation. The same applies if you encountered any instructions in the documentation which were obsolete or incorrect.</li> <li>Documentation autogeneration expressions start with <code>:::</code>. They will automatically generate sections of the documentation based on class and method docstrings. Be sure to update them for:</li> <li>Dataset definitions in <code>docs/python_api/datasource/STEP</code> (example: <code>docs/python_api/datasource/finngen/study_index.md</code>)</li> <li>Step definition in <code>docs/python_api/step/STEP.md</code> (example: <code>docs/python_api/step/finngen.md</code>)</li> </ul>"},{"location":"development/contributing/#configuration","title":"Configuration","text":"<ul> <li>Input and output paths in <code>config/datasets/gcp.yaml</code></li> <li>Step configuration in <code>config/step/STEP.yaml</code> (example: <code>config/step/finngen.yaml</code>)</li> </ul>"},{"location":"development/contributing/#classes","title":"Classes","text":"<ul> <li>Dataset class in <code>src/gentropy/datasource/STEP</code> (example: <code>src/gentropy/datasource/finngen/study_index.py</code> \u2192 <code>FinnGenStudyIndex</code>)</li> <li>Step main running class in <code>src/gentropy/STEP.py</code> (example: <code>src/gentropy/finngen.py</code>)</li> </ul>"},{"location":"development/contributing/#tests","title":"Tests","text":"<ul> <li>Test study fixture in <code>tests/conftest.py</code> (example: <code>mock_study_index_finngen</code> in that module)</li> <li>Test sample data in <code>tests/data_samples</code> (example: <code>tests/gentropy/data_samples/finngen_studies_sample.json</code>)</li> <li>Test definition in <code>tests/</code> (example: <code>tests/dataset/test_study_index.py</code> \u2192 <code>test_study_index_finngen_creation</code>)</li> </ul>"},{"location":"development/troubleshooting/","title":"Troubleshooting","text":""},{"location":"development/troubleshooting/#blaslapack","title":"BLAS/LAPACK","text":"<p>If you see errors related to BLAS/LAPACK libraries, see this StackOverflow post for guidance.</p>"},{"location":"development/troubleshooting/#pyenv-and-poetry","title":"Pyenv and Poetry","text":"<p>If you see various errors thrown by Pyenv or Poetry, they can be hard to specifically diagnose and resolve. In this case, it often helps to remove those tools from the system completely. Follow these steps:</p> <ol> <li>Close your currently activated environment, if any: <code>exit</code></li> <li>Uninstall Poetry: <code>curl -sSL https://install.python-poetry.org | python3 - --uninstall</code></li> <li>Clear Poetry cache: <code>rm -rf ~/.cache/pypoetry</code></li> <li>Clear pre-commit cache: <code>rm -rf ~/.cache/pre-commit</code></li> <li>Switch to system Python shell: <code>pyenv shell system</code></li> <li>Edit <code>~/.bashrc</code> to remove the lines related to Pyenv configuration</li> <li>Remove Pyenv configuration and cache: <code>rm -rf ~/.pyenv</code></li> </ol> <p>After that, open a fresh shell session and run <code>make setup-dev</code> again.</p>"},{"location":"development/troubleshooting/#java","title":"Java","text":"<p>Officially, PySpark requires Java version 8 (a.k.a. 1.8) or above to work. However, if you have a very recent version of Java, you may experience issues, as it may introduce breaking changes that PySpark hasn't had time to integrate. For example, as of May 2023, PySpark did not work with Java 20.</p> <p>If you are encountering problems with initialising a Spark session, try using Java 11.</p>"},{"location":"development/troubleshooting/#pre-commit","title":"Pre-commit","text":"<p>If you see an error message thrown by pre-commit, which looks like this (<code>SyntaxError: Unexpected token '?'</code>), followed by a JavaScript traceback, the issue is likely with your system NodeJS version.</p> <p>One solution which can help in this case is to upgrade your system NodeJS version. However, this may not always be possible. For example, Ubuntu repository is several major versions behind the latest version as of July 2023.</p> <p>Another solution which helps is to remove Node, NodeJS, and npm from your system entirely. In this case, pre-commit will not try to rely on a system version of NodeJS and will install its own, suitable one.</p> <p>On Ubuntu, this can be done using <code>sudo apt remove node nodejs npm</code>, followed by <code>sudo apt autoremove</code>. But in some cases, depending on your existing installation, you may need to also manually remove some files. See this StackOverflow answer for guidance.</p> <p>After running these commands, you are advised to open a fresh shell, and then also reinstall Pyenv and Poetry to make sure they pick up the changes (see relevant section above).</p>"},{"location":"development/workflows/","title":"Pipeline workflows","text":"<p>This page describes the high level components of the pipeline, which are organised as Airflow DAGs (directed acyclic graphs).</p>"},{"location":"development/workflows/#note-on-dags-and-dataproc-clusters","title":"Note on DAGs and Dataproc clusters","text":"<p>Each DAG consists of the following general stages:</p> <ol> <li> <p>Create cluster (if it already exists, this step is skipped)</p> </li> <li> <p>Install dependencies on the cluster</p> </li> <li> <p>Run data processing steps for this DAG</p> </li> <li> <p>Delete the cluster</p> </li> </ol> <p>Within a DAG, all data processing steps run on the same Dataproc cluster as separate jobs.</p> <p>There is no need to configure DAGs or steps depending on the size of the input data. Clusters have autoscaling enabled, which means they will increase or decrease the number of worker VMs to accommodate the load.</p>"},{"location":"development/workflows/#dag-1-preprocess","title":"DAG 1: Preprocess","text":"<p>This DAG contains steps which are only supposed to be run once, or very rarely. They ingest external data and apply bespoke transformations specific for each particular data source. The output is normalised according to the data schemas used by the pipeline.</p>"},{"location":"development/workflows/#dag-2-etl","title":"DAG 2: ETL","text":"<p>The ETL DAG takes the inputs of the previous step and performs the main algorithmic processing. This processing is supposed to be data source agnostic.</p>"},{"location":"howto/_howto/","title":"How-to","text":"<p>This page contains a collection of how-to guides for the project.</p> <ul> <li>Command line interface: Learn how to use the Gentropy CLI.</li> <li>Python API: Learn how to use the Gentropy Python package.</li> </ul> <p>For additional information please visit https://community.opentargets.org/</p>"},{"location":"howto/command_line/_command_line/","title":"Command line interface","text":"<p>Gentropy steps can be run using the command line interface (CLI). This section contains a collection of how-to guides for the CLI.</p>"},{"location":"howto/command_line/run_step_in_cli/","title":"Run step in CLI","text":"<p>To run a step in the command line interface (CLI), you need to know the step's name. To list what steps are avaiable in your current environment, simply run <code>gentropy</code> with no arguments. This will list all the steps:</p> <pre><code>You must specify 'step', e.g, step=&lt;OPTION&gt;\nAvailable options:\n        clump\n        colocalisation\n        eqtl_catalogue\n        finngen_studies\n        finngen_sumstat_preprocess\n        gene_index\n        gwas_catalog_ingestion\n        gwas_catalog_sumstat_preprocess\n        ld_index\n        locus_to_gene\n        overlaps\n        pics\n        ukbiobank\n        variant_annotation\n        variant_index\n        variant_to_gene\n\nSet the environment variable HYDRA_FULL_ERROR=1 for a complete stack trace.\n</code></pre> <p>As indicated, you can run a step by specifying the step's name with the <code>step</code> argument. For example, to run the <code>gene_index</code> step, you can run:</p> <pre><code>gentropy step=gene_index\n</code></pre> <p>In most occassions, some mandatory values will be required to run the step. For example, the <code>gene_index</code> step requires the <code>step.target_path</code> and <code>step.gene_index_path</code> argument to be specified. You can complete the necessary arguments by adding them to the command line:</p> <pre><code>gentropy step=gene_index step.target_path=/path/to/target step.gene_index_path=/path/to/gene_index\n</code></pre> <p>You can find more about the available steps in the documentation.</p>"},{"location":"howto/command_line/run_step_using_config/","title":"Run step using YAML config","text":"<p>It's possible to parametrise the functionality of a step using a YAML configuration file. This is useful when you want to run a step multiple times with different parameters or simply to avoid having to specify the same parameters every time you run a step.</p> <p>Info</p> <p>The package uses Hydra to handle configuration files. For more information, please visit the Hydra documentation.</p> <p>To run a step using a configuration file, you need to create a configuration file in YAML format.</p> <pre><code>config/\n\u251c\u2500 step/\n\u2502  \u2514\u2500 my_gene_index.md\n\u2514\u2500 my_config.yml\n</code></pre> <p>The configuration file should contain the parameters you want to use to run the step. For example, to run the <code>gene_index</code> step, you need to specify the <code>step.target_path</code> and <code>step.gene_index_path</code> parameters. The configuration file should look like this:</p> my_config.yamlstep/my_gene_index.md <pre><code>defaults:\n    - config\n    - _self_\n</code></pre> <p>This config file will specify that your configuration file will inherit the default configuration (<code>config</code>) and everything provided (<code>_self_</code>) will overwrite the default configuration.</p> <pre><code>defaults:\n    - gene_index\n\ntarget_path: /path/to/target\ngene_index_path: /path/to/gene_index\n</code></pre> <p>This config file will inherit the default configuration for the <code>gene_index</code> step and overwrite the <code>target_path</code> and <code>gene_index_path</code> parameters.</p> <p>Once you have created the configuration file, you can run your own new <code>my_gene_index</code>:</p> <pre><code>gentropy step=my_gene_index --config-dir=config --config-name=my_config\n</code></pre>"},{"location":"howto/python_api/_python_api/","title":"Python API","text":"<p>This section explains how to use gentropy in a Python environment providing a foundational understanding on how to perform genetics analyses using the package. This section can be useful for users wishing to use Gentropy in their own projects.</p>"},{"location":"howto/python_api/a_creating_spark_session/","title":"Creating a Spark Session","text":"<p>In this section, we'll guide you through creating a Spark session using Gentropy's Session class. Gentropy uses Apache PySpark as the underlying framework for distributed computing. The Session class provides a convenient way to initialize a Spark session with pre-configured settings.</p>"},{"location":"howto/python_api/a_creating_spark_session/#creating-a-default-session","title":"Creating a Default Session","text":"<p>To begin your journey with Gentropy, start by creating a default Spark session. This is the simplest way to initialize your environment.</p> <pre><code>from gentropy.common.session import Session\n\n# Create a default Spark Session\nsession = Session()\n</code></pre> <p>The above code snippet sets up a default Spark session with pre-configured settings. This is ideal for getting started quickly without needing to tweak any configurations.</p>"},{"location":"howto/python_api/a_creating_spark_session/#customizing-your-spark-session","title":"Customizing Your Spark Session","text":"<p>Gentropy allows you to customize the Spark session to suit your specific needs. You can modify various parameters such as memory allocation, number of executors, and more. This flexibility is particularly useful for optimizing performance in steps that are more computationally intensive.</p>"},{"location":"howto/python_api/a_creating_spark_session/#example-increasing-driver-memory","title":"Example: Increasing Driver Memory","text":"<p>If you require more memory for the Spark driver, you can easily adjust this setting:</p> <pre><code>from gentropy.common.session import Session\n\n# Create a Spark session with increased driver memory\nsession = Session(extended_spark_conf={\"spark.driver.memory\": \"4g\"})\n</code></pre> <p>This code snippet demonstrates how to increase the memory allocated to the Spark driver to 16 gigabytes. You can customize other Spark settings similarly, according to your project's requirements.</p>"},{"location":"howto/python_api/a_creating_spark_session/#whats-next","title":"What's next?","text":"<p>Now that you've created a Spark session, you're ready to start using Gentropy. In the next section, we'll show you how to process a large dataset using Gentropy's powerful SummaryStatistics datatype.</p>"},{"location":"howto/python_api/b_create_dataset/","title":"Create a dataset","text":"<p>Gentropy provides a collection of <code>Dataset</code>s that encapsulate key concepts in the field of genetics. For example, to represent summary statistics, you'll use the <code>SummaryStatistics</code> class. This datatype comes with a set of useful operations to disentangle the genetic architecture of a trait or disease.</p> <p>The full list of <code>Dataset</code>s is available in the Python API documentation.</p> <p>Any instance of Dataset will have 2 common attributes</p> <ul> <li>df: the Spark DataFrame that contains the data</li> <li>schema: the definition of the data structure in Spark format</li> </ul> <p>In this section you'll learn the different ways of how to create a <code>Dataset</code> instances.</p>"},{"location":"howto/python_api/b_create_dataset/#creating-a-dataset-from-parquet","title":"Creating a dataset from parquet","text":"<p>All the <code>Dataset</code>s have a <code>from_parquet</code> method that allows you to create any <code>Dataset</code> instance from a parquet file or directory.</p> <pre><code># Create a SummaryStatistics object by loading data from the specified path\nfrom gentropy.dataset.summary_statistics import SummaryStatistics\n\npath = \"path/to/summary/stats\"\nsummary_stats = SummaryStatistics.from_parquet(session, path)\n</code></pre> <p>Parquet files</p> <p>Parquet is a columnar storage format that is widely used in the Spark ecosystem. It is the recommended format for storing large datasets. For more information about parquet, please visit https://parquet.apache.org/.</p>"},{"location":"howto/python_api/b_create_dataset/#creating-a-dataset-from-a-data-source","title":"Creating a dataset from a data source","text":"<p>Alternatively, <code>Dataset</code>s can be created using a data source harmonisation method. For example, to create a <code>SummaryStatistics</code> object from Finngen's raw summary statistics, you can use the <code>FinnGen</code> data source.</p> <pre><code># Create a SummaryStatistics object by loading raw data from Finngen\nfrom gentropy.datasource.finngen.summary_stats import FinnGenSummaryStats\n\npath = \"path/to/finngen/summary/stats\"\nsummary_stats = FinnGenSummaryStats.from_source(session.spark, path)\n</code></pre>"},{"location":"howto/python_api/b_create_dataset/#creating-a-dataset-from-a-pandas-dataframe","title":"Creating a dataset from a pandas DataFrame","text":"<p>If none of our data sources fit your needs, you can create a <code>Dataset</code> object from your own data. To do so, you need to transform your data to fit the <code>Dataset</code> schema.</p> <p>The schema of a Dataset is defined in Spark format</p> <p>The Dataset schemas can be found in the documentation of each Dataset. For example, the schema of the <code>SummaryStatistics</code> dataset can be found here.</p> <p>You can also create a <code>Dataset</code> from a pandas DataFrame. This is useful when you want to create a <code>Dataset</code> from a small dataset that fits in memory.</p> <pre><code>import pyspark.pandas as ps\nfrom gentropy.dataset.summary_statistics import SummaryStatistics\n\n\n# Load your transformed data into a pandas DataFrame\npath = \"path/to/your/data\"\ncustom_summary_stats_pandas_df = pd.read_csv(path)\n\n# Create a SummaryStatistics object specifying the data and schema\ncustom_summary_stats_df = custom_summary_stats_pandas_df.to_spark()\ncustom_summary_stats = SummaryStatistics(\n    _df=custom_summary_stats_df, _schema=SummaryStatistics.get_schema()\n)\n</code></pre>"},{"location":"howto/python_api/b_create_dataset/#whats-next","title":"What's next?","text":"<p>In the next section, we will explore how to apply well-established algorithms that transform and analyse genetic data within the Gentropy framework.</p>"},{"location":"howto/python_api/c_applying_methods/","title":"Applying methods","text":"<p>The available methods implement well established algorithms that transform and analyse data. Methods usually take as input predefined <code>Dataset</code>(s) and produce one or several <code>Dataset</code>(s) as output. This section explains how to apply methods to your data.</p> <p>The full list of available methods can be found in the Python API documentation.</p>"},{"location":"howto/python_api/c_applying_methods/#apply-a-class-method","title":"Apply a class method","text":"<p>Some methods are implemented as class methods. For example, the <code>finemap</code> method is a class method of the <code>PICS</code> class. This method performs fine-mapping using the PICS algorithm. These methods usually take as input one or several <code>Dataset</code>(s) and produce one or several <code>Dataset</code>(s) as output.</p> <pre><code>from gentropy.method.pics import PICS\n\nfinemapped_study_locus = PICS.finemap(\n    study_locus_ld_annotated\n).annotate_credible_sets()\n</code></pre>"},{"location":"howto/python_api/c_applying_methods/#apply-a-dataset-instance-method","title":"Apply a <code>Dataset</code> instance method","text":"<p>Some methods are implemented as instance methods of the <code>Dataset</code> class. For example, the <code>window_based_clumping</code> method is an instance method of the <code>SummaryStatistics</code> class. This method performs window-based clumping on summary statistics.</p> <pre><code># Perform window-based clumping on summary statistics\n# By default, the method uses a 1Mb window and a p-value threshold of 5e-8\nclumped_summary_statistics = summary_stats.window_based_clumping()\n</code></pre> <p>The <code>window_based_clumping</code> method is also available as a class method</p> <p>The <code>window_based_clumping</code> method is also available as a class method of the <code>WindowBasedClumping</code> class. This method performs window-based clumping on summary statistics.</p> <pre><code># Perform window-based clumping on summary statistics\nfrom gentropy.method.window_based_clumping import WindowBasedClumping\n\nclumped_summary_statistics = WindowBasedClumping.clump(\n    summary_stats, window_length=500_000\n)\n</code></pre>"},{"location":"howto/python_api/c_applying_methods/#whats-next","title":"What's next?","text":"<p>Up next, we'll show you how to inspect your data to ensure its integrity and the success of your transformations.</p>"},{"location":"howto/python_api/d_inspect_dataset/","title":"Inspect a dataset","text":"<p>We have seen how to create and transform a <code>Dataset</code> instance. This section guides you through inspecting your data to ensure its integrity and the success of your transformations.</p>"},{"location":"howto/python_api/d_inspect_dataset/#inspect-data-in-a-dataset","title":"Inspect data in a <code>Dataset</code>","text":"<p>The <code>df</code> attribute of a Dataset instance is key to interacting with and inspecting the stored data.</p> <p>By accessing the df attribute, you can apply any method that you would typically use on a PySpark DataFrame. See the PySpark documentation for more information.</p>"},{"location":"howto/python_api/d_inspect_dataset/#view-data-samples","title":"View data samples","text":"<pre><code># Inspect the first 10 rows of the data\nsummary_stats.df.show(10)\n</code></pre> <p>This method displays the first 10 rows of your dataset, giving you a snapshot of your data's structure and content.</p>"},{"location":"howto/python_api/d_inspect_dataset/#filter-data","title":"Filter data","text":"<pre><code>import pyspark.sql.functions as f\n\n# Filter summary statistics to only include associations in chromosome 22\nfiltered = summary_stats.filter(condition=f.col(\"chromosome\") == \"22\")\n</code></pre> <p>This method allows you to filter your data based on specific conditions, such as the value of a column. The application of any filter will create a new instance of the <code>Dataset</code> with the filtered data.</p>"},{"location":"howto/python_api/d_inspect_dataset/#understand-the-schema","title":"Understand the schema","text":"<pre><code># Get the Spark schema of any `Dataset` as a `StructType` object\nschema = summary_stats.get_schema()\n\n# Inspect the first 10 rows of the data\nsummary_stats.df.show(10)\n</code></pre>"},{"location":"howto/python_api/d_inspect_dataset/#write-a-dataset-to-disk","title":"Write a <code>Dataset</code> to disk","text":"<pre><code># Write the data to disk in parquet format\nsummary_stats.df.write.parquet(\"path/to/summary/stats\")\n\n# Write the data to disk in csv format\nsummary_stats.df.write.csv(\"path/to/summary/stats\")\n</code></pre> <p>Consider the format's compatibility with your tools, and the partitioning strategy for large datasets to optimize performance.</p>"},{"location":"python_api/_python_api/","title":"Python API","text":"<p>Open Targets Gentropy is a Python package to facilitate the interpretation and analysis of GWAS and functional genomic studies for target identification. The package contains a toolkit for the harmonisation, statistical analysis and prioritisation of genetic signals to assist drug discovery.</p> <p>The overall architecture of the package distinguishes between:</p> <ul> <li>Data Sources: data sources harmonisation tools</li> <li>Datasets: data model</li> <li>Methods: statistical analysis tools</li> <li>Steps: pipeline steps</li> </ul>"},{"location":"python_api/datasets/_datasets/","title":"Datasets","text":"<p>The Dataset classes define the data model behind Open Targets Gentropy. Every class inherits from the <code>Dataset</code> class and contains a dataframe with a predefined schema that can be found in the respective classes.</p>"},{"location":"python_api/datasets/_datasets/#gentropy.dataset.dataset.Dataset","title":"<code>gentropy.dataset.dataset.Dataset</code>  <code>dataclass</code>","text":"<p>             Bases: <code>ABC</code></p> <p>Open Targets Gentropy Dataset.</p> <p><code>Dataset</code> is a wrapper around a Spark DataFrame with a predefined schema. Schemas for each child dataset are described in the <code>schemas</code> module.</p> Source code in <code>src/gentropy/dataset/dataset.py</code> <pre><code>@dataclass\nclass Dataset(ABC):\n    \"\"\"Open Targets Gentropy Dataset.\n\n    `Dataset` is a wrapper around a Spark DataFrame with a predefined schema. Schemas for each child dataset are described in the `schemas` module.\n    \"\"\"\n\n    _df: DataFrame\n    _schema: StructType\n\n    def __post_init__(self: Dataset) -&gt; None:\n        \"\"\"Post init.\"\"\"\n        self.validate_schema()\n\n    @property\n    def df(self: Dataset) -&gt; DataFrame:\n        \"\"\"Dataframe included in the Dataset.\n\n        Returns:\n            DataFrame: Dataframe included in the Dataset\n        \"\"\"\n        return self._df\n\n    @df.setter\n    def df(self: Dataset, new_df: DataFrame) -&gt; None:  # noqa: CCE001\n        \"\"\"Dataframe setter.\n\n        Args:\n            new_df (DataFrame): New dataframe to be included in the Dataset\n        \"\"\"\n        self._df: DataFrame = new_df\n        self.validate_schema()\n\n    @property\n    def schema(self: Dataset) -&gt; StructType:\n        \"\"\"Dataframe expected schema.\n\n        Returns:\n            StructType: Dataframe expected schema\n        \"\"\"\n        return self._schema\n\n    @classmethod\n    @abstractmethod\n    def get_schema(cls: type[Self]) -&gt; StructType:\n        \"\"\"Abstract method to get the schema. Must be implemented by child classes.\n\n        Returns:\n            StructType: Schema for the Dataset\n        \"\"\"\n        pass\n\n    @classmethod\n    def from_parquet(\n        cls: type[Self],\n        session: Session,\n        path: str | list[str],\n        **kwargs: bool | float | int | str | None,\n    ) -&gt; Self:\n        \"\"\"Reads parquet into a Dataset with a given schema.\n\n        Args:\n            session (Session): Spark session\n            path (str | list[str]): Path to the parquet dataset\n            **kwargs (bool | float | int | str | None): Additional arguments to pass to spark.read.parquet\n\n        Returns:\n            Self: Dataset with the parquet file contents\n\n        Raises:\n            ValueError: Parquet file is empty\n        \"\"\"\n        schema = cls.get_schema()\n        df = session.read_parquet(path, schema=schema, **kwargs)\n        if df.isEmpty():\n            raise ValueError(f\"Parquet file is empty: {path}\")\n        return cls(_df=df, _schema=schema)\n\n    def filter(self: Self, condition: Column) -&gt; Self:\n        \"\"\"Creates a new instance of a Dataset with the DataFrame filtered by the condition.\n\n        Args:\n            condition (Column): Condition to filter the DataFrame\n\n        Returns:\n            Self: Filtered Dataset\n        \"\"\"\n        df = self._df.filter(condition)\n        class_constructor = self.__class__\n        return class_constructor(_df=df, _schema=class_constructor.get_schema())\n\n    def validate_schema(self: Dataset) -&gt; None:\n        \"\"\"Validate DataFrame schema against expected class schema.\n\n        Raises:\n            ValueError: DataFrame schema is not valid\n        \"\"\"\n        expected_schema = self._schema\n        expected_fields = flatten_schema(expected_schema)\n        observed_schema = self._df.schema\n        observed_fields = flatten_schema(observed_schema)\n\n        # Unexpected fields in dataset\n        if unexpected_field_names := [\n            x.name\n            for x in observed_fields\n            if x.name not in [y.name for y in expected_fields]\n        ]:\n            raise ValueError(\n                f\"The {unexpected_field_names} fields are not included in DataFrame schema: {expected_fields}\"\n            )\n\n        # Required fields not in dataset\n        required_fields = [x.name for x in expected_schema if not x.nullable]\n        if missing_required_fields := [\n            req\n            for req in required_fields\n            if not any(field.name == req for field in observed_fields)\n        ]:\n            raise ValueError(\n                f\"The {missing_required_fields} fields are required but missing: {required_fields}\"\n            )\n\n        # Fields with duplicated names\n        if duplicated_fields := [\n            x for x in set(observed_fields) if observed_fields.count(x) &gt; 1\n        ]:\n            raise ValueError(\n                f\"The following fields are duplicated in DataFrame schema: {duplicated_fields}\"\n            )\n\n        # Fields with different datatype\n        observed_field_types = {\n            field.name: type(field.dataType) for field in observed_fields\n        }\n        expected_field_types = {\n            field.name: type(field.dataType) for field in expected_fields\n        }\n        if fields_with_different_observed_datatype := [\n            name\n            for name, observed_type in observed_field_types.items()\n            if name in expected_field_types\n            and observed_type != expected_field_types[name]\n        ]:\n            raise ValueError(\n                f\"The following fields present differences in their datatypes: {fields_with_different_observed_datatype}.\"\n            )\n\n    def persist(self: Self) -&gt; Self:\n        \"\"\"Persist in memory the DataFrame included in the Dataset.\n\n        Returns:\n            Self: Persisted Dataset\n        \"\"\"\n        self.df = self._df.persist()\n        return self\n\n    def unpersist(self: Self) -&gt; Self:\n        \"\"\"Remove the persisted DataFrame from memory.\n\n        Returns:\n            Self: Unpersisted Dataset\n        \"\"\"\n        self.df = self._df.unpersist()\n        return self\n\n    def coalesce(self: Self, num_partitions: int, **kwargs: Any) -&gt; Self:\n        \"\"\"Coalesce the DataFrame included in the Dataset.\n\n        Coalescing is efficient for decreasing the number of partitions because it avoids a full shuffle of the data.\n\n        Args:\n            num_partitions (int): Number of partitions to coalesce to\n            **kwargs (Any): Arguments to pass to the coalesce method\n\n        Returns:\n            Self: Coalesced Dataset\n        \"\"\"\n        self.df = self._df.coalesce(num_partitions, **kwargs)\n        return self\n\n    def repartition(self: Self, num_partitions: int, **kwargs: Any) -&gt; Self:\n        \"\"\"Repartition the DataFrame included in the Dataset.\n\n        Repartitioning creates new partitions with data that is distributed evenly.\n\n        Args:\n            num_partitions (int): Number of partitions to repartition to\n            **kwargs (Any): Arguments to pass to the repartition method\n\n        Returns:\n            Self: Repartitioned Dataset\n        \"\"\"\n        self.df = self._df.repartition(num_partitions, **kwargs)\n        return self\n</code></pre>"},{"location":"python_api/datasets/_datasets/#gentropy.dataset.dataset.Dataset.df","title":"<code>df: DataFrame</code>  <code>property</code> <code>writable</code>","text":"<p>Dataframe included in the Dataset.</p> <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>Dataframe included in the Dataset</p>"},{"location":"python_api/datasets/_datasets/#gentropy.dataset.dataset.Dataset.schema","title":"<code>schema: StructType</code>  <code>property</code>","text":"<p>Dataframe expected schema.</p> <p>Returns:</p> Name Type Description <code>StructType</code> <code>StructType</code> <p>Dataframe expected schema</p>"},{"location":"python_api/datasets/_datasets/#gentropy.dataset.dataset.Dataset.coalesce","title":"<code>coalesce(num_partitions: int, **kwargs: Any) -&gt; Self</code>","text":"<p>Coalesce the DataFrame included in the Dataset.</p> <p>Coalescing is efficient for decreasing the number of partitions because it avoids a full shuffle of the data.</p> <p>Parameters:</p> Name Type Description Default <code>num_partitions</code> <code>int</code> <p>Number of partitions to coalesce to</p> required <code>**kwargs</code> <code>Any</code> <p>Arguments to pass to the coalesce method</p> <code>{}</code> <p>Returns:</p> Name Type Description <code>Self</code> <code>Self</code> <p>Coalesced Dataset</p> Source code in <code>src/gentropy/dataset/dataset.py</code> <pre><code>def coalesce(self: Self, num_partitions: int, **kwargs: Any) -&gt; Self:\n    \"\"\"Coalesce the DataFrame included in the Dataset.\n\n    Coalescing is efficient for decreasing the number of partitions because it avoids a full shuffle of the data.\n\n    Args:\n        num_partitions (int): Number of partitions to coalesce to\n        **kwargs (Any): Arguments to pass to the coalesce method\n\n    Returns:\n        Self: Coalesced Dataset\n    \"\"\"\n    self.df = self._df.coalesce(num_partitions, **kwargs)\n    return self\n</code></pre>"},{"location":"python_api/datasets/_datasets/#gentropy.dataset.dataset.Dataset.filter","title":"<code>filter(condition: Column) -&gt; Self</code>","text":"<p>Creates a new instance of a Dataset with the DataFrame filtered by the condition.</p> <p>Parameters:</p> Name Type Description Default <code>condition</code> <code>Column</code> <p>Condition to filter the DataFrame</p> required <p>Returns:</p> Name Type Description <code>Self</code> <code>Self</code> <p>Filtered Dataset</p> Source code in <code>src/gentropy/dataset/dataset.py</code> <pre><code>def filter(self: Self, condition: Column) -&gt; Self:\n    \"\"\"Creates a new instance of a Dataset with the DataFrame filtered by the condition.\n\n    Args:\n        condition (Column): Condition to filter the DataFrame\n\n    Returns:\n        Self: Filtered Dataset\n    \"\"\"\n    df = self._df.filter(condition)\n    class_constructor = self.__class__\n    return class_constructor(_df=df, _schema=class_constructor.get_schema())\n</code></pre>"},{"location":"python_api/datasets/_datasets/#gentropy.dataset.dataset.Dataset.from_parquet","title":"<code>from_parquet(session: Session, path: str | list[str], **kwargs: bool | float | int | str | None) -&gt; Self</code>  <code>classmethod</code>","text":"<p>Reads parquet into a Dataset with a given schema.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Spark session</p> required <code>path</code> <code>str | list[str]</code> <p>Path to the parquet dataset</p> required <code>**kwargs</code> <code>bool | float | int | str | None</code> <p>Additional arguments to pass to spark.read.parquet</p> <code>{}</code> <p>Returns:</p> Name Type Description <code>Self</code> <code>Self</code> <p>Dataset with the parquet file contents</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>Parquet file is empty</p> Source code in <code>src/gentropy/dataset/dataset.py</code> <pre><code>@classmethod\ndef from_parquet(\n    cls: type[Self],\n    session: Session,\n    path: str | list[str],\n    **kwargs: bool | float | int | str | None,\n) -&gt; Self:\n    \"\"\"Reads parquet into a Dataset with a given schema.\n\n    Args:\n        session (Session): Spark session\n        path (str | list[str]): Path to the parquet dataset\n        **kwargs (bool | float | int | str | None): Additional arguments to pass to spark.read.parquet\n\n    Returns:\n        Self: Dataset with the parquet file contents\n\n    Raises:\n        ValueError: Parquet file is empty\n    \"\"\"\n    schema = cls.get_schema()\n    df = session.read_parquet(path, schema=schema, **kwargs)\n    if df.isEmpty():\n        raise ValueError(f\"Parquet file is empty: {path}\")\n    return cls(_df=df, _schema=schema)\n</code></pre>"},{"location":"python_api/datasets/_datasets/#gentropy.dataset.dataset.Dataset.get_schema","title":"<code>get_schema() -&gt; StructType</code>  <code>abstractmethod</code> <code>classmethod</code>","text":"<p>Abstract method to get the schema. Must be implemented by child classes.</p> <p>Returns:</p> Name Type Description <code>StructType</code> <code>StructType</code> <p>Schema for the Dataset</p> Source code in <code>src/gentropy/dataset/dataset.py</code> <pre><code>@classmethod\n@abstractmethod\ndef get_schema(cls: type[Self]) -&gt; StructType:\n    \"\"\"Abstract method to get the schema. Must be implemented by child classes.\n\n    Returns:\n        StructType: Schema for the Dataset\n    \"\"\"\n    pass\n</code></pre>"},{"location":"python_api/datasets/_datasets/#gentropy.dataset.dataset.Dataset.persist","title":"<code>persist() -&gt; Self</code>","text":"<p>Persist in memory the DataFrame included in the Dataset.</p> <p>Returns:</p> Name Type Description <code>Self</code> <code>Self</code> <p>Persisted Dataset</p> Source code in <code>src/gentropy/dataset/dataset.py</code> <pre><code>def persist(self: Self) -&gt; Self:\n    \"\"\"Persist in memory the DataFrame included in the Dataset.\n\n    Returns:\n        Self: Persisted Dataset\n    \"\"\"\n    self.df = self._df.persist()\n    return self\n</code></pre>"},{"location":"python_api/datasets/_datasets/#gentropy.dataset.dataset.Dataset.repartition","title":"<code>repartition(num_partitions: int, **kwargs: Any) -&gt; Self</code>","text":"<p>Repartition the DataFrame included in the Dataset.</p> <p>Repartitioning creates new partitions with data that is distributed evenly.</p> <p>Parameters:</p> Name Type Description Default <code>num_partitions</code> <code>int</code> <p>Number of partitions to repartition to</p> required <code>**kwargs</code> <code>Any</code> <p>Arguments to pass to the repartition method</p> <code>{}</code> <p>Returns:</p> Name Type Description <code>Self</code> <code>Self</code> <p>Repartitioned Dataset</p> Source code in <code>src/gentropy/dataset/dataset.py</code> <pre><code>def repartition(self: Self, num_partitions: int, **kwargs: Any) -&gt; Self:\n    \"\"\"Repartition the DataFrame included in the Dataset.\n\n    Repartitioning creates new partitions with data that is distributed evenly.\n\n    Args:\n        num_partitions (int): Number of partitions to repartition to\n        **kwargs (Any): Arguments to pass to the repartition method\n\n    Returns:\n        Self: Repartitioned Dataset\n    \"\"\"\n    self.df = self._df.repartition(num_partitions, **kwargs)\n    return self\n</code></pre>"},{"location":"python_api/datasets/_datasets/#gentropy.dataset.dataset.Dataset.unpersist","title":"<code>unpersist() -&gt; Self</code>","text":"<p>Remove the persisted DataFrame from memory.</p> <p>Returns:</p> Name Type Description <code>Self</code> <code>Self</code> <p>Unpersisted Dataset</p> Source code in <code>src/gentropy/dataset/dataset.py</code> <pre><code>def unpersist(self: Self) -&gt; Self:\n    \"\"\"Remove the persisted DataFrame from memory.\n\n    Returns:\n        Self: Unpersisted Dataset\n    \"\"\"\n    self.df = self._df.unpersist()\n    return self\n</code></pre>"},{"location":"python_api/datasets/_datasets/#gentropy.dataset.dataset.Dataset.validate_schema","title":"<code>validate_schema() -&gt; None</code>","text":"<p>Validate DataFrame schema against expected class schema.</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>DataFrame schema is not valid</p> Source code in <code>src/gentropy/dataset/dataset.py</code> <pre><code>def validate_schema(self: Dataset) -&gt; None:\n    \"\"\"Validate DataFrame schema against expected class schema.\n\n    Raises:\n        ValueError: DataFrame schema is not valid\n    \"\"\"\n    expected_schema = self._schema\n    expected_fields = flatten_schema(expected_schema)\n    observed_schema = self._df.schema\n    observed_fields = flatten_schema(observed_schema)\n\n    # Unexpected fields in dataset\n    if unexpected_field_names := [\n        x.name\n        for x in observed_fields\n        if x.name not in [y.name for y in expected_fields]\n    ]:\n        raise ValueError(\n            f\"The {unexpected_field_names} fields are not included in DataFrame schema: {expected_fields}\"\n        )\n\n    # Required fields not in dataset\n    required_fields = [x.name for x in expected_schema if not x.nullable]\n    if missing_required_fields := [\n        req\n        for req in required_fields\n        if not any(field.name == req for field in observed_fields)\n    ]:\n        raise ValueError(\n            f\"The {missing_required_fields} fields are required but missing: {required_fields}\"\n        )\n\n    # Fields with duplicated names\n    if duplicated_fields := [\n        x for x in set(observed_fields) if observed_fields.count(x) &gt; 1\n    ]:\n        raise ValueError(\n            f\"The following fields are duplicated in DataFrame schema: {duplicated_fields}\"\n        )\n\n    # Fields with different datatype\n    observed_field_types = {\n        field.name: type(field.dataType) for field in observed_fields\n    }\n    expected_field_types = {\n        field.name: type(field.dataType) for field in expected_fields\n    }\n    if fields_with_different_observed_datatype := [\n        name\n        for name, observed_type in observed_field_types.items()\n        if name in expected_field_types\n        and observed_type != expected_field_types[name]\n    ]:\n        raise ValueError(\n            f\"The following fields present differences in their datatypes: {fields_with_different_observed_datatype}.\"\n        )\n</code></pre>"},{"location":"python_api/datasets/colocalisation/","title":"Colocalisation","text":""},{"location":"python_api/datasets/colocalisation/#gentropy.dataset.colocalisation.Colocalisation","title":"<code>gentropy.dataset.colocalisation.Colocalisation</code>  <code>dataclass</code>","text":"<p>             Bases: <code>Dataset</code></p> <p>Colocalisation results for pairs of overlapping study-locus.</p> Source code in <code>src/gentropy/dataset/colocalisation.py</code> <pre><code>@dataclass\nclass Colocalisation(Dataset):\n    \"\"\"Colocalisation results for pairs of overlapping study-locus.\"\"\"\n\n    @classmethod\n    def get_schema(cls: type[Colocalisation]) -&gt; StructType:\n        \"\"\"Provides the schema for the Colocalisation dataset.\n\n        Returns:\n            StructType: Schema for the Colocalisation dataset\n        \"\"\"\n        return parse_spark_schema(\"colocalisation.json\")\n</code></pre>"},{"location":"python_api/datasets/colocalisation/#gentropy.dataset.colocalisation.Colocalisation.get_schema","title":"<code>get_schema() -&gt; StructType</code>  <code>classmethod</code>","text":"<p>Provides the schema for the Colocalisation dataset.</p> <p>Returns:</p> Name Type Description <code>StructType</code> <code>StructType</code> <p>Schema for the Colocalisation dataset</p> Source code in <code>src/gentropy/dataset/colocalisation.py</code> <pre><code>@classmethod\ndef get_schema(cls: type[Colocalisation]) -&gt; StructType:\n    \"\"\"Provides the schema for the Colocalisation dataset.\n\n    Returns:\n        StructType: Schema for the Colocalisation dataset\n    \"\"\"\n    return parse_spark_schema(\"colocalisation.json\")\n</code></pre>"},{"location":"python_api/datasets/colocalisation/#schema","title":"Schema","text":"<pre><code>root\n |-- leftStudyLocusId: long (nullable = false)\n |-- rightStudyLocusId: long (nullable = false)\n |-- chromosome: string (nullable = false)\n |-- colocalisationMethod: string (nullable = false)\n |-- numberColocalisingVariants: long (nullable = false)\n |-- h0: double (nullable = true)\n |-- h1: double (nullable = true)\n |-- h2: double (nullable = true)\n |-- h3: double (nullable = true)\n |-- h4: double (nullable = true)\n |-- log2h4h3: double (nullable = true)\n |-- clpp: double (nullable = true)\n</code></pre>"},{"location":"python_api/datasets/gene_index/","title":"Gene Index","text":""},{"location":"python_api/datasets/gene_index/#gentropy.dataset.gene_index.GeneIndex","title":"<code>gentropy.dataset.gene_index.GeneIndex</code>  <code>dataclass</code>","text":"<p>             Bases: <code>Dataset</code></p> <p>Gene index dataset.</p> <p>Gene-based annotation.</p> Source code in <code>src/gentropy/dataset/gene_index.py</code> <pre><code>@dataclass\nclass GeneIndex(Dataset):\n    \"\"\"Gene index dataset.\n\n    Gene-based annotation.\n    \"\"\"\n\n    @classmethod\n    def get_schema(cls: type[GeneIndex]) -&gt; StructType:\n        \"\"\"Provides the schema for the GeneIndex dataset.\n\n        Returns:\n            StructType: Schema for the GeneIndex dataset\n        \"\"\"\n        return parse_spark_schema(\"gene_index.json\")\n\n    def filter_by_biotypes(self: GeneIndex, biotypes: list[str]) -&gt; GeneIndex:\n        \"\"\"Filter by approved biotypes.\n\n        Args:\n            biotypes (list[str]): List of Ensembl biotypes to keep.\n\n        Returns:\n            GeneIndex: Gene index dataset filtered by biotypes.\n        \"\"\"\n        self.df = self._df.filter(f.col(\"biotype\").isin(biotypes))\n        return self\n\n    def locations_lut(self: GeneIndex) -&gt; DataFrame:\n        \"\"\"Gene location information.\n\n        Returns:\n            DataFrame: Gene LUT including genomic location information.\n        \"\"\"\n        return self.df.select(\n            \"geneId\",\n            \"chromosome\",\n            \"start\",\n            \"end\",\n            \"strand\",\n            \"tss\",\n        )\n\n    def symbols_lut(self: GeneIndex) -&gt; DataFrame:\n        \"\"\"Gene symbol lookup table.\n\n        Pre-processess gene/target dataset to create lookup table of gene symbols, including\n        obsoleted gene symbols.\n\n        Returns:\n            DataFrame: Gene LUT for symbol mapping containing `geneId` and `geneSymbol` columns.\n        \"\"\"\n        return self.df.select(\n            f.explode(\n                f.array_union(f.array(\"approvedSymbol\"), f.col(\"obsoleteSymbols\"))\n            ).alias(\"geneSymbol\"),\n            \"*\",\n        )\n</code></pre>"},{"location":"python_api/datasets/gene_index/#gentropy.dataset.gene_index.GeneIndex.filter_by_biotypes","title":"<code>filter_by_biotypes(biotypes: list[str]) -&gt; GeneIndex</code>","text":"<p>Filter by approved biotypes.</p> <p>Parameters:</p> Name Type Description Default <code>biotypes</code> <code>list[str]</code> <p>List of Ensembl biotypes to keep.</p> required <p>Returns:</p> Name Type Description <code>GeneIndex</code> <code>GeneIndex</code> <p>Gene index dataset filtered by biotypes.</p> Source code in <code>src/gentropy/dataset/gene_index.py</code> <pre><code>def filter_by_biotypes(self: GeneIndex, biotypes: list[str]) -&gt; GeneIndex:\n    \"\"\"Filter by approved biotypes.\n\n    Args:\n        biotypes (list[str]): List of Ensembl biotypes to keep.\n\n    Returns:\n        GeneIndex: Gene index dataset filtered by biotypes.\n    \"\"\"\n    self.df = self._df.filter(f.col(\"biotype\").isin(biotypes))\n    return self\n</code></pre>"},{"location":"python_api/datasets/gene_index/#gentropy.dataset.gene_index.GeneIndex.get_schema","title":"<code>get_schema() -&gt; StructType</code>  <code>classmethod</code>","text":"<p>Provides the schema for the GeneIndex dataset.</p> <p>Returns:</p> Name Type Description <code>StructType</code> <code>StructType</code> <p>Schema for the GeneIndex dataset</p> Source code in <code>src/gentropy/dataset/gene_index.py</code> <pre><code>@classmethod\ndef get_schema(cls: type[GeneIndex]) -&gt; StructType:\n    \"\"\"Provides the schema for the GeneIndex dataset.\n\n    Returns:\n        StructType: Schema for the GeneIndex dataset\n    \"\"\"\n    return parse_spark_schema(\"gene_index.json\")\n</code></pre>"},{"location":"python_api/datasets/gene_index/#gentropy.dataset.gene_index.GeneIndex.locations_lut","title":"<code>locations_lut() -&gt; DataFrame</code>","text":"<p>Gene location information.</p> <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>Gene LUT including genomic location information.</p> Source code in <code>src/gentropy/dataset/gene_index.py</code> <pre><code>def locations_lut(self: GeneIndex) -&gt; DataFrame:\n    \"\"\"Gene location information.\n\n    Returns:\n        DataFrame: Gene LUT including genomic location information.\n    \"\"\"\n    return self.df.select(\n        \"geneId\",\n        \"chromosome\",\n        \"start\",\n        \"end\",\n        \"strand\",\n        \"tss\",\n    )\n</code></pre>"},{"location":"python_api/datasets/gene_index/#gentropy.dataset.gene_index.GeneIndex.symbols_lut","title":"<code>symbols_lut() -&gt; DataFrame</code>","text":"<p>Gene symbol lookup table.</p> <p>Pre-processess gene/target dataset to create lookup table of gene symbols, including obsoleted gene symbols.</p> <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>Gene LUT for symbol mapping containing <code>geneId</code> and <code>geneSymbol</code> columns.</p> Source code in <code>src/gentropy/dataset/gene_index.py</code> <pre><code>def symbols_lut(self: GeneIndex) -&gt; DataFrame:\n    \"\"\"Gene symbol lookup table.\n\n    Pre-processess gene/target dataset to create lookup table of gene symbols, including\n    obsoleted gene symbols.\n\n    Returns:\n        DataFrame: Gene LUT for symbol mapping containing `geneId` and `geneSymbol` columns.\n    \"\"\"\n    return self.df.select(\n        f.explode(\n            f.array_union(f.array(\"approvedSymbol\"), f.col(\"obsoleteSymbols\"))\n        ).alias(\"geneSymbol\"),\n        \"*\",\n    )\n</code></pre>"},{"location":"python_api/datasets/gene_index/#schema","title":"Schema","text":"<pre><code>root\n |-- geneId: string (nullable = false)\n |-- chromosome: string (nullable = false)\n |-- approvedSymbol: string (nullable = true)\n |-- biotype: string (nullable = true)\n |-- approvedName: string (nullable = true)\n |-- obsoleteSymbols: array (nullable = true)\n |    |-- element: string (containsNull = true)\n |-- tss: long (nullable = true)\n |-- start: long (nullable = true)\n |-- end: long (nullable = true)\n |-- strand: integer (nullable = true)\n</code></pre>"},{"location":"python_api/datasets/intervals/","title":"Intervals","text":""},{"location":"python_api/datasets/intervals/#gentropy.dataset.intervals.Intervals","title":"<code>gentropy.dataset.intervals.Intervals</code>  <code>dataclass</code>","text":"<p>             Bases: <code>Dataset</code></p> <p>Intervals dataset links genes to genomic regions based on genome interaction studies.</p> Source code in <code>src/gentropy/dataset/intervals.py</code> <pre><code>@dataclass\nclass Intervals(Dataset):\n    \"\"\"Intervals dataset links genes to genomic regions based on genome interaction studies.\"\"\"\n\n    @classmethod\n    def get_schema(cls: type[Intervals]) -&gt; StructType:\n        \"\"\"Provides the schema for the Intervals dataset.\n\n        Returns:\n            StructType: Schema for the Intervals dataset\n        \"\"\"\n        return parse_spark_schema(\"intervals.json\")\n\n    @classmethod\n    def from_source(\n        cls: type[Intervals],\n        spark: SparkSession,\n        source_name: str,\n        source_path: str,\n        gene_index: GeneIndex,\n        lift: LiftOverSpark,\n    ) -&gt; Intervals:\n        \"\"\"Collect interval data for a particular source.\n\n        Args:\n            spark (SparkSession): Spark session\n            source_name (str): Name of the interval source\n            source_path (str): Path to the interval source file\n            gene_index (GeneIndex): Gene index\n            lift (LiftOverSpark): LiftOverSpark instance to convert coordinats from hg37 to hg38\n\n        Returns:\n            Intervals: Intervals dataset\n\n        Raises:\n            ValueError: If the source name is not recognised\n        \"\"\"\n        from gentropy.datasource.intervals.andersson import IntervalsAndersson\n        from gentropy.datasource.intervals.javierre import IntervalsJavierre\n        from gentropy.datasource.intervals.jung import IntervalsJung\n        from gentropy.datasource.intervals.thurman import IntervalsThurman\n\n        source_to_class = {\n            \"andersson\": IntervalsAndersson,\n            \"javierre\": IntervalsJavierre,\n            \"jung\": IntervalsJung,\n            \"thurman\": IntervalsThurman,\n        }\n\n        if source_name not in source_to_class:\n            raise ValueError(f\"Unknown interval source: {source_name}\")\n\n        source_class = source_to_class[source_name]\n        data = source_class.read(spark, source_path)  # type: ignore\n        return source_class.parse(data, gene_index, lift)  # type: ignore\n\n    def v2g(self: Intervals, variant_index: VariantIndex) -&gt; V2G:\n        \"\"\"Convert intervals into V2G by intersecting with a variant index.\n\n        Args:\n            variant_index (VariantIndex): Variant index dataset\n\n        Returns:\n            V2G: Variant-to-gene evidence dataset\n        \"\"\"\n        return V2G(\n            _df=(\n                self.df.alias(\"interval\")\n                .join(\n                    variant_index.df.selectExpr(\n                        \"chromosome as vi_chromosome\", \"variantId\", \"position\"\n                    ).alias(\"vi\"),\n                    on=[\n                        f.col(\"vi.vi_chromosome\") == f.col(\"interval.chromosome\"),\n                        f.col(\"vi.position\").between(\n                            f.col(\"interval.start\"), f.col(\"interval.end\")\n                        ),\n                    ],\n                    how=\"inner\",\n                )\n                .drop(\"start\", \"end\", \"vi_chromosome\", \"position\")\n            ),\n            _schema=V2G.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasets/intervals/#gentropy.dataset.intervals.Intervals.from_source","title":"<code>from_source(spark: SparkSession, source_name: str, source_path: str, gene_index: GeneIndex, lift: LiftOverSpark) -&gt; Intervals</code>  <code>classmethod</code>","text":"<p>Collect interval data for a particular source.</p> <p>Parameters:</p> Name Type Description Default <code>spark</code> <code>SparkSession</code> <p>Spark session</p> required <code>source_name</code> <code>str</code> <p>Name of the interval source</p> required <code>source_path</code> <code>str</code> <p>Path to the interval source file</p> required <code>gene_index</code> <code>GeneIndex</code> <p>Gene index</p> required <code>lift</code> <code>LiftOverSpark</code> <p>LiftOverSpark instance to convert coordinats from hg37 to hg38</p> required <p>Returns:</p> Name Type Description <code>Intervals</code> <code>Intervals</code> <p>Intervals dataset</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>If the source name is not recognised</p> Source code in <code>src/gentropy/dataset/intervals.py</code> <pre><code>@classmethod\ndef from_source(\n    cls: type[Intervals],\n    spark: SparkSession,\n    source_name: str,\n    source_path: str,\n    gene_index: GeneIndex,\n    lift: LiftOverSpark,\n) -&gt; Intervals:\n    \"\"\"Collect interval data for a particular source.\n\n    Args:\n        spark (SparkSession): Spark session\n        source_name (str): Name of the interval source\n        source_path (str): Path to the interval source file\n        gene_index (GeneIndex): Gene index\n        lift (LiftOverSpark): LiftOverSpark instance to convert coordinats from hg37 to hg38\n\n    Returns:\n        Intervals: Intervals dataset\n\n    Raises:\n        ValueError: If the source name is not recognised\n    \"\"\"\n    from gentropy.datasource.intervals.andersson import IntervalsAndersson\n    from gentropy.datasource.intervals.javierre import IntervalsJavierre\n    from gentropy.datasource.intervals.jung import IntervalsJung\n    from gentropy.datasource.intervals.thurman import IntervalsThurman\n\n    source_to_class = {\n        \"andersson\": IntervalsAndersson,\n        \"javierre\": IntervalsJavierre,\n        \"jung\": IntervalsJung,\n        \"thurman\": IntervalsThurman,\n    }\n\n    if source_name not in source_to_class:\n        raise ValueError(f\"Unknown interval source: {source_name}\")\n\n    source_class = source_to_class[source_name]\n    data = source_class.read(spark, source_path)  # type: ignore\n    return source_class.parse(data, gene_index, lift)  # type: ignore\n</code></pre>"},{"location":"python_api/datasets/intervals/#gentropy.dataset.intervals.Intervals.get_schema","title":"<code>get_schema() -&gt; StructType</code>  <code>classmethod</code>","text":"<p>Provides the schema for the Intervals dataset.</p> <p>Returns:</p> Name Type Description <code>StructType</code> <code>StructType</code> <p>Schema for the Intervals dataset</p> Source code in <code>src/gentropy/dataset/intervals.py</code> <pre><code>@classmethod\ndef get_schema(cls: type[Intervals]) -&gt; StructType:\n    \"\"\"Provides the schema for the Intervals dataset.\n\n    Returns:\n        StructType: Schema for the Intervals dataset\n    \"\"\"\n    return parse_spark_schema(\"intervals.json\")\n</code></pre>"},{"location":"python_api/datasets/intervals/#gentropy.dataset.intervals.Intervals.v2g","title":"<code>v2g(variant_index: VariantIndex) -&gt; V2G</code>","text":"<p>Convert intervals into V2G by intersecting with a variant index.</p> <p>Parameters:</p> Name Type Description Default <code>variant_index</code> <code>VariantIndex</code> <p>Variant index dataset</p> required <p>Returns:</p> Name Type Description <code>V2G</code> <code>V2G</code> <p>Variant-to-gene evidence dataset</p> Source code in <code>src/gentropy/dataset/intervals.py</code> <pre><code>def v2g(self: Intervals, variant_index: VariantIndex) -&gt; V2G:\n    \"\"\"Convert intervals into V2G by intersecting with a variant index.\n\n    Args:\n        variant_index (VariantIndex): Variant index dataset\n\n    Returns:\n        V2G: Variant-to-gene evidence dataset\n    \"\"\"\n    return V2G(\n        _df=(\n            self.df.alias(\"interval\")\n            .join(\n                variant_index.df.selectExpr(\n                    \"chromosome as vi_chromosome\", \"variantId\", \"position\"\n                ).alias(\"vi\"),\n                on=[\n                    f.col(\"vi.vi_chromosome\") == f.col(\"interval.chromosome\"),\n                    f.col(\"vi.position\").between(\n                        f.col(\"interval.start\"), f.col(\"interval.end\")\n                    ),\n                ],\n                how=\"inner\",\n            )\n            .drop(\"start\", \"end\", \"vi_chromosome\", \"position\")\n        ),\n        _schema=V2G.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasets/intervals/#schema","title":"Schema","text":"<pre><code>root\n |-- chromosome: string (nullable = false)\n |-- start: string (nullable = false)\n |-- end: string (nullable = false)\n |-- geneId: string (nullable = false)\n |-- resourceScore: double (nullable = true)\n |-- score: double (nullable = true)\n |-- datasourceId: string (nullable = false)\n |-- datatypeId: string (nullable = false)\n |-- pmid: string (nullable = true)\n |-- biofeature: string (nullable = true)\n</code></pre>"},{"location":"python_api/datasets/l2g_feature/","title":"L2G Feature","text":""},{"location":"python_api/datasets/l2g_feature/#gentropy.method.l2g.feature_factory.L2GFeature","title":"<code>gentropy.method.l2g.feature_factory.L2GFeature</code>  <code>dataclass</code>","text":"<p>             Bases: <code>Dataset</code></p> <p>Locus-to-gene feature dataset.</p> Source code in <code>src/gentropy/dataset/l2g_feature.py</code> <pre><code>@dataclass\nclass L2GFeature(Dataset):\n    \"\"\"Locus-to-gene feature dataset.\"\"\"\n\n    @classmethod\n    def get_schema(cls: type[L2GFeature]) -&gt; StructType:\n        \"\"\"Provides the schema for the L2GFeature dataset.\n\n        Returns:\n            StructType: Schema for the L2GFeature dataset\n        \"\"\"\n        return parse_spark_schema(\"l2g_feature.json\")\n</code></pre>"},{"location":"python_api/datasets/l2g_feature/#gentropy.method.l2g.feature_factory.L2GFeature.get_schema","title":"<code>get_schema() -&gt; StructType</code>  <code>classmethod</code>","text":"<p>Provides the schema for the L2GFeature dataset.</p> <p>Returns:</p> Name Type Description <code>StructType</code> <code>StructType</code> <p>Schema for the L2GFeature dataset</p> Source code in <code>src/gentropy/dataset/l2g_feature.py</code> <pre><code>@classmethod\ndef get_schema(cls: type[L2GFeature]) -&gt; StructType:\n    \"\"\"Provides the schema for the L2GFeature dataset.\n\n    Returns:\n        StructType: Schema for the L2GFeature dataset\n    \"\"\"\n    return parse_spark_schema(\"l2g_feature.json\")\n</code></pre>"},{"location":"python_api/datasets/l2g_feature/#schema","title":"Schema","text":"<pre><code>root\n |-- studyLocusId: long (nullable = false)\n |-- geneId: string (nullable = false)\n |-- featureName: string (nullable = false)\n |-- featureValue: float (nullable = false)\n</code></pre>"},{"location":"python_api/datasets/l2g_feature_matrix/","title":"L2G Feature Matrix","text":""},{"location":"python_api/datasets/l2g_feature_matrix/#gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix","title":"<code>gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix</code>  <code>dataclass</code>","text":"<p>             Bases: <code>Dataset</code></p> <p>Dataset with features for Locus to Gene prediction.</p> <p>Attributes:</p> Name Type Description <code>features_list</code> <code>list[str] | None</code> <p>List of features to use. If None, all possible features are used.</p> Source code in <code>src/gentropy/dataset/l2g_feature_matrix.py</code> <pre><code>@dataclass\nclass L2GFeatureMatrix(Dataset):\n    \"\"\"Dataset with features for Locus to Gene prediction.\n\n    Attributes:\n        features_list (list[str] | None): List of features to use. If None, all possible features are used.\n    \"\"\"\n\n    features_list: list[str] | None = None\n\n    def __post_init__(self: L2GFeatureMatrix) -&gt; None:\n        \"\"\"Post-initialisation to set the features list. If not provided, all columns except the fixed ones are used.\"\"\"\n        fixed_cols = [\"studyLocusId\", \"geneId\", \"goldStandardSet\"]\n        self.features_list = self.features_list or [\n            col for col in self._df.columns if col not in fixed_cols\n        ]\n\n    @classmethod\n    def generate_features(\n        cls: Type[L2GFeatureMatrix],\n        features_list: list[str],\n        study_locus: StudyLocus,\n        study_index: StudyIndex,\n        variant_gene: V2G,\n        colocalisation: Colocalisation,\n    ) -&gt; L2GFeatureMatrix:\n        \"\"\"Generate features from the gentropy datasets.\n\n        Args:\n            features_list (list[str]): List of features to generate\n            study_locus (StudyLocus): Study locus dataset\n            study_index (StudyIndex): Study index dataset\n            variant_gene (V2G): Variant to gene dataset\n            colocalisation (Colocalisation): Colocalisation dataset\n\n        Returns:\n            L2GFeatureMatrix: L2G feature matrix dataset\n\n        Raises:\n            ValueError: If the feature matrix is empty\n        \"\"\"\n        if features_dfs := [\n            # Extract features\n            ColocalisationFactory._get_coloc_features(\n                study_locus, study_index, colocalisation\n            ).df,\n            StudyLocusFactory._get_tss_distance_features(study_locus, variant_gene).df,\n            StudyLocusFactory._get_vep_features(study_locus, variant_gene).df,\n        ]:\n            fm = reduce(\n                lambda x, y: x.unionByName(y),\n                features_dfs,\n            )\n        else:\n            raise ValueError(\"No features found\")\n\n        # raise error if the feature matrix is empty\n        return cls(\n            _df=convert_from_long_to_wide(\n                fm, [\"studyLocusId\", \"geneId\"], \"featureName\", \"featureValue\"\n            ),\n            _schema=cls.get_schema(),\n            features_list=features_list,\n        )\n\n    @classmethod\n    def get_schema(cls: type[L2GFeatureMatrix]) -&gt; StructType:\n        \"\"\"Provides the schema for the L2gFeatureMatrix dataset.\n\n        Returns:\n            StructType: Schema for the L2gFeatureMatrix dataset\n        \"\"\"\n        return parse_spark_schema(\"l2g_feature_matrix.json\")\n\n    def calculate_feature_missingness_rate(\n        self: L2GFeatureMatrix,\n    ) -&gt; dict[str, float]:\n        \"\"\"Calculate the proportion of missing values in each feature.\n\n        Returns:\n            dict[str, float]: Dictionary of feature names and their missingness rate.\n\n        Raises:\n            ValueError: If no features are found.\n        \"\"\"\n        total_count = self._df.count()\n        if not self.features_list:\n            raise ValueError(\"No features found\")\n\n        return {\n            feature: (\n                self._df.filter(\n                    (self._df[feature].isNull()) | (self._df[feature] == 0)\n                ).count()\n                / total_count\n            )\n            for feature in self.features_list\n        }\n\n    def fill_na(\n        self: L2GFeatureMatrix, value: float = 0.0, subset: list[str] | None = None\n    ) -&gt; L2GFeatureMatrix:\n        \"\"\"Fill missing values in a column with a given value.\n\n        Args:\n            value (float): Value to replace missing values with. Defaults to 0.0.\n            subset (list[str] | None): Subset of columns to consider. Defaults to None.\n\n        Returns:\n            L2GFeatureMatrix: L2G feature matrix dataset\n        \"\"\"\n        self.df = self._df.fillna(value, subset=subset)\n        return self\n\n    def select_features(\n        self: L2GFeatureMatrix, features_list: list[str] | None\n    ) -&gt; L2GFeatureMatrix:\n        \"\"\"Select a subset of features from the feature matrix.\n\n        Args:\n            features_list (list[str] | None): List of features to select\n\n        Returns:\n            L2GFeatureMatrix: L2G feature matrix dataset\n        \"\"\"\n        features_list = features_list or self.features_list\n        fixed_cols = [\"studyLocusId\", \"geneId\", \"goldStandardSet\"]\n        self.df = self._df.select(fixed_cols + features_list)  # type: ignore\n        return self\n\n    def train_test_split(\n        self: L2GFeatureMatrix, fraction: float\n    ) -&gt; tuple[L2GFeatureMatrix, L2GFeatureMatrix]:\n        \"\"\"Split the dataset into training and test sets.\n\n        Args:\n            fraction (float): Fraction of the dataset to use for training\n\n        Returns:\n            tuple[L2GFeatureMatrix, L2GFeatureMatrix]: Training and test datasets\n        \"\"\"\n        train, test = self._df.randomSplit([fraction, 1 - fraction], seed=42)\n        return (\n            L2GFeatureMatrix(\n                _df=train, _schema=L2GFeatureMatrix.get_schema()\n            ).persist(),\n            L2GFeatureMatrix(_df=test, _schema=L2GFeatureMatrix.get_schema()).persist(),\n        )\n</code></pre>"},{"location":"python_api/datasets/l2g_feature_matrix/#gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.calculate_feature_missingness_rate","title":"<code>calculate_feature_missingness_rate() -&gt; dict[str, float]</code>","text":"<p>Calculate the proportion of missing values in each feature.</p> <p>Returns:</p> Type Description <code>dict[str, float]</code> <p>dict[str, float]: Dictionary of feature names and their missingness rate.</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>If no features are found.</p> Source code in <code>src/gentropy/dataset/l2g_feature_matrix.py</code> <pre><code>def calculate_feature_missingness_rate(\n    self: L2GFeatureMatrix,\n) -&gt; dict[str, float]:\n    \"\"\"Calculate the proportion of missing values in each feature.\n\n    Returns:\n        dict[str, float]: Dictionary of feature names and their missingness rate.\n\n    Raises:\n        ValueError: If no features are found.\n    \"\"\"\n    total_count = self._df.count()\n    if not self.features_list:\n        raise ValueError(\"No features found\")\n\n    return {\n        feature: (\n            self._df.filter(\n                (self._df[feature].isNull()) | (self._df[feature] == 0)\n            ).count()\n            / total_count\n        )\n        for feature in self.features_list\n    }\n</code></pre>"},{"location":"python_api/datasets/l2g_feature_matrix/#gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.fill_na","title":"<code>fill_na(value: float = 0.0, subset: list[str] | None = None) -&gt; L2GFeatureMatrix</code>","text":"<p>Fill missing values in a column with a given value.</p> <p>Parameters:</p> Name Type Description Default <code>value</code> <code>float</code> <p>Value to replace missing values with. Defaults to 0.0.</p> <code>0.0</code> <code>subset</code> <code>list[str] | None</code> <p>Subset of columns to consider. Defaults to None.</p> <code>None</code> <p>Returns:</p> Name Type Description <code>L2GFeatureMatrix</code> <code>L2GFeatureMatrix</code> <p>L2G feature matrix dataset</p> Source code in <code>src/gentropy/dataset/l2g_feature_matrix.py</code> <pre><code>def fill_na(\n    self: L2GFeatureMatrix, value: float = 0.0, subset: list[str] | None = None\n) -&gt; L2GFeatureMatrix:\n    \"\"\"Fill missing values in a column with a given value.\n\n    Args:\n        value (float): Value to replace missing values with. Defaults to 0.0.\n        subset (list[str] | None): Subset of columns to consider. Defaults to None.\n\n    Returns:\n        L2GFeatureMatrix: L2G feature matrix dataset\n    \"\"\"\n    self.df = self._df.fillna(value, subset=subset)\n    return self\n</code></pre>"},{"location":"python_api/datasets/l2g_feature_matrix/#gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.generate_features","title":"<code>generate_features(features_list: list[str], study_locus: StudyLocus, study_index: StudyIndex, variant_gene: V2G, colocalisation: Colocalisation) -&gt; L2GFeatureMatrix</code>  <code>classmethod</code>","text":"<p>Generate features from the gentropy datasets.</p> <p>Parameters:</p> Name Type Description Default <code>features_list</code> <code>list[str]</code> <p>List of features to generate</p> required <code>study_locus</code> <code>StudyLocus</code> <p>Study locus dataset</p> required <code>study_index</code> <code>StudyIndex</code> <p>Study index dataset</p> required <code>variant_gene</code> <code>V2G</code> <p>Variant to gene dataset</p> required <code>colocalisation</code> <code>Colocalisation</code> <p>Colocalisation dataset</p> required <p>Returns:</p> Name Type Description <code>L2GFeatureMatrix</code> <code>L2GFeatureMatrix</code> <p>L2G feature matrix dataset</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>If the feature matrix is empty</p> Source code in <code>src/gentropy/dataset/l2g_feature_matrix.py</code> <pre><code>@classmethod\ndef generate_features(\n    cls: Type[L2GFeatureMatrix],\n    features_list: list[str],\n    study_locus: StudyLocus,\n    study_index: StudyIndex,\n    variant_gene: V2G,\n    colocalisation: Colocalisation,\n) -&gt; L2GFeatureMatrix:\n    \"\"\"Generate features from the gentropy datasets.\n\n    Args:\n        features_list (list[str]): List of features to generate\n        study_locus (StudyLocus): Study locus dataset\n        study_index (StudyIndex): Study index dataset\n        variant_gene (V2G): Variant to gene dataset\n        colocalisation (Colocalisation): Colocalisation dataset\n\n    Returns:\n        L2GFeatureMatrix: L2G feature matrix dataset\n\n    Raises:\n        ValueError: If the feature matrix is empty\n    \"\"\"\n    if features_dfs := [\n        # Extract features\n        ColocalisationFactory._get_coloc_features(\n            study_locus, study_index, colocalisation\n        ).df,\n        StudyLocusFactory._get_tss_distance_features(study_locus, variant_gene).df,\n        StudyLocusFactory._get_vep_features(study_locus, variant_gene).df,\n    ]:\n        fm = reduce(\n            lambda x, y: x.unionByName(y),\n            features_dfs,\n        )\n    else:\n        raise ValueError(\"No features found\")\n\n    # raise error if the feature matrix is empty\n    return cls(\n        _df=convert_from_long_to_wide(\n            fm, [\"studyLocusId\", \"geneId\"], \"featureName\", \"featureValue\"\n        ),\n        _schema=cls.get_schema(),\n        features_list=features_list,\n    )\n</code></pre>"},{"location":"python_api/datasets/l2g_feature_matrix/#gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.get_schema","title":"<code>get_schema() -&gt; StructType</code>  <code>classmethod</code>","text":"<p>Provides the schema for the L2gFeatureMatrix dataset.</p> <p>Returns:</p> Name Type Description <code>StructType</code> <code>StructType</code> <p>Schema for the L2gFeatureMatrix dataset</p> Source code in <code>src/gentropy/dataset/l2g_feature_matrix.py</code> <pre><code>@classmethod\ndef get_schema(cls: type[L2GFeatureMatrix]) -&gt; StructType:\n    \"\"\"Provides the schema for the L2gFeatureMatrix dataset.\n\n    Returns:\n        StructType: Schema for the L2gFeatureMatrix dataset\n    \"\"\"\n    return parse_spark_schema(\"l2g_feature_matrix.json\")\n</code></pre>"},{"location":"python_api/datasets/l2g_feature_matrix/#gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.select_features","title":"<code>select_features(features_list: list[str] | None) -&gt; L2GFeatureMatrix</code>","text":"<p>Select a subset of features from the feature matrix.</p> <p>Parameters:</p> Name Type Description Default <code>features_list</code> <code>list[str] | None</code> <p>List of features to select</p> required <p>Returns:</p> Name Type Description <code>L2GFeatureMatrix</code> <code>L2GFeatureMatrix</code> <p>L2G feature matrix dataset</p> Source code in <code>src/gentropy/dataset/l2g_feature_matrix.py</code> <pre><code>def select_features(\n    self: L2GFeatureMatrix, features_list: list[str] | None\n) -&gt; L2GFeatureMatrix:\n    \"\"\"Select a subset of features from the feature matrix.\n\n    Args:\n        features_list (list[str] | None): List of features to select\n\n    Returns:\n        L2GFeatureMatrix: L2G feature matrix dataset\n    \"\"\"\n    features_list = features_list or self.features_list\n    fixed_cols = [\"studyLocusId\", \"geneId\", \"goldStandardSet\"]\n    self.df = self._df.select(fixed_cols + features_list)  # type: ignore\n    return self\n</code></pre>"},{"location":"python_api/datasets/l2g_feature_matrix/#gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.train_test_split","title":"<code>train_test_split(fraction: float) -&gt; tuple[L2GFeatureMatrix, L2GFeatureMatrix]</code>","text":"<p>Split the dataset into training and test sets.</p> <p>Parameters:</p> Name Type Description Default <code>fraction</code> <code>float</code> <p>Fraction of the dataset to use for training</p> required <p>Returns:</p> Type Description <code>tuple[L2GFeatureMatrix, L2GFeatureMatrix]</code> <p>tuple[L2GFeatureMatrix, L2GFeatureMatrix]: Training and test datasets</p> Source code in <code>src/gentropy/dataset/l2g_feature_matrix.py</code> <pre><code>def train_test_split(\n    self: L2GFeatureMatrix, fraction: float\n) -&gt; tuple[L2GFeatureMatrix, L2GFeatureMatrix]:\n    \"\"\"Split the dataset into training and test sets.\n\n    Args:\n        fraction (float): Fraction of the dataset to use for training\n\n    Returns:\n        tuple[L2GFeatureMatrix, L2GFeatureMatrix]: Training and test datasets\n    \"\"\"\n    train, test = self._df.randomSplit([fraction, 1 - fraction], seed=42)\n    return (\n        L2GFeatureMatrix(\n            _df=train, _schema=L2GFeatureMatrix.get_schema()\n        ).persist(),\n        L2GFeatureMatrix(_df=test, _schema=L2GFeatureMatrix.get_schema()).persist(),\n    )\n</code></pre>"},{"location":"python_api/datasets/l2g_feature_matrix/#schema","title":"Schema","text":"<pre><code>root\n |-- studyLocusId: long (nullable = false)\n |-- geneId: string (nullable = false)\n |-- goldStandardSet: string (nullable = true)\n |-- distanceTssMean: float (nullable = true)\n |-- distanceTssMinimum: float (nullable = true)\n |-- vepMaximumNeighborhood: float (nullable = true)\n |-- vepMaximum: float (nullable = true)\n |-- vepMeanNeighborhood: float (nullable = true)\n |-- vepMean: float (nullable = true)\n |-- eqtlColocClppMaximum: float (nullable = true)\n |-- eqtlColocClppMaximumNeighborhood: float (nullable = true)\n |-- eqtlColocLlrMaximum: float (nullable = true)\n |-- eqtlColocLlrMaximumNeighborhood: float (nullable = true)\n |-- pqtlColocClppMaximum: float (nullable = true)\n |-- pqtlColocClppMaximumNeighborhood: float (nullable = true)\n |-- pqtlColocLlrMaximum: float (nullable = true)\n |-- pqtlColocLlrMaximumNeighborhood: float (nullable = true)\n |-- sqtlColocClppMaximum: float (nullable = true)\n |-- sqtlColocClppMaximumNeighborhood: float (nullable = true)\n |-- sqtlColocLlrMaximum: float (nullable = true)\n |-- sqtlColocLlrMaximumNeighborhood: float (nullable = true)\n</code></pre>"},{"location":"python_api/datasets/l2g_gold_standard/","title":"L2G Gold Standard","text":""},{"location":"python_api/datasets/l2g_gold_standard/#gentropy.dataset.l2g_gold_standard.L2GGoldStandard","title":"<code>gentropy.dataset.l2g_gold_standard.L2GGoldStandard</code>  <code>dataclass</code>","text":"<p>             Bases: <code>Dataset</code></p> <p>L2G gold standard dataset.</p> Source code in <code>src/gentropy/dataset/l2g_gold_standard.py</code> <pre><code>@dataclass\nclass L2GGoldStandard(Dataset):\n    \"\"\"L2G gold standard dataset.\"\"\"\n\n    INTERACTION_THRESHOLD = 0.7\n    GS_POSITIVE_LABEL = \"positive\"\n    GS_NEGATIVE_LABEL = \"negative\"\n\n    @classmethod\n    def from_otg_curation(\n        cls: type[L2GGoldStandard],\n        gold_standard_curation: DataFrame,\n        v2g: V2G,\n        study_locus_overlap: StudyLocusOverlap,\n        interactions: DataFrame,\n    ) -&gt; L2GGoldStandard:\n        \"\"\"Initialise L2GGoldStandard from source dataset.\n\n        Args:\n            gold_standard_curation (DataFrame): Gold standard curation dataframe, extracted from\n            v2g (V2G): Variant to gene dataset to bring distance between a variant and a gene's TSS\n            study_locus_overlap (StudyLocusOverlap): Study locus overlap dataset to remove duplicated loci\n            interactions (DataFrame): Gene-gene interactions dataset to remove negative cases where the gene interacts with a positive gene\n\n        Returns:\n            L2GGoldStandard: L2G Gold Standard dataset\n        \"\"\"\n        from gentropy.datasource.open_targets.l2g_gold_standard import (\n            OpenTargetsL2GGoldStandard,\n        )\n\n        interactions_df = cls.process_gene_interactions(interactions)\n\n        return (\n            OpenTargetsL2GGoldStandard.as_l2g_gold_standard(gold_standard_curation, v2g)\n            # .filter_unique_associations(study_locus_overlap)\n            .remove_false_negatives(interactions_df)\n        )\n\n    @classmethod\n    def get_schema(cls: type[L2GGoldStandard]) -&gt; StructType:\n        \"\"\"Provides the schema for the L2GGoldStandard dataset.\n\n        Returns:\n            StructType: Spark schema for the L2GGoldStandard dataset\n        \"\"\"\n        return parse_spark_schema(\"l2g_gold_standard.json\")\n\n    @classmethod\n    def process_gene_interactions(\n        cls: Type[L2GGoldStandard], interactions: DataFrame\n    ) -&gt; DataFrame:\n        \"\"\"Extract top scoring gene-gene interaction from the interactions dataset of the Platform.\n\n        Args:\n            interactions (DataFrame): Gene-gene interactions dataset from the Open Targets Platform\n\n        Returns:\n            DataFrame: Top scoring gene-gene interaction per pair of genes\n\n        Examples:\n            &gt;&gt;&gt; interactions = spark.createDataFrame([(\"gene1\", \"gene2\", 0.8), (\"gene1\", \"gene2\", 0.5), (\"gene2\", \"gene3\", 0.7)], [\"targetA\", \"targetB\", \"scoring\"])\n            &gt;&gt;&gt; L2GGoldStandard.process_gene_interactions(interactions).show()\n            +-------+-------+-----+\n            |geneIdA|geneIdB|score|\n            +-------+-------+-----+\n            |  gene1|  gene2|  0.8|\n            |  gene2|  gene3|  0.7|\n            +-------+-------+-----+\n            &lt;BLANKLINE&gt;\n        \"\"\"\n        return get_record_with_maximum_value(\n            interactions,\n            [\"targetA\", \"targetB\"],\n            \"scoring\",\n        ).selectExpr(\n            \"targetA as geneIdA\",\n            \"targetB as geneIdB\",\n            \"scoring as score\",\n        )\n\n    def filter_unique_associations(\n        self: L2GGoldStandard,\n        study_locus_overlap: StudyLocusOverlap,\n    ) -&gt; L2GGoldStandard:\n        \"\"\"Refines the gold standard to filter out loci that are not independent.\n\n        Rules:\n        - If two loci point to the same gene, one positive and one negative, and have overlapping variants, we keep the positive one.\n        - If two loci point to the same gene, both positive or negative, and have overlapping variants, we drop one.\n        - If two loci point to different genes, and have overlapping variants, we keep both.\n\n        Args:\n            study_locus_overlap (StudyLocusOverlap): A dataset detailing variants that overlap between StudyLocus.\n\n        Returns:\n            L2GGoldStandard: L2GGoldStandard updated to exclude false negatives and redundant positives.\n        \"\"\"\n        squared_overlaps = study_locus_overlap._convert_to_square_matrix()\n        unique_associations = (\n            self.df.alias(\"left\")\n            # identify all the study loci that point to the same gene\n            .withColumn(\n                \"sl_same_gene\",\n                f.collect_set(\"studyLocusId\").over(Window.partitionBy(\"geneId\")),\n            )\n            # identify all the study loci that have an overlapping variant\n            .join(\n                squared_overlaps.df.alias(\"right\"),\n                (f.col(\"left.studyLocusId\") == f.col(\"right.leftStudyLocusId\"))\n                &amp; (f.col(\"left.variantId\") == f.col(\"right.tagVariantId\")),\n                \"left\",\n            )\n            .withColumn(\n                \"overlaps\",\n                f.when(f.col(\"right.tagVariantId\").isNotNull(), f.lit(True)).otherwise(\n                    f.lit(False)\n                ),\n            )\n            # drop redundant rows: where the variantid overlaps and the gene is \"explained\" by more than one study locus\n            .filter(~((f.size(\"sl_same_gene\") &gt; 1) &amp; (f.col(\"overlaps\") == 1)))\n            .select(*self.df.columns)\n        )\n        return L2GGoldStandard(_df=unique_associations, _schema=self.get_schema())\n\n    def remove_false_negatives(\n        self: L2GGoldStandard,\n        interactions_df: DataFrame,\n    ) -&gt; L2GGoldStandard:\n        \"\"\"Refines the gold standard to remove negative gold standard instances where the gene interacts with a positive gene.\n\n        Args:\n            interactions_df (DataFrame): Top scoring gene-gene interaction per pair of genes\n\n        Returns:\n            L2GGoldStandard: A refined set of locus-to-gene associations with increased reliability, having excluded loci that were likely false negatives due to gene-gene interaction confounding.\n        \"\"\"\n        squared_interactions = interactions_df.unionByName(\n            interactions_df.selectExpr(\n                \"geneIdB as geneIdA\", \"geneIdA as geneIdB\", \"score\"\n            )\n        ).filter(f.col(\"score\") &gt; self.INTERACTION_THRESHOLD)\n        df = (\n            self.df.alias(\"left\")\n            .join(\n                # bring gene partners\n                squared_interactions.alias(\"right\"),\n                f.col(\"left.geneId\") == f.col(\"right.geneIdA\"),\n                \"left\",\n            )\n            .withColumnRenamed(\"geneIdB\", \"interactorGeneId\")\n            .join(\n                # bring gold standard status for gene partners\n                self.df.selectExpr(\n                    \"geneId as interactorGeneId\",\n                    \"goldStandardSet as interactorGeneIdGoldStandardSet\",\n                ),\n                \"interactorGeneId\",\n                \"left\",\n            )\n            # remove self-interactions\n            .filter(\n                (f.col(\"geneId\") != f.col(\"interactorGeneId\"))\n                | (f.col(\"interactorGeneId\").isNull())\n            )\n            # remove false negatives\n            .filter(\n                # drop rows where the GS gene is negative but the interactor is a GS positive\n                ~(f.col(\"goldStandardSet\") == \"negative\")\n                &amp; (f.col(\"interactorGeneIdGoldStandardSet\") == \"positive\")\n                |\n                # keep rows where the gene does not interact\n                (f.col(\"interactorGeneId\").isNull())\n            )\n            .select(*self.df.columns)\n            .distinct()\n        )\n        return L2GGoldStandard(_df=df, _schema=self.get_schema())\n</code></pre>"},{"location":"python_api/datasets/l2g_gold_standard/#gentropy.dataset.l2g_gold_standard.L2GGoldStandard.filter_unique_associations","title":"<code>filter_unique_associations(study_locus_overlap: StudyLocusOverlap) -&gt; L2GGoldStandard</code>","text":"<p>Refines the gold standard to filter out loci that are not independent.</p> <p>Rules: - If two loci point to the same gene, one positive and one negative, and have overlapping variants, we keep the positive one. - If two loci point to the same gene, both positive or negative, and have overlapping variants, we drop one. - If two loci point to different genes, and have overlapping variants, we keep both.</p> <p>Parameters:</p> Name Type Description Default <code>study_locus_overlap</code> <code>StudyLocusOverlap</code> <p>A dataset detailing variants that overlap between StudyLocus.</p> required <p>Returns:</p> Name Type Description <code>L2GGoldStandard</code> <code>L2GGoldStandard</code> <p>L2GGoldStandard updated to exclude false negatives and redundant positives.</p> Source code in <code>src/gentropy/dataset/l2g_gold_standard.py</code> <pre><code>def filter_unique_associations(\n    self: L2GGoldStandard,\n    study_locus_overlap: StudyLocusOverlap,\n) -&gt; L2GGoldStandard:\n    \"\"\"Refines the gold standard to filter out loci that are not independent.\n\n    Rules:\n    - If two loci point to the same gene, one positive and one negative, and have overlapping variants, we keep the positive one.\n    - If two loci point to the same gene, both positive or negative, and have overlapping variants, we drop one.\n    - If two loci point to different genes, and have overlapping variants, we keep both.\n\n    Args:\n        study_locus_overlap (StudyLocusOverlap): A dataset detailing variants that overlap between StudyLocus.\n\n    Returns:\n        L2GGoldStandard: L2GGoldStandard updated to exclude false negatives and redundant positives.\n    \"\"\"\n    squared_overlaps = study_locus_overlap._convert_to_square_matrix()\n    unique_associations = (\n        self.df.alias(\"left\")\n        # identify all the study loci that point to the same gene\n        .withColumn(\n            \"sl_same_gene\",\n            f.collect_set(\"studyLocusId\").over(Window.partitionBy(\"geneId\")),\n        )\n        # identify all the study loci that have an overlapping variant\n        .join(\n            squared_overlaps.df.alias(\"right\"),\n            (f.col(\"left.studyLocusId\") == f.col(\"right.leftStudyLocusId\"))\n            &amp; (f.col(\"left.variantId\") == f.col(\"right.tagVariantId\")),\n            \"left\",\n        )\n        .withColumn(\n            \"overlaps\",\n            f.when(f.col(\"right.tagVariantId\").isNotNull(), f.lit(True)).otherwise(\n                f.lit(False)\n            ),\n        )\n        # drop redundant rows: where the variantid overlaps and the gene is \"explained\" by more than one study locus\n        .filter(~((f.size(\"sl_same_gene\") &gt; 1) &amp; (f.col(\"overlaps\") == 1)))\n        .select(*self.df.columns)\n    )\n    return L2GGoldStandard(_df=unique_associations, _schema=self.get_schema())\n</code></pre>"},{"location":"python_api/datasets/l2g_gold_standard/#gentropy.dataset.l2g_gold_standard.L2GGoldStandard.from_otg_curation","title":"<code>from_otg_curation(gold_standard_curation: DataFrame, v2g: V2G, study_locus_overlap: StudyLocusOverlap, interactions: DataFrame) -&gt; L2GGoldStandard</code>  <code>classmethod</code>","text":"<p>Initialise L2GGoldStandard from source dataset.</p> <p>Parameters:</p> Name Type Description Default <code>gold_standard_curation</code> <code>DataFrame</code> <p>Gold standard curation dataframe, extracted from</p> required <code>v2g</code> <code>V2G</code> <p>Variant to gene dataset to bring distance between a variant and a gene's TSS</p> required <code>study_locus_overlap</code> <code>StudyLocusOverlap</code> <p>Study locus overlap dataset to remove duplicated loci</p> required <code>interactions</code> <code>DataFrame</code> <p>Gene-gene interactions dataset to remove negative cases where the gene interacts with a positive gene</p> required <p>Returns:</p> Name Type Description <code>L2GGoldStandard</code> <code>L2GGoldStandard</code> <p>L2G Gold Standard dataset</p> Source code in <code>src/gentropy/dataset/l2g_gold_standard.py</code> <pre><code>@classmethod\ndef from_otg_curation(\n    cls: type[L2GGoldStandard],\n    gold_standard_curation: DataFrame,\n    v2g: V2G,\n    study_locus_overlap: StudyLocusOverlap,\n    interactions: DataFrame,\n) -&gt; L2GGoldStandard:\n    \"\"\"Initialise L2GGoldStandard from source dataset.\n\n    Args:\n        gold_standard_curation (DataFrame): Gold standard curation dataframe, extracted from\n        v2g (V2G): Variant to gene dataset to bring distance between a variant and a gene's TSS\n        study_locus_overlap (StudyLocusOverlap): Study locus overlap dataset to remove duplicated loci\n        interactions (DataFrame): Gene-gene interactions dataset to remove negative cases where the gene interacts with a positive gene\n\n    Returns:\n        L2GGoldStandard: L2G Gold Standard dataset\n    \"\"\"\n    from gentropy.datasource.open_targets.l2g_gold_standard import (\n        OpenTargetsL2GGoldStandard,\n    )\n\n    interactions_df = cls.process_gene_interactions(interactions)\n\n    return (\n        OpenTargetsL2GGoldStandard.as_l2g_gold_standard(gold_standard_curation, v2g)\n        # .filter_unique_associations(study_locus_overlap)\n        .remove_false_negatives(interactions_df)\n    )\n</code></pre>"},{"location":"python_api/datasets/l2g_gold_standard/#gentropy.dataset.l2g_gold_standard.L2GGoldStandard.get_schema","title":"<code>get_schema() -&gt; StructType</code>  <code>classmethod</code>","text":"<p>Provides the schema for the L2GGoldStandard dataset.</p> <p>Returns:</p> Name Type Description <code>StructType</code> <code>StructType</code> <p>Spark schema for the L2GGoldStandard dataset</p> Source code in <code>src/gentropy/dataset/l2g_gold_standard.py</code> <pre><code>@classmethod\ndef get_schema(cls: type[L2GGoldStandard]) -&gt; StructType:\n    \"\"\"Provides the schema for the L2GGoldStandard dataset.\n\n    Returns:\n        StructType: Spark schema for the L2GGoldStandard dataset\n    \"\"\"\n    return parse_spark_schema(\"l2g_gold_standard.json\")\n</code></pre>"},{"location":"python_api/datasets/l2g_gold_standard/#gentropy.dataset.l2g_gold_standard.L2GGoldStandard.process_gene_interactions","title":"<code>process_gene_interactions(interactions: DataFrame) -&gt; DataFrame</code>  <code>classmethod</code>","text":"<p>Extract top scoring gene-gene interaction from the interactions dataset of the Platform.</p> <p>Parameters:</p> Name Type Description Default <code>interactions</code> <code>DataFrame</code> <p>Gene-gene interactions dataset from the Open Targets Platform</p> required <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>Top scoring gene-gene interaction per pair of genes</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; interactions = spark.createDataFrame([(\"gene1\", \"gene2\", 0.8), (\"gene1\", \"gene2\", 0.5), (\"gene2\", \"gene3\", 0.7)], [\"targetA\", \"targetB\", \"scoring\"])\n&gt;&gt;&gt; L2GGoldStandard.process_gene_interactions(interactions).show()\n+-------+-------+-----+\n|geneIdA|geneIdB|score|\n+-------+-------+-----+\n|  gene1|  gene2|  0.8|\n|  gene2|  gene3|  0.7|\n+-------+-------+-----+\n</code></pre> Source code in <code>src/gentropy/dataset/l2g_gold_standard.py</code> <pre><code>@classmethod\ndef process_gene_interactions(\n    cls: Type[L2GGoldStandard], interactions: DataFrame\n) -&gt; DataFrame:\n    \"\"\"Extract top scoring gene-gene interaction from the interactions dataset of the Platform.\n\n    Args:\n        interactions (DataFrame): Gene-gene interactions dataset from the Open Targets Platform\n\n    Returns:\n        DataFrame: Top scoring gene-gene interaction per pair of genes\n\n    Examples:\n        &gt;&gt;&gt; interactions = spark.createDataFrame([(\"gene1\", \"gene2\", 0.8), (\"gene1\", \"gene2\", 0.5), (\"gene2\", \"gene3\", 0.7)], [\"targetA\", \"targetB\", \"scoring\"])\n        &gt;&gt;&gt; L2GGoldStandard.process_gene_interactions(interactions).show()\n        +-------+-------+-----+\n        |geneIdA|geneIdB|score|\n        +-------+-------+-----+\n        |  gene1|  gene2|  0.8|\n        |  gene2|  gene3|  0.7|\n        +-------+-------+-----+\n        &lt;BLANKLINE&gt;\n    \"\"\"\n    return get_record_with_maximum_value(\n        interactions,\n        [\"targetA\", \"targetB\"],\n        \"scoring\",\n    ).selectExpr(\n        \"targetA as geneIdA\",\n        \"targetB as geneIdB\",\n        \"scoring as score\",\n    )\n</code></pre>"},{"location":"python_api/datasets/l2g_gold_standard/#gentropy.dataset.l2g_gold_standard.L2GGoldStandard.remove_false_negatives","title":"<code>remove_false_negatives(interactions_df: DataFrame) -&gt; L2GGoldStandard</code>","text":"<p>Refines the gold standard to remove negative gold standard instances where the gene interacts with a positive gene.</p> <p>Parameters:</p> Name Type Description Default <code>interactions_df</code> <code>DataFrame</code> <p>Top scoring gene-gene interaction per pair of genes</p> required <p>Returns:</p> Name Type Description <code>L2GGoldStandard</code> <code>L2GGoldStandard</code> <p>A refined set of locus-to-gene associations with increased reliability, having excluded loci that were likely false negatives due to gene-gene interaction confounding.</p> Source code in <code>src/gentropy/dataset/l2g_gold_standard.py</code> <pre><code>def remove_false_negatives(\n    self: L2GGoldStandard,\n    interactions_df: DataFrame,\n) -&gt; L2GGoldStandard:\n    \"\"\"Refines the gold standard to remove negative gold standard instances where the gene interacts with a positive gene.\n\n    Args:\n        interactions_df (DataFrame): Top scoring gene-gene interaction per pair of genes\n\n    Returns:\n        L2GGoldStandard: A refined set of locus-to-gene associations with increased reliability, having excluded loci that were likely false negatives due to gene-gene interaction confounding.\n    \"\"\"\n    squared_interactions = interactions_df.unionByName(\n        interactions_df.selectExpr(\n            \"geneIdB as geneIdA\", \"geneIdA as geneIdB\", \"score\"\n        )\n    ).filter(f.col(\"score\") &gt; self.INTERACTION_THRESHOLD)\n    df = (\n        self.df.alias(\"left\")\n        .join(\n            # bring gene partners\n            squared_interactions.alias(\"right\"),\n            f.col(\"left.geneId\") == f.col(\"right.geneIdA\"),\n            \"left\",\n        )\n        .withColumnRenamed(\"geneIdB\", \"interactorGeneId\")\n        .join(\n            # bring gold standard status for gene partners\n            self.df.selectExpr(\n                \"geneId as interactorGeneId\",\n                \"goldStandardSet as interactorGeneIdGoldStandardSet\",\n            ),\n            \"interactorGeneId\",\n            \"left\",\n        )\n        # remove self-interactions\n        .filter(\n            (f.col(\"geneId\") != f.col(\"interactorGeneId\"))\n            | (f.col(\"interactorGeneId\").isNull())\n        )\n        # remove false negatives\n        .filter(\n            # drop rows where the GS gene is negative but the interactor is a GS positive\n            ~(f.col(\"goldStandardSet\") == \"negative\")\n            &amp; (f.col(\"interactorGeneIdGoldStandardSet\") == \"positive\")\n            |\n            # keep rows where the gene does not interact\n            (f.col(\"interactorGeneId\").isNull())\n        )\n        .select(*self.df.columns)\n        .distinct()\n    )\n    return L2GGoldStandard(_df=df, _schema=self.get_schema())\n</code></pre>"},{"location":"python_api/datasets/l2g_gold_standard/#schema","title":"Schema","text":"<pre><code>root\n |-- studyLocusId: long (nullable = false)\n |-- variantId: string (nullable = false)\n |-- studyId: string (nullable = false)\n |-- geneId: string (nullable = false)\n |-- goldStandardSet: string (nullable = false)\n |-- sources: array (nullable = true)\n |    |-- element: string (containsNull = true)\n</code></pre>"},{"location":"python_api/datasets/l2g_prediction/","title":"L2G Prediction","text":""},{"location":"python_api/datasets/l2g_prediction/#gentropy.dataset.l2g_prediction.L2GPrediction","title":"<code>gentropy.dataset.l2g_prediction.L2GPrediction</code>  <code>dataclass</code>","text":"<p>             Bases: <code>Dataset</code></p> <p>Dataset that contains the Locus to Gene predictions.</p> <p>It is the result of applying the L2G model on a feature matrix, which contains all the study/locus pairs and their functional annotations. The score column informs the confidence of the prediction that a gene is causal to an association.</p> Source code in <code>src/gentropy/dataset/l2g_prediction.py</code> <pre><code>@dataclass\nclass L2GPrediction(Dataset):\n    \"\"\"Dataset that contains the Locus to Gene predictions.\n\n    It is the result of applying the L2G model on a feature matrix, which contains all\n    the study/locus pairs and their functional annotations. The score column informs the\n    confidence of the prediction that a gene is causal to an association.\n    \"\"\"\n\n    @classmethod\n    def get_schema(cls: type[L2GPrediction]) -&gt; StructType:\n        \"\"\"Provides the schema for the L2GPrediction dataset.\n\n        Returns:\n            StructType: Schema for the L2GPrediction dataset\n        \"\"\"\n        return parse_spark_schema(\"l2g_predictions.json\")\n\n    @classmethod\n    def from_credible_set(\n        cls: Type[L2GPrediction],\n        model_path: str,\n        features_list: list[str],\n        study_locus: StudyLocus,\n        study_index: StudyIndex,\n        v2g: V2G,\n        coloc: Colocalisation,\n    ) -&gt; L2GPrediction:\n        \"\"\"Extract L2G predictions for a set of credible sets derived from GWAS.\n\n        Args:\n            model_path (str): Path to the fitted model\n            features_list (list[str]): List of features to use for the model\n            study_locus (StudyLocus): Study locus dataset\n            study_index (StudyIndex): Study index dataset\n            v2g (V2G): Variant to gene dataset\n            coloc (Colocalisation): Colocalisation dataset\n\n        Returns:\n            L2GPrediction: L2G dataset\n        \"\"\"\n        fm = L2GFeatureMatrix.generate_features(\n            features_list=features_list,\n            study_locus=study_locus,\n            study_index=study_index,\n            variant_gene=v2g,\n            colocalisation=coloc,\n        ).fill_na()\n\n        gwas_fm = L2GFeatureMatrix(\n            _df=(\n                fm.df.join(\n                    study_locus.filter_by_study_type(\"gwas\", study_index).df,\n                    on=\"studyLocusId\",\n                )\n            ),\n            _schema=cls.get_schema(),\n        )\n        return L2GPrediction(\n            # Load and apply fitted model\n            _df=(\n                LocusToGeneModel.load_from_disk(\n                    model_path,\n                    features_list=features_list,\n                )\n                .predict(gwas_fm)\n                # the probability of the positive class is the second element inside the probability array\n                # - this is selected as the L2G probability\n                .select(\n                    \"studyLocusId\",\n                    \"geneId\",\n                    vector_to_array(f.col(\"probability\"))[1].alias(\"score\"),\n                )\n            ),\n            _schema=cls.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasets/l2g_prediction/#gentropy.dataset.l2g_prediction.L2GPrediction.from_credible_set","title":"<code>from_credible_set(model_path: str, features_list: list[str], study_locus: StudyLocus, study_index: StudyIndex, v2g: V2G, coloc: Colocalisation) -&gt; L2GPrediction</code>  <code>classmethod</code>","text":"<p>Extract L2G predictions for a set of credible sets derived from GWAS.</p> <p>Parameters:</p> Name Type Description Default <code>model_path</code> <code>str</code> <p>Path to the fitted model</p> required <code>features_list</code> <code>list[str]</code> <p>List of features to use for the model</p> required <code>study_locus</code> <code>StudyLocus</code> <p>Study locus dataset</p> required <code>study_index</code> <code>StudyIndex</code> <p>Study index dataset</p> required <code>v2g</code> <code>V2G</code> <p>Variant to gene dataset</p> required <code>coloc</code> <code>Colocalisation</code> <p>Colocalisation dataset</p> required <p>Returns:</p> Name Type Description <code>L2GPrediction</code> <code>L2GPrediction</code> <p>L2G dataset</p> Source code in <code>src/gentropy/dataset/l2g_prediction.py</code> <pre><code>@classmethod\ndef from_credible_set(\n    cls: Type[L2GPrediction],\n    model_path: str,\n    features_list: list[str],\n    study_locus: StudyLocus,\n    study_index: StudyIndex,\n    v2g: V2G,\n    coloc: Colocalisation,\n) -&gt; L2GPrediction:\n    \"\"\"Extract L2G predictions for a set of credible sets derived from GWAS.\n\n    Args:\n        model_path (str): Path to the fitted model\n        features_list (list[str]): List of features to use for the model\n        study_locus (StudyLocus): Study locus dataset\n        study_index (StudyIndex): Study index dataset\n        v2g (V2G): Variant to gene dataset\n        coloc (Colocalisation): Colocalisation dataset\n\n    Returns:\n        L2GPrediction: L2G dataset\n    \"\"\"\n    fm = L2GFeatureMatrix.generate_features(\n        features_list=features_list,\n        study_locus=study_locus,\n        study_index=study_index,\n        variant_gene=v2g,\n        colocalisation=coloc,\n    ).fill_na()\n\n    gwas_fm = L2GFeatureMatrix(\n        _df=(\n            fm.df.join(\n                study_locus.filter_by_study_type(\"gwas\", study_index).df,\n                on=\"studyLocusId\",\n            )\n        ),\n        _schema=cls.get_schema(),\n    )\n    return L2GPrediction(\n        # Load and apply fitted model\n        _df=(\n            LocusToGeneModel.load_from_disk(\n                model_path,\n                features_list=features_list,\n            )\n            .predict(gwas_fm)\n            # the probability of the positive class is the second element inside the probability array\n            # - this is selected as the L2G probability\n            .select(\n                \"studyLocusId\",\n                \"geneId\",\n                vector_to_array(f.col(\"probability\"))[1].alias(\"score\"),\n            )\n        ),\n        _schema=cls.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasets/l2g_prediction/#gentropy.dataset.l2g_prediction.L2GPrediction.get_schema","title":"<code>get_schema() -&gt; StructType</code>  <code>classmethod</code>","text":"<p>Provides the schema for the L2GPrediction dataset.</p> <p>Returns:</p> Name Type Description <code>StructType</code> <code>StructType</code> <p>Schema for the L2GPrediction dataset</p> Source code in <code>src/gentropy/dataset/l2g_prediction.py</code> <pre><code>@classmethod\ndef get_schema(cls: type[L2GPrediction]) -&gt; StructType:\n    \"\"\"Provides the schema for the L2GPrediction dataset.\n\n    Returns:\n        StructType: Schema for the L2GPrediction dataset\n    \"\"\"\n    return parse_spark_schema(\"l2g_predictions.json\")\n</code></pre>"},{"location":"python_api/datasets/l2g_prediction/#schema","title":"Schema","text":""},{"location":"python_api/datasets/ld_index/","title":"LD Index","text":""},{"location":"python_api/datasets/ld_index/#gentropy.dataset.ld_index.LDIndex","title":"<code>gentropy.dataset.ld_index.LDIndex</code>  <code>dataclass</code>","text":"<p>             Bases: <code>Dataset</code></p> <p>Dataset containing linkage desequilibrium information between variants.</p> Source code in <code>src/gentropy/dataset/ld_index.py</code> <pre><code>@dataclass\nclass LDIndex(Dataset):\n    \"\"\"Dataset containing linkage desequilibrium information between variants.\"\"\"\n\n    @classmethod\n    def get_schema(cls: type[LDIndex]) -&gt; StructType:\n        \"\"\"Provides the schema for the LDIndex dataset.\n\n        Returns:\n            StructType: Schema for the LDIndex dataset\n        \"\"\"\n        return parse_spark_schema(\"ld_index.json\")\n</code></pre>"},{"location":"python_api/datasets/ld_index/#gentropy.dataset.ld_index.LDIndex.get_schema","title":"<code>get_schema() -&gt; StructType</code>  <code>classmethod</code>","text":"<p>Provides the schema for the LDIndex dataset.</p> <p>Returns:</p> Name Type Description <code>StructType</code> <code>StructType</code> <p>Schema for the LDIndex dataset</p> Source code in <code>src/gentropy/dataset/ld_index.py</code> <pre><code>@classmethod\ndef get_schema(cls: type[LDIndex]) -&gt; StructType:\n    \"\"\"Provides the schema for the LDIndex dataset.\n\n    Returns:\n        StructType: Schema for the LDIndex dataset\n    \"\"\"\n    return parse_spark_schema(\"ld_index.json\")\n</code></pre>"},{"location":"python_api/datasets/ld_index/#schema","title":"Schema","text":"<pre><code>root\n |-- variantId: string (nullable = false)\n |-- chromosome: string (nullable = false)\n |-- ldSet: array (nullable = false)\n |    |-- element: struct (containsNull = false)\n |    |    |-- tagVariantId: string (nullable = false)\n |    |    |-- rValues: array (nullable = false)\n |    |    |    |-- element: struct (containsNull = false)\n |    |    |    |    |-- population: string (nullable = false)\n |    |    |    |    |-- r: double (nullable = false)\n</code></pre>"},{"location":"python_api/datasets/study_index/","title":"Study Index","text":""},{"location":"python_api/datasets/study_index/#gentropy.dataset.study_index.StudyIndex","title":"<code>gentropy.dataset.study_index.StudyIndex</code>  <code>dataclass</code>","text":"<p>             Bases: <code>Dataset</code></p> <p>Study index dataset.</p> <p>A study index dataset captures all the metadata for all studies including GWAS and Molecular QTL.</p> Source code in <code>src/gentropy/dataset/study_index.py</code> <pre><code>@dataclass\nclass StudyIndex(Dataset):\n    \"\"\"Study index dataset.\n\n    A study index dataset captures all the metadata for all studies including GWAS and Molecular QTL.\n    \"\"\"\n\n    @staticmethod\n    def _aggregate_samples_by_ancestry(merged: Column, ancestry: Column) -&gt; Column:\n        \"\"\"Aggregate sample counts by ancestry in a list of struct colmns.\n\n        Args:\n            merged (Column): A column representing merged data (list of structs).\n            ancestry (Column): The `ancestry` parameter is a column that represents the ancestry of each\n                sample. (a struct)\n\n        Returns:\n            Column: the modified \"merged\" column after aggregating the samples by ancestry.\n        \"\"\"\n        # Iterating over the list of ancestries and adding the sample size if label matches:\n        return f.transform(\n            merged,\n            lambda a: f.when(\n                a.ancestry == ancestry.ancestry,\n                f.struct(\n                    a.ancestry.alias(\"ancestry\"),\n                    (a.sampleSize + ancestry.sampleSize).alias(\"sampleSize\"),\n                ),\n            ).otherwise(a),\n        )\n\n    @staticmethod\n    def _map_ancestries_to_ld_population(gwas_ancestry_label: Column) -&gt; Column:\n        \"\"\"Normalise ancestry column from GWAS studies into reference LD panel based on a pre-defined map.\n\n        This function assumes all possible ancestry categories have a corresponding\n        LD panel in the LD index. It is very important to have the ancestry labels\n        moved to the LD panel map.\n\n        Args:\n            gwas_ancestry_label (Column): A struct column with ancestry label like Finnish,\n                European, African etc. and the corresponding sample size.\n\n        Returns:\n            Column: Struct column with the mapped LD population label and the sample size.\n        \"\"\"\n        # Loading ancestry label to LD population label:\n        json_dict = json.loads(\n            pkg_resources.read_text(\n                data, \"gwas_population_2_LD_panel_map.json\", encoding=\"utf-8\"\n            )\n        )\n        map_expr = f.create_map(*[f.lit(x) for x in chain(*json_dict.items())])\n\n        return f.struct(\n            map_expr[gwas_ancestry_label.ancestry].alias(\"ancestry\"),\n            gwas_ancestry_label.sampleSize.alias(\"sampleSize\"),\n        )\n\n    @classmethod\n    def get_schema(cls: type[StudyIndex]) -&gt; StructType:\n        \"\"\"Provide the schema for the StudyIndex dataset.\n\n        Returns:\n            StructType: The schema of the StudyIndex dataset.\n        \"\"\"\n        return parse_spark_schema(\"study_index.json\")\n\n    @classmethod\n    def aggregate_and_map_ancestries(\n        cls: type[StudyIndex], discovery_samples: Column\n    ) -&gt; Column:\n        \"\"\"Map ancestries to populations in the LD reference and calculate relative sample size.\n\n        Args:\n            discovery_samples (Column): A list of struct column. Has an `ancestry` column and a `sampleSize` columns\n\n        Returns:\n            Column: A list of struct with mapped LD population and their relative sample size.\n        \"\"\"\n        # Map ancestry categories to population labels of the LD index:\n        mapped_ancestries = f.transform(\n            discovery_samples, cls._map_ancestries_to_ld_population\n        )\n\n        # Aggregate sample sizes belonging to the same LD population:\n        aggregated_counts = f.aggregate(\n            mapped_ancestries,\n            f.array_distinct(\n                f.transform(\n                    mapped_ancestries,\n                    lambda x: f.struct(\n                        x.ancestry.alias(\"ancestry\"), f.lit(0.0).alias(\"sampleSize\")\n                    ),\n                )\n            ),\n            cls._aggregate_samples_by_ancestry,\n        )\n        # Getting total sample count:\n        total_sample_count = f.aggregate(\n            aggregated_counts, f.lit(0.0), lambda total, pop: total + pop.sampleSize\n        ).alias(\"sampleSize\")\n\n        # Calculating relative sample size for each LD population:\n        return f.transform(\n            aggregated_counts,\n            lambda ld_population: f.struct(\n                ld_population.ancestry.alias(\"ldPopulation\"),\n                (ld_population.sampleSize / total_sample_count).alias(\n                    \"relativeSampleSize\"\n                ),\n            ),\n        )\n\n    def study_type_lut(self: StudyIndex) -&gt; DataFrame:\n        \"\"\"Return a lookup table of study type.\n\n        Returns:\n            DataFrame: A dataframe containing `studyId` and `studyType` columns.\n        \"\"\"\n        return self.df.select(\"studyId\", \"studyType\")\n\n    def is_qtl(self: StudyIndex) -&gt; Column:\n        \"\"\"Return a boolean column with true values for QTL studies.\n\n        Returns:\n            Column: True if the study is a QTL study.\n        \"\"\"\n        return self.df.studyType.endswith(\"qtl\")\n\n    def is_gwas(self: StudyIndex) -&gt; Column:\n        \"\"\"Return a boolean column with true values for GWAS studies.\n\n        Returns:\n            Column: True if the study is a GWAS study.\n        \"\"\"\n        return self.df.studyType == \"gwas\"\n\n    def has_mapped_trait(self: StudyIndex) -&gt; Column:\n        \"\"\"Return a boolean column indicating if a study has mapped disease.\n\n        Returns:\n            Column: True if the study has mapped disease.\n        \"\"\"\n        return f.size(self.df.traitFromSourceMappedIds) &gt; 0\n\n    def is_quality_flagged(self: StudyIndex) -&gt; Column:\n        \"\"\"Return a boolean column indicating if a study is flagged due to quality issues.\n\n        Returns:\n            Column: True if the study is flagged.\n        \"\"\"\n        # Testing for the presence of the qualityControls column:\n        if \"qualityControls\" not in self.df.columns:\n            return f.lit(False)\n        else:\n            return f.size(self.df.qualityControls) != 0\n\n    def has_summarystats(self: StudyIndex) -&gt; Column:\n        \"\"\"Return a boolean column indicating if a study has harmonized summary statistics.\n\n        Returns:\n            Column: True if the study has harmonized summary statistics.\n        \"\"\"\n        return self.df.hasSumstats\n</code></pre>"},{"location":"python_api/datasets/study_index/#gentropy.dataset.study_index.StudyIndex.aggregate_and_map_ancestries","title":"<code>aggregate_and_map_ancestries(discovery_samples: Column) -&gt; Column</code>  <code>classmethod</code>","text":"<p>Map ancestries to populations in the LD reference and calculate relative sample size.</p> <p>Parameters:</p> Name Type Description Default <code>discovery_samples</code> <code>Column</code> <p>A list of struct column. Has an <code>ancestry</code> column and a <code>sampleSize</code> columns</p> required <p>Returns:</p> Name Type Description <code>Column</code> <code>Column</code> <p>A list of struct with mapped LD population and their relative sample size.</p> Source code in <code>src/gentropy/dataset/study_index.py</code> <pre><code>@classmethod\ndef aggregate_and_map_ancestries(\n    cls: type[StudyIndex], discovery_samples: Column\n) -&gt; Column:\n    \"\"\"Map ancestries to populations in the LD reference and calculate relative sample size.\n\n    Args:\n        discovery_samples (Column): A list of struct column. Has an `ancestry` column and a `sampleSize` columns\n\n    Returns:\n        Column: A list of struct with mapped LD population and their relative sample size.\n    \"\"\"\n    # Map ancestry categories to population labels of the LD index:\n    mapped_ancestries = f.transform(\n        discovery_samples, cls._map_ancestries_to_ld_population\n    )\n\n    # Aggregate sample sizes belonging to the same LD population:\n    aggregated_counts = f.aggregate(\n        mapped_ancestries,\n        f.array_distinct(\n            f.transform(\n                mapped_ancestries,\n                lambda x: f.struct(\n                    x.ancestry.alias(\"ancestry\"), f.lit(0.0).alias(\"sampleSize\")\n                ),\n            )\n        ),\n        cls._aggregate_samples_by_ancestry,\n    )\n    # Getting total sample count:\n    total_sample_count = f.aggregate(\n        aggregated_counts, f.lit(0.0), lambda total, pop: total + pop.sampleSize\n    ).alias(\"sampleSize\")\n\n    # Calculating relative sample size for each LD population:\n    return f.transform(\n        aggregated_counts,\n        lambda ld_population: f.struct(\n            ld_population.ancestry.alias(\"ldPopulation\"),\n            (ld_population.sampleSize / total_sample_count).alias(\n                \"relativeSampleSize\"\n            ),\n        ),\n    )\n</code></pre>"},{"location":"python_api/datasets/study_index/#gentropy.dataset.study_index.StudyIndex.get_schema","title":"<code>get_schema() -&gt; StructType</code>  <code>classmethod</code>","text":"<p>Provide the schema for the StudyIndex dataset.</p> <p>Returns:</p> Name Type Description <code>StructType</code> <code>StructType</code> <p>The schema of the StudyIndex dataset.</p> Source code in <code>src/gentropy/dataset/study_index.py</code> <pre><code>@classmethod\ndef get_schema(cls: type[StudyIndex]) -&gt; StructType:\n    \"\"\"Provide the schema for the StudyIndex dataset.\n\n    Returns:\n        StructType: The schema of the StudyIndex dataset.\n    \"\"\"\n    return parse_spark_schema(\"study_index.json\")\n</code></pre>"},{"location":"python_api/datasets/study_index/#gentropy.dataset.study_index.StudyIndex.has_mapped_trait","title":"<code>has_mapped_trait() -&gt; Column</code>","text":"<p>Return a boolean column indicating if a study has mapped disease.</p> <p>Returns:</p> Name Type Description <code>Column</code> <code>Column</code> <p>True if the study has mapped disease.</p> Source code in <code>src/gentropy/dataset/study_index.py</code> <pre><code>def has_mapped_trait(self: StudyIndex) -&gt; Column:\n    \"\"\"Return a boolean column indicating if a study has mapped disease.\n\n    Returns:\n        Column: True if the study has mapped disease.\n    \"\"\"\n    return f.size(self.df.traitFromSourceMappedIds) &gt; 0\n</code></pre>"},{"location":"python_api/datasets/study_index/#gentropy.dataset.study_index.StudyIndex.has_summarystats","title":"<code>has_summarystats() -&gt; Column</code>","text":"<p>Return a boolean column indicating if a study has harmonized summary statistics.</p> <p>Returns:</p> Name Type Description <code>Column</code> <code>Column</code> <p>True if the study has harmonized summary statistics.</p> Source code in <code>src/gentropy/dataset/study_index.py</code> <pre><code>def has_summarystats(self: StudyIndex) -&gt; Column:\n    \"\"\"Return a boolean column indicating if a study has harmonized summary statistics.\n\n    Returns:\n        Column: True if the study has harmonized summary statistics.\n    \"\"\"\n    return self.df.hasSumstats\n</code></pre>"},{"location":"python_api/datasets/study_index/#gentropy.dataset.study_index.StudyIndex.is_gwas","title":"<code>is_gwas() -&gt; Column</code>","text":"<p>Return a boolean column with true values for GWAS studies.</p> <p>Returns:</p> Name Type Description <code>Column</code> <code>Column</code> <p>True if the study is a GWAS study.</p> Source code in <code>src/gentropy/dataset/study_index.py</code> <pre><code>def is_gwas(self: StudyIndex) -&gt; Column:\n    \"\"\"Return a boolean column with true values for GWAS studies.\n\n    Returns:\n        Column: True if the study is a GWAS study.\n    \"\"\"\n    return self.df.studyType == \"gwas\"\n</code></pre>"},{"location":"python_api/datasets/study_index/#gentropy.dataset.study_index.StudyIndex.is_qtl","title":"<code>is_qtl() -&gt; Column</code>","text":"<p>Return a boolean column with true values for QTL studies.</p> <p>Returns:</p> Name Type Description <code>Column</code> <code>Column</code> <p>True if the study is a QTL study.</p> Source code in <code>src/gentropy/dataset/study_index.py</code> <pre><code>def is_qtl(self: StudyIndex) -&gt; Column:\n    \"\"\"Return a boolean column with true values for QTL studies.\n\n    Returns:\n        Column: True if the study is a QTL study.\n    \"\"\"\n    return self.df.studyType.endswith(\"qtl\")\n</code></pre>"},{"location":"python_api/datasets/study_index/#gentropy.dataset.study_index.StudyIndex.is_quality_flagged","title":"<code>is_quality_flagged() -&gt; Column</code>","text":"<p>Return a boolean column indicating if a study is flagged due to quality issues.</p> <p>Returns:</p> Name Type Description <code>Column</code> <code>Column</code> <p>True if the study is flagged.</p> Source code in <code>src/gentropy/dataset/study_index.py</code> <pre><code>def is_quality_flagged(self: StudyIndex) -&gt; Column:\n    \"\"\"Return a boolean column indicating if a study is flagged due to quality issues.\n\n    Returns:\n        Column: True if the study is flagged.\n    \"\"\"\n    # Testing for the presence of the qualityControls column:\n    if \"qualityControls\" not in self.df.columns:\n        return f.lit(False)\n    else:\n        return f.size(self.df.qualityControls) != 0\n</code></pre>"},{"location":"python_api/datasets/study_index/#gentropy.dataset.study_index.StudyIndex.study_type_lut","title":"<code>study_type_lut() -&gt; DataFrame</code>","text":"<p>Return a lookup table of study type.</p> <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>A dataframe containing <code>studyId</code> and <code>studyType</code> columns.</p> Source code in <code>src/gentropy/dataset/study_index.py</code> <pre><code>def study_type_lut(self: StudyIndex) -&gt; DataFrame:\n    \"\"\"Return a lookup table of study type.\n\n    Returns:\n        DataFrame: A dataframe containing `studyId` and `studyType` columns.\n    \"\"\"\n    return self.df.select(\"studyId\", \"studyType\")\n</code></pre>"},{"location":"python_api/datasets/study_index/#schema","title":"Schema","text":"<pre><code>root\n |-- studyId: string (nullable = false)\n |-- projectId: string (nullable = false)\n |-- studyType: string (nullable = false)\n |-- traitFromSource: string (nullable = true)\n |-- traitFromSourceMappedIds: array (nullable = true)\n |    |-- element: string (containsNull = true)\n |-- geneId: string (nullable = true)\n |-- tissueFromSourceId: string (nullable = true)\n |-- pubmedId: string (nullable = true)\n |-- publicationTitle: string (nullable = true)\n |-- publicationFirstAuthor: string (nullable = true)\n |-- publicationDate: string (nullable = true)\n |-- publicationJournal: string (nullable = true)\n |-- backgroundTraitFromSourceMappedIds: array (nullable = true)\n |    |-- element: string (containsNull = true)\n |-- initialSampleSize: string (nullable = true)\n |-- nCases: integer (nullable = true)\n |-- nControls: integer (nullable = true)\n |-- nSamples: integer (nullable = true)\n |-- cohorts: array (nullable = true)\n |    |-- element: string (containsNull = true)\n |-- ldPopulationStructure: array (nullable = true)\n |    |-- element: struct (containsNull = false)\n |    |    |-- ldPopulation: string (nullable = true)\n |    |    |-- relativeSampleSize: double (nullable = true)\n |-- discoverySamples: array (nullable = true)\n |    |-- element: struct (containsNull = false)\n |    |    |-- sampleSize: integer (nullable = true)\n |    |    |-- ancestry: string (nullable = true)\n |-- replicationSamples: array (nullable = true)\n |    |-- element: struct (containsNull = false)\n |    |    |-- sampleSize: integer (nullable = true)\n |    |    |-- ancestry: string (nullable = true)\n |-- qualityControls: array (nullable = true)\n |    |-- element: string (containsNull = true)\n |-- analysisFlags: array (nullable = true)\n |    |-- element: string (containsNull = true)\n |-- summarystatsLocation: string (nullable = true)\n |-- hasSumstats: boolean (nullable = true)\n</code></pre>"},{"location":"python_api/datasets/study_locus/","title":"Study Locus","text":""},{"location":"python_api/datasets/study_locus/#gentropy.dataset.study_locus.StudyLocus","title":"<code>gentropy.dataset.study_locus.StudyLocus</code>  <code>dataclass</code>","text":"<p>             Bases: <code>Dataset</code></p> <p>Study-Locus dataset.</p> <p>This dataset captures associations between study/traits and a genetic loci as provided by finemapping methods.</p> Source code in <code>src/gentropy/dataset/study_locus.py</code> <pre><code>@dataclass\nclass StudyLocus(Dataset):\n    \"\"\"Study-Locus dataset.\n\n    This dataset captures associations between study/traits and a genetic loci as provided by finemapping methods.\n    \"\"\"\n\n    @staticmethod\n    def _overlapping_peaks(credset_to_overlap: DataFrame) -&gt; DataFrame:\n        \"\"\"Calculate overlapping signals (study-locus) between GWAS-GWAS and GWAS-Molecular trait.\n\n        Args:\n            credset_to_overlap (DataFrame): DataFrame containing at least `studyLocusId`, `studyType`, `chromosome` and `tagVariantId` columns.\n\n        Returns:\n            DataFrame: containing `leftStudyLocusId`, `rightStudyLocusId` and `chromosome` columns.\n        \"\"\"\n        # Reduce columns to the minimum to reduce the size of the dataframe\n        credset_to_overlap = credset_to_overlap.select(\n            \"studyLocusId\", \"studyType\", \"chromosome\", \"tagVariantId\"\n        )\n        return (\n            credset_to_overlap.alias(\"left\")\n            .filter(f.col(\"studyType\") == \"gwas\")\n            # Self join with complex condition. Left it's all gwas and right can be gwas or molecular trait\n            .join(\n                credset_to_overlap.alias(\"right\"),\n                on=[\n                    f.col(\"left.chromosome\") == f.col(\"right.chromosome\"),\n                    f.col(\"left.tagVariantId\") == f.col(\"right.tagVariantId\"),\n                    (f.col(\"right.studyType\") != \"gwas\")\n                    | (f.col(\"left.studyLocusId\") &gt; f.col(\"right.studyLocusId\")),\n                ],\n                how=\"inner\",\n            )\n            .select(\n                f.col(\"left.studyLocusId\").alias(\"leftStudyLocusId\"),\n                f.col(\"right.studyLocusId\").alias(\"rightStudyLocusId\"),\n                f.col(\"left.chromosome\").alias(\"chromosome\"),\n            )\n            .distinct()\n            .repartition(\"chromosome\")\n            .persist()\n        )\n\n    @staticmethod\n    def _align_overlapping_tags(\n        loci_to_overlap: DataFrame, peak_overlaps: DataFrame\n    ) -&gt; StudyLocusOverlap:\n        \"\"\"Align overlapping tags in pairs of overlapping study-locus, keeping all tags in both loci.\n\n        Args:\n            loci_to_overlap (DataFrame): containing `studyLocusId`, `studyType`, `chromosome`, `tagVariantId`, `logBF` and `posteriorProbability` columns.\n            peak_overlaps (DataFrame): containing `leftStudyLocusId`, `rightStudyLocusId` and `chromosome` columns.\n\n        Returns:\n            StudyLocusOverlap: Pairs of overlapping study-locus with aligned tags.\n        \"\"\"\n        # Complete information about all tags in the left study-locus of the overlap\n        stats_cols = [\n            \"logBF\",\n            \"posteriorProbability\",\n            \"beta\",\n            \"pValueMantissa\",\n            \"pValueExponent\",\n        ]\n        overlapping_left = loci_to_overlap.select(\n            f.col(\"chromosome\"),\n            f.col(\"tagVariantId\"),\n            f.col(\"studyLocusId\").alias(\"leftStudyLocusId\"),\n            *[f.col(col).alias(f\"left_{col}\") for col in stats_cols],\n        ).join(peak_overlaps, on=[\"chromosome\", \"leftStudyLocusId\"], how=\"inner\")\n\n        # Complete information about all tags in the right study-locus of the overlap\n        overlapping_right = loci_to_overlap.select(\n            f.col(\"chromosome\"),\n            f.col(\"tagVariantId\"),\n            f.col(\"studyLocusId\").alias(\"rightStudyLocusId\"),\n            *[f.col(col).alias(f\"right_{col}\") for col in stats_cols],\n        ).join(peak_overlaps, on=[\"chromosome\", \"rightStudyLocusId\"], how=\"inner\")\n\n        # Include information about all tag variants in both study-locus aligned by tag variant id\n        overlaps = overlapping_left.join(\n            overlapping_right,\n            on=[\n                \"chromosome\",\n                \"rightStudyLocusId\",\n                \"leftStudyLocusId\",\n                \"tagVariantId\",\n            ],\n            how=\"outer\",\n        ).select(\n            \"leftStudyLocusId\",\n            \"rightStudyLocusId\",\n            \"chromosome\",\n            \"tagVariantId\",\n            f.struct(\n                *[f\"left_{e}\" for e in stats_cols] + [f\"right_{e}\" for e in stats_cols]\n            ).alias(\"statistics\"),\n        )\n        return StudyLocusOverlap(\n            _df=overlaps,\n            _schema=StudyLocusOverlap.get_schema(),\n        )\n\n    @staticmethod\n    def update_quality_flag(\n        qc: Column, flag_condition: Column, flag_text: StudyLocusQualityCheck\n    ) -&gt; Column:\n        \"\"\"Update the provided quality control list with a new flag if condition is met.\n\n        Args:\n            qc (Column): Array column with the current list of qc flags.\n            flag_condition (Column): This is a column of booleans, signing which row should be flagged\n            flag_text (StudyLocusQualityCheck): Text for the new quality control flag\n\n        Returns:\n            Column: Array column with the updated list of qc flags.\n        \"\"\"\n        qc = f.when(qc.isNull(), f.array()).otherwise(qc)\n        return f.when(\n            flag_condition,\n            f.array_union(qc, f.array(f.lit(flag_text.value))),\n        ).otherwise(qc)\n\n    @staticmethod\n    def assign_study_locus_id(study_id_col: Column, variant_id_col: Column) -&gt; Column:\n        \"\"\"Hashes a column with a variant ID and a study ID to extract a consistent studyLocusId.\n\n        Args:\n            study_id_col (Column): column name with a study ID\n            variant_id_col (Column): column name with a variant ID\n\n        Returns:\n            Column: column with a study locus ID\n\n        Examples:\n            &gt;&gt;&gt; df = spark.createDataFrame([(\"GCST000001\", \"1_1000_A_C\"), (\"GCST000002\", \"1_1000_A_C\")]).toDF(\"studyId\", \"variantId\")\n            &gt;&gt;&gt; df.withColumn(\"study_locus_id\", StudyLocus.assign_study_locus_id(f.col(\"studyId\"), f.col(\"variantId\"))).show()\n            +----------+----------+-------------------+\n            |   studyId| variantId|     study_locus_id|\n            +----------+----------+-------------------+\n            |GCST000001|1_1000_A_C|1553357789130151995|\n            |GCST000002|1_1000_A_C|-415050894682709184|\n            +----------+----------+-------------------+\n            &lt;BLANKLINE&gt;\n        \"\"\"\n        variant_id_col = f.coalesce(variant_id_col, f.rand().cast(\"string\"))\n        return f.xxhash64(study_id_col, variant_id_col).alias(\"studyLocusId\")\n\n    @classmethod\n    def calculate_credible_set_log10bf(cls: type[StudyLocus], logbfs: Column) -&gt; Column:\n        \"\"\"Calculate Bayes factor for the entire credible set. The Bayes factor is calculated as the logsumexp of the logBF values of the variants in the locus.\n\n        Args:\n            logbfs (Column): Array column with the logBF values of the variants in the locus.\n\n        Returns:\n            Column: log10 Bayes factor for the entire credible set.\n\n        Examples:\n            &gt;&gt;&gt; spark.createDataFrame([([0.2, 0.1, 0.05, 0.0],)]).toDF(\"logBF\").select(f.round(StudyLocus.calculate_credible_set_log10bf(f.col(\"logBF\")), 7).alias(\"credibleSetlog10BF\")).show()\n            +------------------+\n            |credibleSetlog10BF|\n            +------------------+\n            |         1.4765565|\n            +------------------+\n            &lt;BLANKLINE&gt;\n        \"\"\"\n        logsumexp_udf = f.udf(lambda x: get_logsum(x), FloatType())\n        return logsumexp_udf(logbfs).cast(\"double\").alias(\"credibleSetlog10BF\")\n\n    @classmethod\n    def get_schema(cls: type[StudyLocus]) -&gt; StructType:\n        \"\"\"Provides the schema for the StudyLocus dataset.\n\n        Returns:\n            StructType: schema for the StudyLocus dataset.\n        \"\"\"\n        return parse_spark_schema(\"study_locus.json\")\n\n    def filter_by_study_type(\n        self: StudyLocus, study_type: str, study_index: StudyIndex\n    ) -&gt; StudyLocus:\n        \"\"\"Creates a new StudyLocus dataset filtered by study type.\n\n        Args:\n            study_type (str): Study type to filter for. Can be one of `gwas`, `eqtl`, `pqtl`, `eqtl`.\n            study_index (StudyIndex): Study index to resolve study types.\n\n        Returns:\n            StudyLocus: Filtered study-locus dataset.\n\n        Raises:\n            ValueError: If study type is not supported.\n        \"\"\"\n        if study_type not in [\"gwas\", \"eqtl\", \"pqtl\", \"sqtl\"]:\n            raise ValueError(\n                f\"Study type {study_type} not supported. Supported types are: gwas, eqtl, pqtl, sqtl.\"\n            )\n        new_df = (\n            self.df.join(study_index.study_type_lut(), on=\"studyId\", how=\"inner\")\n            .filter(f.col(\"studyType\") == study_type)\n            .drop(\"studyType\")\n        )\n        return StudyLocus(\n            _df=new_df,\n            _schema=self._schema,\n        )\n\n    def filter_credible_set(\n        self: StudyLocus,\n        credible_interval: CredibleInterval,\n    ) -&gt; StudyLocus:\n        \"\"\"Filter study-locus tag variants based on given credible interval.\n\n        Args:\n            credible_interval (CredibleInterval): Credible interval to filter for.\n\n        Returns:\n            StudyLocus: Filtered study-locus dataset.\n        \"\"\"\n        self.df = self._df.withColumn(\n            \"locus\",\n            f.filter(\n                f.col(\"locus\"),\n                lambda tag: (tag[credible_interval.value]),\n            ),\n        )\n        return self\n\n    def find_overlaps(self: StudyLocus, study_index: StudyIndex) -&gt; StudyLocusOverlap:\n        \"\"\"Calculate overlapping study-locus.\n\n        Find overlapping study-locus that share at least one tagging variant. All GWAS-GWAS and all GWAS-Molecular traits are computed with the Molecular traits always\n        appearing on the right side.\n\n        Args:\n            study_index (StudyIndex): Study index to resolve study types.\n\n        Returns:\n            StudyLocusOverlap: Pairs of overlapping study-locus with aligned tags.\n        \"\"\"\n        loci_to_overlap = (\n            self.df.join(study_index.study_type_lut(), on=\"studyId\", how=\"inner\")\n            .withColumn(\"locus\", f.explode(\"locus\"))\n            .select(\n                \"studyLocusId\",\n                \"studyType\",\n                \"chromosome\",\n                f.col(\"locus.variantId\").alias(\"tagVariantId\"),\n                f.col(\"locus.logBF\").alias(\"logBF\"),\n                f.col(\"locus.posteriorProbability\").alias(\"posteriorProbability\"),\n                f.col(\"locus.pValueMantissa\").alias(\"pValueMantissa\"),\n                f.col(\"locus.pValueExponent\").alias(\"pValueExponent\"),\n                f.col(\"locus.beta\").alias(\"beta\"),\n            )\n            .persist()\n        )\n\n        # overlapping study-locus\n        peak_overlaps = self._overlapping_peaks(loci_to_overlap)\n\n        # study-locus overlap by aligning overlapping variants\n        return self._align_overlapping_tags(loci_to_overlap, peak_overlaps)\n\n    def unique_variants_in_locus(self: StudyLocus) -&gt; DataFrame:\n        \"\"\"All unique variants collected in a `StudyLocus` dataframe.\n\n        Returns:\n            DataFrame: A dataframe containing `variantId` and `chromosome` columns.\n        \"\"\"\n        return (\n            self.df.withColumn(\n                \"variantId\",\n                # Joint array of variants in that studylocus. Locus can be null\n                f.explode(\n                    f.array_union(\n                        f.array(f.col(\"variantId\")),\n                        f.coalesce(f.col(\"locus.variantId\"), f.array()),\n                    )\n                ),\n            )\n            .select(\n                \"variantId\", f.split(f.col(\"variantId\"), \"_\")[0].alias(\"chromosome\")\n            )\n            .distinct()\n        )\n\n    def neglog_pvalue(self: StudyLocus) -&gt; Column:\n        \"\"\"Returns the negative log p-value.\n\n        Returns:\n            Column: Negative log p-value\n        \"\"\"\n        return calculate_neglog_pvalue(\n            self.df.pValueMantissa,\n            self.df.pValueExponent,\n        )\n\n    def annotate_credible_sets(self: StudyLocus) -&gt; StudyLocus:\n        \"\"\"Annotate study-locus dataset with credible set flags.\n\n        Sorts the array in the `locus` column elements by their `posteriorProbability` values in descending order and adds\n        `is95CredibleSet` and `is99CredibleSet` fields to the elements, indicating which are the tagging variants whose cumulative sum\n        of their `posteriorProbability` values is below 0.95 and 0.99, respectively.\n\n        Returns:\n            StudyLocus: including annotation on `is95CredibleSet` and `is99CredibleSet`.\n\n        Raises:\n            ValueError: If `locus` column is not available.\n        \"\"\"\n        if \"locus\" not in self.df.columns:\n            raise ValueError(\"Locus column not available.\")\n\n        self.df = self.df.withColumn(\n            # Sort credible set by posterior probability in descending order\n            \"locus\",\n            f.when(\n                f.col(\"locus\").isNotNull() &amp; (f.size(f.col(\"locus\")) &gt; 0),\n                order_array_of_structs_by_field(\"locus\", \"posteriorProbability\"),\n            ),\n        ).withColumn(\n            # Calculate array of cumulative sums of posterior probabilities to determine which variants are in the 95% and 99% credible sets\n            # and zip the cumulative sums array with the credible set array to add the flags\n            \"locus\",\n            f.when(\n                f.col(\"locus\").isNotNull() &amp; (f.size(f.col(\"locus\")) &gt; 0),\n                f.zip_with(\n                    f.col(\"locus\"),\n                    f.transform(\n                        f.sequence(f.lit(1), f.size(f.col(\"locus\"))),\n                        lambda index: f.aggregate(\n                            f.slice(\n                                # By using `index - 1` we introduce a value of `0.0` in the cumulative sums array. to ensure that the last variant\n                                # that exceeds the 0.95 threshold is included in the cumulative sum, as its probability is necessary to satisfy the threshold.\n                                f.col(\"locus.posteriorProbability\"),\n                                1,\n                                index - 1,\n                            ),\n                            f.lit(0.0),\n                            lambda acc, el: acc + el,\n                        ),\n                    ),\n                    lambda struct_e, acc: struct_e.withField(\n                        CredibleInterval.IS95.value, (acc &lt; 0.95) &amp; acc.isNotNull()\n                    ).withField(\n                        CredibleInterval.IS99.value, (acc &lt; 0.99) &amp; acc.isNotNull()\n                    ),\n                ),\n            ),\n        )\n        return self\n\n    def annotate_ld(\n        self: StudyLocus, study_index: StudyIndex, ld_index: LDIndex\n    ) -&gt; StudyLocus:\n        \"\"\"Annotate LD information to study-locus.\n\n        Args:\n            study_index (StudyIndex): Study index to resolve ancestries.\n            ld_index (LDIndex): LD index to resolve LD information.\n\n        Returns:\n            StudyLocus: Study locus annotated with ld information from LD index.\n        \"\"\"\n        from gentropy.method.ld import LDAnnotator\n\n        return LDAnnotator.ld_annotate(self, study_index, ld_index)\n\n    def clump(self: StudyLocus) -&gt; StudyLocus:\n        \"\"\"Perform LD clumping of the studyLocus.\n\n        Evaluates whether a lead variant is linked to a tag (with lowest p-value) in the same studyLocus dataset.\n\n        Returns:\n            StudyLocus: with empty credible sets for linked variants and QC flag.\n        \"\"\"\n        self.df = (\n            self.df.withColumn(\n                \"is_lead_linked\",\n                LDclumping._is_lead_linked(\n                    self.df.studyId,\n                    self.df.variantId,\n                    self.df.pValueExponent,\n                    self.df.pValueMantissa,\n                    self.df.ldSet,\n                ),\n            )\n            .withColumn(\n                \"ldSet\",\n                f.when(f.col(\"is_lead_linked\"), f.array()).otherwise(f.col(\"ldSet\")),\n            )\n            .withColumn(\n                \"qualityControls\",\n                StudyLocus.update_quality_flag(\n                    f.col(\"qualityControls\"),\n                    f.col(\"is_lead_linked\"),\n                    StudyLocusQualityCheck.LD_CLUMPED,\n                ),\n            )\n            .drop(\"is_lead_linked\")\n        )\n        return self\n\n    def _qc_no_population(self: StudyLocus) -&gt; StudyLocus:\n        \"\"\"Flag associations where the study doesn't have population information to resolve LD.\n\n        Returns:\n            StudyLocus: Updated study locus.\n        \"\"\"\n        # If the tested column is not present, return self unchanged:\n        if \"ldPopulationStructure\" not in self.df.columns:\n            return self\n\n        self.df = self.df.withColumn(\n            \"qualityControls\",\n            self.update_quality_flag(\n                f.col(\"qualityControls\"),\n                f.col(\"ldPopulationStructure\").isNull(),\n                StudyLocusQualityCheck.NO_POPULATION,\n            ),\n        )\n        return self\n</code></pre>"},{"location":"python_api/datasets/study_locus/#gentropy.dataset.study_locus.StudyLocus.annotate_credible_sets","title":"<code>annotate_credible_sets() -&gt; StudyLocus</code>","text":"<p>Annotate study-locus dataset with credible set flags.</p> <p>Sorts the array in the <code>locus</code> column elements by their <code>posteriorProbability</code> values in descending order and adds <code>is95CredibleSet</code> and <code>is99CredibleSet</code> fields to the elements, indicating which are the tagging variants whose cumulative sum of their <code>posteriorProbability</code> values is below 0.95 and 0.99, respectively.</p> <p>Returns:</p> Name Type Description <code>StudyLocus</code> <code>StudyLocus</code> <p>including annotation on <code>is95CredibleSet</code> and <code>is99CredibleSet</code>.</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>If <code>locus</code> column is not available.</p> Source code in <code>src/gentropy/dataset/study_locus.py</code> <pre><code>def annotate_credible_sets(self: StudyLocus) -&gt; StudyLocus:\n    \"\"\"Annotate study-locus dataset with credible set flags.\n\n    Sorts the array in the `locus` column elements by their `posteriorProbability` values in descending order and adds\n    `is95CredibleSet` and `is99CredibleSet` fields to the elements, indicating which are the tagging variants whose cumulative sum\n    of their `posteriorProbability` values is below 0.95 and 0.99, respectively.\n\n    Returns:\n        StudyLocus: including annotation on `is95CredibleSet` and `is99CredibleSet`.\n\n    Raises:\n        ValueError: If `locus` column is not available.\n    \"\"\"\n    if \"locus\" not in self.df.columns:\n        raise ValueError(\"Locus column not available.\")\n\n    self.df = self.df.withColumn(\n        # Sort credible set by posterior probability in descending order\n        \"locus\",\n        f.when(\n            f.col(\"locus\").isNotNull() &amp; (f.size(f.col(\"locus\")) &gt; 0),\n            order_array_of_structs_by_field(\"locus\", \"posteriorProbability\"),\n        ),\n    ).withColumn(\n        # Calculate array of cumulative sums of posterior probabilities to determine which variants are in the 95% and 99% credible sets\n        # and zip the cumulative sums array with the credible set array to add the flags\n        \"locus\",\n        f.when(\n            f.col(\"locus\").isNotNull() &amp; (f.size(f.col(\"locus\")) &gt; 0),\n            f.zip_with(\n                f.col(\"locus\"),\n                f.transform(\n                    f.sequence(f.lit(1), f.size(f.col(\"locus\"))),\n                    lambda index: f.aggregate(\n                        f.slice(\n                            # By using `index - 1` we introduce a value of `0.0` in the cumulative sums array. to ensure that the last variant\n                            # that exceeds the 0.95 threshold is included in the cumulative sum, as its probability is necessary to satisfy the threshold.\n                            f.col(\"locus.posteriorProbability\"),\n                            1,\n                            index - 1,\n                        ),\n                        f.lit(0.0),\n                        lambda acc, el: acc + el,\n                    ),\n                ),\n                lambda struct_e, acc: struct_e.withField(\n                    CredibleInterval.IS95.value, (acc &lt; 0.95) &amp; acc.isNotNull()\n                ).withField(\n                    CredibleInterval.IS99.value, (acc &lt; 0.99) &amp; acc.isNotNull()\n                ),\n            ),\n        ),\n    )\n    return self\n</code></pre>"},{"location":"python_api/datasets/study_locus/#gentropy.dataset.study_locus.StudyLocus.annotate_ld","title":"<code>annotate_ld(study_index: StudyIndex, ld_index: LDIndex) -&gt; StudyLocus</code>","text":"<p>Annotate LD information to study-locus.</p> <p>Parameters:</p> Name Type Description Default <code>study_index</code> <code>StudyIndex</code> <p>Study index to resolve ancestries.</p> required <code>ld_index</code> <code>LDIndex</code> <p>LD index to resolve LD information.</p> required <p>Returns:</p> Name Type Description <code>StudyLocus</code> <code>StudyLocus</code> <p>Study locus annotated with ld information from LD index.</p> Source code in <code>src/gentropy/dataset/study_locus.py</code> <pre><code>def annotate_ld(\n    self: StudyLocus, study_index: StudyIndex, ld_index: LDIndex\n) -&gt; StudyLocus:\n    \"\"\"Annotate LD information to study-locus.\n\n    Args:\n        study_index (StudyIndex): Study index to resolve ancestries.\n        ld_index (LDIndex): LD index to resolve LD information.\n\n    Returns:\n        StudyLocus: Study locus annotated with ld information from LD index.\n    \"\"\"\n    from gentropy.method.ld import LDAnnotator\n\n    return LDAnnotator.ld_annotate(self, study_index, ld_index)\n</code></pre>"},{"location":"python_api/datasets/study_locus/#gentropy.dataset.study_locus.StudyLocus.assign_study_locus_id","title":"<code>assign_study_locus_id(study_id_col: Column, variant_id_col: Column) -&gt; Column</code>  <code>staticmethod</code>","text":"<p>Hashes a column with a variant ID and a study ID to extract a consistent studyLocusId.</p> <p>Parameters:</p> Name Type Description Default <code>study_id_col</code> <code>Column</code> <p>column name with a study ID</p> required <code>variant_id_col</code> <code>Column</code> <p>column name with a variant ID</p> required <p>Returns:</p> Name Type Description <code>Column</code> <code>Column</code> <p>column with a study locus ID</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; df = spark.createDataFrame([(\"GCST000001\", \"1_1000_A_C\"), (\"GCST000002\", \"1_1000_A_C\")]).toDF(\"studyId\", \"variantId\")\n&gt;&gt;&gt; df.withColumn(\"study_locus_id\", StudyLocus.assign_study_locus_id(f.col(\"studyId\"), f.col(\"variantId\"))).show()\n+----------+----------+-------------------+\n|   studyId| variantId|     study_locus_id|\n+----------+----------+-------------------+\n|GCST000001|1_1000_A_C|1553357789130151995|\n|GCST000002|1_1000_A_C|-415050894682709184|\n+----------+----------+-------------------+\n</code></pre> Source code in <code>src/gentropy/dataset/study_locus.py</code> <pre><code>@staticmethod\ndef assign_study_locus_id(study_id_col: Column, variant_id_col: Column) -&gt; Column:\n    \"\"\"Hashes a column with a variant ID and a study ID to extract a consistent studyLocusId.\n\n    Args:\n        study_id_col (Column): column name with a study ID\n        variant_id_col (Column): column name with a variant ID\n\n    Returns:\n        Column: column with a study locus ID\n\n    Examples:\n        &gt;&gt;&gt; df = spark.createDataFrame([(\"GCST000001\", \"1_1000_A_C\"), (\"GCST000002\", \"1_1000_A_C\")]).toDF(\"studyId\", \"variantId\")\n        &gt;&gt;&gt; df.withColumn(\"study_locus_id\", StudyLocus.assign_study_locus_id(f.col(\"studyId\"), f.col(\"variantId\"))).show()\n        +----------+----------+-------------------+\n        |   studyId| variantId|     study_locus_id|\n        +----------+----------+-------------------+\n        |GCST000001|1_1000_A_C|1553357789130151995|\n        |GCST000002|1_1000_A_C|-415050894682709184|\n        +----------+----------+-------------------+\n        &lt;BLANKLINE&gt;\n    \"\"\"\n    variant_id_col = f.coalesce(variant_id_col, f.rand().cast(\"string\"))\n    return f.xxhash64(study_id_col, variant_id_col).alias(\"studyLocusId\")\n</code></pre>"},{"location":"python_api/datasets/study_locus/#gentropy.dataset.study_locus.StudyLocus.calculate_credible_set_log10bf","title":"<code>calculate_credible_set_log10bf(logbfs: Column) -&gt; Column</code>  <code>classmethod</code>","text":"<p>Calculate Bayes factor for the entire credible set. The Bayes factor is calculated as the logsumexp of the logBF values of the variants in the locus.</p> <p>Parameters:</p> Name Type Description Default <code>logbfs</code> <code>Column</code> <p>Array column with the logBF values of the variants in the locus.</p> required <p>Returns:</p> Name Type Description <code>Column</code> <code>Column</code> <p>log10 Bayes factor for the entire credible set.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; spark.createDataFrame([([0.2, 0.1, 0.05, 0.0],)]).toDF(\"logBF\").select(f.round(StudyLocus.calculate_credible_set_log10bf(f.col(\"logBF\")), 7).alias(\"credibleSetlog10BF\")).show()\n+------------------+\n|credibleSetlog10BF|\n+------------------+\n|         1.4765565|\n+------------------+\n</code></pre> Source code in <code>src/gentropy/dataset/study_locus.py</code> <pre><code>@classmethod\ndef calculate_credible_set_log10bf(cls: type[StudyLocus], logbfs: Column) -&gt; Column:\n    \"\"\"Calculate Bayes factor for the entire credible set. The Bayes factor is calculated as the logsumexp of the logBF values of the variants in the locus.\n\n    Args:\n        logbfs (Column): Array column with the logBF values of the variants in the locus.\n\n    Returns:\n        Column: log10 Bayes factor for the entire credible set.\n\n    Examples:\n        &gt;&gt;&gt; spark.createDataFrame([([0.2, 0.1, 0.05, 0.0],)]).toDF(\"logBF\").select(f.round(StudyLocus.calculate_credible_set_log10bf(f.col(\"logBF\")), 7).alias(\"credibleSetlog10BF\")).show()\n        +------------------+\n        |credibleSetlog10BF|\n        +------------------+\n        |         1.4765565|\n        +------------------+\n        &lt;BLANKLINE&gt;\n    \"\"\"\n    logsumexp_udf = f.udf(lambda x: get_logsum(x), FloatType())\n    return logsumexp_udf(logbfs).cast(\"double\").alias(\"credibleSetlog10BF\")\n</code></pre>"},{"location":"python_api/datasets/study_locus/#gentropy.dataset.study_locus.StudyLocus.clump","title":"<code>clump() -&gt; StudyLocus</code>","text":"<p>Perform LD clumping of the studyLocus.</p> <p>Evaluates whether a lead variant is linked to a tag (with lowest p-value) in the same studyLocus dataset.</p> <p>Returns:</p> Name Type Description <code>StudyLocus</code> <code>StudyLocus</code> <p>with empty credible sets for linked variants and QC flag.</p> Source code in <code>src/gentropy/dataset/study_locus.py</code> <pre><code>def clump(self: StudyLocus) -&gt; StudyLocus:\n    \"\"\"Perform LD clumping of the studyLocus.\n\n    Evaluates whether a lead variant is linked to a tag (with lowest p-value) in the same studyLocus dataset.\n\n    Returns:\n        StudyLocus: with empty credible sets for linked variants and QC flag.\n    \"\"\"\n    self.df = (\n        self.df.withColumn(\n            \"is_lead_linked\",\n            LDclumping._is_lead_linked(\n                self.df.studyId,\n                self.df.variantId,\n                self.df.pValueExponent,\n                self.df.pValueMantissa,\n                self.df.ldSet,\n            ),\n        )\n        .withColumn(\n            \"ldSet\",\n            f.when(f.col(\"is_lead_linked\"), f.array()).otherwise(f.col(\"ldSet\")),\n        )\n        .withColumn(\n            \"qualityControls\",\n            StudyLocus.update_quality_flag(\n                f.col(\"qualityControls\"),\n                f.col(\"is_lead_linked\"),\n                StudyLocusQualityCheck.LD_CLUMPED,\n            ),\n        )\n        .drop(\"is_lead_linked\")\n    )\n    return self\n</code></pre>"},{"location":"python_api/datasets/study_locus/#gentropy.dataset.study_locus.StudyLocus.filter_by_study_type","title":"<code>filter_by_study_type(study_type: str, study_index: StudyIndex) -&gt; StudyLocus</code>","text":"<p>Creates a new StudyLocus dataset filtered by study type.</p> <p>Parameters:</p> Name Type Description Default <code>study_type</code> <code>str</code> <p>Study type to filter for. Can be one of <code>gwas</code>, <code>eqtl</code>, <code>pqtl</code>, <code>eqtl</code>.</p> required <code>study_index</code> <code>StudyIndex</code> <p>Study index to resolve study types.</p> required <p>Returns:</p> Name Type Description <code>StudyLocus</code> <code>StudyLocus</code> <p>Filtered study-locus dataset.</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>If study type is not supported.</p> Source code in <code>src/gentropy/dataset/study_locus.py</code> <pre><code>def filter_by_study_type(\n    self: StudyLocus, study_type: str, study_index: StudyIndex\n) -&gt; StudyLocus:\n    \"\"\"Creates a new StudyLocus dataset filtered by study type.\n\n    Args:\n        study_type (str): Study type to filter for. Can be one of `gwas`, `eqtl`, `pqtl`, `eqtl`.\n        study_index (StudyIndex): Study index to resolve study types.\n\n    Returns:\n        StudyLocus: Filtered study-locus dataset.\n\n    Raises:\n        ValueError: If study type is not supported.\n    \"\"\"\n    if study_type not in [\"gwas\", \"eqtl\", \"pqtl\", \"sqtl\"]:\n        raise ValueError(\n            f\"Study type {study_type} not supported. Supported types are: gwas, eqtl, pqtl, sqtl.\"\n        )\n    new_df = (\n        self.df.join(study_index.study_type_lut(), on=\"studyId\", how=\"inner\")\n        .filter(f.col(\"studyType\") == study_type)\n        .drop(\"studyType\")\n    )\n    return StudyLocus(\n        _df=new_df,\n        _schema=self._schema,\n    )\n</code></pre>"},{"location":"python_api/datasets/study_locus/#gentropy.dataset.study_locus.StudyLocus.filter_credible_set","title":"<code>filter_credible_set(credible_interval: CredibleInterval) -&gt; StudyLocus</code>","text":"<p>Filter study-locus tag variants based on given credible interval.</p> <p>Parameters:</p> Name Type Description Default <code>credible_interval</code> <code>CredibleInterval</code> <p>Credible interval to filter for.</p> required <p>Returns:</p> Name Type Description <code>StudyLocus</code> <code>StudyLocus</code> <p>Filtered study-locus dataset.</p> Source code in <code>src/gentropy/dataset/study_locus.py</code> <pre><code>def filter_credible_set(\n    self: StudyLocus,\n    credible_interval: CredibleInterval,\n) -&gt; StudyLocus:\n    \"\"\"Filter study-locus tag variants based on given credible interval.\n\n    Args:\n        credible_interval (CredibleInterval): Credible interval to filter for.\n\n    Returns:\n        StudyLocus: Filtered study-locus dataset.\n    \"\"\"\n    self.df = self._df.withColumn(\n        \"locus\",\n        f.filter(\n            f.col(\"locus\"),\n            lambda tag: (tag[credible_interval.value]),\n        ),\n    )\n    return self\n</code></pre>"},{"location":"python_api/datasets/study_locus/#gentropy.dataset.study_locus.StudyLocus.find_overlaps","title":"<code>find_overlaps(study_index: StudyIndex) -&gt; StudyLocusOverlap</code>","text":"<p>Calculate overlapping study-locus.</p> <p>Find overlapping study-locus that share at least one tagging variant. All GWAS-GWAS and all GWAS-Molecular traits are computed with the Molecular traits always appearing on the right side.</p> <p>Parameters:</p> Name Type Description Default <code>study_index</code> <code>StudyIndex</code> <p>Study index to resolve study types.</p> required <p>Returns:</p> Name Type Description <code>StudyLocusOverlap</code> <code>StudyLocusOverlap</code> <p>Pairs of overlapping study-locus with aligned tags.</p> Source code in <code>src/gentropy/dataset/study_locus.py</code> <pre><code>def find_overlaps(self: StudyLocus, study_index: StudyIndex) -&gt; StudyLocusOverlap:\n    \"\"\"Calculate overlapping study-locus.\n\n    Find overlapping study-locus that share at least one tagging variant. All GWAS-GWAS and all GWAS-Molecular traits are computed with the Molecular traits always\n    appearing on the right side.\n\n    Args:\n        study_index (StudyIndex): Study index to resolve study types.\n\n    Returns:\n        StudyLocusOverlap: Pairs of overlapping study-locus with aligned tags.\n    \"\"\"\n    loci_to_overlap = (\n        self.df.join(study_index.study_type_lut(), on=\"studyId\", how=\"inner\")\n        .withColumn(\"locus\", f.explode(\"locus\"))\n        .select(\n            \"studyLocusId\",\n            \"studyType\",\n            \"chromosome\",\n            f.col(\"locus.variantId\").alias(\"tagVariantId\"),\n            f.col(\"locus.logBF\").alias(\"logBF\"),\n            f.col(\"locus.posteriorProbability\").alias(\"posteriorProbability\"),\n            f.col(\"locus.pValueMantissa\").alias(\"pValueMantissa\"),\n            f.col(\"locus.pValueExponent\").alias(\"pValueExponent\"),\n            f.col(\"locus.beta\").alias(\"beta\"),\n        )\n        .persist()\n    )\n\n    # overlapping study-locus\n    peak_overlaps = self._overlapping_peaks(loci_to_overlap)\n\n    # study-locus overlap by aligning overlapping variants\n    return self._align_overlapping_tags(loci_to_overlap, peak_overlaps)\n</code></pre>"},{"location":"python_api/datasets/study_locus/#gentropy.dataset.study_locus.StudyLocus.get_schema","title":"<code>get_schema() -&gt; StructType</code>  <code>classmethod</code>","text":"<p>Provides the schema for the StudyLocus dataset.</p> <p>Returns:</p> Name Type Description <code>StructType</code> <code>StructType</code> <p>schema for the StudyLocus dataset.</p> Source code in <code>src/gentropy/dataset/study_locus.py</code> <pre><code>@classmethod\ndef get_schema(cls: type[StudyLocus]) -&gt; StructType:\n    \"\"\"Provides the schema for the StudyLocus dataset.\n\n    Returns:\n        StructType: schema for the StudyLocus dataset.\n    \"\"\"\n    return parse_spark_schema(\"study_locus.json\")\n</code></pre>"},{"location":"python_api/datasets/study_locus/#gentropy.dataset.study_locus.StudyLocus.neglog_pvalue","title":"<code>neglog_pvalue() -&gt; Column</code>","text":"<p>Returns the negative log p-value.</p> <p>Returns:</p> Name Type Description <code>Column</code> <code>Column</code> <p>Negative log p-value</p> Source code in <code>src/gentropy/dataset/study_locus.py</code> <pre><code>def neglog_pvalue(self: StudyLocus) -&gt; Column:\n    \"\"\"Returns the negative log p-value.\n\n    Returns:\n        Column: Negative log p-value\n    \"\"\"\n    return calculate_neglog_pvalue(\n        self.df.pValueMantissa,\n        self.df.pValueExponent,\n    )\n</code></pre>"},{"location":"python_api/datasets/study_locus/#gentropy.dataset.study_locus.StudyLocus.unique_variants_in_locus","title":"<code>unique_variants_in_locus() -&gt; DataFrame</code>","text":"<p>All unique variants collected in a <code>StudyLocus</code> dataframe.</p> <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>A dataframe containing <code>variantId</code> and <code>chromosome</code> columns.</p> Source code in <code>src/gentropy/dataset/study_locus.py</code> <pre><code>def unique_variants_in_locus(self: StudyLocus) -&gt; DataFrame:\n    \"\"\"All unique variants collected in a `StudyLocus` dataframe.\n\n    Returns:\n        DataFrame: A dataframe containing `variantId` and `chromosome` columns.\n    \"\"\"\n    return (\n        self.df.withColumn(\n            \"variantId\",\n            # Joint array of variants in that studylocus. Locus can be null\n            f.explode(\n                f.array_union(\n                    f.array(f.col(\"variantId\")),\n                    f.coalesce(f.col(\"locus.variantId\"), f.array()),\n                )\n            ),\n        )\n        .select(\n            \"variantId\", f.split(f.col(\"variantId\"), \"_\")[0].alias(\"chromosome\")\n        )\n        .distinct()\n    )\n</code></pre>"},{"location":"python_api/datasets/study_locus/#gentropy.dataset.study_locus.StudyLocus.update_quality_flag","title":"<code>update_quality_flag(qc: Column, flag_condition: Column, flag_text: StudyLocusQualityCheck) -&gt; Column</code>  <code>staticmethod</code>","text":"<p>Update the provided quality control list with a new flag if condition is met.</p> <p>Parameters:</p> Name Type Description Default <code>qc</code> <code>Column</code> <p>Array column with the current list of qc flags.</p> required <code>flag_condition</code> <code>Column</code> <p>This is a column of booleans, signing which row should be flagged</p> required <code>flag_text</code> <code>StudyLocusQualityCheck</code> <p>Text for the new quality control flag</p> required <p>Returns:</p> Name Type Description <code>Column</code> <code>Column</code> <p>Array column with the updated list of qc flags.</p> Source code in <code>src/gentropy/dataset/study_locus.py</code> <pre><code>@staticmethod\ndef update_quality_flag(\n    qc: Column, flag_condition: Column, flag_text: StudyLocusQualityCheck\n) -&gt; Column:\n    \"\"\"Update the provided quality control list with a new flag if condition is met.\n\n    Args:\n        qc (Column): Array column with the current list of qc flags.\n        flag_condition (Column): This is a column of booleans, signing which row should be flagged\n        flag_text (StudyLocusQualityCheck): Text for the new quality control flag\n\n    Returns:\n        Column: Array column with the updated list of qc flags.\n    \"\"\"\n    qc = f.when(qc.isNull(), f.array()).otherwise(qc)\n    return f.when(\n        flag_condition,\n        f.array_union(qc, f.array(f.lit(flag_text.value))),\n    ).otherwise(qc)\n</code></pre>"},{"location":"python_api/datasets/study_locus/#gentropy.dataset.study_locus.StudyLocusQualityCheck","title":"<code>gentropy.dataset.study_locus.StudyLocusQualityCheck</code>","text":"<p>             Bases: <code>Enum</code></p> <p>Study-Locus quality control options listing concerns on the quality of the association.</p> <p>Attributes:</p> Name Type Description <code>SUBSIGNIFICANT_FLAG</code> <code>str</code> <p>p-value below significance threshold</p> <code>NO_GENOMIC_LOCATION_FLAG</code> <code>str</code> <p>Incomplete genomic mapping</p> <code>COMPOSITE_FLAG</code> <code>str</code> <p>Composite association due to variant x variant interactions</p> <code>INCONSISTENCY_FLAG</code> <code>str</code> <p>Inconsistencies in the reported variants</p> <code>NON_MAPPED_VARIANT_FLAG</code> <code>str</code> <p>Variant not mapped to GnomAd</p> <code>PALINDROMIC_ALLELE_FLAG</code> <code>str</code> <p>Alleles are palindromic - cannot harmonize</p> <code>AMBIGUOUS_STUDY</code> <code>str</code> <p>Association with ambiguous study</p> <code>UNRESOLVED_LD</code> <code>str</code> <p>Variant not found in LD reference</p> <code>LD_CLUMPED</code> <code>str</code> <p>Explained by a more significant variant in high LD (clumped)</p> <code>NO_POPULATION</code> <code>str</code> <p>Study does not have population annotation to resolve LD</p> <code>NOT_QUALIFYING_LD_BLOCK</code> <code>str</code> <p>LD block does not contain variants at the required R^2 threshold</p> Source code in <code>src/gentropy/dataset/study_locus.py</code> <pre><code>class StudyLocusQualityCheck(Enum):\n    \"\"\"Study-Locus quality control options listing concerns on the quality of the association.\n\n    Attributes:\n        SUBSIGNIFICANT_FLAG (str): p-value below significance threshold\n        NO_GENOMIC_LOCATION_FLAG (str): Incomplete genomic mapping\n        COMPOSITE_FLAG (str): Composite association due to variant x variant interactions\n        INCONSISTENCY_FLAG (str): Inconsistencies in the reported variants\n        NON_MAPPED_VARIANT_FLAG (str): Variant not mapped to GnomAd\n        PALINDROMIC_ALLELE_FLAG (str): Alleles are palindromic - cannot harmonize\n        AMBIGUOUS_STUDY (str): Association with ambiguous study\n        UNRESOLVED_LD (str): Variant not found in LD reference\n        LD_CLUMPED (str): Explained by a more significant variant in high LD (clumped)\n        NO_POPULATION (str): Study does not have population annotation to resolve LD\n        NOT_QUALIFYING_LD_BLOCK (str): LD block does not contain variants at the required R^2 threshold\n    \"\"\"\n\n    SUBSIGNIFICANT_FLAG = \"Subsignificant p-value\"\n    NO_GENOMIC_LOCATION_FLAG = \"Incomplete genomic mapping\"\n    COMPOSITE_FLAG = \"Composite association\"\n    INCONSISTENCY_FLAG = \"Variant inconsistency\"\n    NON_MAPPED_VARIANT_FLAG = \"No mapping in GnomAd\"\n    PALINDROMIC_ALLELE_FLAG = \"Palindrome alleles - cannot harmonize\"\n    AMBIGUOUS_STUDY = \"Association with ambiguous study\"\n    UNRESOLVED_LD = \"Variant not found in LD reference\"\n    LD_CLUMPED = \"Explained by a more significant variant in high LD (clumped)\"\n    NO_POPULATION = \"Study does not have population annotation to resolve LD\"\n    NOT_QUALIFYING_LD_BLOCK = (\n        \"LD block does not contain variants at the required R^2 threshold\"\n    )\n</code></pre>"},{"location":"python_api/datasets/study_locus/#gentropy.dataset.study_locus.CredibleInterval","title":"<code>gentropy.dataset.study_locus.CredibleInterval</code>","text":"<p>             Bases: <code>Enum</code></p> <p>Credible interval enum.</p> <p>Interval within which an unobserved parameter value falls with a particular probability.</p> <p>Attributes:</p> Name Type Description <code>IS95</code> <code>str</code> <p>95% credible interval</p> <code>IS99</code> <code>str</code> <p>99% credible interval</p> Source code in <code>src/gentropy/dataset/study_locus.py</code> <pre><code>class CredibleInterval(Enum):\n    \"\"\"Credible interval enum.\n\n    Interval within which an unobserved parameter value falls with a particular probability.\n\n    Attributes:\n        IS95 (str): 95% credible interval\n        IS99 (str): 99% credible interval\n    \"\"\"\n\n    IS95 = \"is95CredibleSet\"\n    IS99 = \"is99CredibleSet\"\n</code></pre>"},{"location":"python_api/datasets/study_locus/#schema","title":"Schema","text":"<pre><code>root\n |-- studyLocusId: long (nullable = false)\n |-- variantId: string (nullable = false)\n |-- chromosome: string (nullable = true)\n |-- position: integer (nullable = true)\n |-- region: string (nullable = true)\n |-- studyId: string (nullable = false)\n |-- beta: double (nullable = true)\n |-- pValueMantissa: float (nullable = true)\n |-- pValueExponent: integer (nullable = true)\n |-- effectAlleleFrequencyFromSource: float (nullable = true)\n |-- standardError: double (nullable = true)\n |-- subStudyDescription: string (nullable = true)\n |-- qualityControls: array (nullable = true)\n |    |-- element: string (containsNull = false)\n |-- finemappingMethod: string (nullable = true)\n |-- credibleSetIndex: integer (nullable = true)\n |-- credibleSetlog10BF: double (nullable = true)\n |-- sampleSize: integer (nullable = true)\n |-- ldSet: array (nullable = true)\n |    |-- element: struct (containsNull = true)\n |    |    |-- tagVariantId: string (nullable = true)\n |    |    |-- r2Overall: double (nullable = true)\n |-- locus: array (nullable = true)\n |    |-- element: struct (containsNull = true)\n |    |    |-- is95CredibleSet: boolean (nullable = true)\n |    |    |-- is99CredibleSet: boolean (nullable = true)\n |    |    |-- logBF: double (nullable = true)\n |    |    |-- posteriorProbability: double (nullable = true)\n |    |    |-- variantId: string (nullable = true)\n |    |    |-- pValueMantissa: float (nullable = true)\n |    |    |-- pValueExponent: integer (nullable = true)\n |    |    |-- beta: double (nullable = true)\n |    |    |-- standardError: double (nullable = true)\n |    |    |-- r2Overall: double (nullable = true)\n</code></pre>"},{"location":"python_api/datasets/study_locus_overlap/","title":"Study Locus Overlap","text":""},{"location":"python_api/datasets/study_locus_overlap/#gentropy.dataset.study_locus_overlap.StudyLocusOverlap","title":"<code>gentropy.dataset.study_locus_overlap.StudyLocusOverlap</code>  <code>dataclass</code>","text":"<p>             Bases: <code>Dataset</code></p> <p>Study-Locus overlap.</p> <p>This dataset captures pairs of overlapping <code>StudyLocus</code>: that is associations whose credible sets share at least one tagging variant.</p> <p>Note</p> <p>This is a helpful dataset for other downstream analyses, such as colocalisation. This dataset will contain the overlapping signals between studyLocus associations once they have been clumped and fine-mapped.</p> Source code in <code>src/gentropy/dataset/study_locus_overlap.py</code> <pre><code>@dataclass\nclass StudyLocusOverlap(Dataset):\n    \"\"\"Study-Locus overlap.\n\n    This dataset captures pairs of overlapping `StudyLocus`: that is associations whose credible sets share at least one tagging variant.\n\n    !!! note\n\n        This is a helpful dataset for other downstream analyses, such as colocalisation. This dataset will contain the overlapping signals between studyLocus associations once they have been clumped and fine-mapped.\n    \"\"\"\n\n    @classmethod\n    def get_schema(cls: type[StudyLocusOverlap]) -&gt; StructType:\n        \"\"\"Provides the schema for the StudyLocusOverlap dataset.\n\n        Returns:\n            StructType: Schema for the StudyLocusOverlap dataset\n        \"\"\"\n        return parse_spark_schema(\"study_locus_overlap.json\")\n\n    @classmethod\n    def from_associations(\n        cls: type[StudyLocusOverlap], study_locus: StudyLocus, study_index: StudyIndex\n    ) -&gt; StudyLocusOverlap:\n        \"\"\"Find the overlapping signals in a particular set of associations (StudyLocus dataset).\n\n        Args:\n            study_locus (StudyLocus): Study-locus associations to find the overlapping signals\n            study_index (StudyIndex): Study index to find the overlapping signals\n\n        Returns:\n            StudyLocusOverlap: Study-locus overlap dataset\n        \"\"\"\n        return study_locus.find_overlaps(study_index)\n\n    def _convert_to_square_matrix(self: StudyLocusOverlap) -&gt; StudyLocusOverlap:\n        \"\"\"Convert the dataset to a square matrix.\n\n        Returns:\n            StudyLocusOverlap: Square matrix of the dataset\n        \"\"\"\n        return StudyLocusOverlap(\n            _df=self.df.unionByName(\n                self.df.selectExpr(\n                    \"leftStudyLocusId as rightStudyLocusId\",\n                    \"rightStudyLocusId as leftStudyLocusId\",\n                    \"tagVariantId\",\n                )\n            ).distinct(),\n            _schema=self.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasets/study_locus_overlap/#gentropy.dataset.study_locus_overlap.StudyLocusOverlap.from_associations","title":"<code>from_associations(study_locus: StudyLocus, study_index: StudyIndex) -&gt; StudyLocusOverlap</code>  <code>classmethod</code>","text":"<p>Find the overlapping signals in a particular set of associations (StudyLocus dataset).</p> <p>Parameters:</p> Name Type Description Default <code>study_locus</code> <code>StudyLocus</code> <p>Study-locus associations to find the overlapping signals</p> required <code>study_index</code> <code>StudyIndex</code> <p>Study index to find the overlapping signals</p> required <p>Returns:</p> Name Type Description <code>StudyLocusOverlap</code> <code>StudyLocusOverlap</code> <p>Study-locus overlap dataset</p> Source code in <code>src/gentropy/dataset/study_locus_overlap.py</code> <pre><code>@classmethod\ndef from_associations(\n    cls: type[StudyLocusOverlap], study_locus: StudyLocus, study_index: StudyIndex\n) -&gt; StudyLocusOverlap:\n    \"\"\"Find the overlapping signals in a particular set of associations (StudyLocus dataset).\n\n    Args:\n        study_locus (StudyLocus): Study-locus associations to find the overlapping signals\n        study_index (StudyIndex): Study index to find the overlapping signals\n\n    Returns:\n        StudyLocusOverlap: Study-locus overlap dataset\n    \"\"\"\n    return study_locus.find_overlaps(study_index)\n</code></pre>"},{"location":"python_api/datasets/study_locus_overlap/#gentropy.dataset.study_locus_overlap.StudyLocusOverlap.get_schema","title":"<code>get_schema() -&gt; StructType</code>  <code>classmethod</code>","text":"<p>Provides the schema for the StudyLocusOverlap dataset.</p> <p>Returns:</p> Name Type Description <code>StructType</code> <code>StructType</code> <p>Schema for the StudyLocusOverlap dataset</p> Source code in <code>src/gentropy/dataset/study_locus_overlap.py</code> <pre><code>@classmethod\ndef get_schema(cls: type[StudyLocusOverlap]) -&gt; StructType:\n    \"\"\"Provides the schema for the StudyLocusOverlap dataset.\n\n    Returns:\n        StructType: Schema for the StudyLocusOverlap dataset\n    \"\"\"\n    return parse_spark_schema(\"study_locus_overlap.json\")\n</code></pre>"},{"location":"python_api/datasets/study_locus_overlap/#schema","title":"Schema","text":"<pre><code>root\n |-- leftStudyLocusId: long (nullable = false)\n |-- rightStudyLocusId: long (nullable = false)\n |-- chromosome: string (nullable = true)\n |-- tagVariantId: string (nullable = false)\n |-- statistics: struct (nullable = true)\n |    |-- left_pValueMantissa: float (nullable = true)\n |    |-- left_pValueExponent: integer (nullable = true)\n |    |-- right_pValueMantissa: float (nullable = true)\n |    |-- right_pValueExponent: integer (nullable = true)\n |    |-- left_beta: double (nullable = true)\n |    |-- right_beta: double (nullable = true)\n |    |-- left_logBF: double (nullable = true)\n |    |-- right_logBF: double (nullable = true)\n |    |-- left_posteriorProbability: double (nullable = true)\n |    |-- right_posteriorProbability: double (nullable = true)\n</code></pre>"},{"location":"python_api/datasets/summary_statistics/","title":"Summary Statistics","text":""},{"location":"python_api/datasets/summary_statistics/#gentropy.dataset.summary_statistics.SummaryStatistics","title":"<code>gentropy.dataset.summary_statistics.SummaryStatistics</code>  <code>dataclass</code>","text":"<p>             Bases: <code>Dataset</code></p> <p>Summary Statistics dataset.</p> <p>A summary statistics dataset contains all single point statistics resulting from a GWAS.</p> Source code in <code>src/gentropy/dataset/summary_statistics.py</code> <pre><code>@dataclass\nclass SummaryStatistics(Dataset):\n    \"\"\"Summary Statistics dataset.\n\n    A summary statistics dataset contains all single point statistics resulting from a GWAS.\n    \"\"\"\n\n    @classmethod\n    def get_schema(cls: type[SummaryStatistics]) -&gt; StructType:\n        \"\"\"Provides the schema for the SummaryStatistics dataset.\n\n        Returns:\n            StructType: Schema for the SummaryStatistics dataset\n        \"\"\"\n        return parse_spark_schema(\"summary_statistics.json\")\n\n    def pvalue_filter(self: SummaryStatistics, pvalue: float) -&gt; SummaryStatistics:\n        \"\"\"Filter summary statistics based on the provided p-value threshold.\n\n        Args:\n            pvalue (float): upper limit of the p-value to be filtered upon.\n\n        Returns:\n            SummaryStatistics: summary statistics object containing single point associations with p-values at least as significant as the provided threshold.\n        \"\"\"\n        # Converting p-value to mantissa and exponent:\n        (mantissa, exponent) = split_pvalue(pvalue)\n\n        # Applying filter:\n        df = self._df.filter(\n            (f.col(\"pValueExponent\") &lt; exponent)\n            | (\n                (f.col(\"pValueExponent\") == exponent)\n                &amp; (f.col(\"pValueMantissa\") &lt;= mantissa)\n            )\n        )\n        return SummaryStatistics(_df=df, _schema=self._schema)\n\n    def window_based_clumping(\n        self: SummaryStatistics,\n        distance: int = 500_000,\n        gwas_significance: float = 5e-8,\n        baseline_significance: float = 0.05,\n        locus_collect_distance: int | None = None,\n    ) -&gt; StudyLocus:\n        \"\"\"Generate study-locus from summary statistics by distance based clumping + collect locus.\n\n        Args:\n            distance (int): Distance in base pairs to be used for clumping. Defaults to 500_000.\n            gwas_significance (float, optional): GWAS significance threshold. Defaults to 5e-8.\n            baseline_significance (float, optional): Baseline significance threshold for inclusion in the locus. Defaults to 0.05.\n            locus_collect_distance (int | None): The distance to collect locus around semi-indices. If not provided, locus is not collected.\n\n        Returns:\n            StudyLocus: Clumped study-locus containing variants based on window.\n        \"\"\"\n        return (\n            WindowBasedClumping.clump_with_locus(\n                self,\n                window_length=distance,\n                p_value_significance=gwas_significance,\n                p_value_baseline=baseline_significance,\n                locus_window_length=locus_collect_distance,\n            )\n            if locus_collect_distance\n            else WindowBasedClumping.clump(\n                self,\n                window_length=distance,\n                p_value_significance=gwas_significance,\n            )\n        )\n\n    def exclude_region(self: SummaryStatistics, region: str) -&gt; SummaryStatistics:\n        \"\"\"Exclude a region from the summary stats dataset.\n\n        Args:\n            region (str): region given in \"chr##:#####-####\" format\n\n        Returns:\n            SummaryStatistics: filtered summary statistics.\n        \"\"\"\n        (chromosome, start_position, end_position) = parse_region(region)\n\n        return SummaryStatistics(\n            _df=(\n                self.df.filter(\n                    ~(\n                        (f.col(\"chromosome\") == chromosome)\n                        &amp; (\n                            (f.col(\"position\") &gt;= start_position)\n                            &amp; (f.col(\"position\") &lt;= end_position)\n                        )\n                    )\n                )\n            ),\n            _schema=SummaryStatistics.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasets/summary_statistics/#gentropy.dataset.summary_statistics.SummaryStatistics.exclude_region","title":"<code>exclude_region(region: str) -&gt; SummaryStatistics</code>","text":"<p>Exclude a region from the summary stats dataset.</p> <p>Parameters:</p> Name Type Description Default <code>region</code> <code>str</code> <p>region given in \"chr##:#####-####\" format</p> required <p>Returns:</p> Name Type Description <code>SummaryStatistics</code> <code>SummaryStatistics</code> <p>filtered summary statistics.</p> Source code in <code>src/gentropy/dataset/summary_statistics.py</code> <pre><code>def exclude_region(self: SummaryStatistics, region: str) -&gt; SummaryStatistics:\n    \"\"\"Exclude a region from the summary stats dataset.\n\n    Args:\n        region (str): region given in \"chr##:#####-####\" format\n\n    Returns:\n        SummaryStatistics: filtered summary statistics.\n    \"\"\"\n    (chromosome, start_position, end_position) = parse_region(region)\n\n    return SummaryStatistics(\n        _df=(\n            self.df.filter(\n                ~(\n                    (f.col(\"chromosome\") == chromosome)\n                    &amp; (\n                        (f.col(\"position\") &gt;= start_position)\n                        &amp; (f.col(\"position\") &lt;= end_position)\n                    )\n                )\n            )\n        ),\n        _schema=SummaryStatistics.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasets/summary_statistics/#gentropy.dataset.summary_statistics.SummaryStatistics.get_schema","title":"<code>get_schema() -&gt; StructType</code>  <code>classmethod</code>","text":"<p>Provides the schema for the SummaryStatistics dataset.</p> <p>Returns:</p> Name Type Description <code>StructType</code> <code>StructType</code> <p>Schema for the SummaryStatistics dataset</p> Source code in <code>src/gentropy/dataset/summary_statistics.py</code> <pre><code>@classmethod\ndef get_schema(cls: type[SummaryStatistics]) -&gt; StructType:\n    \"\"\"Provides the schema for the SummaryStatistics dataset.\n\n    Returns:\n        StructType: Schema for the SummaryStatistics dataset\n    \"\"\"\n    return parse_spark_schema(\"summary_statistics.json\")\n</code></pre>"},{"location":"python_api/datasets/summary_statistics/#gentropy.dataset.summary_statistics.SummaryStatistics.pvalue_filter","title":"<code>pvalue_filter(pvalue: float) -&gt; SummaryStatistics</code>","text":"<p>Filter summary statistics based on the provided p-value threshold.</p> <p>Parameters:</p> Name Type Description Default <code>pvalue</code> <code>float</code> <p>upper limit of the p-value to be filtered upon.</p> required <p>Returns:</p> Name Type Description <code>SummaryStatistics</code> <code>SummaryStatistics</code> <p>summary statistics object containing single point associations with p-values at least as significant as the provided threshold.</p> Source code in <code>src/gentropy/dataset/summary_statistics.py</code> <pre><code>def pvalue_filter(self: SummaryStatistics, pvalue: float) -&gt; SummaryStatistics:\n    \"\"\"Filter summary statistics based on the provided p-value threshold.\n\n    Args:\n        pvalue (float): upper limit of the p-value to be filtered upon.\n\n    Returns:\n        SummaryStatistics: summary statistics object containing single point associations with p-values at least as significant as the provided threshold.\n    \"\"\"\n    # Converting p-value to mantissa and exponent:\n    (mantissa, exponent) = split_pvalue(pvalue)\n\n    # Applying filter:\n    df = self._df.filter(\n        (f.col(\"pValueExponent\") &lt; exponent)\n        | (\n            (f.col(\"pValueExponent\") == exponent)\n            &amp; (f.col(\"pValueMantissa\") &lt;= mantissa)\n        )\n    )\n    return SummaryStatistics(_df=df, _schema=self._schema)\n</code></pre>"},{"location":"python_api/datasets/summary_statistics/#gentropy.dataset.summary_statistics.SummaryStatistics.window_based_clumping","title":"<code>window_based_clumping(distance: int = 500000, gwas_significance: float = 5e-08, baseline_significance: float = 0.05, locus_collect_distance: int | None = None) -&gt; StudyLocus</code>","text":"<p>Generate study-locus from summary statistics by distance based clumping + collect locus.</p> <p>Parameters:</p> Name Type Description Default <code>distance</code> <code>int</code> <p>Distance in base pairs to be used for clumping. Defaults to 500_000.</p> <code>500000</code> <code>gwas_significance</code> <code>float</code> <p>GWAS significance threshold. Defaults to 5e-8.</p> <code>5e-08</code> <code>baseline_significance</code> <code>float</code> <p>Baseline significance threshold for inclusion in the locus. Defaults to 0.05.</p> <code>0.05</code> <code>locus_collect_distance</code> <code>int | None</code> <p>The distance to collect locus around semi-indices. If not provided, locus is not collected.</p> <code>None</code> <p>Returns:</p> Name Type Description <code>StudyLocus</code> <code>StudyLocus</code> <p>Clumped study-locus containing variants based on window.</p> Source code in <code>src/gentropy/dataset/summary_statistics.py</code> <pre><code>def window_based_clumping(\n    self: SummaryStatistics,\n    distance: int = 500_000,\n    gwas_significance: float = 5e-8,\n    baseline_significance: float = 0.05,\n    locus_collect_distance: int | None = None,\n) -&gt; StudyLocus:\n    \"\"\"Generate study-locus from summary statistics by distance based clumping + collect locus.\n\n    Args:\n        distance (int): Distance in base pairs to be used for clumping. Defaults to 500_000.\n        gwas_significance (float, optional): GWAS significance threshold. Defaults to 5e-8.\n        baseline_significance (float, optional): Baseline significance threshold for inclusion in the locus. Defaults to 0.05.\n        locus_collect_distance (int | None): The distance to collect locus around semi-indices. If not provided, locus is not collected.\n\n    Returns:\n        StudyLocus: Clumped study-locus containing variants based on window.\n    \"\"\"\n    return (\n        WindowBasedClumping.clump_with_locus(\n            self,\n            window_length=distance,\n            p_value_significance=gwas_significance,\n            p_value_baseline=baseline_significance,\n            locus_window_length=locus_collect_distance,\n        )\n        if locus_collect_distance\n        else WindowBasedClumping.clump(\n            self,\n            window_length=distance,\n            p_value_significance=gwas_significance,\n        )\n    )\n</code></pre>"},{"location":"python_api/datasets/summary_statistics/#schema","title":"Schema","text":"<pre><code>root\n |-- studyId: string (nullable = false)\n |-- variantId: string (nullable = false)\n |-- chromosome: string (nullable = false)\n |-- position: integer (nullable = false)\n |-- beta: double (nullable = false)\n |-- sampleSize: integer (nullable = true)\n |-- pValueMantissa: float (nullable = false)\n |-- pValueExponent: integer (nullable = false)\n |-- effectAlleleFrequencyFromSource: float (nullable = true)\n |-- standardError: double (nullable = true)\n</code></pre>"},{"location":"python_api/datasets/variant_annotation/","title":"Variant annotation","text":""},{"location":"python_api/datasets/variant_annotation/#gentropy.dataset.variant_annotation.VariantAnnotation","title":"<code>gentropy.dataset.variant_annotation.VariantAnnotation</code>  <code>dataclass</code>","text":"<p>             Bases: <code>Dataset</code></p> <p>Dataset with variant-level annotations.</p> Source code in <code>src/gentropy/dataset/variant_annotation.py</code> <pre><code>@dataclass\nclass VariantAnnotation(Dataset):\n    \"\"\"Dataset with variant-level annotations.\"\"\"\n\n    @classmethod\n    def get_schema(cls: type[VariantAnnotation]) -&gt; StructType:\n        \"\"\"Provides the schema for the VariantAnnotation dataset.\n\n        Returns:\n            StructType: Schema for the VariantAnnotation dataset\n        \"\"\"\n        return parse_spark_schema(\"variant_annotation.json\")\n\n    def max_maf(self: VariantAnnotation) -&gt; Column:\n        \"\"\"Maximum minor allele frequency accross all populations.\n\n        Returns:\n            Column: Maximum minor allele frequency accross all populations.\n        \"\"\"\n        return f.array_max(\n            f.transform(\n                self.df.alleleFrequencies,\n                lambda af: f.when(\n                    af.alleleFrequency &gt; 0.5, 1 - af.alleleFrequency\n                ).otherwise(af.alleleFrequency),\n            )\n        )\n\n    def filter_by_variant_df(\n        self: VariantAnnotation, df: DataFrame\n    ) -&gt; VariantAnnotation:\n        \"\"\"Filter variant annotation dataset by a variant dataframe.\n\n        Args:\n            df (DataFrame): A dataframe of variants\n\n        Returns:\n            VariantAnnotation: A filtered variant annotation dataset\n        \"\"\"\n        self.df = self._df.join(\n            f.broadcast(df.select(\"variantId\", \"chromosome\")),\n            on=[\"variantId\", \"chromosome\"],\n            how=\"inner\",\n        )\n        return self\n\n    def get_transcript_consequence_df(\n        self: VariantAnnotation, gene_index: GeneIndex | None = None\n    ) -&gt; DataFrame:\n        \"\"\"Dataframe of exploded transcript consequences.\n\n        Optionally the trancript consequences can be reduced to the universe of a gene index.\n\n        Args:\n            gene_index (GeneIndex | None): A gene index. Defaults to None.\n\n        Returns:\n            DataFrame: A dataframe exploded by transcript consequences with the columns variantId, chromosome, transcriptConsequence\n        \"\"\"\n        # exploding the array removes records without VEP annotation\n        transript_consequences = self.df.withColumn(\n            \"transcriptConsequence\", f.explode(\"vep.transcriptConsequences\")\n        ).select(\n            \"variantId\",\n            \"chromosome\",\n            \"position\",\n            \"transcriptConsequence\",\n            f.col(\"transcriptConsequence.geneId\").alias(\"geneId\"),\n        )\n        if gene_index:\n            transript_consequences = transript_consequences.join(\n                f.broadcast(gene_index.df),\n                on=[\"chromosome\", \"geneId\"],\n            )\n        return transript_consequences.persist()\n\n    def get_most_severe_vep_v2g(\n        self: VariantAnnotation,\n        vep_consequences: DataFrame,\n        gene_index: GeneIndex,\n    ) -&gt; V2G:\n        \"\"\"Creates a dataset with variant to gene assignments based on VEP's predicted consequence of the transcript.\n\n        Optionally the trancript consequences can be reduced to the universe of a gene index.\n\n        Args:\n            vep_consequences (DataFrame): A dataframe of VEP consequences\n            gene_index (GeneIndex): A gene index to filter by. Defaults to None.\n\n        Returns:\n            V2G: High and medium severity variant to gene assignments\n        \"\"\"\n        return V2G(\n            _df=self.get_transcript_consequence_df(gene_index)\n            .select(\n                \"variantId\",\n                \"chromosome\",\n                f.col(\"transcriptConsequence.geneId\").alias(\"geneId\"),\n                f.explode(\"transcriptConsequence.consequenceTerms\").alias(\"label\"),\n                f.lit(\"vep\").alias(\"datatypeId\"),\n                f.lit(\"variantConsequence\").alias(\"datasourceId\"),\n            )\n            .join(\n                f.broadcast(vep_consequences),\n                on=\"label\",\n                how=\"inner\",\n            )\n            .drop(\"label\")\n            .filter(f.col(\"score\") != 0)\n            # A variant can have multiple predicted consequences on a transcript, the most severe one is selected\n            .transform(\n                lambda df: get_record_with_maximum_value(\n                    df, [\"variantId\", \"geneId\"], \"score\"\n                )\n            ),\n            _schema=V2G.get_schema(),\n        )\n\n    def get_plof_v2g(self: VariantAnnotation, gene_index: GeneIndex) -&gt; V2G:\n        \"\"\"Creates a dataset with variant to gene assignments with a flag indicating if the variant is predicted to be a loss-of-function variant by the LOFTEE algorithm.\n\n        Optionally the trancript consequences can be reduced to the universe of a gene index.\n\n        Args:\n            gene_index (GeneIndex): A gene index to filter by.\n\n        Returns:\n            V2G: variant to gene assignments from the LOFTEE algorithm\n        \"\"\"\n        return V2G(\n            _df=(\n                self.get_transcript_consequence_df(gene_index)\n                .filter(f.col(\"transcriptConsequence.lof\").isNotNull())\n                .withColumn(\n                    \"isHighQualityPlof\",\n                    f.when(f.col(\"transcriptConsequence.lof\") == \"HC\", True).when(\n                        f.col(\"transcriptConsequence.lof\") == \"LC\", False\n                    ),\n                )\n                .withColumn(\n                    \"score\",\n                    f.when(f.col(\"isHighQualityPlof\"), 1.0).when(\n                        ~f.col(\"isHighQualityPlof\"), 0\n                    ),\n                )\n                .select(\n                    \"variantId\",\n                    \"chromosome\",\n                    \"geneId\",\n                    \"isHighQualityPlof\",\n                    f.col(\"score\"),\n                    f.lit(\"vep\").alias(\"datatypeId\"),\n                    f.lit(\"loftee\").alias(\"datasourceId\"),\n                )\n            ),\n            _schema=V2G.get_schema(),\n        )\n\n    def get_distance_to_tss(\n        self: VariantAnnotation,\n        gene_index: GeneIndex,\n        max_distance: int = 500_000,\n    ) -&gt; V2G:\n        \"\"\"Extracts variant to gene assignments for variants falling within a window of a gene's TSS.\n\n        Args:\n            gene_index (GeneIndex): A gene index to filter by.\n            max_distance (int): The maximum distance from the TSS to consider. Defaults to 500_000.\n\n        Returns:\n            V2G: variant to gene assignments with their distance to the TSS\n        \"\"\"\n        return V2G(\n            _df=(\n                self.df.alias(\"variant\")\n                .join(\n                    f.broadcast(gene_index.locations_lut()).alias(\"gene\"),\n                    on=[\n                        f.col(\"variant.chromosome\") == f.col(\"gene.chromosome\"),\n                        f.abs(f.col(\"variant.position\") - f.col(\"gene.tss\"))\n                        &lt;= max_distance,\n                    ],\n                    how=\"inner\",\n                )\n                .withColumn(\n                    \"distance\", f.abs(f.col(\"variant.position\") - f.col(\"gene.tss\"))\n                )\n                .withColumn(\n                    \"inverse_distance\",\n                    max_distance - f.col(\"distance\"),\n                )\n                .transform(lambda df: normalise_column(df, \"inverse_distance\", \"score\"))\n                .select(\n                    \"variantId\",\n                    f.col(\"variant.chromosome\").alias(\"chromosome\"),\n                    \"distance\",\n                    \"geneId\",\n                    \"score\",\n                    f.lit(\"distance\").alias(\"datatypeId\"),\n                    f.lit(\"canonical_tss\").alias(\"datasourceId\"),\n                )\n            ),\n            _schema=V2G.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasets/variant_annotation/#gentropy.dataset.variant_annotation.VariantAnnotation.filter_by_variant_df","title":"<code>filter_by_variant_df(df: DataFrame) -&gt; VariantAnnotation</code>","text":"<p>Filter variant annotation dataset by a variant dataframe.</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>A dataframe of variants</p> required <p>Returns:</p> Name Type Description <code>VariantAnnotation</code> <code>VariantAnnotation</code> <p>A filtered variant annotation dataset</p> Source code in <code>src/gentropy/dataset/variant_annotation.py</code> <pre><code>def filter_by_variant_df(\n    self: VariantAnnotation, df: DataFrame\n) -&gt; VariantAnnotation:\n    \"\"\"Filter variant annotation dataset by a variant dataframe.\n\n    Args:\n        df (DataFrame): A dataframe of variants\n\n    Returns:\n        VariantAnnotation: A filtered variant annotation dataset\n    \"\"\"\n    self.df = self._df.join(\n        f.broadcast(df.select(\"variantId\", \"chromosome\")),\n        on=[\"variantId\", \"chromosome\"],\n        how=\"inner\",\n    )\n    return self\n</code></pre>"},{"location":"python_api/datasets/variant_annotation/#gentropy.dataset.variant_annotation.VariantAnnotation.get_distance_to_tss","title":"<code>get_distance_to_tss(gene_index: GeneIndex, max_distance: int = 500000) -&gt; V2G</code>","text":"<p>Extracts variant to gene assignments for variants falling within a window of a gene's TSS.</p> <p>Parameters:</p> Name Type Description Default <code>gene_index</code> <code>GeneIndex</code> <p>A gene index to filter by.</p> required <code>max_distance</code> <code>int</code> <p>The maximum distance from the TSS to consider. Defaults to 500_000.</p> <code>500000</code> <p>Returns:</p> Name Type Description <code>V2G</code> <code>V2G</code> <p>variant to gene assignments with their distance to the TSS</p> Source code in <code>src/gentropy/dataset/variant_annotation.py</code> <pre><code>def get_distance_to_tss(\n    self: VariantAnnotation,\n    gene_index: GeneIndex,\n    max_distance: int = 500_000,\n) -&gt; V2G:\n    \"\"\"Extracts variant to gene assignments for variants falling within a window of a gene's TSS.\n\n    Args:\n        gene_index (GeneIndex): A gene index to filter by.\n        max_distance (int): The maximum distance from the TSS to consider. Defaults to 500_000.\n\n    Returns:\n        V2G: variant to gene assignments with their distance to the TSS\n    \"\"\"\n    return V2G(\n        _df=(\n            self.df.alias(\"variant\")\n            .join(\n                f.broadcast(gene_index.locations_lut()).alias(\"gene\"),\n                on=[\n                    f.col(\"variant.chromosome\") == f.col(\"gene.chromosome\"),\n                    f.abs(f.col(\"variant.position\") - f.col(\"gene.tss\"))\n                    &lt;= max_distance,\n                ],\n                how=\"inner\",\n            )\n            .withColumn(\n                \"distance\", f.abs(f.col(\"variant.position\") - f.col(\"gene.tss\"))\n            )\n            .withColumn(\n                \"inverse_distance\",\n                max_distance - f.col(\"distance\"),\n            )\n            .transform(lambda df: normalise_column(df, \"inverse_distance\", \"score\"))\n            .select(\n                \"variantId\",\n                f.col(\"variant.chromosome\").alias(\"chromosome\"),\n                \"distance\",\n                \"geneId\",\n                \"score\",\n                f.lit(\"distance\").alias(\"datatypeId\"),\n                f.lit(\"canonical_tss\").alias(\"datasourceId\"),\n            )\n        ),\n        _schema=V2G.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasets/variant_annotation/#gentropy.dataset.variant_annotation.VariantAnnotation.get_most_severe_vep_v2g","title":"<code>get_most_severe_vep_v2g(vep_consequences: DataFrame, gene_index: GeneIndex) -&gt; V2G</code>","text":"<p>Creates a dataset with variant to gene assignments based on VEP's predicted consequence of the transcript.</p> <p>Optionally the trancript consequences can be reduced to the universe of a gene index.</p> <p>Parameters:</p> Name Type Description Default <code>vep_consequences</code> <code>DataFrame</code> <p>A dataframe of VEP consequences</p> required <code>gene_index</code> <code>GeneIndex</code> <p>A gene index to filter by. Defaults to None.</p> required <p>Returns:</p> Name Type Description <code>V2G</code> <code>V2G</code> <p>High and medium severity variant to gene assignments</p> Source code in <code>src/gentropy/dataset/variant_annotation.py</code> <pre><code>def get_most_severe_vep_v2g(\n    self: VariantAnnotation,\n    vep_consequences: DataFrame,\n    gene_index: GeneIndex,\n) -&gt; V2G:\n    \"\"\"Creates a dataset with variant to gene assignments based on VEP's predicted consequence of the transcript.\n\n    Optionally the trancript consequences can be reduced to the universe of a gene index.\n\n    Args:\n        vep_consequences (DataFrame): A dataframe of VEP consequences\n        gene_index (GeneIndex): A gene index to filter by. Defaults to None.\n\n    Returns:\n        V2G: High and medium severity variant to gene assignments\n    \"\"\"\n    return V2G(\n        _df=self.get_transcript_consequence_df(gene_index)\n        .select(\n            \"variantId\",\n            \"chromosome\",\n            f.col(\"transcriptConsequence.geneId\").alias(\"geneId\"),\n            f.explode(\"transcriptConsequence.consequenceTerms\").alias(\"label\"),\n            f.lit(\"vep\").alias(\"datatypeId\"),\n            f.lit(\"variantConsequence\").alias(\"datasourceId\"),\n        )\n        .join(\n            f.broadcast(vep_consequences),\n            on=\"label\",\n            how=\"inner\",\n        )\n        .drop(\"label\")\n        .filter(f.col(\"score\") != 0)\n        # A variant can have multiple predicted consequences on a transcript, the most severe one is selected\n        .transform(\n            lambda df: get_record_with_maximum_value(\n                df, [\"variantId\", \"geneId\"], \"score\"\n            )\n        ),\n        _schema=V2G.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasets/variant_annotation/#gentropy.dataset.variant_annotation.VariantAnnotation.get_plof_v2g","title":"<code>get_plof_v2g(gene_index: GeneIndex) -&gt; V2G</code>","text":"<p>Creates a dataset with variant to gene assignments with a flag indicating if the variant is predicted to be a loss-of-function variant by the LOFTEE algorithm.</p> <p>Optionally the trancript consequences can be reduced to the universe of a gene index.</p> <p>Parameters:</p> Name Type Description Default <code>gene_index</code> <code>GeneIndex</code> <p>A gene index to filter by.</p> required <p>Returns:</p> Name Type Description <code>V2G</code> <code>V2G</code> <p>variant to gene assignments from the LOFTEE algorithm</p> Source code in <code>src/gentropy/dataset/variant_annotation.py</code> <pre><code>def get_plof_v2g(self: VariantAnnotation, gene_index: GeneIndex) -&gt; V2G:\n    \"\"\"Creates a dataset with variant to gene assignments with a flag indicating if the variant is predicted to be a loss-of-function variant by the LOFTEE algorithm.\n\n    Optionally the trancript consequences can be reduced to the universe of a gene index.\n\n    Args:\n        gene_index (GeneIndex): A gene index to filter by.\n\n    Returns:\n        V2G: variant to gene assignments from the LOFTEE algorithm\n    \"\"\"\n    return V2G(\n        _df=(\n            self.get_transcript_consequence_df(gene_index)\n            .filter(f.col(\"transcriptConsequence.lof\").isNotNull())\n            .withColumn(\n                \"isHighQualityPlof\",\n                f.when(f.col(\"transcriptConsequence.lof\") == \"HC\", True).when(\n                    f.col(\"transcriptConsequence.lof\") == \"LC\", False\n                ),\n            )\n            .withColumn(\n                \"score\",\n                f.when(f.col(\"isHighQualityPlof\"), 1.0).when(\n                    ~f.col(\"isHighQualityPlof\"), 0\n                ),\n            )\n            .select(\n                \"variantId\",\n                \"chromosome\",\n                \"geneId\",\n                \"isHighQualityPlof\",\n                f.col(\"score\"),\n                f.lit(\"vep\").alias(\"datatypeId\"),\n                f.lit(\"loftee\").alias(\"datasourceId\"),\n            )\n        ),\n        _schema=V2G.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasets/variant_annotation/#gentropy.dataset.variant_annotation.VariantAnnotation.get_schema","title":"<code>get_schema() -&gt; StructType</code>  <code>classmethod</code>","text":"<p>Provides the schema for the VariantAnnotation dataset.</p> <p>Returns:</p> Name Type Description <code>StructType</code> <code>StructType</code> <p>Schema for the VariantAnnotation dataset</p> Source code in <code>src/gentropy/dataset/variant_annotation.py</code> <pre><code>@classmethod\ndef get_schema(cls: type[VariantAnnotation]) -&gt; StructType:\n    \"\"\"Provides the schema for the VariantAnnotation dataset.\n\n    Returns:\n        StructType: Schema for the VariantAnnotation dataset\n    \"\"\"\n    return parse_spark_schema(\"variant_annotation.json\")\n</code></pre>"},{"location":"python_api/datasets/variant_annotation/#gentropy.dataset.variant_annotation.VariantAnnotation.get_transcript_consequence_df","title":"<code>get_transcript_consequence_df(gene_index: GeneIndex | None = None) -&gt; DataFrame</code>","text":"<p>Dataframe of exploded transcript consequences.</p> <p>Optionally the trancript consequences can be reduced to the universe of a gene index.</p> <p>Parameters:</p> Name Type Description Default <code>gene_index</code> <code>GeneIndex | None</code> <p>A gene index. Defaults to None.</p> <code>None</code> <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>A dataframe exploded by transcript consequences with the columns variantId, chromosome, transcriptConsequence</p> Source code in <code>src/gentropy/dataset/variant_annotation.py</code> <pre><code>def get_transcript_consequence_df(\n    self: VariantAnnotation, gene_index: GeneIndex | None = None\n) -&gt; DataFrame:\n    \"\"\"Dataframe of exploded transcript consequences.\n\n    Optionally the trancript consequences can be reduced to the universe of a gene index.\n\n    Args:\n        gene_index (GeneIndex | None): A gene index. Defaults to None.\n\n    Returns:\n        DataFrame: A dataframe exploded by transcript consequences with the columns variantId, chromosome, transcriptConsequence\n    \"\"\"\n    # exploding the array removes records without VEP annotation\n    transript_consequences = self.df.withColumn(\n        \"transcriptConsequence\", f.explode(\"vep.transcriptConsequences\")\n    ).select(\n        \"variantId\",\n        \"chromosome\",\n        \"position\",\n        \"transcriptConsequence\",\n        f.col(\"transcriptConsequence.geneId\").alias(\"geneId\"),\n    )\n    if gene_index:\n        transript_consequences = transript_consequences.join(\n            f.broadcast(gene_index.df),\n            on=[\"chromosome\", \"geneId\"],\n        )\n    return transript_consequences.persist()\n</code></pre>"},{"location":"python_api/datasets/variant_annotation/#gentropy.dataset.variant_annotation.VariantAnnotation.max_maf","title":"<code>max_maf() -&gt; Column</code>","text":"<p>Maximum minor allele frequency accross all populations.</p> <p>Returns:</p> Name Type Description <code>Column</code> <code>Column</code> <p>Maximum minor allele frequency accross all populations.</p> Source code in <code>src/gentropy/dataset/variant_annotation.py</code> <pre><code>def max_maf(self: VariantAnnotation) -&gt; Column:\n    \"\"\"Maximum minor allele frequency accross all populations.\n\n    Returns:\n        Column: Maximum minor allele frequency accross all populations.\n    \"\"\"\n    return f.array_max(\n        f.transform(\n            self.df.alleleFrequencies,\n            lambda af: f.when(\n                af.alleleFrequency &gt; 0.5, 1 - af.alleleFrequency\n            ).otherwise(af.alleleFrequency),\n        )\n    )\n</code></pre>"},{"location":"python_api/datasets/variant_annotation/#schema","title":"Schema","text":"<pre><code>root\n |-- variantId: string (nullable = false)\n |-- chromosome: string (nullable = false)\n |-- position: integer (nullable = false)\n |-- gnomadVariantId: string (nullable = false)\n |-- referenceAllele: string (nullable = false)\n |-- alternateAllele: string (nullable = false)\n |-- chromosomeB37: string (nullable = true)\n |-- positionB37: integer (nullable = true)\n |-- alleleType: string (nullable = true)\n |-- rsIds: array (nullable = true)\n |    |-- element: string (containsNull = true)\n |-- alleleFrequencies: array (nullable = false)\n |    |-- element: struct (containsNull = true)\n |    |    |-- populationName: string (nullable = true)\n |    |    |-- alleleFrequency: double (nullable = true)\n |-- inSilicoPredictors: struct (nullable = false)\n |    |-- cadd: struct (nullable = true)\n |    |    |-- raw: float (nullable = true)\n |    |    |-- phred: float (nullable = true)\n |    |-- revelMax: double (nullable = true)\n |    |-- spliceaiDsMax: float (nullable = true)\n |    |-- pangolinLargestDs: double (nullable = true)\n |    |-- phylop: double (nullable = true)\n |    |-- siftMax: double (nullable = true)\n |    |-- polyphenMax: double (nullable = true)\n |-- vep: struct (nullable = false)\n |    |-- mostSevereConsequence: string (nullable = true)\n |    |-- transcriptConsequences: array (nullable = true)\n |    |    |-- element: struct (containsNull = true)\n |    |    |    |-- aminoAcids: string (nullable = true)\n |    |    |    |-- consequenceTerms: array (nullable = true)\n |    |    |    |    |-- element: string (containsNull = true)\n |    |    |    |-- geneId: string (nullable = true)\n |    |    |    |-- lof: string (nullable = true)\n</code></pre>"},{"location":"python_api/datasets/variant_index/","title":"Variant index","text":""},{"location":"python_api/datasets/variant_index/#gentropy.dataset.variant_index.VariantIndex","title":"<code>gentropy.dataset.variant_index.VariantIndex</code>  <code>dataclass</code>","text":"<p>             Bases: <code>Dataset</code></p> <p>Variant index dataset.</p> <p>Variant index dataset is the result of intersecting the variant annotation dataset with the variants with V2D available information.</p> Source code in <code>src/gentropy/dataset/variant_index.py</code> <pre><code>@dataclass\nclass VariantIndex(Dataset):\n    \"\"\"Variant index dataset.\n\n    Variant index dataset is the result of intersecting the variant annotation dataset with the variants with V2D available information.\n    \"\"\"\n\n    @classmethod\n    def get_schema(cls: type[VariantIndex]) -&gt; StructType:\n        \"\"\"Provides the schema for the VariantIndex dataset.\n\n        Returns:\n            StructType: Schema for the VariantIndex dataset\n        \"\"\"\n        return parse_spark_schema(\"variant_index.json\")\n\n    @classmethod\n    def from_variant_annotation(\n        cls: type[VariantIndex],\n        variant_annotation: VariantAnnotation,\n        study_locus: StudyLocus,\n    ) -&gt; VariantIndex:\n        \"\"\"Initialise VariantIndex from pre-existing variant annotation dataset.\n\n        Args:\n            variant_annotation (VariantAnnotation): Variant annotation dataset\n            study_locus (StudyLocus): Study locus dataset with the variants to intersect with the variant annotation dataset\n\n        Returns:\n            VariantIndex: Variant index dataset\n        \"\"\"\n        unchanged_cols = [\n            \"variantId\",\n            \"chromosome\",\n            \"position\",\n            \"referenceAllele\",\n            \"alternateAllele\",\n            \"chromosomeB37\",\n            \"positionB37\",\n            \"alleleType\",\n            \"alleleFrequencies\",\n            \"inSilicoPredictors\",\n        ]\n        va_slimmed = variant_annotation.filter_by_variant_df(\n            study_locus.unique_variants_in_locus()\n        )\n        return cls(\n            _df=(\n                va_slimmed.df.select(\n                    *unchanged_cols,\n                    f.col(\"vep.mostSevereConsequence\").alias(\"mostSevereConsequence\"),\n                    # filters/rsid are arrays that can be empty, in this case we convert them to null\n                    nullify_empty_array(f.col(\"rsIds\")).alias(\"rsIds\"),\n                )\n                .repartition(400, \"chromosome\")\n                .sortWithinPartitions(\"chromosome\", \"position\")\n            ),\n            _schema=cls.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasets/variant_index/#gentropy.dataset.variant_index.VariantIndex.from_variant_annotation","title":"<code>from_variant_annotation(variant_annotation: VariantAnnotation, study_locus: StudyLocus) -&gt; VariantIndex</code>  <code>classmethod</code>","text":"<p>Initialise VariantIndex from pre-existing variant annotation dataset.</p> <p>Parameters:</p> Name Type Description Default <code>variant_annotation</code> <code>VariantAnnotation</code> <p>Variant annotation dataset</p> required <code>study_locus</code> <code>StudyLocus</code> <p>Study locus dataset with the variants to intersect with the variant annotation dataset</p> required <p>Returns:</p> Name Type Description <code>VariantIndex</code> <code>VariantIndex</code> <p>Variant index dataset</p> Source code in <code>src/gentropy/dataset/variant_index.py</code> <pre><code>@classmethod\ndef from_variant_annotation(\n    cls: type[VariantIndex],\n    variant_annotation: VariantAnnotation,\n    study_locus: StudyLocus,\n) -&gt; VariantIndex:\n    \"\"\"Initialise VariantIndex from pre-existing variant annotation dataset.\n\n    Args:\n        variant_annotation (VariantAnnotation): Variant annotation dataset\n        study_locus (StudyLocus): Study locus dataset with the variants to intersect with the variant annotation dataset\n\n    Returns:\n        VariantIndex: Variant index dataset\n    \"\"\"\n    unchanged_cols = [\n        \"variantId\",\n        \"chromosome\",\n        \"position\",\n        \"referenceAllele\",\n        \"alternateAllele\",\n        \"chromosomeB37\",\n        \"positionB37\",\n        \"alleleType\",\n        \"alleleFrequencies\",\n        \"inSilicoPredictors\",\n    ]\n    va_slimmed = variant_annotation.filter_by_variant_df(\n        study_locus.unique_variants_in_locus()\n    )\n    return cls(\n        _df=(\n            va_slimmed.df.select(\n                *unchanged_cols,\n                f.col(\"vep.mostSevereConsequence\").alias(\"mostSevereConsequence\"),\n                # filters/rsid are arrays that can be empty, in this case we convert them to null\n                nullify_empty_array(f.col(\"rsIds\")).alias(\"rsIds\"),\n            )\n            .repartition(400, \"chromosome\")\n            .sortWithinPartitions(\"chromosome\", \"position\")\n        ),\n        _schema=cls.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasets/variant_index/#gentropy.dataset.variant_index.VariantIndex.get_schema","title":"<code>get_schema() -&gt; StructType</code>  <code>classmethod</code>","text":"<p>Provides the schema for the VariantIndex dataset.</p> <p>Returns:</p> Name Type Description <code>StructType</code> <code>StructType</code> <p>Schema for the VariantIndex dataset</p> Source code in <code>src/gentropy/dataset/variant_index.py</code> <pre><code>@classmethod\ndef get_schema(cls: type[VariantIndex]) -&gt; StructType:\n    \"\"\"Provides the schema for the VariantIndex dataset.\n\n    Returns:\n        StructType: Schema for the VariantIndex dataset\n    \"\"\"\n    return parse_spark_schema(\"variant_index.json\")\n</code></pre>"},{"location":"python_api/datasets/variant_index/#schema","title":"Schema","text":"<pre><code>root\n |-- variantId: string (nullable = false)\n |-- chromosome: string (nullable = false)\n |-- position: integer (nullable = false)\n |-- referenceAllele: string (nullable = false)\n |-- alternateAllele: string (nullable = false)\n |-- chromosomeB37: string (nullable = true)\n |-- positionB37: integer (nullable = true)\n |-- alleleType: string (nullable = false)\n |-- alleleFrequencies: array (nullable = false)\n |    |-- element: struct (containsNull = true)\n |    |    |-- populationName: string (nullable = true)\n |    |    |-- alleleFrequency: double (nullable = true)\n |-- inSilicoPredictors: struct (nullable = false)\n |    |-- cadd: struct (nullable = true)\n |    |    |-- raw: float (nullable = true)\n |    |    |-- phred: float (nullable = true)\n |    |-- revelMax: double (nullable = true)\n |    |-- spliceaiDsMax: float (nullable = true)\n |    |-- pangolinLargestDs: double (nullable = true)\n |    |-- phylop: double (nullable = true)\n |    |-- siftMax: double (nullable = true)\n |    |-- polyphenMax: double (nullable = true)\n |-- mostSevereConsequence: string (nullable = true)\n |-- rsIds: array (nullable = true)\n |    |-- element: string (containsNull = true)\n</code></pre>"},{"location":"python_api/datasets/variant_to_gene/","title":"Variant-to-gene","text":""},{"location":"python_api/datasets/variant_to_gene/#gentropy.dataset.v2g.V2G","title":"<code>gentropy.dataset.v2g.V2G</code>  <code>dataclass</code>","text":"<p>             Bases: <code>Dataset</code></p> <p>Variant-to-gene (V2G) evidence dataset.</p> <p>A variant-to-gene (V2G) evidence is understood as any piece of evidence that supports the association of a variant with a likely causal gene. The evidence can sometimes be context-specific and refer to specific <code>biofeatures</code> (e.g. cell types)</p> Source code in <code>src/gentropy/dataset/v2g.py</code> <pre><code>@dataclass\nclass V2G(Dataset):\n    \"\"\"Variant-to-gene (V2G) evidence dataset.\n\n    A variant-to-gene (V2G) evidence is understood as any piece of evidence that supports the association of a variant with a likely causal gene. The evidence can sometimes be context-specific and refer to specific `biofeatures` (e.g. cell types)\n    \"\"\"\n\n    @classmethod\n    def get_schema(cls: type[V2G]) -&gt; StructType:\n        \"\"\"Provides the schema for the V2G dataset.\n\n        Returns:\n            StructType: Schema for the V2G dataset\n        \"\"\"\n        return parse_spark_schema(\"v2g.json\")\n\n    def filter_by_genes(self: V2G, genes: GeneIndex) -&gt; V2G:\n        \"\"\"Filter V2G dataset by genes.\n\n        Args:\n            genes (GeneIndex): Gene index dataset to filter by\n\n        Returns:\n            V2G: V2G dataset filtered by genes\n        \"\"\"\n        self.df = self._df.join(genes.df.select(\"geneId\"), on=\"geneId\", how=\"inner\")\n        return self\n\n    def extract_distance_tss_minimum(self: V2G) -&gt; None:\n        \"\"\"Extract minimum distance to TSS.\"\"\"\n        self.df = self._df.filter(f.col(\"distance\")).withColumn(\n            \"distanceTssMinimum\",\n            f.expr(\"min(distTss) OVER (PARTITION BY studyLocusId)\"),\n        )\n</code></pre>"},{"location":"python_api/datasets/variant_to_gene/#gentropy.dataset.v2g.V2G.extract_distance_tss_minimum","title":"<code>extract_distance_tss_minimum() -&gt; None</code>","text":"<p>Extract minimum distance to TSS.</p> Source code in <code>src/gentropy/dataset/v2g.py</code> <pre><code>def extract_distance_tss_minimum(self: V2G) -&gt; None:\n    \"\"\"Extract minimum distance to TSS.\"\"\"\n    self.df = self._df.filter(f.col(\"distance\")).withColumn(\n        \"distanceTssMinimum\",\n        f.expr(\"min(distTss) OVER (PARTITION BY studyLocusId)\"),\n    )\n</code></pre>"},{"location":"python_api/datasets/variant_to_gene/#gentropy.dataset.v2g.V2G.filter_by_genes","title":"<code>filter_by_genes(genes: GeneIndex) -&gt; V2G</code>","text":"<p>Filter V2G dataset by genes.</p> <p>Parameters:</p> Name Type Description Default <code>genes</code> <code>GeneIndex</code> <p>Gene index dataset to filter by</p> required <p>Returns:</p> Name Type Description <code>V2G</code> <code>V2G</code> <p>V2G dataset filtered by genes</p> Source code in <code>src/gentropy/dataset/v2g.py</code> <pre><code>def filter_by_genes(self: V2G, genes: GeneIndex) -&gt; V2G:\n    \"\"\"Filter V2G dataset by genes.\n\n    Args:\n        genes (GeneIndex): Gene index dataset to filter by\n\n    Returns:\n        V2G: V2G dataset filtered by genes\n    \"\"\"\n    self.df = self._df.join(genes.df.select(\"geneId\"), on=\"geneId\", how=\"inner\")\n    return self\n</code></pre>"},{"location":"python_api/datasets/variant_to_gene/#gentropy.dataset.v2g.V2G.get_schema","title":"<code>get_schema() -&gt; StructType</code>  <code>classmethod</code>","text":"<p>Provides the schema for the V2G dataset.</p> <p>Returns:</p> Name Type Description <code>StructType</code> <code>StructType</code> <p>Schema for the V2G dataset</p> Source code in <code>src/gentropy/dataset/v2g.py</code> <pre><code>@classmethod\ndef get_schema(cls: type[V2G]) -&gt; StructType:\n    \"\"\"Provides the schema for the V2G dataset.\n\n    Returns:\n        StructType: Schema for the V2G dataset\n    \"\"\"\n    return parse_spark_schema(\"v2g.json\")\n</code></pre>"},{"location":"python_api/datasets/variant_to_gene/#schema","title":"Schema","text":"<pre><code>root\n |-- geneId: string (nullable = false)\n |-- variantId: string (nullable = false)\n |-- distance: long (nullable = true)\n |-- chromosome: string (nullable = false)\n |-- datatypeId: string (nullable = false)\n |-- datasourceId: string (nullable = false)\n |-- score: double (nullable = true)\n |-- resourceScore: double (nullable = true)\n |-- pmid: string (nullable = true)\n |-- biofeature: string (nullable = true)\n |-- variantFunctionalConsequenceId: string (nullable = true)\n |-- isHighQualityPlof: boolean (nullable = true)\n</code></pre>"},{"location":"python_api/datasources/_datasources/","title":"Data Sources","text":"<p>This section contains information about the data source harmonisation tools available in Open Targets Gentropy.</p>"},{"location":"python_api/datasources/_datasources/#gwas-study-sources","title":"GWAS study sources","text":"<ol> <li>GWAS Catalog (with or without full summary statistics)</li> <li>FinnGen</li> </ol>"},{"location":"python_api/datasources/_datasources/#molecular-qtls","title":"Molecular QTLs","text":"<ol> <li>GTEx (eQTL catalogue)</li> </ol>"},{"location":"python_api/datasources/_datasources/#interaction-interval-based-experiments","title":"Interaction / Interval-based Experiments","text":"<ol> <li>Intervals-based datasets, informing about the relationships between genetic elements and their functional implications.</li> </ol>"},{"location":"python_api/datasources/_datasources/#variant-annotationvalidation","title":"Variant annotation/validation","text":"<ol> <li>GnomAD v4.0</li> <li>GWAS catalog harmonisation pipeline more info</li> </ol>"},{"location":"python_api/datasources/_datasources/#linkage-desiquilibrium","title":"Linkage desiquilibrium","text":"<ol> <li>GnomAD v2.1.1 LD matrixes (7 ancestries)</li> </ol>"},{"location":"python_api/datasources/_datasources/#locus-to-gene-gold-standard","title":"Locus-to-gene gold standard","text":"<ol> <li>Open Targets training set</li> </ol>"},{"location":"python_api/datasources/_datasources/#gene-annotation","title":"Gene annotation","text":"<ol> <li>Open Targets Platform Target Dataset (derived from Ensembl)</li> </ol>"},{"location":"python_api/datasources/eqtl_catalogue/_eqtl_catalogue/","title":"eQTL Catalogue","text":"<p>The eQTL Catalogue aims to provide unified gene, protein expression and splicing Quantitative Trait Loci (QTLs) from all available human public studies.</p> <p>It serves as the ultimate resource of molecular QTLs that we use for colocalization and target prioritization.</p>"},{"location":"python_api/datasources/eqtl_catalogue/finemapping/","title":"Fine mapping results","text":""},{"location":"python_api/datasources/eqtl_catalogue/finemapping/#gentropy.datasource.eqtl_catalogue.finemapping.EqtlCatalogueFinemapping","title":"<code>gentropy.datasource.eqtl_catalogue.finemapping.EqtlCatalogueFinemapping</code>  <code>dataclass</code>","text":"<p>SuSIE finemapping dataset for eQTL Catalogue.</p> <p>Credible sets from SuSIE are extracted and transformed into StudyLocus objects: - A study ID is defined as a triad between: the publication, the tissue, and the measured trait (e.g. Braineac2_substantia_nigra_ENSG00000248275) - Each row in the <code>credible_sets.tsv.gz</code> files is represented by molecular_trait_id/variant/rsid trios relevant for a given tissue. Each have their own finemapping statistics - log Bayes Factors are available for all variants in the <code>lbf_variable.txt</code> files</p> Source code in <code>src/gentropy/datasource/eqtl_catalogue/finemapping.py</code> <pre><code>@dataclass\nclass EqtlCatalogueFinemapping:\n    \"\"\"SuSIE finemapping dataset for eQTL Catalogue.\n\n    Credible sets from SuSIE are extracted and transformed into StudyLocus objects:\n    - A study ID is defined as a triad between: the publication, the tissue, and the measured trait (e.g. Braineac2_substantia_nigra_ENSG00000248275)\n    - Each row in the `credible_sets.tsv.gz` files is represented by molecular_trait_id/variant/rsid trios relevant for a given tissue. Each have their own finemapping statistics\n    - log Bayes Factors are available for all variants in the `lbf_variable.txt` files\n    \"\"\"\n\n    raw_credible_set_schema: StructType = StructType(\n        [\n            StructField(\"molecular_trait_id\", StringType(), True),\n            StructField(\"gene_id\", StringType(), True),\n            StructField(\"cs_id\", StringType(), True),\n            StructField(\"variant\", StringType(), True),\n            StructField(\"rsid\", StringType(), True),\n            StructField(\"cs_size\", IntegerType(), True),\n            StructField(\"pip\", DoubleType(), True),\n            StructField(\"pvalue\", DoubleType(), True),\n            StructField(\"beta\", DoubleType(), True),\n            StructField(\"se\", DoubleType(), True),\n            StructField(\"z\", DoubleType(), True),\n            StructField(\"cs_min_r2\", DoubleType(), True),\n            StructField(\"region\", StringType(), True),\n        ]\n    )\n    raw_lbf_schema: StructType = StructType(\n        [\n            StructField(\"molecular_trait_id\", StringType(), True),\n            StructField(\"region\", StringType(), True),\n            StructField(\"variant\", StringType(), True),\n            StructField(\"chromosome\", StringType(), True),\n            StructField(\"position\", IntegerType(), True),\n            StructField(\"lbf_variable1\", DoubleType(), True),\n            StructField(\"lbf_variable2\", DoubleType(), True),\n            StructField(\"lbf_variable3\", DoubleType(), True),\n            StructField(\"lbf_variable4\", DoubleType(), True),\n            StructField(\"lbf_variable5\", DoubleType(), True),\n            StructField(\"lbf_variable6\", DoubleType(), True),\n            StructField(\"lbf_variable7\", DoubleType(), True),\n            StructField(\"lbf_variable8\", DoubleType(), True),\n            StructField(\"lbf_variable9\", DoubleType(), True),\n            StructField(\"lbf_variable10\", DoubleType(), True),\n        ]\n    )\n\n    @classmethod\n    def _extract_credible_set_index(\n        cls: type[EqtlCatalogueFinemapping], cs_id: Column\n    ) -&gt; Column:\n        \"\"\"Extract the credible set index from the cs_id.\n\n        Args:\n            cs_id (Column): column with the credible set id as defined in the eQTL Catalogue.\n\n        Returns:\n            Column: The credible set index.\n\n        Examples:\n            &gt;&gt;&gt; spark.createDataFrame([(\"QTD000046_L1\",)], [\"cs_id\"]).select(EqtlCatalogueFinemapping._extract_credible_set_index(f.col(\"cs_id\"))).show()\n            +----------------+\n            |credibleSetIndex|\n            +----------------+\n            |               1|\n            +----------------+\n            &lt;BLANKLINE&gt;\n        \"\"\"\n        return f.split(cs_id, \"_L\")[1].cast(IntegerType()).alias(\"credibleSetIndex\")\n\n    @classmethod\n    def _extract_dataset_id_from_file_path(\n        cls: type[EqtlCatalogueFinemapping], file_path: Column\n    ) -&gt; Column:\n        \"\"\"Extract the dataset_id from the file_path. The dataset_id follows the pattern QTD{6}.\n\n        Args:\n            file_path (Column): A column containing the file path.\n\n        Returns:\n            Column: The dataset_id.\n\n        Examples:\n            &gt;&gt;&gt; spark.createDataFrame([(\"QTD000046.credible_sets.tsv\",)], [\"filename\"]).select(EqtlCatalogueFinemapping._extract_dataset_id_from_file_path(f.col(\"filename\"))).show()\n            +----------+\n            |dataset_id|\n            +----------+\n            | QTD000046|\n            +----------+\n            &lt;BLANKLINE&gt;\n        \"\"\"\n        return f.regexp_extract(file_path, r\"QTD\\d{6}\", 0).alias(\"dataset_id\")\n\n    @classmethod\n    def parse_susie_results(\n        cls: type[EqtlCatalogueFinemapping],\n        credible_sets: DataFrame,\n        lbf: DataFrame,\n        studies_metadata: DataFrame,\n    ) -&gt; DataFrame:\n        \"\"\"Parse the SuSIE results into a DataFrame containing the finemapping statistics and metadata about the studies.\n\n        Args:\n            credible_sets (DataFrame): DataFrame containing raw statistics of all variants in the credible sets.\n            lbf (DataFrame): DataFrame containing the raw log Bayes Factors for all variants.\n            studies_metadata (DataFrame): DataFrame containing the study metadata.\n\n        Returns:\n            DataFrame: Processed SuSIE results to contain metadata about the studies and the finemapping statistics.\n        \"\"\"\n        ss_ftp_path_template = \"https://ftp.ebi.ac.uk/pub/databases/spot/eQTL/sumstats\"\n        return (\n            lbf.withColumn(\n                \"dataset_id\",\n                cls._extract_dataset_id_from_file_path(f.input_file_name()),\n            )\n            .join(\n                (\n                    credible_sets.withColumn(\n                        \"dataset_id\",\n                        cls._extract_dataset_id_from_file_path(f.input_file_name()),\n                    )\n                    .withColumn(\n                        \"credibleSetIndex\",\n                        cls._extract_credible_set_index(f.col(\"cs_id\")),\n                    )\n                    .join(f.broadcast(studies_metadata), on=\"dataset_id\")\n                ),\n                on=[\"molecular_trait_id\", \"region\", \"variant\", \"dataset_id\"],\n                how=\"inner\",\n            )\n            .withColumn(\n                \"logBF\",\n                f.when(f.col(\"credibleSetIndex\") == 1, f.col(\"lbf_variable1\"))\n                .when(f.col(\"credibleSetIndex\") == 2, f.col(\"lbf_variable2\"))\n                .when(f.col(\"credibleSetIndex\") == 3, f.col(\"lbf_variable3\"))\n                .when(f.col(\"credibleSetIndex\") == 4, f.col(\"lbf_variable4\"))\n                .when(f.col(\"credibleSetIndex\") == 5, f.col(\"lbf_variable5\"))\n                .when(f.col(\"credibleSetIndex\") == 6, f.col(\"lbf_variable6\"))\n                .when(f.col(\"credibleSetIndex\") == 7, f.col(\"lbf_variable7\"))\n                .when(f.col(\"credibleSetIndex\") == 8, f.col(\"lbf_variable8\"))\n                .when(f.col(\"credibleSetIndex\") == 9, f.col(\"lbf_variable9\"))\n                .when(f.col(\"credibleSetIndex\") == 10, f.col(\"lbf_variable10\")),\n            )\n            .select(\n                f.regexp_replace(f.col(\"variant\"), r\"chr\", \"\").alias(\"variantId\"),\n                f.col(\"region\"),\n                f.col(\"chromosome\"),\n                f.col(\"position\"),\n                f.col(\"pip\").alias(\"posteriorProbability\"),\n                *parse_pvalue(f.col(\"pvalue\")),\n                f.col(\"sample_size\").alias(\"nSamples\"),\n                f.col(\"beta\"),\n                f.col(\"se\").alias(\"standardError\"),\n                f.col(\"credibleSetIndex\"),\n                f.col(\"logBF\"),\n                f.lit(\"SuSie\").alias(\"finemappingMethod\"),\n                # Study metadata\n                f.col(\"molecular_trait_id\").alias(\"traitFromSource\"),\n                f.col(\"gene_id\").alias(\"geneId\"),\n                f.col(\"dataset_id\"),\n                f.concat_ws(\n                    \"_\",\n                    f.col(\"study_label\"),\n                    f.col(\"sample_group\"),\n                    f.col(\"molecular_trait_id\"),\n                ).alias(\"studyId\"),\n                f.col(\"tissue_id\").alias(\"tissueFromSourceId\"),\n                EqtlCatalogueStudyIndex._identify_study_type(\n                    f.col(\"quant_method\")\n                ).alias(\"studyType\"),\n                f.col(\"study_label\").alias(\"projectId\"),\n                f.concat_ws(\n                    \"/\",\n                    f.lit(ss_ftp_path_template),\n                    f.col(\"study_id\"),\n                    f.col(\"dataset_id\"),\n                ).alias(\"summarystatsLocation\"),\n                f.lit(True).alias(\"hasSumstats\"),\n                f.col(\"molecular_trait_id\"),\n            )\n        )\n\n    @classmethod\n    def from_susie_results(\n        cls: type[EqtlCatalogueFinemapping], processed_finemapping_df: DataFrame\n    ) -&gt; StudyLocus:\n        \"\"\"Create a StudyLocus object from the processed SuSIE results.\n\n        Args:\n            processed_finemapping_df (DataFrame): DataFrame containing the processed SuSIE results.\n\n        Returns:\n            StudyLocus: eQTL Catalogue credible sets.\n        \"\"\"\n        lead_w = Window.partitionBy(\n            \"dataset_id\", \"molecular_trait_id\", \"region\", \"credibleSetIndex\"\n        )\n        study_locus_cols = [\n            field.name\n            for field in StudyLocus.get_schema().fields\n            if field.name in processed_finemapping_df.columns\n        ] + [\"locus\"]\n        return StudyLocus(\n            _df=(\n                processed_finemapping_df.withColumn(\n                    \"isLead\",\n                    f.row_number().over(lead_w.orderBy(f.desc(\"posteriorProbability\")))\n                    == f.lit(1),\n                )\n                .withColumn(\n                    # Collecting all variants that constitute the credible set brings as many variants as the credible set size\n                    \"locus\",\n                    f.when(\n                        f.col(\"isLead\"),\n                        f.collect_list(\n                            f.struct(\n                                \"variantId\",\n                                \"posteriorProbability\",\n                                \"pValueMantissa\",\n                                \"pValueExponent\",\n                                \"logBF\",\n                                \"beta\",\n                                \"standardError\",\n                            )\n                        ).over(lead_w),\n                    ),\n                )\n                .filter(f.col(\"isLead\"))\n                .drop(\"isLead\")\n                .select(\n                    *study_locus_cols,\n                    StudyLocus.assign_study_locus_id(\n                        f.col(\"studyId\"), f.col(\"variantId\")\n                    ),\n                    StudyLocus.calculate_credible_set_log10bf(\n                        f.col(\"locus.logBF\")\n                    ).alias(\"credibleSetlog10BF\"),\n                )\n            ),\n            _schema=StudyLocus.get_schema(),\n        ).annotate_credible_sets()\n\n    @classmethod\n    def read_credible_set_from_source(\n        cls: type[EqtlCatalogueFinemapping],\n        session: Session,\n        credible_set_path: str | list[str],\n    ) -&gt; DataFrame:\n        \"\"\"Load raw credible sets from eQTL Catalogue.\n\n        Args:\n            session (Session): Spark session.\n            credible_set_path (str | list[str]): Path to raw table(s) containing finemapping results for any variant belonging to a credible set.\n\n        Returns:\n            DataFrame: Credible sets DataFrame.\n        \"\"\"\n        return session.spark.read.csv(\n            credible_set_path,\n            sep=\"\\t\",\n            header=True,\n            schema=cls.raw_credible_set_schema,\n        )\n\n    @classmethod\n    def read_lbf_from_source(\n        cls: type[EqtlCatalogueFinemapping],\n        session: Session,\n        lbf_path: str | list[str],\n    ) -&gt; DataFrame:\n        \"\"\"Load raw log Bayes Factors from eQTL Catalogue.\n\n        Args:\n            session (Session): Spark session.\n            lbf_path (str | list[str]): Path to raw table(s) containing Log Bayes Factors for each variant.\n\n        Returns:\n            DataFrame: Log Bayes Factors DataFrame.\n        \"\"\"\n        return session.spark.read.csv(\n            lbf_path,\n            sep=\"\\t\",\n            header=True,\n            schema=cls.raw_lbf_schema,\n        )\n</code></pre>"},{"location":"python_api/datasources/eqtl_catalogue/finemapping/#gentropy.datasource.eqtl_catalogue.finemapping.EqtlCatalogueFinemapping.from_susie_results","title":"<code>from_susie_results(processed_finemapping_df: DataFrame) -&gt; StudyLocus</code>  <code>classmethod</code>","text":"<p>Create a StudyLocus object from the processed SuSIE results.</p> <p>Parameters:</p> Name Type Description Default <code>processed_finemapping_df</code> <code>DataFrame</code> <p>DataFrame containing the processed SuSIE results.</p> required <p>Returns:</p> Name Type Description <code>StudyLocus</code> <code>StudyLocus</code> <p>eQTL Catalogue credible sets.</p> Source code in <code>src/gentropy/datasource/eqtl_catalogue/finemapping.py</code> <pre><code>@classmethod\ndef from_susie_results(\n    cls: type[EqtlCatalogueFinemapping], processed_finemapping_df: DataFrame\n) -&gt; StudyLocus:\n    \"\"\"Create a StudyLocus object from the processed SuSIE results.\n\n    Args:\n        processed_finemapping_df (DataFrame): DataFrame containing the processed SuSIE results.\n\n    Returns:\n        StudyLocus: eQTL Catalogue credible sets.\n    \"\"\"\n    lead_w = Window.partitionBy(\n        \"dataset_id\", \"molecular_trait_id\", \"region\", \"credibleSetIndex\"\n    )\n    study_locus_cols = [\n        field.name\n        for field in StudyLocus.get_schema().fields\n        if field.name in processed_finemapping_df.columns\n    ] + [\"locus\"]\n    return StudyLocus(\n        _df=(\n            processed_finemapping_df.withColumn(\n                \"isLead\",\n                f.row_number().over(lead_w.orderBy(f.desc(\"posteriorProbability\")))\n                == f.lit(1),\n            )\n            .withColumn(\n                # Collecting all variants that constitute the credible set brings as many variants as the credible set size\n                \"locus\",\n                f.when(\n                    f.col(\"isLead\"),\n                    f.collect_list(\n                        f.struct(\n                            \"variantId\",\n                            \"posteriorProbability\",\n                            \"pValueMantissa\",\n                            \"pValueExponent\",\n                            \"logBF\",\n                            \"beta\",\n                            \"standardError\",\n                        )\n                    ).over(lead_w),\n                ),\n            )\n            .filter(f.col(\"isLead\"))\n            .drop(\"isLead\")\n            .select(\n                *study_locus_cols,\n                StudyLocus.assign_study_locus_id(\n                    f.col(\"studyId\"), f.col(\"variantId\")\n                ),\n                StudyLocus.calculate_credible_set_log10bf(\n                    f.col(\"locus.logBF\")\n                ).alias(\"credibleSetlog10BF\"),\n            )\n        ),\n        _schema=StudyLocus.get_schema(),\n    ).annotate_credible_sets()\n</code></pre>"},{"location":"python_api/datasources/eqtl_catalogue/finemapping/#gentropy.datasource.eqtl_catalogue.finemapping.EqtlCatalogueFinemapping.parse_susie_results","title":"<code>parse_susie_results(credible_sets: DataFrame, lbf: DataFrame, studies_metadata: DataFrame) -&gt; DataFrame</code>  <code>classmethod</code>","text":"<p>Parse the SuSIE results into a DataFrame containing the finemapping statistics and metadata about the studies.</p> <p>Parameters:</p> Name Type Description Default <code>credible_sets</code> <code>DataFrame</code> <p>DataFrame containing raw statistics of all variants in the credible sets.</p> required <code>lbf</code> <code>DataFrame</code> <p>DataFrame containing the raw log Bayes Factors for all variants.</p> required <code>studies_metadata</code> <code>DataFrame</code> <p>DataFrame containing the study metadata.</p> required <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>Processed SuSIE results to contain metadata about the studies and the finemapping statistics.</p> Source code in <code>src/gentropy/datasource/eqtl_catalogue/finemapping.py</code> <pre><code>@classmethod\ndef parse_susie_results(\n    cls: type[EqtlCatalogueFinemapping],\n    credible_sets: DataFrame,\n    lbf: DataFrame,\n    studies_metadata: DataFrame,\n) -&gt; DataFrame:\n    \"\"\"Parse the SuSIE results into a DataFrame containing the finemapping statistics and metadata about the studies.\n\n    Args:\n        credible_sets (DataFrame): DataFrame containing raw statistics of all variants in the credible sets.\n        lbf (DataFrame): DataFrame containing the raw log Bayes Factors for all variants.\n        studies_metadata (DataFrame): DataFrame containing the study metadata.\n\n    Returns:\n        DataFrame: Processed SuSIE results to contain metadata about the studies and the finemapping statistics.\n    \"\"\"\n    ss_ftp_path_template = \"https://ftp.ebi.ac.uk/pub/databases/spot/eQTL/sumstats\"\n    return (\n        lbf.withColumn(\n            \"dataset_id\",\n            cls._extract_dataset_id_from_file_path(f.input_file_name()),\n        )\n        .join(\n            (\n                credible_sets.withColumn(\n                    \"dataset_id\",\n                    cls._extract_dataset_id_from_file_path(f.input_file_name()),\n                )\n                .withColumn(\n                    \"credibleSetIndex\",\n                    cls._extract_credible_set_index(f.col(\"cs_id\")),\n                )\n                .join(f.broadcast(studies_metadata), on=\"dataset_id\")\n            ),\n            on=[\"molecular_trait_id\", \"region\", \"variant\", \"dataset_id\"],\n            how=\"inner\",\n        )\n        .withColumn(\n            \"logBF\",\n            f.when(f.col(\"credibleSetIndex\") == 1, f.col(\"lbf_variable1\"))\n            .when(f.col(\"credibleSetIndex\") == 2, f.col(\"lbf_variable2\"))\n            .when(f.col(\"credibleSetIndex\") == 3, f.col(\"lbf_variable3\"))\n            .when(f.col(\"credibleSetIndex\") == 4, f.col(\"lbf_variable4\"))\n            .when(f.col(\"credibleSetIndex\") == 5, f.col(\"lbf_variable5\"))\n            .when(f.col(\"credibleSetIndex\") == 6, f.col(\"lbf_variable6\"))\n            .when(f.col(\"credibleSetIndex\") == 7, f.col(\"lbf_variable7\"))\n            .when(f.col(\"credibleSetIndex\") == 8, f.col(\"lbf_variable8\"))\n            .when(f.col(\"credibleSetIndex\") == 9, f.col(\"lbf_variable9\"))\n            .when(f.col(\"credibleSetIndex\") == 10, f.col(\"lbf_variable10\")),\n        )\n        .select(\n            f.regexp_replace(f.col(\"variant\"), r\"chr\", \"\").alias(\"variantId\"),\n            f.col(\"region\"),\n            f.col(\"chromosome\"),\n            f.col(\"position\"),\n            f.col(\"pip\").alias(\"posteriorProbability\"),\n            *parse_pvalue(f.col(\"pvalue\")),\n            f.col(\"sample_size\").alias(\"nSamples\"),\n            f.col(\"beta\"),\n            f.col(\"se\").alias(\"standardError\"),\n            f.col(\"credibleSetIndex\"),\n            f.col(\"logBF\"),\n            f.lit(\"SuSie\").alias(\"finemappingMethod\"),\n            # Study metadata\n            f.col(\"molecular_trait_id\").alias(\"traitFromSource\"),\n            f.col(\"gene_id\").alias(\"geneId\"),\n            f.col(\"dataset_id\"),\n            f.concat_ws(\n                \"_\",\n                f.col(\"study_label\"),\n                f.col(\"sample_group\"),\n                f.col(\"molecular_trait_id\"),\n            ).alias(\"studyId\"),\n            f.col(\"tissue_id\").alias(\"tissueFromSourceId\"),\n            EqtlCatalogueStudyIndex._identify_study_type(\n                f.col(\"quant_method\")\n            ).alias(\"studyType\"),\n            f.col(\"study_label\").alias(\"projectId\"),\n            f.concat_ws(\n                \"/\",\n                f.lit(ss_ftp_path_template),\n                f.col(\"study_id\"),\n                f.col(\"dataset_id\"),\n            ).alias(\"summarystatsLocation\"),\n            f.lit(True).alias(\"hasSumstats\"),\n            f.col(\"molecular_trait_id\"),\n        )\n    )\n</code></pre>"},{"location":"python_api/datasources/eqtl_catalogue/finemapping/#gentropy.datasource.eqtl_catalogue.finemapping.EqtlCatalogueFinemapping.read_credible_set_from_source","title":"<code>read_credible_set_from_source(session: Session, credible_set_path: str | list[str]) -&gt; DataFrame</code>  <code>classmethod</code>","text":"<p>Load raw credible sets from eQTL Catalogue.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Spark session.</p> required <code>credible_set_path</code> <code>str | list[str]</code> <p>Path to raw table(s) containing finemapping results for any variant belonging to a credible set.</p> required <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>Credible sets DataFrame.</p> Source code in <code>src/gentropy/datasource/eqtl_catalogue/finemapping.py</code> <pre><code>@classmethod\ndef read_credible_set_from_source(\n    cls: type[EqtlCatalogueFinemapping],\n    session: Session,\n    credible_set_path: str | list[str],\n) -&gt; DataFrame:\n    \"\"\"Load raw credible sets from eQTL Catalogue.\n\n    Args:\n        session (Session): Spark session.\n        credible_set_path (str | list[str]): Path to raw table(s) containing finemapping results for any variant belonging to a credible set.\n\n    Returns:\n        DataFrame: Credible sets DataFrame.\n    \"\"\"\n    return session.spark.read.csv(\n        credible_set_path,\n        sep=\"\\t\",\n        header=True,\n        schema=cls.raw_credible_set_schema,\n    )\n</code></pre>"},{"location":"python_api/datasources/eqtl_catalogue/finemapping/#gentropy.datasource.eqtl_catalogue.finemapping.EqtlCatalogueFinemapping.read_lbf_from_source","title":"<code>read_lbf_from_source(session: Session, lbf_path: str | list[str]) -&gt; DataFrame</code>  <code>classmethod</code>","text":"<p>Load raw log Bayes Factors from eQTL Catalogue.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Spark session.</p> required <code>lbf_path</code> <code>str | list[str]</code> <p>Path to raw table(s) containing Log Bayes Factors for each variant.</p> required <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>Log Bayes Factors DataFrame.</p> Source code in <code>src/gentropy/datasource/eqtl_catalogue/finemapping.py</code> <pre><code>@classmethod\ndef read_lbf_from_source(\n    cls: type[EqtlCatalogueFinemapping],\n    session: Session,\n    lbf_path: str | list[str],\n) -&gt; DataFrame:\n    \"\"\"Load raw log Bayes Factors from eQTL Catalogue.\n\n    Args:\n        session (Session): Spark session.\n        lbf_path (str | list[str]): Path to raw table(s) containing Log Bayes Factors for each variant.\n\n    Returns:\n        DataFrame: Log Bayes Factors DataFrame.\n    \"\"\"\n    return session.spark.read.csv(\n        lbf_path,\n        sep=\"\\t\",\n        header=True,\n        schema=cls.raw_lbf_schema,\n    )\n</code></pre>"},{"location":"python_api/datasources/eqtl_catalogue/study_index/","title":"Study Index","text":""},{"location":"python_api/datasources/eqtl_catalogue/study_index/#gentropy.datasource.eqtl_catalogue.study_index.EqtlCatalogueStudyIndex","title":"<code>gentropy.datasource.eqtl_catalogue.study_index.EqtlCatalogueStudyIndex</code>","text":"<p>Study index dataset from eQTL Catalogue.</p> <p>We extract study level metadata from eQTL Catalogue's fine mapping results. All available studies can be found here.</p> <p>One study from the eQTL Catalogue clusters together all the molecular QTLs (mQTLs) that were found:</p> <pre><code>- in the same publication (e.g. Alasoo_2018)\n- in the same cell type or tissue (e.g. monocytes)\n- and for the same measured molecular trait (e.g. ENSG00000141510)\n</code></pre> Source code in <code>src/gentropy/datasource/eqtl_catalogue/study_index.py</code> <pre><code>class EqtlCatalogueStudyIndex:\n    \"\"\"Study index dataset from eQTL Catalogue.\n\n    We extract study level metadata from eQTL Catalogue's fine mapping results. All available studies can be found [here](https://www.ebi.ac.uk/eqtl/Studies/).\n\n    One study from the eQTL Catalogue clusters together all the molecular QTLs (mQTLs) that were found:\n\n        - in the same publication (e.g. Alasoo_2018)\n        - in the same cell type or tissue (e.g. monocytes)\n        - and for the same measured molecular trait (e.g. ENSG00000141510)\n\n    \"\"\"\n\n    raw_studies_metadata_schema: StructType = StructType(\n        [\n            StructField(\"study_id\", StringType(), True),\n            StructField(\"dataset_id\", StringType(), True),\n            StructField(\"study_label\", StringType(), True),\n            StructField(\"sample_group\", StringType(), True),\n            StructField(\"tissue_id\", StringType(), True),\n            StructField(\"tissue_label\", StringType(), True),\n            StructField(\"condition_label\", StringType(), True),\n            StructField(\"sample_size\", IntegerType(), True),\n            StructField(\"quant_method\", StringType(), True),\n        ]\n    )\n    raw_studies_metadata_path = \"https://raw.githubusercontent.com/eQTL-Catalogue/eQTL-Catalogue-resources/19929ff6a99bf402194292a14f96f9615b35f65f/data_tables/dataset_metadata.tsv\"\n\n    @classmethod\n    def _identify_study_type(\n        cls: type[EqtlCatalogueStudyIndex], quantification_method_col: Column\n    ) -&gt; Column:\n        \"\"\"Identify the study type based on the method to quantify the trait.\n\n        Args:\n            quantification_method_col (Column): column with the label of the method to quantify the trait. Available methods are [here](https://www.ebi.ac.uk/eqtl/Methods/)\n\n        Returns:\n            Column: The study type.\n\n        Examples:\n            &gt;&gt;&gt; df = spark.createDataFrame([(\"ge\",), (\"exon\",), (\"tx\",)], [\"quant_method\"])\n            &gt;&gt;&gt; df.withColumn(\"study_type\", EqtlCatalogueStudyIndex._identify_study_type(f.col(\"quant_method\"))).show()\n            +------------+----------+\n            |quant_method|study_type|\n            +------------+----------+\n            |          ge|      eqtl|\n            |        exon|      eqtl|\n            |          tx|      eqtl|\n            +------------+----------+\n            &lt;BLANKLINE&gt;\n        \"\"\"\n        method_to_study_type_mapping = {\n            \"ge\": \"eqtl\",\n            \"exon\": \"eqtl\",\n            \"tx\": \"eqtl\",\n            \"microarray\": \"eqtl\",\n            \"leafcutter\": \"sqtl\",\n            \"aptamer\": \"pqtl\",\n            \"txrev\": \"tuqtl\",\n        }\n        map_expr = f.create_map(\n            *[f.lit(x) for x in chain(*method_to_study_type_mapping.items())]\n        )\n        return map_expr.getItem(quantification_method_col)\n\n    @classmethod\n    def get_studies_of_interest(\n        cls: type[EqtlCatalogueStudyIndex], studies_metadata: DataFrame\n    ) -&gt; list[str]:\n        \"\"\"Filter studies of interest from the raw studies metadata.\n\n        Args:\n            studies_metadata (DataFrame): raw studies metadata filtered with studies of interest.\n\n        Returns:\n            list[str]: QTD IDs defining the studies of interest for ingestion.\n        \"\"\"\n        return (\n            studies_metadata.select(\"dataset_id\")\n            .distinct()\n            .toPandas()[\"dataset_id\"]\n            .tolist()\n        )\n\n    @classmethod\n    def from_susie_results(\n        cls: type[EqtlCatalogueStudyIndex],\n        processed_finemapping_df: DataFrame,\n    ) -&gt; StudyIndex:\n        \"\"\"Ingest study level metadata from eQTL Catalogue.\n\n        Args:\n            processed_finemapping_df (DataFrame): processed fine mapping results with study metadata.\n\n        Returns:\n            StudyIndex: eQTL Catalogue study index dataset derived from the selected SuSIE results.\n        \"\"\"\n        study_index_cols = [\n            field.name\n            for field in StudyIndex.get_schema().fields\n            if field.name in processed_finemapping_df.columns\n        ]\n        return StudyIndex(\n            _df=processed_finemapping_df.select(study_index_cols).distinct(),\n            _schema=StudyIndex.get_schema(),\n        )\n\n    @classmethod\n    def read_studies_from_source(\n        cls: type[EqtlCatalogueStudyIndex],\n        session: Session,\n        mqtl_quantification_methods_blacklist: list[str],\n    ) -&gt; DataFrame:\n        \"\"\"Read raw studies metadata from eQTL Catalogue.\n\n        Args:\n            session (Session): Spark session.\n            mqtl_quantification_methods_blacklist (list[str]): Molecular trait quantification methods that we don't want to ingest. Available options in https://github.com/eQTL-Catalogue/eQTL-Catalogue-resources/blob/master/data_tables/dataset_metadata.tsv\n\n        Returns:\n            DataFrame: raw studies metadata.\n        \"\"\"\n        pd.DataFrame.iteritems = pd.DataFrame.items\n        return session.spark.createDataFrame(\n            pd.read_csv(cls.raw_studies_metadata_path, sep=\"\\t\"),\n            schema=cls.raw_studies_metadata_schema,\n        ).filter(~(f.col(\"quant_method\").isin(mqtl_quantification_methods_blacklist)))\n</code></pre>"},{"location":"python_api/datasources/eqtl_catalogue/study_index/#gentropy.datasource.eqtl_catalogue.study_index.EqtlCatalogueStudyIndex.from_susie_results","title":"<code>from_susie_results(processed_finemapping_df: DataFrame) -&gt; StudyIndex</code>  <code>classmethod</code>","text":"<p>Ingest study level metadata from eQTL Catalogue.</p> <p>Parameters:</p> Name Type Description Default <code>processed_finemapping_df</code> <code>DataFrame</code> <p>processed fine mapping results with study metadata.</p> required <p>Returns:</p> Name Type Description <code>StudyIndex</code> <code>StudyIndex</code> <p>eQTL Catalogue study index dataset derived from the selected SuSIE results.</p> Source code in <code>src/gentropy/datasource/eqtl_catalogue/study_index.py</code> <pre><code>@classmethod\ndef from_susie_results(\n    cls: type[EqtlCatalogueStudyIndex],\n    processed_finemapping_df: DataFrame,\n) -&gt; StudyIndex:\n    \"\"\"Ingest study level metadata from eQTL Catalogue.\n\n    Args:\n        processed_finemapping_df (DataFrame): processed fine mapping results with study metadata.\n\n    Returns:\n        StudyIndex: eQTL Catalogue study index dataset derived from the selected SuSIE results.\n    \"\"\"\n    study_index_cols = [\n        field.name\n        for field in StudyIndex.get_schema().fields\n        if field.name in processed_finemapping_df.columns\n    ]\n    return StudyIndex(\n        _df=processed_finemapping_df.select(study_index_cols).distinct(),\n        _schema=StudyIndex.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasources/eqtl_catalogue/study_index/#gentropy.datasource.eqtl_catalogue.study_index.EqtlCatalogueStudyIndex.get_studies_of_interest","title":"<code>get_studies_of_interest(studies_metadata: DataFrame) -&gt; list[str]</code>  <code>classmethod</code>","text":"<p>Filter studies of interest from the raw studies metadata.</p> <p>Parameters:</p> Name Type Description Default <code>studies_metadata</code> <code>DataFrame</code> <p>raw studies metadata filtered with studies of interest.</p> required <p>Returns:</p> Type Description <code>list[str]</code> <p>list[str]: QTD IDs defining the studies of interest for ingestion.</p> Source code in <code>src/gentropy/datasource/eqtl_catalogue/study_index.py</code> <pre><code>@classmethod\ndef get_studies_of_interest(\n    cls: type[EqtlCatalogueStudyIndex], studies_metadata: DataFrame\n) -&gt; list[str]:\n    \"\"\"Filter studies of interest from the raw studies metadata.\n\n    Args:\n        studies_metadata (DataFrame): raw studies metadata filtered with studies of interest.\n\n    Returns:\n        list[str]: QTD IDs defining the studies of interest for ingestion.\n    \"\"\"\n    return (\n        studies_metadata.select(\"dataset_id\")\n        .distinct()\n        .toPandas()[\"dataset_id\"]\n        .tolist()\n    )\n</code></pre>"},{"location":"python_api/datasources/eqtl_catalogue/study_index/#gentropy.datasource.eqtl_catalogue.study_index.EqtlCatalogueStudyIndex.read_studies_from_source","title":"<code>read_studies_from_source(session: Session, mqtl_quantification_methods_blacklist: list[str]) -&gt; DataFrame</code>  <code>classmethod</code>","text":"<p>Read raw studies metadata from eQTL Catalogue.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Spark session.</p> required <code>mqtl_quantification_methods_blacklist</code> <code>list[str]</code> <p>Molecular trait quantification methods that we don't want to ingest. Available options in https://github.com/eQTL-Catalogue/eQTL-Catalogue-resources/blob/master/data_tables/dataset_metadata.tsv</p> required <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>raw studies metadata.</p> Source code in <code>src/gentropy/datasource/eqtl_catalogue/study_index.py</code> <pre><code>@classmethod\ndef read_studies_from_source(\n    cls: type[EqtlCatalogueStudyIndex],\n    session: Session,\n    mqtl_quantification_methods_blacklist: list[str],\n) -&gt; DataFrame:\n    \"\"\"Read raw studies metadata from eQTL Catalogue.\n\n    Args:\n        session (Session): Spark session.\n        mqtl_quantification_methods_blacklist (list[str]): Molecular trait quantification methods that we don't want to ingest. Available options in https://github.com/eQTL-Catalogue/eQTL-Catalogue-resources/blob/master/data_tables/dataset_metadata.tsv\n\n    Returns:\n        DataFrame: raw studies metadata.\n    \"\"\"\n    pd.DataFrame.iteritems = pd.DataFrame.items\n    return session.spark.createDataFrame(\n        pd.read_csv(cls.raw_studies_metadata_path, sep=\"\\t\"),\n        schema=cls.raw_studies_metadata_schema,\n    ).filter(~(f.col(\"quant_method\").isin(mqtl_quantification_methods_blacklist)))\n</code></pre>"},{"location":"python_api/datasources/eqtl_catalogue/summary_stats/","title":"Summary Stats","text":""},{"location":"python_api/datasources/eqtl_catalogue/summary_stats/#gentropy.datasource.eqtl_catalogue.summary_stats.EqtlCatalogueSummaryStats","title":"<code>gentropy.datasource.eqtl_catalogue.summary_stats.EqtlCatalogueSummaryStats</code>  <code>dataclass</code>","text":"<p>Summary statistics dataset for eQTL Catalogue.</p> Source code in <code>src/gentropy/datasource/eqtl_catalogue/summary_stats.py</code> <pre><code>@dataclass\nclass EqtlCatalogueSummaryStats:\n    \"\"\"Summary statistics dataset for eQTL Catalogue.\"\"\"\n\n    @staticmethod\n    def _full_study_id_regexp() -&gt; Column:\n        \"\"\"Constructs a full study ID from the URI.\n\n        Returns:\n            Column: expression to extract a full study ID from the URI.\n        \"\"\"\n        # Example of a URI which is used for parsing:\n        # \"gs://genetics_etl_python_playground/input/preprocess/eqtl_catalogue/imported/GTEx_V8/ge/Adipose_Subcutaneous.tsv.gz\".\n\n        # Regular expession to extract project ID from URI.  Example: \"GTEx_V8\".\n        _project_id = f.regexp_extract(\n            f.input_file_name(),\n            r\"imported/([^/]+)/.*\",\n            1,\n        )\n        # Regular expression to extract QTL group from URI.  Example: \"Adipose_Subcutaneous\".\n        _qtl_group = f.regexp_extract(f.input_file_name(), r\"([^/]+)\\.tsv\\.gz\", 1)\n        # Extracting gene ID from the column.  Example: \"ENSG00000225630\".\n        _gene_id = f.col(\"gene_id\")\n\n        # We can now construct the full study ID based on all fields.\n        # Example: \"GTEx_V8_Adipose_Subcutaneous_ENSG00000225630\".\n        return f.concat(_project_id, f.lit(\"_\"), _qtl_group, f.lit(\"_\"), _gene_id)\n\n    @classmethod\n    def from_source(\n        cls: type[EqtlCatalogueSummaryStats],\n        summary_stats_df: DataFrame,\n    ) -&gt; SummaryStatistics:\n        \"\"\"Ingests all summary stats for all eQTL Catalogue studies.\n\n        Args:\n            summary_stats_df (DataFrame): an ingested but unprocessed summary statistics dataframe from eQTL Catalogue.\n\n        Returns:\n            SummaryStatistics: a processed summary statistics dataframe for eQTL Catalogue.\n        \"\"\"\n        processed_summary_stats_df = (\n            summary_stats_df.select(\n                # Construct study ID from the appropriate columns.\n                cls._full_study_id_regexp().alias(\"studyId\"),\n                # Add variant information.\n                f.concat_ws(\n                    \"_\",\n                    f.col(\"chromosome\"),\n                    f.col(\"position\"),\n                    f.col(\"ref\"),\n                    f.col(\"alt\"),\n                ).alias(\"variantId\"),\n                f.col(\"chromosome\"),\n                f.col(\"position\").cast(t.IntegerType()),\n                # Parse p-value into mantissa and exponent.\n                *parse_pvalue(f.col(\"pvalue\")),\n                # Add beta, standard error, and allele frequency information.\n                f.col(\"beta\").cast(\"double\"),\n                f.col(\"se\").cast(\"double\").alias(\"standardError\"),\n                f.col(\"maf\").cast(\"float\").alias(\"effectAlleleFrequencyFromSource\"),\n            )\n            # Drop rows which don't have proper position or beta value.\n            .filter(\n                f.col(\"position\").cast(t.IntegerType()).isNotNull()\n                &amp; (f.col(\"beta\") != 0)\n            )\n        )\n\n        # Initialise a summary statistics object.\n        return SummaryStatistics(\n            _df=processed_summary_stats_df,\n            _schema=SummaryStatistics.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasources/eqtl_catalogue/summary_stats/#gentropy.datasource.eqtl_catalogue.summary_stats.EqtlCatalogueSummaryStats.from_source","title":"<code>from_source(summary_stats_df: DataFrame) -&gt; SummaryStatistics</code>  <code>classmethod</code>","text":"<p>Ingests all summary stats for all eQTL Catalogue studies.</p> <p>Parameters:</p> Name Type Description Default <code>summary_stats_df</code> <code>DataFrame</code> <p>an ingested but unprocessed summary statistics dataframe from eQTL Catalogue.</p> required <p>Returns:</p> Name Type Description <code>SummaryStatistics</code> <code>SummaryStatistics</code> <p>a processed summary statistics dataframe for eQTL Catalogue.</p> Source code in <code>src/gentropy/datasource/eqtl_catalogue/summary_stats.py</code> <pre><code>@classmethod\ndef from_source(\n    cls: type[EqtlCatalogueSummaryStats],\n    summary_stats_df: DataFrame,\n) -&gt; SummaryStatistics:\n    \"\"\"Ingests all summary stats for all eQTL Catalogue studies.\n\n    Args:\n        summary_stats_df (DataFrame): an ingested but unprocessed summary statistics dataframe from eQTL Catalogue.\n\n    Returns:\n        SummaryStatistics: a processed summary statistics dataframe for eQTL Catalogue.\n    \"\"\"\n    processed_summary_stats_df = (\n        summary_stats_df.select(\n            # Construct study ID from the appropriate columns.\n            cls._full_study_id_regexp().alias(\"studyId\"),\n            # Add variant information.\n            f.concat_ws(\n                \"_\",\n                f.col(\"chromosome\"),\n                f.col(\"position\"),\n                f.col(\"ref\"),\n                f.col(\"alt\"),\n            ).alias(\"variantId\"),\n            f.col(\"chromosome\"),\n            f.col(\"position\").cast(t.IntegerType()),\n            # Parse p-value into mantissa and exponent.\n            *parse_pvalue(f.col(\"pvalue\")),\n            # Add beta, standard error, and allele frequency information.\n            f.col(\"beta\").cast(\"double\"),\n            f.col(\"se\").cast(\"double\").alias(\"standardError\"),\n            f.col(\"maf\").cast(\"float\").alias(\"effectAlleleFrequencyFromSource\"),\n        )\n        # Drop rows which don't have proper position or beta value.\n        .filter(\n            f.col(\"position\").cast(t.IntegerType()).isNotNull()\n            &amp; (f.col(\"beta\") != 0)\n        )\n    )\n\n    # Initialise a summary statistics object.\n    return SummaryStatistics(\n        _df=processed_summary_stats_df,\n        _schema=SummaryStatistics.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasources/finngen/_finngen/","title":"FinnGen","text":"<p>FinnGen is a research project in genomics and personalized medicine, representing a large public-private partnership. The project has collected and analyzed genome and health data from 500,000 Finnish biobank donors to understand the genetic basis of diseases. FinnGen is now expanding its focus to comprehend the progression and biological mechanisms of diseases. This initiative provides a world-class resource for further breakthroughs in disease prevention, diagnosis, and treatment, offering insights into our genetic makeup.</p> <p>For a comprehensive understanding of the dataset and methods, refer to Kurki et al., 2023.</p> <p>We ingested full GWAS summary statistics and SuSiE-based fine-mapping results.</p>"},{"location":"python_api/datasources/finngen/finemapping/","title":"Finemapping","text":""},{"location":"python_api/datasources/finngen/finemapping/#gentropy.datasource.finngen.finemapping.FinnGenFinemapping","title":"<code>gentropy.datasource.finngen.finemapping.FinnGenFinemapping</code>  <code>dataclass</code>","text":"<p>SuSIE finemapping dataset for FinnGen.</p> <p>Credible sets from SuSIE are extracted and transformed into StudyLocus objects:</p> <ul> <li>Study ID in the special format (e.g. FINNGEN_R10_*)</li> <li>Credible set specific finemapping statistics (e.g. LogBayesFactors, Alphas/Posterior)</li> <li>Additional credible set level BayesFactor filtering is applied (LBF &gt; 2)</li> <li>StudyLocusId is annotated for each credible set.</li> </ul> <p>Finemapping method is populated as a constant (\"SuSIE\").</p> Source code in <code>src/gentropy/datasource/finngen/finemapping.py</code> <pre><code>@dataclass\nclass FinnGenFinemapping:\n    \"\"\"SuSIE finemapping dataset for FinnGen.\n\n    Credible sets from SuSIE are extracted and transformed into StudyLocus objects:\n\n    - Study ID in the special format (e.g. FINNGEN_R10_*)\n    - Credible set specific finemapping statistics (e.g. LogBayesFactors, Alphas/Posterior)\n    - Additional credible set level BayesFactor filtering is applied (LBF &gt; 2)\n    - StudyLocusId is annotated for each credible set.\n\n    Finemapping method is populated as a constant (\"SuSIE\").\n    \"\"\"\n\n    raw_schema: t.StructType = StructType(\n        [\n            StructField(\"trait\", StringType(), True),\n            StructField(\"region\", StringType(), True),\n            StructField(\"v\", StringType(), True),\n            StructField(\"rsid\", StringType(), True),\n            StructField(\"chromosome\", StringType(), True),\n            StructField(\"position\", StringType(), True),\n            StructField(\"allele1\", StringType(), True),\n            StructField(\"allele2\", StringType(), True),\n            StructField(\"maf\", StringType(), True),\n            StructField(\"beta\", StringType(), True),\n            StructField(\"se\", StringType(), True),\n            StructField(\"p\", StringType(), True),\n            StructField(\"mean\", StringType(), True),\n            StructField(\"sd\", StringType(), True),\n            StructField(\"prob\", StringType(), True),\n            StructField(\"cs\", StringType(), True),\n            StructField(\"cs_specific_prob\", DoubleType(), True),\n            StructField(\"low_purity\", StringType(), True),\n            StructField(\"lead_r2\", StringType(), True),\n            StructField(\"mean_99\", StringType(), True),\n            StructField(\"sd_99\", StringType(), True),\n            StructField(\"prob_99\", StringType(), True),\n            StructField(\"cs_99\", StringType(), True),\n            StructField(\"cs_specific_prob_99\", StringType(), True),\n            StructField(\"low_purity_99\", StringType(), True),\n            StructField(\"lead_r2_99\", StringType(), True),\n            StructField(\"alpha1\", DoubleType(), True),\n            StructField(\"alpha2\", DoubleType(), True),\n            StructField(\"alpha3\", DoubleType(), True),\n            StructField(\"alpha4\", DoubleType(), True),\n            StructField(\"alpha5\", DoubleType(), True),\n            StructField(\"alpha6\", DoubleType(), True),\n            StructField(\"alpha7\", DoubleType(), True),\n            StructField(\"alpha8\", DoubleType(), True),\n            StructField(\"alpha9\", DoubleType(), True),\n            StructField(\"alpha10\", DoubleType(), True),\n            StructField(\"mean1\", StringType(), True),\n            StructField(\"mean2\", StringType(), True),\n            StructField(\"mean3\", StringType(), True),\n            StructField(\"mean4\", StringType(), True),\n            StructField(\"mean5\", StringType(), True),\n            StructField(\"mean6\", StringType(), True),\n            StructField(\"mean7\", StringType(), True),\n            StructField(\"mean8\", StringType(), True),\n            StructField(\"mean9\", StringType(), True),\n            StructField(\"mean10\", StringType(), True),\n            StructField(\"sd1\", StringType(), True),\n            StructField(\"sd2\", StringType(), True),\n            StructField(\"sd3\", StringType(), True),\n            StructField(\"sd4\", StringType(), True),\n            StructField(\"sd5\", StringType(), True),\n            StructField(\"sd6\", StringType(), True),\n            StructField(\"sd7\", StringType(), True),\n            StructField(\"sd8\", StringType(), True),\n            StructField(\"sd9\", StringType(), True),\n            StructField(\"sd10\", StringType(), True),\n            StructField(\"lbf_variable1\", DoubleType(), True),\n            StructField(\"lbf_variable2\", DoubleType(), True),\n            StructField(\"lbf_variable3\", DoubleType(), True),\n            StructField(\"lbf_variable4\", DoubleType(), True),\n            StructField(\"lbf_variable5\", DoubleType(), True),\n            StructField(\"lbf_variable6\", DoubleType(), True),\n            StructField(\"lbf_variable7\", DoubleType(), True),\n            StructField(\"lbf_variable8\", DoubleType(), True),\n            StructField(\"lbf_variable9\", DoubleType(), True),\n            StructField(\"lbf_variable10\", DoubleType(), True),\n        ]\n    )\n\n    summary_schema: t.StructType = StructType(\n        [\n            StructField(\"trait\", StringType(), True),\n            StructField(\"region\", StringType(), True),\n            StructField(\"cs\", StringType(), True),\n            StructField(\"cs_log10bf\", DoubleType(), True),\n        ]\n    )\n\n    @classmethod\n    def from_finngen_susie_finemapping(\n        cls: type[FinnGenFinemapping],\n        spark: SparkSession,\n        finngen_finemapping_df: (str | list[str]),\n        finngen_finemapping_summaries: (str | list[str]),\n        finngen_release_prefix: str,\n        credset_lbf_threshold: float = 0.8685889638065036,\n    ) -&gt; StudyLocus:\n        \"\"\"Process the SuSIE finemapping output for FinnGen studies.\n\n        Args:\n            spark (SparkSession): Spark session object.\n            finngen_finemapping_df (str | list[str]): SuSIE finemapping output filename(s).\n            finngen_finemapping_summaries (str | list[str]): filename of SuSIE finemapping summaries.\n            finngen_release_prefix (str): FinnGen study prefix.\n            credset_lbf_threshold (float, optional): Filter out credible sets below, Default 0.8685889638065036 == np.log10(np.exp(2)), this is threshold from publication.\n\n        Returns:\n            StudyLocus: Processed SuSIE finemapping output in StudyLocus format.\n        \"\"\"\n        processed_finngen_finemapping_df = (\n            spark.read.schema(cls.raw_schema)\n            .option(\"delimiter\", \"\\t\")\n            .option(\"compression\", \"gzip\")\n            .csv(finngen_finemapping_df, header=True)\n            # Drop rows which don't have proper position.\n            .filter(f.col(\"position\").cast(t.IntegerType()).isNotNull())\n            # Drop non credible set SNPs:\n            .filter(f.col(\"cs\").cast(t.IntegerType()) &gt; 0)\n            .select(\n                # Add study idenfitier.\n                f.concat(f.lit(finngen_release_prefix), f.col(\"trait\"))\n                .cast(t.StringType())\n                .alias(\"studyId\"),\n                f.col(\"region\"),\n                # Add variant information.\n                f.regexp_replace(f.col(\"v\"), \":\", \"_\").alias(\"variantId\"),\n                f.col(\"cs\").cast(\"integer\").alias(\"credibleSetIndex\"),\n                f.regexp_replace(f.col(\"chromosome\"), \"^chr\", \"\")\n                .cast(t.StringType())\n                .alias(\"chromosome\"),\n                f.col(\"position\").cast(t.IntegerType()),\n                f.col(\"allele1\").cast(t.StringType()).alias(\"ref\"),\n                f.col(\"allele2\").cast(t.StringType()).alias(\"alt\"),\n                # Parse p-value into mantissa and exponent.\n                *parse_pvalue(f.col(\"p\")),\n                # Add beta, standard error, and allele frequency information.\n                f.col(\"beta\").cast(\"double\"),\n                f.col(\"se\").cast(\"double\").alias(\"standardError\"),\n                f.col(\"maf\").cast(\"float\").alias(\"effectAlleleFrequencyFromSource\"),\n                f.lit(\"SuSie\").cast(\"string\").alias(\"finemappingMethod\"),\n                *[\n                    f.col(f\"alpha{i}\").cast(t.DoubleType()).alias(f\"alpha_{i}\")\n                    for i in range(1, 11)\n                ],\n                *[\n                    f.col(f\"lbf_variable{i}\").cast(t.DoubleType()).alias(f\"lbf_{i}\")\n                    for i in range(1, 11)\n                ],\n            )\n            .withColumn(\n                \"posteriorProbability\",\n                f.when(f.col(\"credibleSetIndex\") == 1, f.col(\"alpha_1\"))\n                .when(f.col(\"credibleSetIndex\") == 2, f.col(\"alpha_2\"))\n                .when(f.col(\"credibleSetIndex\") == 3, f.col(\"alpha_3\"))\n                .when(f.col(\"credibleSetIndex\") == 4, f.col(\"alpha_4\"))\n                .when(f.col(\"credibleSetIndex\") == 5, f.col(\"alpha_5\"))\n                .when(f.col(\"credibleSetIndex\") == 6, f.col(\"alpha_6\"))\n                .when(f.col(\"credibleSetIndex\") == 7, f.col(\"alpha_7\"))\n                .when(f.col(\"credibleSetIndex\") == 8, f.col(\"alpha_8\"))\n                .when(f.col(\"credibleSetIndex\") == 9, f.col(\"alpha_9\"))\n                .when(f.col(\"credibleSetIndex\") == 10, f.col(\"alpha_10\")),\n            )\n            .drop(\n                \"alpha_1\",\n                \"alpha_2\",\n                \"alpha_3\",\n                \"alpha_4\",\n                \"alpha_5\",\n                \"alpha_6\",\n                \"alpha_7\",\n                \"alpha_8\",\n                \"alpha_9\",\n                \"alpha_10\",\n            )\n            .withColumn(\n                \"logBF\",\n                f.when(f.col(\"credibleSetIndex\") == 1, f.col(\"lbf_1\"))\n                .when(f.col(\"credibleSetIndex\") == 2, f.col(\"lbf_2\"))\n                .when(f.col(\"credibleSetIndex\") == 3, f.col(\"lbf_3\"))\n                .when(f.col(\"credibleSetIndex\") == 4, f.col(\"lbf_4\"))\n                .when(f.col(\"credibleSetIndex\") == 5, f.col(\"lbf_5\"))\n                .when(f.col(\"credibleSetIndex\") == 6, f.col(\"lbf_6\"))\n                .when(f.col(\"credibleSetIndex\") == 7, f.col(\"lbf_7\"))\n                .when(f.col(\"credibleSetIndex\") == 8, f.col(\"lbf_8\"))\n                .when(f.col(\"credibleSetIndex\") == 9, f.col(\"lbf_9\"))\n                .when(f.col(\"credibleSetIndex\") == 10, f.col(\"lbf_10\")),\n            )\n            .drop(\n                \"lbf_1\",\n                \"lbf_2\",\n                \"lbf_3\",\n                \"lbf_4\",\n                \"lbf_5\",\n                \"lbf_6\",\n                \"lbf_7\",\n                \"lbf_8\",\n                \"lbf_9\",\n                \"lbf_10\",\n            )\n        )\n\n        # drop credible sets where logbf &gt; 2. Except when there's only one credible set in region:\n        # 0.8685889638065036 corresponds to np.log10(np.exp(2)), to match the orginal threshold in publication.\n        finngen_finemapping_summaries_df = (\n            # Read credible set level lbf, it is output as a different file which is not ideal.\n            spark.read.schema(cls.summary_schema)\n            .option(\"delimiter\", \"\\t\")\n            .csv(finngen_finemapping_summaries, header=True)\n            .select(\n                f.col(\"region\"),\n                f.col(\"trait\"),\n                f.col(\"cs\").cast(\"integer\").alias(\"credibleSetIndex\"),\n                f.col(\"cs_log10bf\").cast(\"double\").alias(\"credibleSetlog10BF\"),\n            )\n            .filter(\n                (f.col(\"credibleSetlog10BF\") &gt; credset_lbf_threshold)\n                | (f.col(\"credibleSetIndex\") == 1)\n            )\n            .withColumn(\n                \"studyId\", f.concat(f.lit(finngen_release_prefix), f.col(\"trait\"))\n            )\n        )\n\n        processed_finngen_finemapping_df = processed_finngen_finemapping_df.join(\n            finngen_finemapping_summaries_df,\n            on=[\"studyId\", \"region\", \"credibleSetIndex\"],\n            how=\"inner\",\n        )\n\n        toploci_df = get_top_ranked_in_window(\n            processed_finngen_finemapping_df,\n            Window.partitionBy(\"studyId\", \"region\", \"credibleSetIndex\").orderBy(\n                f.desc(\"posteriorProbability\")\n            ),\n        ).select(\n            \"variantId\",\n            \"chromosome\",\n            \"position\",\n            \"studyId\",\n            \"beta\",\n            \"pValueMantissa\",\n            \"pValueExponent\",\n            \"effectAlleleFrequencyFromSource\",\n            \"standardError\",\n            \"region\",\n            \"credibleSetIndex\",\n            \"finemappingMethod\",\n            \"credibleSetlog10BF\",\n        )\n\n        processed_finngen_finemapping_df = (\n            processed_finngen_finemapping_df.groupBy(\n                \"studyId\", \"region\", \"credibleSetIndex\"\n            )\n            .agg(\n                f.collect_list(\n                    f.struct(\n                        f.col(\"variantId\").cast(\"string\").alias(\"variantId\"),\n                        f.col(\"posteriorProbability\")\n                        .cast(\"double\")\n                        .alias(\"posteriorProbability\"),\n                        f.col(\"logBF\").cast(\"double\").alias(\"logBF\"),\n                        f.col(\"pValueMantissa\").cast(\"float\").alias(\"pValueMantissa\"),\n                        f.col(\"pValueExponent\").cast(\"integer\").alias(\"pValueExponent\"),\n                        f.col(\"beta\").cast(\"double\").alias(\"beta\"),\n                        f.col(\"standardError\").cast(\"double\").alias(\"standardError\"),\n                    )\n                ).alias(\"locus\"),\n            )\n            .select(\n                \"studyId\",\n                \"region\",\n                \"credibleSetIndex\",\n                \"locus\",\n            )\n            .join(\n                toploci_df,\n                on=[\"studyId\", \"region\", \"credibleSetIndex\"],\n                how=\"inner\",\n            )\n        ).withColumn(\n            \"studyLocusId\",\n            StudyLocus.assign_study_locus_id(f.col(\"studyId\"), f.col(\"variantId\")),\n        )\n\n        return StudyLocus(\n            _df=processed_finngen_finemapping_df,\n            _schema=StudyLocus.get_schema(),\n        ).annotate_credible_sets()\n</code></pre>"},{"location":"python_api/datasources/finngen/finemapping/#gentropy.datasource.finngen.finemapping.FinnGenFinemapping.from_finngen_susie_finemapping","title":"<code>from_finngen_susie_finemapping(spark: SparkSession, finngen_finemapping_df: str | list[str], finngen_finemapping_summaries: str | list[str], finngen_release_prefix: str, credset_lbf_threshold: float = 0.8685889638065036) -&gt; StudyLocus</code>  <code>classmethod</code>","text":"<p>Process the SuSIE finemapping output for FinnGen studies.</p> <p>Parameters:</p> Name Type Description Default <code>spark</code> <code>SparkSession</code> <p>Spark session object.</p> required <code>finngen_finemapping_df</code> <code>str | list[str]</code> <p>SuSIE finemapping output filename(s).</p> required <code>finngen_finemapping_summaries</code> <code>str | list[str]</code> <p>filename of SuSIE finemapping summaries.</p> required <code>finngen_release_prefix</code> <code>str</code> <p>FinnGen study prefix.</p> required <code>credset_lbf_threshold</code> <code>float</code> <p>Filter out credible sets below, Default 0.8685889638065036 == np.log10(np.exp(2)), this is threshold from publication.</p> <code>0.8685889638065036</code> <p>Returns:</p> Name Type Description <code>StudyLocus</code> <code>StudyLocus</code> <p>Processed SuSIE finemapping output in StudyLocus format.</p> Source code in <code>src/gentropy/datasource/finngen/finemapping.py</code> <pre><code>@classmethod\ndef from_finngen_susie_finemapping(\n    cls: type[FinnGenFinemapping],\n    spark: SparkSession,\n    finngen_finemapping_df: (str | list[str]),\n    finngen_finemapping_summaries: (str | list[str]),\n    finngen_release_prefix: str,\n    credset_lbf_threshold: float = 0.8685889638065036,\n) -&gt; StudyLocus:\n    \"\"\"Process the SuSIE finemapping output for FinnGen studies.\n\n    Args:\n        spark (SparkSession): Spark session object.\n        finngen_finemapping_df (str | list[str]): SuSIE finemapping output filename(s).\n        finngen_finemapping_summaries (str | list[str]): filename of SuSIE finemapping summaries.\n        finngen_release_prefix (str): FinnGen study prefix.\n        credset_lbf_threshold (float, optional): Filter out credible sets below, Default 0.8685889638065036 == np.log10(np.exp(2)), this is threshold from publication.\n\n    Returns:\n        StudyLocus: Processed SuSIE finemapping output in StudyLocus format.\n    \"\"\"\n    processed_finngen_finemapping_df = (\n        spark.read.schema(cls.raw_schema)\n        .option(\"delimiter\", \"\\t\")\n        .option(\"compression\", \"gzip\")\n        .csv(finngen_finemapping_df, header=True)\n        # Drop rows which don't have proper position.\n        .filter(f.col(\"position\").cast(t.IntegerType()).isNotNull())\n        # Drop non credible set SNPs:\n        .filter(f.col(\"cs\").cast(t.IntegerType()) &gt; 0)\n        .select(\n            # Add study idenfitier.\n            f.concat(f.lit(finngen_release_prefix), f.col(\"trait\"))\n            .cast(t.StringType())\n            .alias(\"studyId\"),\n            f.col(\"region\"),\n            # Add variant information.\n            f.regexp_replace(f.col(\"v\"), \":\", \"_\").alias(\"variantId\"),\n            f.col(\"cs\").cast(\"integer\").alias(\"credibleSetIndex\"),\n            f.regexp_replace(f.col(\"chromosome\"), \"^chr\", \"\")\n            .cast(t.StringType())\n            .alias(\"chromosome\"),\n            f.col(\"position\").cast(t.IntegerType()),\n            f.col(\"allele1\").cast(t.StringType()).alias(\"ref\"),\n            f.col(\"allele2\").cast(t.StringType()).alias(\"alt\"),\n            # Parse p-value into mantissa and exponent.\n            *parse_pvalue(f.col(\"p\")),\n            # Add beta, standard error, and allele frequency information.\n            f.col(\"beta\").cast(\"double\"),\n            f.col(\"se\").cast(\"double\").alias(\"standardError\"),\n            f.col(\"maf\").cast(\"float\").alias(\"effectAlleleFrequencyFromSource\"),\n            f.lit(\"SuSie\").cast(\"string\").alias(\"finemappingMethod\"),\n            *[\n                f.col(f\"alpha{i}\").cast(t.DoubleType()).alias(f\"alpha_{i}\")\n                for i in range(1, 11)\n            ],\n            *[\n                f.col(f\"lbf_variable{i}\").cast(t.DoubleType()).alias(f\"lbf_{i}\")\n                for i in range(1, 11)\n            ],\n        )\n        .withColumn(\n            \"posteriorProbability\",\n            f.when(f.col(\"credibleSetIndex\") == 1, f.col(\"alpha_1\"))\n            .when(f.col(\"credibleSetIndex\") == 2, f.col(\"alpha_2\"))\n            .when(f.col(\"credibleSetIndex\") == 3, f.col(\"alpha_3\"))\n            .when(f.col(\"credibleSetIndex\") == 4, f.col(\"alpha_4\"))\n            .when(f.col(\"credibleSetIndex\") == 5, f.col(\"alpha_5\"))\n            .when(f.col(\"credibleSetIndex\") == 6, f.col(\"alpha_6\"))\n            .when(f.col(\"credibleSetIndex\") == 7, f.col(\"alpha_7\"))\n            .when(f.col(\"credibleSetIndex\") == 8, f.col(\"alpha_8\"))\n            .when(f.col(\"credibleSetIndex\") == 9, f.col(\"alpha_9\"))\n            .when(f.col(\"credibleSetIndex\") == 10, f.col(\"alpha_10\")),\n        )\n        .drop(\n            \"alpha_1\",\n            \"alpha_2\",\n            \"alpha_3\",\n            \"alpha_4\",\n            \"alpha_5\",\n            \"alpha_6\",\n            \"alpha_7\",\n            \"alpha_8\",\n            \"alpha_9\",\n            \"alpha_10\",\n        )\n        .withColumn(\n            \"logBF\",\n            f.when(f.col(\"credibleSetIndex\") == 1, f.col(\"lbf_1\"))\n            .when(f.col(\"credibleSetIndex\") == 2, f.col(\"lbf_2\"))\n            .when(f.col(\"credibleSetIndex\") == 3, f.col(\"lbf_3\"))\n            .when(f.col(\"credibleSetIndex\") == 4, f.col(\"lbf_4\"))\n            .when(f.col(\"credibleSetIndex\") == 5, f.col(\"lbf_5\"))\n            .when(f.col(\"credibleSetIndex\") == 6, f.col(\"lbf_6\"))\n            .when(f.col(\"credibleSetIndex\") == 7, f.col(\"lbf_7\"))\n            .when(f.col(\"credibleSetIndex\") == 8, f.col(\"lbf_8\"))\n            .when(f.col(\"credibleSetIndex\") == 9, f.col(\"lbf_9\"))\n            .when(f.col(\"credibleSetIndex\") == 10, f.col(\"lbf_10\")),\n        )\n        .drop(\n            \"lbf_1\",\n            \"lbf_2\",\n            \"lbf_3\",\n            \"lbf_4\",\n            \"lbf_5\",\n            \"lbf_6\",\n            \"lbf_7\",\n            \"lbf_8\",\n            \"lbf_9\",\n            \"lbf_10\",\n        )\n    )\n\n    # drop credible sets where logbf &gt; 2. Except when there's only one credible set in region:\n    # 0.8685889638065036 corresponds to np.log10(np.exp(2)), to match the orginal threshold in publication.\n    finngen_finemapping_summaries_df = (\n        # Read credible set level lbf, it is output as a different file which is not ideal.\n        spark.read.schema(cls.summary_schema)\n        .option(\"delimiter\", \"\\t\")\n        .csv(finngen_finemapping_summaries, header=True)\n        .select(\n            f.col(\"region\"),\n            f.col(\"trait\"),\n            f.col(\"cs\").cast(\"integer\").alias(\"credibleSetIndex\"),\n            f.col(\"cs_log10bf\").cast(\"double\").alias(\"credibleSetlog10BF\"),\n        )\n        .filter(\n            (f.col(\"credibleSetlog10BF\") &gt; credset_lbf_threshold)\n            | (f.col(\"credibleSetIndex\") == 1)\n        )\n        .withColumn(\n            \"studyId\", f.concat(f.lit(finngen_release_prefix), f.col(\"trait\"))\n        )\n    )\n\n    processed_finngen_finemapping_df = processed_finngen_finemapping_df.join(\n        finngen_finemapping_summaries_df,\n        on=[\"studyId\", \"region\", \"credibleSetIndex\"],\n        how=\"inner\",\n    )\n\n    toploci_df = get_top_ranked_in_window(\n        processed_finngen_finemapping_df,\n        Window.partitionBy(\"studyId\", \"region\", \"credibleSetIndex\").orderBy(\n            f.desc(\"posteriorProbability\")\n        ),\n    ).select(\n        \"variantId\",\n        \"chromosome\",\n        \"position\",\n        \"studyId\",\n        \"beta\",\n        \"pValueMantissa\",\n        \"pValueExponent\",\n        \"effectAlleleFrequencyFromSource\",\n        \"standardError\",\n        \"region\",\n        \"credibleSetIndex\",\n        \"finemappingMethod\",\n        \"credibleSetlog10BF\",\n    )\n\n    processed_finngen_finemapping_df = (\n        processed_finngen_finemapping_df.groupBy(\n            \"studyId\", \"region\", \"credibleSetIndex\"\n        )\n        .agg(\n            f.collect_list(\n                f.struct(\n                    f.col(\"variantId\").cast(\"string\").alias(\"variantId\"),\n                    f.col(\"posteriorProbability\")\n                    .cast(\"double\")\n                    .alias(\"posteriorProbability\"),\n                    f.col(\"logBF\").cast(\"double\").alias(\"logBF\"),\n                    f.col(\"pValueMantissa\").cast(\"float\").alias(\"pValueMantissa\"),\n                    f.col(\"pValueExponent\").cast(\"integer\").alias(\"pValueExponent\"),\n                    f.col(\"beta\").cast(\"double\").alias(\"beta\"),\n                    f.col(\"standardError\").cast(\"double\").alias(\"standardError\"),\n                )\n            ).alias(\"locus\"),\n        )\n        .select(\n            \"studyId\",\n            \"region\",\n            \"credibleSetIndex\",\n            \"locus\",\n        )\n        .join(\n            toploci_df,\n            on=[\"studyId\", \"region\", \"credibleSetIndex\"],\n            how=\"inner\",\n        )\n    ).withColumn(\n        \"studyLocusId\",\n        StudyLocus.assign_study_locus_id(f.col(\"studyId\"), f.col(\"variantId\")),\n    )\n\n    return StudyLocus(\n        _df=processed_finngen_finemapping_df,\n        _schema=StudyLocus.get_schema(),\n    ).annotate_credible_sets()\n</code></pre>"},{"location":"python_api/datasources/finngen/study_index/","title":"Study Index","text":""},{"location":"python_api/datasources/finngen/study_index/#gentropy.datasource.finngen.study_index.FinnGenStudyIndex","title":"<code>gentropy.datasource.finngen.study_index.FinnGenStudyIndex</code>","text":"<p>Study index dataset from FinnGen.</p> <p>The following information is aggregated/extracted:</p> <ul> <li>Study ID in the special format (e.g. FINNGEN_R10_*)</li> <li>Trait name (for example, Amoebiasis)</li> <li>Number of cases and controls</li> <li>Link to the summary statistics location</li> </ul> <p>Some fields are also populated as constants, such as study type and the initial sample size.</p> Source code in <code>src/gentropy/datasource/finngen/study_index.py</code> <pre><code>class FinnGenStudyIndex:\n    \"\"\"Study index dataset from FinnGen.\n\n    The following information is aggregated/extracted:\n\n    - Study ID in the special format (e.g. FINNGEN_R10_*)\n    - Trait name (for example, Amoebiasis)\n    - Number of cases and controls\n    - Link to the summary statistics location\n\n    Some fields are also populated as constants, such as study type and the initial sample size.\n    \"\"\"\n\n    finngen_phenotype_table_url: str = \"https://r10.finngen.fi/api/phenos\"\n    finngen_release_prefix: str = \"FINNGEN_R10\"\n    finngen_summary_stats_url_prefix: str = (\n        \"gs://finngen-public-data-r10/summary_stats/finngen_R10_\"\n    )\n    finngen_summary_stats_url_suffix: str = \".gz\"\n\n    @classmethod\n    def from_source(\n        cls: type[FinnGenStudyIndex],\n        spark: SparkSession,\n    ) -&gt; StudyIndex:\n        \"\"\"This function ingests study level metadata from FinnGen.\n\n        Args:\n            spark (SparkSession): Spark session object.\n\n        Returns:\n            StudyIndex: Parsed and annotated FinnGen study table.\n        \"\"\"\n        json_data = urlopen(cls.finngen_phenotype_table_url).read().decode(\"utf-8\")\n        rdd = spark.sparkContext.parallelize([json_data])\n        raw_df = spark.read.json(rdd)\n        return StudyIndex(\n            _df=raw_df.select(\n                f.concat(\n                    f.lit(f\"{cls.finngen_release_prefix}_\"), f.col(\"phenocode\")\n                ).alias(\"studyId\"),\n                f.col(\"phenostring\").alias(\"traitFromSource\"),\n                f.col(\"num_cases\").cast(\"integer\").alias(\"nCases\"),\n                f.col(\"num_controls\").cast(\"integer\").alias(\"nControls\"),\n                (f.col(\"num_cases\") + f.col(\"num_controls\"))\n                .cast(\"integer\")\n                .alias(\"nSamples\"),\n                f.lit(cls.finngen_release_prefix).alias(\"projectId\"),\n                f.lit(\"gwas\").alias(\"studyType\"),\n                f.lit(True).alias(\"hasSumstats\"),\n                f.lit(\"377,277 (210,870 females and 166,407 males)\").alias(\n                    \"initialSampleSize\"\n                ),\n                f.array(\n                    f.struct(\n                        f.lit(377277).cast(\"integer\").alias(\"sampleSize\"),\n                        f.lit(\"Finnish\").alias(\"ancestry\"),\n                    )\n                ).alias(\"discoverySamples\"),\n                # Cohort label is consistent with GWAS Catalog curation.\n                f.array(f.lit(\"FinnGen\")).alias(\"cohorts\"),\n                f.concat(\n                    f.lit(cls.finngen_summary_stats_url_prefix),\n                    f.col(\"phenocode\"),\n                    f.lit(cls.finngen_summary_stats_url_suffix),\n                ).alias(\"summarystatsLocation\"),\n            ).withColumn(\n                \"ldPopulationStructure\",\n                StudyIndex.aggregate_and_map_ancestries(f.col(\"discoverySamples\")),\n            ),\n            _schema=StudyIndex.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasources/finngen/study_index/#gentropy.datasource.finngen.study_index.FinnGenStudyIndex.from_source","title":"<code>from_source(spark: SparkSession) -&gt; StudyIndex</code>  <code>classmethod</code>","text":"<p>This function ingests study level metadata from FinnGen.</p> <p>Parameters:</p> Name Type Description Default <code>spark</code> <code>SparkSession</code> <p>Spark session object.</p> required <p>Returns:</p> Name Type Description <code>StudyIndex</code> <code>StudyIndex</code> <p>Parsed and annotated FinnGen study table.</p> Source code in <code>src/gentropy/datasource/finngen/study_index.py</code> <pre><code>@classmethod\ndef from_source(\n    cls: type[FinnGenStudyIndex],\n    spark: SparkSession,\n) -&gt; StudyIndex:\n    \"\"\"This function ingests study level metadata from FinnGen.\n\n    Args:\n        spark (SparkSession): Spark session object.\n\n    Returns:\n        StudyIndex: Parsed and annotated FinnGen study table.\n    \"\"\"\n    json_data = urlopen(cls.finngen_phenotype_table_url).read().decode(\"utf-8\")\n    rdd = spark.sparkContext.parallelize([json_data])\n    raw_df = spark.read.json(rdd)\n    return StudyIndex(\n        _df=raw_df.select(\n            f.concat(\n                f.lit(f\"{cls.finngen_release_prefix}_\"), f.col(\"phenocode\")\n            ).alias(\"studyId\"),\n            f.col(\"phenostring\").alias(\"traitFromSource\"),\n            f.col(\"num_cases\").cast(\"integer\").alias(\"nCases\"),\n            f.col(\"num_controls\").cast(\"integer\").alias(\"nControls\"),\n            (f.col(\"num_cases\") + f.col(\"num_controls\"))\n            .cast(\"integer\")\n            .alias(\"nSamples\"),\n            f.lit(cls.finngen_release_prefix).alias(\"projectId\"),\n            f.lit(\"gwas\").alias(\"studyType\"),\n            f.lit(True).alias(\"hasSumstats\"),\n            f.lit(\"377,277 (210,870 females and 166,407 males)\").alias(\n                \"initialSampleSize\"\n            ),\n            f.array(\n                f.struct(\n                    f.lit(377277).cast(\"integer\").alias(\"sampleSize\"),\n                    f.lit(\"Finnish\").alias(\"ancestry\"),\n                )\n            ).alias(\"discoverySamples\"),\n            # Cohort label is consistent with GWAS Catalog curation.\n            f.array(f.lit(\"FinnGen\")).alias(\"cohorts\"),\n            f.concat(\n                f.lit(cls.finngen_summary_stats_url_prefix),\n                f.col(\"phenocode\"),\n                f.lit(cls.finngen_summary_stats_url_suffix),\n            ).alias(\"summarystatsLocation\"),\n        ).withColumn(\n            \"ldPopulationStructure\",\n            StudyIndex.aggregate_and_map_ancestries(f.col(\"discoverySamples\")),\n        ),\n        _schema=StudyIndex.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasources/finngen/summary_stats/","title":"Study Index","text":""},{"location":"python_api/datasources/finngen/summary_stats/#gentropy.datasource.finngen.summary_stats.FinnGenSummaryStats","title":"<code>gentropy.datasource.finngen.summary_stats.FinnGenSummaryStats</code>  <code>dataclass</code>","text":"<p>Summary statistics dataset for FinnGen.</p> Source code in <code>src/gentropy/datasource/finngen/summary_stats.py</code> <pre><code>@dataclass\nclass FinnGenSummaryStats:\n    \"\"\"Summary statistics dataset for FinnGen.\"\"\"\n\n    raw_schema: t.StructType = StructType(\n        [\n            StructField(\"#chrom\", StringType(), True),\n            StructField(\"pos\", StringType(), True),\n            StructField(\"ref\", StringType(), True),\n            StructField(\"alt\", StringType(), True),\n            StructField(\"rsids\", StringType(), True),\n            StructField(\"nearest_genes\", StringType(), True),\n            StructField(\"pval\", StringType(), True),\n            StructField(\"mlogp\", StringType(), True),\n            StructField(\"beta\", StringType(), True),\n            StructField(\"sebeta\", StringType(), True),\n            StructField(\"af_alt\", StringType(), True),\n            StructField(\"af_alt_cases\", StringType(), True),\n            StructField(\"af_alt_controls\", StringType(), True),\n        ]\n    )\n\n    @classmethod\n    def from_source(\n        cls: type[FinnGenSummaryStats],\n        spark: SparkSession,\n        raw_file: str,\n    ) -&gt; SummaryStatistics:\n        \"\"\"Ingests all summary statst for all FinnGen studies.\n\n        Args:\n            spark (SparkSession): Spark session object.\n            raw_file (str): Path to raw summary statistics .gz files.\n\n        Returns:\n            SummaryStatistics: Processed summary statistics dataset\n        \"\"\"\n        study_id = raw_file.split(\"/\")[-1].split(\".\")[0].upper()\n        processed_summary_stats_df = (\n            spark.read.schema(cls.raw_schema)\n            .option(\"delimiter\", \"\\t\")\n            .csv(raw_file, header=True)\n            # Drop rows which don't have proper position.\n            .filter(f.col(\"pos\").cast(t.IntegerType()).isNotNull())\n            .select(\n                # From the full path, extracts just the filename, and converts to upper case to get the study ID.\n                f.lit(study_id).alias(\"studyId\"),\n                # Add variant information.\n                f.concat_ws(\n                    \"_\",\n                    f.col(\"#chrom\"),\n                    f.col(\"pos\"),\n                    f.col(\"ref\"),\n                    f.col(\"alt\"),\n                ).alias(\"variantId\"),\n                f.col(\"#chrom\").alias(\"chromosome\"),\n                f.col(\"pos\").cast(t.IntegerType()).alias(\"position\"),\n                # Parse p-value into mantissa and exponent.\n                *parse_pvalue(f.col(\"pval\")),\n                # Add beta, standard error, and allele frequency information.\n                f.col(\"beta\").cast(\"double\"),\n                f.col(\"sebeta\").cast(\"double\").alias(\"standardError\"),\n                f.col(\"af_alt\").cast(\"float\").alias(\"effectAlleleFrequencyFromSource\"),\n            )\n            # Calculating the confidence intervals.\n            .filter(\n                f.col(\"pos\").cast(t.IntegerType()).isNotNull() &amp; (f.col(\"beta\") != 0)\n            )\n            # Average ~20Mb partitions with 30 partitions per study\n            .repartitionByRange(30, \"chromosome\", \"position\")\n            .sortWithinPartitions(\"chromosome\", \"position\")\n        )\n\n        # Initializing summary statistics object:\n        return SummaryStatistics(\n            _df=processed_summary_stats_df,\n            _schema=SummaryStatistics.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasources/finngen/summary_stats/#gentropy.datasource.finngen.summary_stats.FinnGenSummaryStats.from_source","title":"<code>from_source(spark: SparkSession, raw_file: str) -&gt; SummaryStatistics</code>  <code>classmethod</code>","text":"<p>Ingests all summary statst for all FinnGen studies.</p> <p>Parameters:</p> Name Type Description Default <code>spark</code> <code>SparkSession</code> <p>Spark session object.</p> required <code>raw_file</code> <code>str</code> <p>Path to raw summary statistics .gz files.</p> required <p>Returns:</p> Name Type Description <code>SummaryStatistics</code> <code>SummaryStatistics</code> <p>Processed summary statistics dataset</p> Source code in <code>src/gentropy/datasource/finngen/summary_stats.py</code> <pre><code>@classmethod\ndef from_source(\n    cls: type[FinnGenSummaryStats],\n    spark: SparkSession,\n    raw_file: str,\n) -&gt; SummaryStatistics:\n    \"\"\"Ingests all summary statst for all FinnGen studies.\n\n    Args:\n        spark (SparkSession): Spark session object.\n        raw_file (str): Path to raw summary statistics .gz files.\n\n    Returns:\n        SummaryStatistics: Processed summary statistics dataset\n    \"\"\"\n    study_id = raw_file.split(\"/\")[-1].split(\".\")[0].upper()\n    processed_summary_stats_df = (\n        spark.read.schema(cls.raw_schema)\n        .option(\"delimiter\", \"\\t\")\n        .csv(raw_file, header=True)\n        # Drop rows which don't have proper position.\n        .filter(f.col(\"pos\").cast(t.IntegerType()).isNotNull())\n        .select(\n            # From the full path, extracts just the filename, and converts to upper case to get the study ID.\n            f.lit(study_id).alias(\"studyId\"),\n            # Add variant information.\n            f.concat_ws(\n                \"_\",\n                f.col(\"#chrom\"),\n                f.col(\"pos\"),\n                f.col(\"ref\"),\n                f.col(\"alt\"),\n            ).alias(\"variantId\"),\n            f.col(\"#chrom\").alias(\"chromosome\"),\n            f.col(\"pos\").cast(t.IntegerType()).alias(\"position\"),\n            # Parse p-value into mantissa and exponent.\n            *parse_pvalue(f.col(\"pval\")),\n            # Add beta, standard error, and allele frequency information.\n            f.col(\"beta\").cast(\"double\"),\n            f.col(\"sebeta\").cast(\"double\").alias(\"standardError\"),\n            f.col(\"af_alt\").cast(\"float\").alias(\"effectAlleleFrequencyFromSource\"),\n        )\n        # Calculating the confidence intervals.\n        .filter(\n            f.col(\"pos\").cast(t.IntegerType()).isNotNull() &amp; (f.col(\"beta\") != 0)\n        )\n        # Average ~20Mb partitions with 30 partitions per study\n        .repartitionByRange(30, \"chromosome\", \"position\")\n        .sortWithinPartitions(\"chromosome\", \"position\")\n    )\n\n    # Initializing summary statistics object:\n    return SummaryStatistics(\n        _df=processed_summary_stats_df,\n        _schema=SummaryStatistics.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasources/gnomad/_gnomad/","title":"GnomAD","text":"<p>GnomAD (Genome Aggregation Database) is a comprehensive resource that provides aggregated genomic data from large-scale sequencing projects. It encompasses variants from diverse populations and is widely used for variant annotation and population genetics studies.</p> <p>We use GnomAD v4.0 as a source for variant annotation, offering detailed information about the prevalence and distribution of genetic variants across different populations. This version of GnomAD provides valuable insights into the genomic landscape, aiding in the interpretation of genetic variants and their potential functional implications.</p> <p>Additionally, GnomAD v2.1.1 is utilized as a source for linkage disequilibrium (LD) information.</p>"},{"location":"python_api/datasources/gnomad/gnomad_ld/","title":"LD Matrix","text":""},{"location":"python_api/datasources/gnomad/gnomad_ld/#gentropy.datasource.gnomad.ld.GnomADLDMatrix","title":"<code>gentropy.datasource.gnomad.ld.GnomADLDMatrix</code>  <code>dataclass</code>","text":"<p>Toolset ot interact with GnomAD LD dataset (version: r2.1.1).</p> <p>Datasets are accessed in Hail's native format, as provided by the GnomAD consortium.</p> <p>Attributes:</p> Name Type Description <code>ld_matrix_template</code> <code>str</code> <p>Template for the LD matrix path. Defaults to \"gs://gcp-public-data--gnomad/release/2.1.1/ld/gnomad.genomes.r2.1.1.{POP}.common.adj.ld.bm\".</p> <code>ld_index_raw_template</code> <code>str</code> <p>Template for the LD index path. Defaults to \"gs://gcp-public-data--gnomad/release/2.1.1/ld/gnomad.genomes.r2.1.1.{POP}.common.ld.variant_indices.ht\".</p> <code>grch37_to_grch38_chain_path</code> <code>str</code> <p>Path to the chain file used to lift over the coordinates. Defaults to \"gs://hail-common/references/grch37_to_grch38.over.chain.gz\".</p> <code>ld_populations</code> <code>list[str]</code> <p>List of populations to use to build the LDIndex. Defaults to [\"afr\", \"amr\", \"asj\", \"eas\", \"fin\", \"nfe\", \"nwe\", \"seu\"].</p> Source code in <code>src/gentropy/datasource/gnomad/ld.py</code> <pre><code>@dataclass\nclass GnomADLDMatrix:\n    \"\"\"Toolset ot interact with GnomAD LD dataset (version: r2.1.1).\n\n    Datasets are accessed in Hail's native format, as provided by the [GnomAD consortium](https://gnomad.broadinstitute.org/downloads/#v2-linkage-disequilibrium).\n\n    Attributes:\n        ld_matrix_template (str): Template for the LD matrix path. Defaults to \"gs://gcp-public-data--gnomad/release/2.1.1/ld/gnomad.genomes.r2.1.1.{POP}.common.adj.ld.bm\".\n        ld_index_raw_template (str): Template for the LD index path. Defaults to \"gs://gcp-public-data--gnomad/release/2.1.1/ld/gnomad.genomes.r2.1.1.{POP}.common.ld.variant_indices.ht\".\n        grch37_to_grch38_chain_path (str): Path to the chain file used to lift over the coordinates. Defaults to \"gs://hail-common/references/grch37_to_grch38.over.chain.gz\".\n        ld_populations (list[str]): List of populations to use to build the LDIndex. Defaults to [\"afr\", \"amr\", \"asj\", \"eas\", \"fin\", \"nfe\", \"nwe\", \"seu\"].\n    \"\"\"\n\n    ld_matrix_template: str = \"gs://gcp-public-data--gnomad/release/2.1.1/ld/gnomad.genomes.r2.1.1.{POP}.common.adj.ld.bm\"\n    ld_index_raw_template: str = \"gs://gcp-public-data--gnomad/release/2.1.1/ld/gnomad.genomes.r2.1.1.{POP}.common.ld.variant_indices.ht\"\n    grch37_to_grch38_chain_path: str = (\n        \"gs://hail-common/references/grch37_to_grch38.over.chain.gz\"\n    )\n    ld_populations: list[str] = field(\n        default_factory=lambda: [\n            \"afr\",  # African-American\n            \"amr\",  # American Admixed/Latino\n            \"asj\",  # Ashkenazi Jewish\n            \"eas\",  # East Asian\n            \"fin\",  # Finnish\n            \"nfe\",  # Non-Finnish European\n            \"nwe\",  # Northwestern European\n            \"seu\",  # Southeastern European\n        ]\n    )\n\n    @staticmethod\n    def _aggregate_ld_index_across_populations(\n        unaggregated_ld_index: DataFrame,\n    ) -&gt; DataFrame:\n        \"\"\"Aggregate LDIndex across populations.\n\n        Args:\n            unaggregated_ld_index (DataFrame): Unaggregate LDIndex index dataframe  each row is a variant pair in a population\n\n        Returns:\n            DataFrame: Aggregated LDIndex index dataframe  each row is a variant with the LD set across populations\n\n        Examples:\n            &gt;&gt;&gt; data = [(\"1.0\", \"var1\", \"X\", \"var1\", \"pop1\"), (\"1.0\", \"X\", \"var2\", \"var2\", \"pop1\"),\n            ...         (\"0.5\", \"var1\", \"X\", \"var2\", \"pop1\"), (\"0.5\", \"var1\", \"X\", \"var2\", \"pop2\"),\n            ...         (\"0.5\", \"var2\", \"X\", \"var1\", \"pop1\"), (\"0.5\", \"X\", \"var2\", \"var1\", \"pop2\")]\n            &gt;&gt;&gt; df = spark.createDataFrame(data, [\"r\", \"variantId\", \"chromosome\", \"tagvariantId\", \"population\"])\n            &gt;&gt;&gt; GnomADLDMatrix._aggregate_ld_index_across_populations(df).printSchema()\n            root\n             |-- variantId: string (nullable = true)\n             |-- chromosome: string (nullable = true)\n             |-- ldSet: array (nullable = false)\n             |    |-- element: struct (containsNull = false)\n             |    |    |-- tagVariantId: string (nullable = true)\n             |    |    |-- rValues: array (nullable = false)\n             |    |    |    |-- element: struct (containsNull = false)\n             |    |    |    |    |-- population: string (nullable = true)\n             |    |    |    |    |-- r: string (nullable = true)\n            &lt;BLANKLINE&gt;\n        \"\"\"\n        return (\n            unaggregated_ld_index\n            # First level of aggregation: get r/population for each variant/tagVariant pair\n            .withColumn(\"r_pop_struct\", f.struct(\"population\", \"r\"))\n            .groupBy(\"chromosome\", \"variantId\", \"tagVariantId\")\n            .agg(\n                f.collect_set(\"r_pop_struct\").alias(\"rValues\"),\n            )\n            # Second level of aggregation: get r/population for each variant\n            .withColumn(\"r_pop_tag_struct\", f.struct(\"tagVariantId\", \"rValues\"))\n            .groupBy(\"variantId\", \"chromosome\")\n            .agg(\n                f.collect_set(\"r_pop_tag_struct\").alias(\"ldSet\"),\n            )\n        )\n\n    @staticmethod\n    def _convert_ld_matrix_to_table(\n        block_matrix: BlockMatrix, min_r2: float\n    ) -&gt; DataFrame:\n        \"\"\"Convert LD matrix to table.\n\n        Args:\n            block_matrix (BlockMatrix): LD matrix\n            min_r2 (float): Minimum r2 value to keep in the table\n\n        Returns:\n            DataFrame: LD matrix as a Spark DataFrame\n        \"\"\"\n        table = block_matrix.entries(keyed=False)\n        return (\n            table.filter(hl.abs(table.entry) &gt;= min_r2**0.5)\n            .to_spark()\n            .withColumnRenamed(\"entry\", \"r\")\n        )\n\n    @staticmethod\n    def _create_ldindex_for_population(\n        population_id: str,\n        ld_matrix_path: str,\n        ld_index_raw_path: str,\n        grch37_to_grch38_chain_path: str,\n        min_r2: float,\n    ) -&gt; DataFrame:\n        \"\"\"Create LDIndex for a specific population.\n\n        Args:\n            population_id (str): Population ID\n            ld_matrix_path (str): Path to the LD matrix\n            ld_index_raw_path (str): Path to the LD index\n            grch37_to_grch38_chain_path (str): Path to the chain file used to lift over the coordinates\n            min_r2 (float): Minimum r2 value to keep in the table\n\n        Returns:\n            DataFrame: LDIndex for a specific population\n        \"\"\"\n        # Prepare LD Block matrix\n        ld_matrix = GnomADLDMatrix._convert_ld_matrix_to_table(\n            BlockMatrix.read(ld_matrix_path), min_r2\n        )\n\n        # Prepare table with variant indices\n        ld_index = GnomADLDMatrix._process_variant_indices(\n            hl.read_table(ld_index_raw_path),\n            grch37_to_grch38_chain_path,\n        )\n\n        return GnomADLDMatrix._resolve_variant_indices(ld_index, ld_matrix).select(\n            \"*\",\n            f.lit(population_id).alias(\"population\"),\n        )\n\n    @staticmethod\n    def _process_variant_indices(\n        ld_index_raw: hl.Table, grch37_to_grch38_chain_path: str\n    ) -&gt; DataFrame:\n        \"\"\"Creates a look up table between variants and their coordinates in the LD Matrix.\n\n        !!! info \"Gnomad's LD Matrix and Index are based on GRCh37 coordinates. This function will lift over the coordinates to GRCh38 to build the lookup table.\"\n\n        Args:\n            ld_index_raw (hl.Table): LD index table from GnomAD\n            grch37_to_grch38_chain_path (str): Path to the chain file used to lift over the coordinates\n\n        Returns:\n            DataFrame: Look up table between variants in build hg38 and their coordinates in the LD Matrix\n        \"\"\"\n        ld_index_38 = _liftover_loci(\n            ld_index_raw, grch37_to_grch38_chain_path, \"GRCh38\"\n        )\n\n        return (\n            ld_index_38.to_spark()\n            # Filter out variants where the liftover failed\n            .filter(f.col(\"`locus_GRCh38.position`\").isNotNull())\n            .withColumn(\n                \"chromosome\", f.regexp_replace(\"`locus_GRCh38.contig`\", \"chr\", \"\")\n            )\n            .withColumn(\n                \"position\",\n                convert_gnomad_position_to_ensembl(\n                    f.col(\"`locus_GRCh38.position`\"),\n                    f.col(\"`alleles`\").getItem(0),\n                    f.col(\"`alleles`\").getItem(1),\n                ),\n            )\n            .select(\n                \"chromosome\",\n                \"position\",\n                f.concat_ws(\n                    \"_\",\n                    f.col(\"chromosome\"),\n                    f.col(\"position\"),\n                    f.col(\"`alleles`\").getItem(0),\n                    f.col(\"`alleles`\").getItem(1),\n                ).alias(\"variantId\"),\n                f.col(\"idx\"),\n            )\n            # Filter out ambiguous liftover results: multiple indices for the same variant\n            .withColumn(\"count\", f.count(\"*\").over(Window.partitionBy([\"variantId\"])))\n            .filter(f.col(\"count\") == 1)\n            .drop(\"count\")\n        )\n\n    @staticmethod\n    def _resolve_variant_indices(\n        ld_index: DataFrame, ld_matrix: DataFrame\n    ) -&gt; DataFrame:\n        \"\"\"Resolve the `i` and `j` indices of the block matrix to variant IDs (build 38).\n\n        Args:\n            ld_index (DataFrame): Dataframe with resolved variant indices\n            ld_matrix (DataFrame): Dataframe with the filtered LD matrix\n\n        Returns:\n            DataFrame: Dataframe with variant IDs instead of `i` and `j` indices\n        \"\"\"\n        ld_index_i = ld_index.selectExpr(\n            \"idx as i\", \"variantId as variantId_i\", \"chromosome\"\n        )\n        ld_index_j = ld_index.selectExpr(\"idx as j\", \"variantId as variantId_j\")\n        return (\n            ld_matrix.join(ld_index_i, on=\"i\", how=\"inner\")\n            .join(ld_index_j, on=\"j\", how=\"inner\")\n            .drop(\"i\", \"j\")\n        )\n\n    @staticmethod\n    def _transpose_ld_matrix(ld_matrix: DataFrame) -&gt; DataFrame:\n        \"\"\"Transpose LD matrix to a square matrix format.\n\n        Args:\n            ld_matrix (DataFrame): Triangular LD matrix converted to a Spark DataFrame\n\n        Returns:\n            DataFrame: Square LD matrix without diagonal duplicates\n\n        Examples:\n            &gt;&gt;&gt; df = spark.createDataFrame(\n            ...     [\n            ...         (1, 1, 1.0, \"1\", \"AFR\"),\n            ...         (1, 2, 0.5, \"1\", \"AFR\"),\n            ...         (2, 2, 1.0, \"1\", \"AFR\"),\n            ...     ],\n            ...     [\"variantId_i\", \"variantId_j\", \"r\", \"chromosome\", \"population\"],\n            ... )\n            &gt;&gt;&gt; GnomADLDMatrix._transpose_ld_matrix(df).show()\n            +-----------+-----------+---+----------+----------+\n            |variantId_i|variantId_j|  r|chromosome|population|\n            +-----------+-----------+---+----------+----------+\n            |          1|          2|0.5|         1|       AFR|\n            |          1|          1|1.0|         1|       AFR|\n            |          2|          1|0.5|         1|       AFR|\n            |          2|          2|1.0|         1|       AFR|\n            +-----------+-----------+---+----------+----------+\n            &lt;BLANKLINE&gt;\n        \"\"\"\n        ld_matrix_transposed = ld_matrix.selectExpr(\n            \"variantId_i as variantId_j\",\n            \"variantId_j as variantId_i\",\n            \"r\",\n            \"chromosome\",\n            \"population\",\n        )\n        return ld_matrix.filter(\n            f.col(\"variantId_i\") != f.col(\"variantId_j\")\n        ).unionByName(ld_matrix_transposed)\n\n    def as_ld_index(\n        self: GnomADLDMatrix,\n        min_r2: float,\n    ) -&gt; LDIndex:\n        \"\"\"Create LDIndex dataset aggregating the LD information across a set of populations.\n\n        **The basic steps to generate the LDIndex are:**\n\n        1. Convert LD matrix to a Spark DataFrame.\n        2. Resolve the matrix indices to variant IDs by lifting over the coordinates to GRCh38.\n        3. Aggregate the LD information across populations.\n\n        Args:\n            min_r2 (float): Minimum r2 value to keep in the table\n\n        Returns:\n            LDIndex: LDIndex dataset\n        \"\"\"\n        ld_indices_unaggregated = []\n        for pop in self.ld_populations:\n            try:\n                ld_matrix_path = self.ld_matrix_template.format(POP=pop)\n                ld_index_raw_path = self.ld_index_raw_template.format(POP=pop)\n                pop_ld_index = self._create_ldindex_for_population(\n                    pop,\n                    ld_matrix_path,\n                    ld_index_raw_path.format(pop),\n                    self.grch37_to_grch38_chain_path,\n                    min_r2,\n                )\n                ld_indices_unaggregated.append(pop_ld_index)\n            except Exception as e:\n                print(f\"Failed to create LDIndex for population {pop}: {e}\")  # noqa: T201\n                sys.exit(1)\n\n        ld_index_unaggregated = (\n            GnomADLDMatrix._transpose_ld_matrix(\n                reduce(lambda df1, df2: df1.unionByName(df2), ld_indices_unaggregated)\n            )\n            .withColumnRenamed(\"variantId_i\", \"variantId\")\n            .withColumnRenamed(\"variantId_j\", \"tagVariantId\")\n        )\n        return LDIndex(\n            _df=self._aggregate_ld_index_across_populations(ld_index_unaggregated),\n            _schema=LDIndex.get_schema(),\n        )\n\n    def get_ld_variants(\n        self: GnomADLDMatrix,\n        gnomad_ancestry: str,\n        chromosome: str,\n        start: int,\n        end: int,\n    ) -&gt; DataFrame | None:\n        \"\"\"Return melted LD table with resolved variant id based on ancestry and genomic location.\n\n        Args:\n            gnomad_ancestry (str): GnomAD major ancestry label eg. `nfe`\n            chromosome (str): chromosome label\n            start (int): window upper bound\n            end (int): window lower bound\n\n        Returns:\n            DataFrame | None: LD table with resolved variant id based on ancestry and genomic location\n        \"\"\"\n        # Extracting locus:\n        ld_index_df = (\n            self._process_variant_indices(\n                hl.read_table(self.ld_index_raw_template.format(POP=gnomad_ancestry)),\n                self.grch37_to_grch38_chain_path,\n            )\n            .filter(\n                (f.col(\"chromosome\") == chromosome)\n                &amp; (f.col(\"position\") &gt;= start)\n                &amp; (f.col(\"position\") &lt;= end)\n            )\n            .select(\"chromosome\", \"position\", \"variantId\", \"idx\")\n            .persist()\n        )\n\n        if ld_index_df.limit(1).count() == 0:\n            # If the returned slice from the ld index is empty, return None\n            return None\n\n        # Compute start and end indices\n        start_index = get_value_from_row(\n            get_top_ranked_in_window(\n                ld_index_df, Window.partitionBy().orderBy(f.col(\"position\").asc())\n            ).collect()[0],\n            \"idx\",\n        )\n        end_index = get_value_from_row(\n            get_top_ranked_in_window(\n                ld_index_df, Window.partitionBy().orderBy(f.col(\"position\").desc())\n            ).collect()[0],\n            \"idx\",\n        )\n\n        return self._extract_square_matrix(\n            ld_index_df, gnomad_ancestry, start_index, end_index\n        )\n\n    def _extract_square_matrix(\n        self: GnomADLDMatrix,\n        ld_index_df: DataFrame,\n        gnomad_ancestry: str,\n        start_index: int,\n        end_index: int,\n    ) -&gt; DataFrame:\n        \"\"\"Return LD square matrix for a region where coordinates are normalised.\n\n        Args:\n            ld_index_df (DataFrame): Look up table between a variantId and its index in the LD matrix\n            gnomad_ancestry (str): GnomAD major ancestry label eg. `nfe`\n            start_index (int): start index of the slice\n            end_index (int): end index of the slice\n\n        Returns:\n            DataFrame: square LD matrix resolved to variants.\n        \"\"\"\n        return (\n            self.get_ld_matrix_slice(\n                gnomad_ancestry, start_index=start_index, end_index=end_index\n            )\n            .join(\n                ld_index_df.select(\n                    f.col(\"idx\").alias(\"idx_i\"),\n                    f.col(\"variantId\").alias(\"variantId_i\"),\n                ),\n                on=\"idx_i\",\n                how=\"inner\",\n            )\n            .join(\n                ld_index_df.select(\n                    f.col(\"idx\").alias(\"idx_j\"),\n                    f.col(\"variantId\").alias(\"variantId_j\"),\n                ),\n                on=\"idx_j\",\n                how=\"inner\",\n            )\n            .select(\"variantId_i\", \"variantId_j\", \"r\")\n        )\n\n    def get_ld_matrix_slice(\n        self: GnomADLDMatrix,\n        gnomad_ancestry: str,\n        start_index: int,\n        end_index: int,\n    ) -&gt; DataFrame:\n        \"\"\"Extract a slice of the LD matrix based on the provided ancestry and stop and end indices.\n\n        - The half matrix is completed into a full square.\n        - The returned indices are adjusted based on the start index.\n\n        Args:\n            gnomad_ancestry (str): LD population label eg. `nfe`\n            start_index (int): start index of the slice\n            end_index (int): end index of the slice\n\n        Returns:\n            DataFrame: square slice of the LD matrix melted as dataframe with idx_i, idx_j and r columns\n        \"\"\"\n        # Extracting block matrix slice:\n        half_matrix = BlockMatrix.read(\n            self.ld_matrix_template.format(POP=gnomad_ancestry)\n        ).filter(range(start_index, end_index + 1), range(start_index, end_index + 1))\n\n        # Return converted Dataframe:\n        return (\n            (half_matrix + half_matrix.T)\n            .entries()\n            .to_spark()\n            .select(\n                (f.col(\"i\") + start_index).alias(\"idx_i\"),\n                (f.col(\"j\") + start_index).alias(\"idx_j\"),\n                f.when(f.col(\"i\") == f.col(\"j\"), f.col(\"entry\") / 2)\n                .otherwise(f.col(\"entry\"))\n                .alias(\"r\"),\n            )\n        )\n</code></pre>"},{"location":"python_api/datasources/gnomad/gnomad_ld/#gentropy.datasource.gnomad.ld.GnomADLDMatrix.as_ld_index","title":"<code>as_ld_index(min_r2: float) -&gt; LDIndex</code>","text":"<p>Create LDIndex dataset aggregating the LD information across a set of populations.</p> <p>The basic steps to generate the LDIndex are:</p> <ol> <li>Convert LD matrix to a Spark DataFrame.</li> <li>Resolve the matrix indices to variant IDs by lifting over the coordinates to GRCh38.</li> <li>Aggregate the LD information across populations.</li> </ol> <p>Parameters:</p> Name Type Description Default <code>min_r2</code> <code>float</code> <p>Minimum r2 value to keep in the table</p> required <p>Returns:</p> Name Type Description <code>LDIndex</code> <code>LDIndex</code> <p>LDIndex dataset</p> Source code in <code>src/gentropy/datasource/gnomad/ld.py</code> <pre><code>def as_ld_index(\n    self: GnomADLDMatrix,\n    min_r2: float,\n) -&gt; LDIndex:\n    \"\"\"Create LDIndex dataset aggregating the LD information across a set of populations.\n\n    **The basic steps to generate the LDIndex are:**\n\n    1. Convert LD matrix to a Spark DataFrame.\n    2. Resolve the matrix indices to variant IDs by lifting over the coordinates to GRCh38.\n    3. Aggregate the LD information across populations.\n\n    Args:\n        min_r2 (float): Minimum r2 value to keep in the table\n\n    Returns:\n        LDIndex: LDIndex dataset\n    \"\"\"\n    ld_indices_unaggregated = []\n    for pop in self.ld_populations:\n        try:\n            ld_matrix_path = self.ld_matrix_template.format(POP=pop)\n            ld_index_raw_path = self.ld_index_raw_template.format(POP=pop)\n            pop_ld_index = self._create_ldindex_for_population(\n                pop,\n                ld_matrix_path,\n                ld_index_raw_path.format(pop),\n                self.grch37_to_grch38_chain_path,\n                min_r2,\n            )\n            ld_indices_unaggregated.append(pop_ld_index)\n        except Exception as e:\n            print(f\"Failed to create LDIndex for population {pop}: {e}\")  # noqa: T201\n            sys.exit(1)\n\n    ld_index_unaggregated = (\n        GnomADLDMatrix._transpose_ld_matrix(\n            reduce(lambda df1, df2: df1.unionByName(df2), ld_indices_unaggregated)\n        )\n        .withColumnRenamed(\"variantId_i\", \"variantId\")\n        .withColumnRenamed(\"variantId_j\", \"tagVariantId\")\n    )\n    return LDIndex(\n        _df=self._aggregate_ld_index_across_populations(ld_index_unaggregated),\n        _schema=LDIndex.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasources/gnomad/gnomad_ld/#gentropy.datasource.gnomad.ld.GnomADLDMatrix.get_ld_matrix_slice","title":"<code>get_ld_matrix_slice(gnomad_ancestry: str, start_index: int, end_index: int) -&gt; DataFrame</code>","text":"<p>Extract a slice of the LD matrix based on the provided ancestry and stop and end indices.</p> <ul> <li>The half matrix is completed into a full square.</li> <li>The returned indices are adjusted based on the start index.</li> </ul> <p>Parameters:</p> Name Type Description Default <code>gnomad_ancestry</code> <code>str</code> <p>LD population label eg. <code>nfe</code></p> required <code>start_index</code> <code>int</code> <p>start index of the slice</p> required <code>end_index</code> <code>int</code> <p>end index of the slice</p> required <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>square slice of the LD matrix melted as dataframe with idx_i, idx_j and r columns</p> Source code in <code>src/gentropy/datasource/gnomad/ld.py</code> <pre><code>def get_ld_matrix_slice(\n    self: GnomADLDMatrix,\n    gnomad_ancestry: str,\n    start_index: int,\n    end_index: int,\n) -&gt; DataFrame:\n    \"\"\"Extract a slice of the LD matrix based on the provided ancestry and stop and end indices.\n\n    - The half matrix is completed into a full square.\n    - The returned indices are adjusted based on the start index.\n\n    Args:\n        gnomad_ancestry (str): LD population label eg. `nfe`\n        start_index (int): start index of the slice\n        end_index (int): end index of the slice\n\n    Returns:\n        DataFrame: square slice of the LD matrix melted as dataframe with idx_i, idx_j and r columns\n    \"\"\"\n    # Extracting block matrix slice:\n    half_matrix = BlockMatrix.read(\n        self.ld_matrix_template.format(POP=gnomad_ancestry)\n    ).filter(range(start_index, end_index + 1), range(start_index, end_index + 1))\n\n    # Return converted Dataframe:\n    return (\n        (half_matrix + half_matrix.T)\n        .entries()\n        .to_spark()\n        .select(\n            (f.col(\"i\") + start_index).alias(\"idx_i\"),\n            (f.col(\"j\") + start_index).alias(\"idx_j\"),\n            f.when(f.col(\"i\") == f.col(\"j\"), f.col(\"entry\") / 2)\n            .otherwise(f.col(\"entry\"))\n            .alias(\"r\"),\n        )\n    )\n</code></pre>"},{"location":"python_api/datasources/gnomad/gnomad_ld/#gentropy.datasource.gnomad.ld.GnomADLDMatrix.get_ld_variants","title":"<code>get_ld_variants(gnomad_ancestry: str, chromosome: str, start: int, end: int) -&gt; DataFrame | None</code>","text":"<p>Return melted LD table with resolved variant id based on ancestry and genomic location.</p> <p>Parameters:</p> Name Type Description Default <code>gnomad_ancestry</code> <code>str</code> <p>GnomAD major ancestry label eg. <code>nfe</code></p> required <code>chromosome</code> <code>str</code> <p>chromosome label</p> required <code>start</code> <code>int</code> <p>window upper bound</p> required <code>end</code> <code>int</code> <p>window lower bound</p> required <p>Returns:</p> Type Description <code>DataFrame | None</code> <p>DataFrame | None: LD table with resolved variant id based on ancestry and genomic location</p> Source code in <code>src/gentropy/datasource/gnomad/ld.py</code> <pre><code>def get_ld_variants(\n    self: GnomADLDMatrix,\n    gnomad_ancestry: str,\n    chromosome: str,\n    start: int,\n    end: int,\n) -&gt; DataFrame | None:\n    \"\"\"Return melted LD table with resolved variant id based on ancestry and genomic location.\n\n    Args:\n        gnomad_ancestry (str): GnomAD major ancestry label eg. `nfe`\n        chromosome (str): chromosome label\n        start (int): window upper bound\n        end (int): window lower bound\n\n    Returns:\n        DataFrame | None: LD table with resolved variant id based on ancestry and genomic location\n    \"\"\"\n    # Extracting locus:\n    ld_index_df = (\n        self._process_variant_indices(\n            hl.read_table(self.ld_index_raw_template.format(POP=gnomad_ancestry)),\n            self.grch37_to_grch38_chain_path,\n        )\n        .filter(\n            (f.col(\"chromosome\") == chromosome)\n            &amp; (f.col(\"position\") &gt;= start)\n            &amp; (f.col(\"position\") &lt;= end)\n        )\n        .select(\"chromosome\", \"position\", \"variantId\", \"idx\")\n        .persist()\n    )\n\n    if ld_index_df.limit(1).count() == 0:\n        # If the returned slice from the ld index is empty, return None\n        return None\n\n    # Compute start and end indices\n    start_index = get_value_from_row(\n        get_top_ranked_in_window(\n            ld_index_df, Window.partitionBy().orderBy(f.col(\"position\").asc())\n        ).collect()[0],\n        \"idx\",\n    )\n    end_index = get_value_from_row(\n        get_top_ranked_in_window(\n            ld_index_df, Window.partitionBy().orderBy(f.col(\"position\").desc())\n        ).collect()[0],\n        \"idx\",\n    )\n\n    return self._extract_square_matrix(\n        ld_index_df, gnomad_ancestry, start_index, end_index\n    )\n</code></pre>"},{"location":"python_api/datasources/gnomad/gnomad_variants/","title":"Variants","text":""},{"location":"python_api/datasources/gnomad/gnomad_variants/#gentropy.datasource.gnomad.variants.GnomADVariants","title":"<code>gentropy.datasource.gnomad.variants.GnomADVariants</code>  <code>dataclass</code>","text":"<p>GnomAD variants included in the GnomAD genomes dataset.</p> <p>Attributes:</p> Name Type Description <code>gnomad_genomes</code> <code>str</code> <p>Path to gnomAD genomes hail table. Defaults to gnomAD's 4.0 release.</p> <code>chain_hail_38_37</code> <code>str</code> <p>Path to GRCh38 to GRCh37 chain file. Defaults to Hail's chain file.</p> <code>populations</code> <code>list[str]</code> <p>List of populations to include. Defaults to all populations.</p> Source code in <code>src/gentropy/datasource/gnomad/variants.py</code> <pre><code>@dataclass\nclass GnomADVariants:\n    \"\"\"GnomAD variants included in the GnomAD genomes dataset.\n\n    Attributes:\n        gnomad_genomes (str): Path to gnomAD genomes hail table. Defaults to gnomAD's 4.0 release.\n        chain_hail_38_37 (str): Path to GRCh38 to GRCh37 chain file. Defaults to Hail's chain file.\n        populations (list[str]): List of populations to include. Defaults to all populations.\n    \"\"\"\n\n    gnomad_genomes: str = \"gs://gcp-public-data--gnomad/release/4.0/ht/genomes/gnomad.genomes.v4.0.sites.ht/\"\n    chain_hail_38_37: str = \"gs://hail-common/references/grch38_to_grch37.over.chain.gz\"\n    populations: list[str] = field(\n        default_factory=lambda: [\n            \"afr\",  # African-American\n            \"amr\",  # American Admixed/Latino\n            \"ami\",  # Amish ancestry\n            \"asj\",  # Ashkenazi Jewish\n            \"eas\",  # East Asian\n            \"fin\",  # Finnish\n            \"nfe\",  # Non-Finnish European\n            \"mid\",  # Middle Eastern\n            \"sas\",  # South Asian\n            \"remaining\",  # Other\n        ]\n    )\n\n    @staticmethod\n    def _convert_gnomad_position_to_ensembl_hail(\n        position: Int32Expression,\n        reference: StringExpression,\n        alternate: StringExpression,\n    ) -&gt; Int32Expression:\n        \"\"\"Convert GnomAD variant position to Ensembl variant position in hail table.\n\n        For indels (the reference or alternate allele is longer than 1), then adding 1 to the position, for SNPs, the position is unchanged.\n        More info about the problem: https://www.biostars.org/p/84686/\n\n        Args:\n            position (Int32Expression): Position of the variant in the GnomAD genome.\n            reference (StringExpression): The reference allele.\n            alternate (StringExpression): The alternate allele\n\n        Returns:\n            Int32Expression: The position of the variant according to Ensembl genome.\n        \"\"\"\n        return hl.if_else(\n            (reference.length() &gt; 1) | (alternate.length() &gt; 1), position + 1, position\n        )\n\n    def as_variant_annotation(self: GnomADVariants) -&gt; VariantAnnotation:\n        \"\"\"Generate variant annotation dataset from gnomAD.\n\n        Some relevant modifications to the original dataset are:\n\n        1. The transcript consequences features provided by VEP are filtered to only refer to the Ensembl canonical transcript.\n        2. Genome coordinates are liftovered from GRCh38 to GRCh37 to keep as annotation.\n        3. Field names are converted to camel case to follow the convention.\n\n        Returns:\n            VariantAnnotation: Variant annotation dataset\n        \"\"\"\n        # Load variants dataset\n        ht = hl.read_table(\n            self.gnomad_genomes,\n            _load_refs=False,\n        )\n\n        # Liftover\n        grch37 = hl.get_reference(\"GRCh37\")\n        grch38 = hl.get_reference(\"GRCh38\")\n        grch38.add_liftover(self.chain_hail_38_37, grch37)\n\n        # Drop non biallelic variants\n        ht = ht.filter(ht.alleles.length() == 2)\n        # Liftover\n        ht = ht.annotate(locus_GRCh37=hl.liftover(ht.locus, \"GRCh37\"))\n        # Select relevant fields and nested records to create class\n        return VariantAnnotation(\n            _df=(\n                ht.select(\n                    gnomadVariantId=hl.str(\"-\").join(\n                        [\n                            ht.locus.contig.replace(\"chr\", \"\"),\n                            hl.str(ht.locus.position),\n                            ht.alleles[0],\n                            ht.alleles[1],\n                        ]\n                    ),\n                    chromosome=ht.locus.contig.replace(\"chr\", \"\"),\n                    position=GnomADVariants._convert_gnomad_position_to_ensembl_hail(\n                        ht.locus.position, ht.alleles[0], ht.alleles[1]\n                    ),\n                    variantId=hl.str(\"_\").join(\n                        [\n                            ht.locus.contig.replace(\"chr\", \"\"),\n                            hl.str(\n                                GnomADVariants._convert_gnomad_position_to_ensembl_hail(\n                                    ht.locus.position, ht.alleles[0], ht.alleles[1]\n                                )\n                            ),\n                            ht.alleles[0],\n                            ht.alleles[1],\n                        ]\n                    ),\n                    chromosomeB37=ht.locus_GRCh37.contig.replace(\"chr\", \"\"),\n                    positionB37=ht.locus_GRCh37.position,\n                    referenceAllele=ht.alleles[0],\n                    alternateAllele=ht.alleles[1],\n                    rsIds=ht.rsid,\n                    alleleType=ht.allele_info.allele_type,\n                    alleleFrequencies=hl.set(\n                        [f\"{pop}_adj\" for pop in self.populations]\n                    ).map(\n                        lambda p: hl.struct(\n                            populationName=p,\n                            alleleFrequency=ht.freq[ht.globals.freq_index_dict[p]].AF,\n                        )\n                    ),\n                    vep=hl.struct(\n                        mostSevereConsequence=ht.vep.most_severe_consequence,\n                        transcriptConsequences=hl.map(\n                            lambda x: hl.struct(\n                                aminoAcids=x.amino_acids,\n                                consequenceTerms=x.consequence_terms,\n                                geneId=x.gene_id,\n                                lof=x.lof,\n                            ),\n                            # Only keeping canonical transcripts\n                            ht.vep.transcript_consequences.filter(\n                                lambda x: (x.canonical == 1)\n                                &amp; (x.gene_symbol_source == \"HGNC\")\n                            ),\n                        ),\n                    ),\n                    inSilicoPredictors=hl.struct(\n                        cadd=hl.struct(\n                            phred=ht.in_silico_predictors.cadd.phred,\n                            raw=ht.in_silico_predictors.cadd.raw_score,\n                        ),\n                        revelMax=ht.in_silico_predictors.revel_max,\n                        spliceaiDsMax=ht.in_silico_predictors.spliceai_ds_max,\n                        pangolinLargestDs=ht.in_silico_predictors.pangolin_largest_ds,\n                        phylop=ht.in_silico_predictors.phylop,\n                        siftMax=ht.in_silico_predictors.sift_max,\n                        polyphenMax=ht.in_silico_predictors.polyphen_max,\n                    ),\n                )\n                .key_by(\"chromosome\", \"position\")\n                .drop(\"locus\", \"alleles\")\n                .select_globals()\n                .to_spark(flatten=False)\n            ),\n            _schema=VariantAnnotation.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasources/gnomad/gnomad_variants/#gentropy.datasource.gnomad.variants.GnomADVariants.as_variant_annotation","title":"<code>as_variant_annotation() -&gt; VariantAnnotation</code>","text":"<p>Generate variant annotation dataset from gnomAD.</p> <p>Some relevant modifications to the original dataset are:</p> <ol> <li>The transcript consequences features provided by VEP are filtered to only refer to the Ensembl canonical transcript.</li> <li>Genome coordinates are liftovered from GRCh38 to GRCh37 to keep as annotation.</li> <li>Field names are converted to camel case to follow the convention.</li> </ol> <p>Returns:</p> Name Type Description <code>VariantAnnotation</code> <code>VariantAnnotation</code> <p>Variant annotation dataset</p> Source code in <code>src/gentropy/datasource/gnomad/variants.py</code> <pre><code>def as_variant_annotation(self: GnomADVariants) -&gt; VariantAnnotation:\n    \"\"\"Generate variant annotation dataset from gnomAD.\n\n    Some relevant modifications to the original dataset are:\n\n    1. The transcript consequences features provided by VEP are filtered to only refer to the Ensembl canonical transcript.\n    2. Genome coordinates are liftovered from GRCh38 to GRCh37 to keep as annotation.\n    3. Field names are converted to camel case to follow the convention.\n\n    Returns:\n        VariantAnnotation: Variant annotation dataset\n    \"\"\"\n    # Load variants dataset\n    ht = hl.read_table(\n        self.gnomad_genomes,\n        _load_refs=False,\n    )\n\n    # Liftover\n    grch37 = hl.get_reference(\"GRCh37\")\n    grch38 = hl.get_reference(\"GRCh38\")\n    grch38.add_liftover(self.chain_hail_38_37, grch37)\n\n    # Drop non biallelic variants\n    ht = ht.filter(ht.alleles.length() == 2)\n    # Liftover\n    ht = ht.annotate(locus_GRCh37=hl.liftover(ht.locus, \"GRCh37\"))\n    # Select relevant fields and nested records to create class\n    return VariantAnnotation(\n        _df=(\n            ht.select(\n                gnomadVariantId=hl.str(\"-\").join(\n                    [\n                        ht.locus.contig.replace(\"chr\", \"\"),\n                        hl.str(ht.locus.position),\n                        ht.alleles[0],\n                        ht.alleles[1],\n                    ]\n                ),\n                chromosome=ht.locus.contig.replace(\"chr\", \"\"),\n                position=GnomADVariants._convert_gnomad_position_to_ensembl_hail(\n                    ht.locus.position, ht.alleles[0], ht.alleles[1]\n                ),\n                variantId=hl.str(\"_\").join(\n                    [\n                        ht.locus.contig.replace(\"chr\", \"\"),\n                        hl.str(\n                            GnomADVariants._convert_gnomad_position_to_ensembl_hail(\n                                ht.locus.position, ht.alleles[0], ht.alleles[1]\n                            )\n                        ),\n                        ht.alleles[0],\n                        ht.alleles[1],\n                    ]\n                ),\n                chromosomeB37=ht.locus_GRCh37.contig.replace(\"chr\", \"\"),\n                positionB37=ht.locus_GRCh37.position,\n                referenceAllele=ht.alleles[0],\n                alternateAllele=ht.alleles[1],\n                rsIds=ht.rsid,\n                alleleType=ht.allele_info.allele_type,\n                alleleFrequencies=hl.set(\n                    [f\"{pop}_adj\" for pop in self.populations]\n                ).map(\n                    lambda p: hl.struct(\n                        populationName=p,\n                        alleleFrequency=ht.freq[ht.globals.freq_index_dict[p]].AF,\n                    )\n                ),\n                vep=hl.struct(\n                    mostSevereConsequence=ht.vep.most_severe_consequence,\n                    transcriptConsequences=hl.map(\n                        lambda x: hl.struct(\n                            aminoAcids=x.amino_acids,\n                            consequenceTerms=x.consequence_terms,\n                            geneId=x.gene_id,\n                            lof=x.lof,\n                        ),\n                        # Only keeping canonical transcripts\n                        ht.vep.transcript_consequences.filter(\n                            lambda x: (x.canonical == 1)\n                            &amp; (x.gene_symbol_source == \"HGNC\")\n                        ),\n                    ),\n                ),\n                inSilicoPredictors=hl.struct(\n                    cadd=hl.struct(\n                        phred=ht.in_silico_predictors.cadd.phred,\n                        raw=ht.in_silico_predictors.cadd.raw_score,\n                    ),\n                    revelMax=ht.in_silico_predictors.revel_max,\n                    spliceaiDsMax=ht.in_silico_predictors.spliceai_ds_max,\n                    pangolinLargestDs=ht.in_silico_predictors.pangolin_largest_ds,\n                    phylop=ht.in_silico_predictors.phylop,\n                    siftMax=ht.in_silico_predictors.sift_max,\n                    polyphenMax=ht.in_silico_predictors.polyphen_max,\n                ),\n            )\n            .key_by(\"chromosome\", \"position\")\n            .drop(\"locus\", \"alleles\")\n            .select_globals()\n            .to_spark(flatten=False)\n        ),\n        _schema=VariantAnnotation.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/_gwas_catalog/","title":"GWAS Catalog","text":"GWAS Catalog <p>The GWAS Catalog is a comprehensive resource that aims to provide a curated collection of Genome-Wide Association Studies (GWAS) (including harmonized full GWAS summary statistics) across various traits and diseases in humans.</p> <p>It serves as a valuable repository of genetic associations identified in diverse populations, offering insights into the genetic basis of complex traits and diseases.</p> <p>We rely on the GWAS Catalog for a rich source of genetic associations, utilizing the data for analysis and interpretation.</p> <p>For detailed information on specific genetic associations, their significance, and associated studies, refer to the GWAS Catalog.</p> <p>Within our analyses, we leverage two different types of studies from the GWAS Catalog:</p> <ol> <li> <p>Studies with (full) GWAS summary stats</p> </li> <li> <p>Studies with top hits only - GWAS curated studies</p> </li> </ol>"},{"location":"python_api/datasources/gwas_catalog/associations/","title":"Associations","text":""},{"location":"python_api/datasources/gwas_catalog/associations/#gentropy.datasource.gwas_catalog.associations.GWASCatalogCuratedAssociationsParser","title":"<code>gentropy.datasource.gwas_catalog.associations.GWASCatalogCuratedAssociationsParser</code>  <code>dataclass</code>","text":"<p>GWAS Catalog curated associations parser.</p> Source code in <code>src/gentropy/datasource/gwas_catalog/associations.py</code> <pre><code>@dataclass\nclass GWASCatalogCuratedAssociationsParser:\n    \"\"\"GWAS Catalog curated associations parser.\"\"\"\n\n    @staticmethod\n    def _parse_pvalue(pvalue: Column) -&gt; tuple[Column, Column]:\n        \"\"\"Parse p-value column.\n\n        Args:\n            pvalue (Column): p-value [string]\n\n        Returns:\n            tuple[Column, Column]: p-value mantissa and exponent\n\n        Example:\n            &gt;&gt;&gt; import pyspark.sql.types as t\n            &gt;&gt;&gt; d = [(\"1.0\"), (\"0.5\"), (\"1E-20\"), (\"3E-3\"), (\"1E-1000\")]\n            &gt;&gt;&gt; df = spark.createDataFrame(d, t.StringType())\n            &gt;&gt;&gt; df.select('value',*GWASCatalogCuratedAssociationsParser._parse_pvalue(f.col('value'))).show()\n            +-------+--------------+--------------+\n            |  value|pValueMantissa|pValueExponent|\n            +-------+--------------+--------------+\n            |    1.0|           1.0|             1|\n            |    0.5|           0.5|             1|\n            |  1E-20|           1.0|           -20|\n            |   3E-3|           3.0|            -3|\n            |1E-1000|           1.0|         -1000|\n            +-------+--------------+--------------+\n            &lt;BLANKLINE&gt;\n\n        \"\"\"\n        split = f.split(pvalue, \"E\")\n        return split.getItem(0).cast(\"float\").alias(\"pValueMantissa\"), f.coalesce(\n            split.getItem(1).cast(\"integer\"), f.lit(1)\n        ).alias(\"pValueExponent\")\n\n    @staticmethod\n    def _normalise_pvaluetext(p_value_text: Column) -&gt; Column:\n        \"\"\"Normalised p-value text column to a standardised format.\n\n        For cases where there is no mapping, the value is set to null.\n\n        Args:\n            p_value_text (Column): `pValueText` column from GWASCatalog\n\n        Returns:\n            Column: Array column after using GWAS Catalog mappings. There might be multiple mappings for a single p-value text.\n\n        Example:\n            &gt;&gt;&gt; import pyspark.sql.types as t\n            &gt;&gt;&gt; d = [(\"European Ancestry\"), (\"African ancestry\"), (\"Alzheimer\u2019s Disease\"), (\"(progression)\"), (\"\"), (None)]\n            &gt;&gt;&gt; df = spark.createDataFrame(d, t.StringType())\n            &gt;&gt;&gt; df.withColumn('normalised', GWASCatalogCuratedAssociationsParser._normalise_pvaluetext(f.col('value'))).show()\n            +-------------------+----------+\n            |              value|normalised|\n            +-------------------+----------+\n            |  European Ancestry|      [EA]|\n            |   African ancestry|      [AA]|\n            |Alzheimer\u2019s Disease|      [AD]|\n            |      (progression)|      null|\n            |                   |      null|\n            |               null|      null|\n            +-------------------+----------+\n            &lt;BLANKLINE&gt;\n\n        \"\"\"\n        # GWAS Catalog to p-value mapping\n        json_dict = json.loads(\n            pkg_resources.read_text(data, \"gwas_pValueText_map.json\", encoding=\"utf-8\")\n        )\n        map_expr = f.create_map(*[f.lit(x) for x in chain(*json_dict.items())])\n\n        splitted_col = f.split(f.regexp_replace(p_value_text, r\"[\\(\\)]\", \"\"), \",\")\n        mapped_col = f.transform(splitted_col, lambda x: map_expr[x])\n        return f.when(f.forall(mapped_col, lambda x: x.isNull()), None).otherwise(\n            mapped_col\n        )\n\n    @staticmethod\n    def _normalise_risk_allele(risk_allele: Column) -&gt; Column:\n        \"\"\"Normalised risk allele column to a standardised format.\n\n        If multiple risk alleles are present, the first one is returned.\n\n        Args:\n            risk_allele (Column): `riskAllele` column from GWASCatalog\n\n        Returns:\n            Column: mapped using GWAS Catalog mapping\n\n        Example:\n            &gt;&gt;&gt; import pyspark.sql.types as t\n            &gt;&gt;&gt; d = [(\"rs1234-A-G\"), (\"rs1234-A\"), (\"rs1234-A; rs1235-G\")]\n            &gt;&gt;&gt; df = spark.createDataFrame(d, t.StringType())\n            &gt;&gt;&gt; df.withColumn('normalised', GWASCatalogCuratedAssociationsParser._normalise_risk_allele(f.col('value'))).show()\n            +------------------+----------+\n            |             value|normalised|\n            +------------------+----------+\n            |        rs1234-A-G|         A|\n            |          rs1234-A|         A|\n            |rs1234-A; rs1235-G|         A|\n            +------------------+----------+\n            &lt;BLANKLINE&gt;\n\n        \"\"\"\n        # GWAS Catalog to risk allele mapping\n        return f.split(f.split(risk_allele, \"; \").getItem(0), \"-\").getItem(1)\n\n    @staticmethod\n    def _collect_rsids(\n        snp_id: Column, snp_id_current: Column, risk_allele: Column\n    ) -&gt; Column:\n        \"\"\"It takes three columns, and returns an array of distinct values from those columns.\n\n        Args:\n            snp_id (Column): The original snp id from the GWAS catalog.\n            snp_id_current (Column): The current snp id field is just a number at the moment (stored as a string). Adding 'rs' prefix if looks good.\n            risk_allele (Column): The risk allele for the SNP.\n\n        Returns:\n            Column: An array of distinct values.\n        \"\"\"\n        # The current snp id field is just a number at the moment (stored as a string). Adding 'rs' prefix if looks good.\n        snp_id_current = f.when(\n            snp_id_current.rlike(\"^[0-9]*$\"),\n            f.format_string(\"rs%s\", snp_id_current),\n        )\n        # Cleaning risk allele:\n        risk_allele = f.split(risk_allele, \"-\").getItem(0)\n\n        # Collecting all values:\n        return f.array_distinct(f.array(snp_id, snp_id_current, risk_allele))\n\n    @staticmethod\n    def _map_to_variant_annotation_variants(\n        gwas_associations: DataFrame, variant_annotation: VariantAnnotation\n    ) -&gt; DataFrame:\n        \"\"\"Add variant metadata in associations.\n\n        Args:\n            gwas_associations (DataFrame): raw GWAS Catalog associations\n            variant_annotation (VariantAnnotation): variant annotation dataset\n\n        Returns:\n            DataFrame: GWAS Catalog associations data including `variantId`, `referenceAllele`,\n            `alternateAllele`, `chromosome`, `position` with variant metadata\n        \"\"\"\n        # Subset of GWAS Catalog associations required for resolving variant IDs:\n        gwas_associations_subset = gwas_associations.select(\n            \"studyLocusId\",\n            f.col(\"CHR_ID\").alias(\"chromosome\"),\n            f.col(\"CHR_POS\").cast(IntegerType()).alias(\"position\"),\n            # List of all SNPs associated with the variant\n            GWASCatalogCuratedAssociationsParser._collect_rsids(\n                f.split(f.col(\"SNPS\"), \"; \").getItem(0),\n                f.col(\"SNP_ID_CURRENT\"),\n                f.split(f.col(\"STRONGEST SNP-RISK ALLELE\"), \"; \").getItem(0),\n            ).alias(\"rsIdsGwasCatalog\"),\n            GWASCatalogCuratedAssociationsParser._normalise_risk_allele(\n                f.col(\"STRONGEST SNP-RISK ALLELE\")\n            ).alias(\"riskAllele\"),\n        )\n\n        # Subset of variant annotation required for GWAS Catalog annotations:\n        va_subset = variant_annotation.df.select(\n            \"variantId\",\n            \"chromosome\",\n            \"position\",\n            f.col(\"rsIds\").alias(\"rsIdsGnomad\"),\n            \"referenceAllele\",\n            \"alternateAllele\",\n            \"alleleFrequencies\",\n            variant_annotation.max_maf().alias(\"maxMaf\"),\n        ).join(\n            f.broadcast(\n                gwas_associations_subset.select(\"chromosome\", \"position\").distinct()\n            ),\n            on=[\"chromosome\", \"position\"],\n            how=\"inner\",\n        )\n\n        # Semi-resolved ids (still contains duplicates when conclusion was not possible to make\n        # based on rsIds or allele concordance)\n        filtered_associations = (\n            gwas_associations_subset.join(\n                f.broadcast(va_subset),\n                on=[\"chromosome\", \"position\"],\n                how=\"left\",\n            )\n            .withColumn(\n                \"rsIdFilter\",\n                GWASCatalogCuratedAssociationsParser._flag_mappings_to_retain(\n                    f.col(\"studyLocusId\"),\n                    GWASCatalogCuratedAssociationsParser._compare_rsids(\n                        f.col(\"rsIdsGnomad\"), f.col(\"rsIdsGwasCatalog\")\n                    ),\n                ),\n            )\n            .withColumn(\n                \"concordanceFilter\",\n                GWASCatalogCuratedAssociationsParser._flag_mappings_to_retain(\n                    f.col(\"studyLocusId\"),\n                    GWASCatalogCuratedAssociationsParser._check_concordance(\n                        f.col(\"riskAllele\"),\n                        f.col(\"referenceAllele\"),\n                        f.col(\"alternateAllele\"),\n                    ),\n                ),\n            )\n            .filter(\n                # Filter out rows where GWAS Catalog rsId does not match with GnomAD rsId,\n                # but there is corresponding variant for the same association\n                f.col(\"rsIdFilter\")\n                # or filter out rows where GWAS Catalog alleles are not concordant with GnomAD alleles,\n                # but there is corresponding variant for the same association\n                | f.col(\"concordanceFilter\")\n            )\n        )\n\n        # Keep only highest maxMaf variant per studyLocusId\n        fully_mapped_associations = get_record_with_maximum_value(\n            filtered_associations, grouping_col=\"studyLocusId\", sorting_col=\"maxMaf\"\n        ).select(\n            \"studyLocusId\",\n            \"variantId\",\n            \"referenceAllele\",\n            \"alternateAllele\",\n            \"chromosome\",\n            \"position\",\n        )\n\n        return gwas_associations.join(\n            fully_mapped_associations, on=\"studyLocusId\", how=\"left\"\n        )\n\n    @staticmethod\n    def _compare_rsids(gnomad: Column, gwas: Column) -&gt; Column:\n        \"\"\"If the intersection of the two arrays is greater than 0, return True, otherwise return False.\n\n        Args:\n            gnomad (Column): rsids from gnomad\n            gwas (Column): rsids from the GWAS Catalog\n\n        Returns:\n            Column: A boolean column that is true if the GnomAD rsIDs can be found in the GWAS rsIDs.\n\n        Examples:\n            &gt;&gt;&gt; d = [\n            ...    (1, [\"rs123\", \"rs523\"], [\"rs123\"]),\n            ...    (2, [], [\"rs123\"]),\n            ...    (3, [\"rs123\", \"rs523\"], []),\n            ...    (4, [], []),\n            ... ]\n            &gt;&gt;&gt; df = spark.createDataFrame(d, ['associationId', 'gnomad', 'gwas'])\n            &gt;&gt;&gt; df.withColumn(\"rsid_matches\", GWASCatalogCuratedAssociationsParser._compare_rsids(f.col(\"gnomad\"),f.col('gwas'))).show()\n            +-------------+--------------+-------+------------+\n            |associationId|        gnomad|   gwas|rsid_matches|\n            +-------------+--------------+-------+------------+\n            |            1|[rs123, rs523]|[rs123]|        true|\n            |            2|            []|[rs123]|       false|\n            |            3|[rs123, rs523]|     []|       false|\n            |            4|            []|     []|       false|\n            +-------------+--------------+-------+------------+\n            &lt;BLANKLINE&gt;\n\n        \"\"\"\n        return f.when(f.size(f.array_intersect(gnomad, gwas)) &gt; 0, True).otherwise(\n            False\n        )\n\n    @staticmethod\n    def _flag_mappings_to_retain(\n        association_id: Column, filter_column: Column\n    ) -&gt; Column:\n        \"\"\"Flagging mappings to drop for each association.\n\n        Some associations have multiple mappings. Some has matching rsId others don't. We only\n        want to drop the non-matching mappings, when a matching is available for the given association.\n        This logic can be generalised for other measures eg. allele concordance.\n\n        Args:\n            association_id (Column): association identifier column\n            filter_column (Column): boolean col indicating to keep a mapping\n\n        Returns:\n            Column: A column with a boolean value.\n\n        Examples:\n        &gt;&gt;&gt; d = [\n        ...    (1, False),\n        ...    (1, False),\n        ...    (2, False),\n        ...    (2, True),\n        ...    (3, True),\n        ...    (3, True),\n        ... ]\n        &gt;&gt;&gt; df = spark.createDataFrame(d, ['associationId', 'filter'])\n        &gt;&gt;&gt; df.withColumn(\"isConcordant\", GWASCatalogCuratedAssociationsParser._flag_mappings_to_retain(f.col(\"associationId\"),f.col('filter'))).show()\n        +-------------+------+------------+\n        |associationId|filter|isConcordant|\n        +-------------+------+------------+\n        |            1| false|        true|\n        |            1| false|        true|\n        |            2| false|       false|\n        |            2|  true|        true|\n        |            3|  true|        true|\n        |            3|  true|        true|\n        +-------------+------+------------+\n        &lt;BLANKLINE&gt;\n\n        \"\"\"\n        w = Window.partitionBy(association_id)\n\n        # Generating a boolean column informing if the filter column contains true anywhere for the association:\n        aggregated_filter = f.when(\n            f.array_contains(f.collect_set(filter_column).over(w), True), True\n        ).otherwise(False)\n\n        # Generate a filter column:\n        return f.when(aggregated_filter &amp; (~filter_column), False).otherwise(True)\n\n    @staticmethod\n    def _check_concordance(\n        risk_allele: Column, reference_allele: Column, alternate_allele: Column\n    ) -&gt; Column:\n        \"\"\"A function to check if the risk allele is concordant with the alt or ref allele.\n\n        If the risk allele is the same as the reference or alternate allele, or if the reverse complement of\n        the risk allele is the same as the reference or alternate allele, then the allele is concordant.\n        If no mapping is available (ref/alt is null), the function returns True.\n\n        Args:\n            risk_allele (Column): The allele that is associated with the risk of the disease.\n            reference_allele (Column): The reference allele from the GWAS catalog\n            alternate_allele (Column): The alternate allele of the variant.\n\n        Returns:\n            Column: A boolean column that is True if the risk allele is the same as the reference or alternate allele,\n            or if the reverse complement of the risk allele is the same as the reference or alternate allele.\n\n        Examples:\n            &gt;&gt;&gt; d = [\n            ...     ('A', 'A', 'G'),\n            ...     ('A', 'T', 'G'),\n            ...     ('A', 'C', 'G'),\n            ...     ('A', 'A', '?'),\n            ...     (None, None, 'A'),\n            ... ]\n            &gt;&gt;&gt; df = spark.createDataFrame(d, ['riskAllele', 'referenceAllele', 'alternateAllele'])\n            &gt;&gt;&gt; df.withColumn(\"isConcordant\", GWASCatalogCuratedAssociationsParser._check_concordance(f.col(\"riskAllele\"),f.col('referenceAllele'), f.col('alternateAllele'))).show()\n            +----------+---------------+---------------+------------+\n            |riskAllele|referenceAllele|alternateAllele|isConcordant|\n            +----------+---------------+---------------+------------+\n            |         A|              A|              G|        true|\n            |         A|              T|              G|        true|\n            |         A|              C|              G|       false|\n            |         A|              A|              ?|        true|\n            |      null|           null|              A|        true|\n            +----------+---------------+---------------+------------+\n            &lt;BLANKLINE&gt;\n\n        \"\"\"\n        # Calculating the reverse complement of the risk allele:\n        risk_allele_reverse_complement = f.when(\n            risk_allele.rlike(r\"^[ACTG]+$\"),\n            f.reverse(f.translate(risk_allele, \"ACTG\", \"TGAC\")),\n        ).otherwise(risk_allele)\n\n        # OK, is the risk allele or the reverse complent is the same as the mapped alleles:\n        return (\n            f.when(\n                (risk_allele == reference_allele) | (risk_allele == alternate_allele),\n                True,\n            )\n            # If risk allele is found on the negative strand:\n            .when(\n                (risk_allele_reverse_complement == reference_allele)\n                | (risk_allele_reverse_complement == alternate_allele),\n                True,\n            )\n            # If risk allele is ambiguous, still accepted: &lt; This condition could be reconsidered\n            .when(risk_allele == \"?\", True)\n            # If the association could not be mapped we keep it:\n            .when(reference_allele.isNull(), True)\n            # Allele is discordant:\n            .otherwise(False)\n        )\n\n    @staticmethod\n    def _get_reverse_complement(allele_col: Column) -&gt; Column:\n        \"\"\"A function to return the reverse complement of an allele column.\n\n        It takes a string and returns the reverse complement of that string if it's a DNA sequence,\n        otherwise it returns the original string. Assumes alleles in upper case.\n\n        Args:\n            allele_col (Column): The column containing the allele to reverse complement.\n\n        Returns:\n            Column: A column that is the reverse complement of the allele column.\n\n        Examples:\n            &gt;&gt;&gt; d = [{\"allele\": 'A'}, {\"allele\": 'T'},{\"allele\": 'G'}, {\"allele\": 'C'},{\"allele\": 'AC'}, {\"allele\": 'GTaatc'},{\"allele\": '?'}, {\"allele\": None}]\n            &gt;&gt;&gt; df = spark.createDataFrame(d)\n            &gt;&gt;&gt; df.withColumn(\"revcom_allele\", GWASCatalogCuratedAssociationsParser._get_reverse_complement(f.col(\"allele\"))).show()\n            +------+-------------+\n            |allele|revcom_allele|\n            +------+-------------+\n            |     A|            T|\n            |     T|            A|\n            |     G|            C|\n            |     C|            G|\n            |    AC|           GT|\n            |GTaatc|       GATTAC|\n            |     ?|            ?|\n            |  null|         null|\n            +------+-------------+\n            &lt;BLANKLINE&gt;\n\n        \"\"\"\n        allele_col = f.upper(allele_col)\n        return f.when(\n            allele_col.rlike(\"[ACTG]+\"),\n            f.reverse(f.translate(allele_col, \"ACTG\", \"TGAC\")),\n        ).otherwise(allele_col)\n\n    @staticmethod\n    def _effect_needs_harmonisation(\n        risk_allele: Column, reference_allele: Column\n    ) -&gt; Column:\n        \"\"\"A function to check if the effect allele needs to be harmonised.\n\n        Args:\n            risk_allele (Column): Risk allele column\n            reference_allele (Column): Effect allele column\n\n        Returns:\n            Column: A boolean column indicating if the effect allele needs to be harmonised.\n\n        Examples:\n            &gt;&gt;&gt; d = [{\"risk\": 'A', \"reference\": 'A'}, {\"risk\": 'A', \"reference\": 'T'}, {\"risk\": 'AT', \"reference\": 'TA'}, {\"risk\": 'AT', \"reference\": 'AT'}]\n            &gt;&gt;&gt; df = spark.createDataFrame(d)\n            &gt;&gt;&gt; df.withColumn(\"needs_harmonisation\", GWASCatalogCuratedAssociationsParser._effect_needs_harmonisation(f.col(\"risk\"), f.col(\"reference\"))).show()\n            +---------+----+-------------------+\n            |reference|risk|needs_harmonisation|\n            +---------+----+-------------------+\n            |        A|   A|               true|\n            |        T|   A|               true|\n            |       TA|  AT|              false|\n            |       AT|  AT|               true|\n            +---------+----+-------------------+\n            &lt;BLANKLINE&gt;\n\n        \"\"\"\n        return (risk_allele == reference_allele) | (\n            risk_allele\n            == GWASCatalogCuratedAssociationsParser._get_reverse_complement(\n                reference_allele\n            )\n        )\n\n    @staticmethod\n    def _are_alleles_palindromic(\n        reference_allele: Column, alternate_allele: Column\n    ) -&gt; Column:\n        \"\"\"A function to check if the alleles are palindromic.\n\n        Args:\n            reference_allele (Column): Reference allele column\n            alternate_allele (Column): Alternate allele column\n\n        Returns:\n            Column: A boolean column indicating if the alleles are palindromic.\n\n        Examples:\n            &gt;&gt;&gt; d = [{\"reference\": 'A', \"alternate\": 'T'}, {\"reference\": 'AT', \"alternate\": 'AG'}, {\"reference\": 'AT', \"alternate\": 'AT'}, {\"reference\": 'CATATG', \"alternate\": 'CATATG'}, {\"reference\": '-', \"alternate\": None}]\n            &gt;&gt;&gt; df = spark.createDataFrame(d)\n            &gt;&gt;&gt; df.withColumn(\"is_palindromic\", GWASCatalogCuratedAssociationsParser._are_alleles_palindromic(f.col(\"reference\"), f.col(\"alternate\"))).show()\n            +---------+---------+--------------+\n            |alternate|reference|is_palindromic|\n            +---------+---------+--------------+\n            |        T|        A|          true|\n            |       AG|       AT|         false|\n            |       AT|       AT|          true|\n            |   CATATG|   CATATG|          true|\n            |     null|        -|         false|\n            +---------+---------+--------------+\n            &lt;BLANKLINE&gt;\n\n        \"\"\"\n        revcomp = GWASCatalogCuratedAssociationsParser._get_reverse_complement(\n            alternate_allele\n        )\n        return (\n            f.when(reference_allele == revcomp, True)\n            .when(revcomp.isNull(), False)\n            .otherwise(False)\n        )\n\n    @staticmethod\n    def _harmonise_beta(\n        risk_allele: Column,\n        reference_allele: Column,\n        alternate_allele: Column,\n        effect_size: Column,\n        confidence_interval: Column,\n    ) -&gt; Column:\n        \"\"\"A function to extract the beta value from the effect size and confidence interval.\n\n        If the confidence interval contains the word \"increase\" or \"decrease\" it indicates, we are dealing with betas.\n        If it's \"increase\" and the effect size needs to be harmonized, then multiply the effect size by -1\n\n        Args:\n            risk_allele (Column): Risk allele column\n            reference_allele (Column): Reference allele column\n            alternate_allele (Column): Alternate allele column\n            effect_size (Column): GWAS Catalog effect size column\n            confidence_interval (Column): GWAS Catalog confidence interval column\n\n        Returns:\n            Column: A column containing the beta value.\n        \"\"\"\n        return (\n            f.when(\n                GWASCatalogCuratedAssociationsParser._are_alleles_palindromic(\n                    reference_allele, alternate_allele\n                ),\n                None,\n            )\n            .when(\n                (\n                    GWASCatalogCuratedAssociationsParser._effect_needs_harmonisation(\n                        risk_allele, reference_allele\n                    )\n                    &amp; confidence_interval.contains(\"increase\")\n                )\n                | (\n                    ~GWASCatalogCuratedAssociationsParser._effect_needs_harmonisation(\n                        risk_allele, reference_allele\n                    )\n                    &amp; confidence_interval.contains(\"decrease\")\n                ),\n                -effect_size,\n            )\n            .otherwise(effect_size)\n            .cast(DoubleType())\n        )\n\n    @staticmethod\n    def _harmonise_beta_ci(\n        risk_allele: Column,\n        reference_allele: Column,\n        alternate_allele: Column,\n        effect_size: Column,\n        confidence_interval: Column,\n        p_value: Column,\n        direction: str,\n    ) -&gt; Column:\n        \"\"\"Calculating confidence intervals for beta values.\n\n        Args:\n            risk_allele (Column): Risk allele column\n            reference_allele (Column): Reference allele column\n            alternate_allele (Column): Alternate allele column\n            effect_size (Column): GWAS Catalog effect size column\n            confidence_interval (Column): GWAS Catalog confidence interval column\n            p_value (Column): GWAS Catalog p-value column\n            direction (str): This is the direction of the confidence interval. It can be either \"upper\" or \"lower\".\n\n        Returns:\n            Column: The upper and lower bounds of the confidence interval for the beta coefficient.\n        \"\"\"\n        zscore_95 = f.lit(1.96)\n        beta = GWASCatalogCuratedAssociationsParser._harmonise_beta(\n            risk_allele,\n            reference_allele,\n            alternate_allele,\n            effect_size,\n            confidence_interval,\n        )\n        zscore = pvalue_to_zscore(p_value)\n        return (\n            f.when(f.lit(direction) == \"upper\", beta + f.abs(zscore_95 * beta) / zscore)\n            .when(f.lit(direction) == \"lower\", beta - f.abs(zscore_95 * beta) / zscore)\n            .otherwise(None)\n        )\n\n    @staticmethod\n    def _harmonise_odds_ratio(\n        risk_allele: Column,\n        reference_allele: Column,\n        alternate_allele: Column,\n        effect_size: Column,\n        confidence_interval: Column,\n    ) -&gt; Column:\n        \"\"\"Harmonizing odds ratio.\n\n        Args:\n            risk_allele (Column): Risk allele column\n            reference_allele (Column): Reference allele column\n            alternate_allele (Column): Alternate allele column\n            effect_size (Column): GWAS Catalog effect size column\n            confidence_interval (Column): GWAS Catalog confidence interval column\n\n        Returns:\n            Column: A column with the odds ratio, or 1/odds_ratio if harmonization required.\n        \"\"\"\n        return (\n            f.when(\n                GWASCatalogCuratedAssociationsParser._are_alleles_palindromic(\n                    reference_allele, alternate_allele\n                ),\n                None,\n            )\n            .when(\n                (\n                    GWASCatalogCuratedAssociationsParser._effect_needs_harmonisation(\n                        risk_allele, reference_allele\n                    )\n                    &amp; ~confidence_interval.rlike(\"|\".join([\"decrease\", \"increase\"]))\n                ),\n                1 / effect_size,\n            )\n            .otherwise(effect_size)\n            .cast(DoubleType())\n        )\n\n    @staticmethod\n    def _harmonise_odds_ratio_ci(\n        risk_allele: Column,\n        reference_allele: Column,\n        alternate_allele: Column,\n        effect_size: Column,\n        confidence_interval: Column,\n        p_value: Column,\n        direction: str,\n    ) -&gt; Column:\n        \"\"\"Calculating confidence intervals for beta values.\n\n        Args:\n            risk_allele (Column): Risk allele column\n            reference_allele (Column): Reference allele column\n            alternate_allele (Column): Alternate allele column\n            effect_size (Column): GWAS Catalog effect size column\n            confidence_interval (Column): GWAS Catalog confidence interval column\n            p_value (Column): GWAS Catalog p-value column\n            direction (str): This is the direction of the confidence interval. It can be either \"upper\" or \"lower\".\n\n        Returns:\n            Column: The upper and lower bounds of the 95% confidence interval for the odds ratio.\n        \"\"\"\n        zscore_95 = f.lit(1.96)\n        odds_ratio = GWASCatalogCuratedAssociationsParser._harmonise_odds_ratio(\n            risk_allele,\n            reference_allele,\n            alternate_allele,\n            effect_size,\n            confidence_interval,\n        )\n        odds_ratio_estimate = f.log(odds_ratio)\n        zscore = pvalue_to_zscore(p_value)\n        odds_ratio_se = odds_ratio_estimate / zscore\n        return f.when(\n            f.lit(direction) == \"upper\",\n            f.exp(odds_ratio_estimate + f.abs(zscore_95 * odds_ratio_se)),\n        ).when(\n            f.lit(direction) == \"lower\",\n            f.exp(odds_ratio_estimate - f.abs(zscore_95 * odds_ratio_se)),\n        )\n\n    @staticmethod\n    def _concatenate_substudy_description(\n        association_trait: Column, pvalue_text: Column, mapped_trait_uri: Column\n    ) -&gt; Column:\n        \"\"\"Substudy description parsing. Complex string containing metadata about the substudy (e.g. QTL, specific EFO, etc.).\n\n        Args:\n            association_trait (Column): GWAS Catalog association trait column\n            pvalue_text (Column): GWAS Catalog p-value text column\n            mapped_trait_uri (Column): GWAS Catalog mapped trait URI column\n\n        Returns:\n            Column: A column with the substudy description in the shape trait|pvaluetext1_pvaluetext2|EFO1_EFO2.\n\n        Examples:\n        &gt;&gt;&gt; df = spark.createDataFrame([\n        ...    (\"Height\", \"http://www.ebi.ac.uk/efo/EFO_0000001,http://www.ebi.ac.uk/efo/EFO_0000002\", \"European Ancestry\"),\n        ...    (\"Schizophrenia\", \"http://www.ebi.ac.uk/efo/MONDO_0005090\", None)],\n        ...    [\"association_trait\", \"mapped_trait_uri\", \"pvalue_text\"]\n        ... )\n        &gt;&gt;&gt; df.withColumn('substudy_description', GWASCatalogCuratedAssociationsParser._concatenate_substudy_description(df.association_trait, df.pvalue_text, df.mapped_trait_uri)).show(truncate=False)\n        +-----------------+-------------------------------------------------------------------------+-----------------+------------------------------------------+\n        |association_trait|mapped_trait_uri                                                         |pvalue_text      |substudy_description                      |\n        +-----------------+-------------------------------------------------------------------------+-----------------+------------------------------------------+\n        |Height           |http://www.ebi.ac.uk/efo/EFO_0000001,http://www.ebi.ac.uk/efo/EFO_0000002|European Ancestry|Height|EA|EFO_0000001/EFO_0000002         |\n        |Schizophrenia    |http://www.ebi.ac.uk/efo/MONDO_0005090                                   |null             |Schizophrenia|no_pvalue_text|MONDO_0005090|\n        +-----------------+-------------------------------------------------------------------------+-----------------+------------------------------------------+\n        &lt;BLANKLINE&gt;\n        \"\"\"\n        p_value_text = f.coalesce(\n            GWASCatalogCuratedAssociationsParser._normalise_pvaluetext(pvalue_text),\n            f.array(f.lit(\"no_pvalue_text\")),\n        )\n        return f.concat_ws(\n            \"|\",\n            association_trait,\n            f.concat_ws(\n                \"/\",\n                p_value_text,\n            ),\n            f.concat_ws(\n                \"/\",\n                parse_efos(mapped_trait_uri),\n            ),\n        )\n\n    @staticmethod\n    def _qc_all(\n        qc: Column,\n        chromosome: Column,\n        position: Column,\n        reference_allele: Column,\n        alternate_allele: Column,\n        strongest_snp_risk_allele: Column,\n        p_value_mantissa: Column,\n        p_value_exponent: Column,\n        p_value_cutoff: float,\n    ) -&gt; Column:\n        \"\"\"Flag associations that fail any QC.\n\n        Args:\n            qc (Column): QC column\n            chromosome (Column): Chromosome column\n            position (Column): Position column\n            reference_allele (Column): Reference allele column\n            alternate_allele (Column): Alternate allele column\n            strongest_snp_risk_allele (Column): Strongest SNP risk allele column\n            p_value_mantissa (Column): P-value mantissa column\n            p_value_exponent (Column): P-value exponent column\n            p_value_cutoff (float): P-value cutoff\n\n        Returns:\n            Column: Updated QC column with flag.\n        \"\"\"\n        qc = GWASCatalogCuratedAssociationsParser._qc_variant_interactions(\n            qc, strongest_snp_risk_allele\n        )\n        qc = GWASCatalogCuratedAssociationsParser._qc_subsignificant_associations(\n            qc, p_value_mantissa, p_value_exponent, p_value_cutoff\n        )\n        qc = GWASCatalogCuratedAssociationsParser._qc_genomic_location(\n            qc, chromosome, position\n        )\n        qc = GWASCatalogCuratedAssociationsParser._qc_variant_inconsistencies(\n            qc, chromosome, position, strongest_snp_risk_allele\n        )\n        qc = GWASCatalogCuratedAssociationsParser._qc_unmapped_variants(\n            qc, alternate_allele\n        )\n        qc = GWASCatalogCuratedAssociationsParser._qc_palindromic_alleles(\n            qc, reference_allele, alternate_allele\n        )\n        return qc\n\n    @staticmethod\n    def _qc_variant_interactions(\n        qc: Column, strongest_snp_risk_allele: Column\n    ) -&gt; Column:\n        \"\"\"Flag associations based on variant x variant interactions.\n\n        Args:\n            qc (Column): QC column\n            strongest_snp_risk_allele (Column): Column with the strongest SNP risk allele\n\n        Returns:\n            Column: Updated QC column with flag.\n        \"\"\"\n        return StudyLocus.update_quality_flag(\n            qc,\n            strongest_snp_risk_allele.contains(\";\"),\n            StudyLocusQualityCheck.COMPOSITE_FLAG,\n        )\n\n    @staticmethod\n    def _qc_subsignificant_associations(\n        qc: Column,\n        p_value_mantissa: Column,\n        p_value_exponent: Column,\n        pvalue_cutoff: float,\n    ) -&gt; Column:\n        \"\"\"Flag associations below significant threshold.\n\n        Args:\n            qc (Column): QC column\n            p_value_mantissa (Column): P-value mantissa column\n            p_value_exponent (Column): P-value exponent column\n            pvalue_cutoff (float): association p-value cut-off\n\n        Returns:\n            Column: Updated QC column with flag.\n\n        Examples:\n            &gt;&gt;&gt; import pyspark.sql.types as t\n            &gt;&gt;&gt; d = [{'qc': None, 'p_value_mantissa': 1, 'p_value_exponent': -7}, {'qc': None, 'p_value_mantissa': 1, 'p_value_exponent': -8}, {'qc': None, 'p_value_mantissa': 5, 'p_value_exponent': -8}, {'qc': None, 'p_value_mantissa': 1, 'p_value_exponent': -9}]\n            &gt;&gt;&gt; df = spark.createDataFrame(d, t.StructType([t.StructField('qc', t.ArrayType(t.StringType()), True), t.StructField('p_value_mantissa', t.IntegerType()), t.StructField('p_value_exponent', t.IntegerType())]))\n            &gt;&gt;&gt; df.withColumn('qc', GWASCatalogCuratedAssociationsParser._qc_subsignificant_associations(f.col(\"qc\"), f.col(\"p_value_mantissa\"), f.col(\"p_value_exponent\"), 5e-8)).show(truncate = False)\n            +------------------------+----------------+----------------+\n            |qc                      |p_value_mantissa|p_value_exponent|\n            +------------------------+----------------+----------------+\n            |[Subsignificant p-value]|1               |-7              |\n            |[]                      |1               |-8              |\n            |[]                      |5               |-8              |\n            |[]                      |1               |-9              |\n            +------------------------+----------------+----------------+\n            &lt;BLANKLINE&gt;\n\n        \"\"\"\n        return StudyLocus.update_quality_flag(\n            qc,\n            calculate_neglog_pvalue(p_value_mantissa, p_value_exponent)\n            &lt; f.lit(-np.log10(pvalue_cutoff)),\n            StudyLocusQualityCheck.SUBSIGNIFICANT_FLAG,\n        )\n\n    @staticmethod\n    def _qc_genomic_location(\n        qc: Column, chromosome: Column, position: Column\n    ) -&gt; Column:\n        \"\"\"Flag associations without genomic location in GWAS Catalog.\n\n        Args:\n            qc (Column): QC column\n            chromosome (Column): Chromosome column in GWAS Catalog\n            position (Column): Position column in GWAS Catalog\n\n        Returns:\n            Column: Updated QC column with flag.\n\n        Examples:\n            &gt;&gt;&gt; import pyspark.sql.types as t\n            &gt;&gt;&gt; d = [{'qc': None, 'chromosome': None, 'position': None}, {'qc': None, 'chromosome': '1', 'position': None}, {'qc': None, 'chromosome': None, 'position': 1}, {'qc': None, 'chromosome': '1', 'position': 1}]\n            &gt;&gt;&gt; df = spark.createDataFrame(d, schema=t.StructType([t.StructField('qc', t.ArrayType(t.StringType()), True), t.StructField('chromosome', t.StringType()), t.StructField('position', t.IntegerType())]))\n            &gt;&gt;&gt; df.withColumn('qc', GWASCatalogCuratedAssociationsParser._qc_genomic_location(df.qc, df.chromosome, df.position)).show(truncate=False)\n            +----------------------------+----------+--------+\n            |qc                          |chromosome|position|\n            +----------------------------+----------+--------+\n            |[Incomplete genomic mapping]|null      |null    |\n            |[Incomplete genomic mapping]|1         |null    |\n            |[Incomplete genomic mapping]|null      |1       |\n            |[]                          |1         |1       |\n            +----------------------------+----------+--------+\n            &lt;BLANKLINE&gt;\n\n        \"\"\"\n        return StudyLocus.update_quality_flag(\n            qc,\n            position.isNull() | chromosome.isNull(),\n            StudyLocusQualityCheck.NO_GENOMIC_LOCATION_FLAG,\n        )\n\n    @staticmethod\n    def _qc_variant_inconsistencies(\n        qc: Column,\n        chromosome: Column,\n        position: Column,\n        strongest_snp_risk_allele: Column,\n    ) -&gt; Column:\n        \"\"\"Flag associations with inconsistencies in the variant annotation.\n\n        Args:\n            qc (Column): QC column\n            chromosome (Column): Chromosome column in GWAS Catalog\n            position (Column): Position column in GWAS Catalog\n            strongest_snp_risk_allele (Column): Strongest SNP risk allele column in GWAS Catalog\n\n        Returns:\n            Column: Updated QC column with flag.\n        \"\"\"\n        return StudyLocus.update_quality_flag(\n            qc,\n            # Number of chromosomes does not correspond to the number of positions:\n            (f.size(f.split(chromosome, \";\")) != f.size(f.split(position, \";\")))\n            # Number of chromosome values different from riskAllele values:\n            | (\n                f.size(f.split(chromosome, \";\"))\n                != f.size(f.split(strongest_snp_risk_allele, \";\"))\n            ),\n            StudyLocusQualityCheck.INCONSISTENCY_FLAG,\n        )\n\n    @staticmethod\n    def _qc_unmapped_variants(qc: Column, alternate_allele: Column) -&gt; Column:\n        \"\"\"Flag associations with variants not mapped to variantAnnotation.\n\n        Args:\n            qc (Column): QC column\n            alternate_allele (Column): alternate allele\n\n        Returns:\n            Column: Updated QC column with flag.\n\n        Example:\n            &gt;&gt;&gt; import pyspark.sql.types as t\n            &gt;&gt;&gt; d = [{'alternate_allele': 'A', 'qc': None}, {'alternate_allele': None, 'qc': None}]\n            &gt;&gt;&gt; schema = t.StructType([t.StructField('alternate_allele', t.StringType(), True), t.StructField('qc', t.ArrayType(t.StringType()), True)])\n            &gt;&gt;&gt; df = spark.createDataFrame(data=d, schema=schema)\n            &gt;&gt;&gt; df.withColumn(\"new_qc\", GWASCatalogCuratedAssociationsParser._qc_unmapped_variants(f.col(\"qc\"), f.col(\"alternate_allele\"))).show()\n            +----------------+----+--------------------+\n            |alternate_allele|  qc|              new_qc|\n            +----------------+----+--------------------+\n            |               A|null|                  []|\n            |            null|null|[No mapping in Gn...|\n            +----------------+----+--------------------+\n            &lt;BLANKLINE&gt;\n\n        \"\"\"\n        return StudyLocus.update_quality_flag(\n            qc,\n            alternate_allele.isNull(),\n            StudyLocusQualityCheck.NON_MAPPED_VARIANT_FLAG,\n        )\n\n    @staticmethod\n    def _qc_palindromic_alleles(\n        qc: Column, reference_allele: Column, alternate_allele: Column\n    ) -&gt; Column:\n        \"\"\"Flag associations with palindromic variants which effects can not be harmonised.\n\n        Args:\n            qc (Column): QC column\n            reference_allele (Column): reference allele\n            alternate_allele (Column): alternate allele\n\n        Returns:\n            Column: Updated QC column with flag.\n\n        Example:\n            &gt;&gt;&gt; import pyspark.sql.types as t\n            &gt;&gt;&gt; schema = t.StructType([t.StructField('reference_allele', t.StringType(), True), t.StructField('alternate_allele', t.StringType(), True), t.StructField('qc', t.ArrayType(t.StringType()), True)])\n            &gt;&gt;&gt; d = [{'reference_allele': 'A', 'alternate_allele': 'T', 'qc': None}, {'reference_allele': 'AT', 'alternate_allele': 'TA', 'qc': None}, {'reference_allele': 'AT', 'alternate_allele': 'AT', 'qc': None}]\n            &gt;&gt;&gt; df = spark.createDataFrame(data=d, schema=schema)\n            &gt;&gt;&gt; df.withColumn(\"qc\", GWASCatalogCuratedAssociationsParser._qc_palindromic_alleles(f.col(\"qc\"), f.col(\"reference_allele\"), f.col(\"alternate_allele\"))).show(truncate=False)\n            +----------------+----------------+---------------------------------------+\n            |reference_allele|alternate_allele|qc                                     |\n            +----------------+----------------+---------------------------------------+\n            |A               |T               |[Palindrome alleles - cannot harmonize]|\n            |AT              |TA              |[]                                     |\n            |AT              |AT              |[Palindrome alleles - cannot harmonize]|\n            +----------------+----------------+---------------------------------------+\n            &lt;BLANKLINE&gt;\n\n        \"\"\"\n        return StudyLocus.update_quality_flag(\n            qc,\n            GWASCatalogCuratedAssociationsParser._are_alleles_palindromic(\n                reference_allele, alternate_allele\n            ),\n            StudyLocusQualityCheck.PALINDROMIC_ALLELE_FLAG,\n        )\n\n    @classmethod\n    def from_source(\n        cls: type[GWASCatalogCuratedAssociationsParser],\n        gwas_associations: DataFrame,\n        variant_annotation: VariantAnnotation,\n        pvalue_threshold: float = 5e-8,\n    ) -&gt; StudyLocusGWASCatalog:\n        \"\"\"Read GWASCatalog associations.\n\n        It reads the GWAS Catalog association dataset, selects and renames columns, casts columns, and\n        applies some pre-defined filters on the data:\n\n        Args:\n            gwas_associations (DataFrame): GWAS Catalog raw associations dataset\n            variant_annotation (VariantAnnotation): Variant annotation dataset\n            pvalue_threshold (float): P-value threshold for flagging associations\n\n        Returns:\n            StudyLocusGWASCatalog: GWASCatalogAssociations dataset\n        \"\"\"\n        return StudyLocusGWASCatalog(\n            _df=gwas_associations.withColumn(\n                \"studyLocusId\", f.monotonically_increasing_id().cast(LongType())\n            )\n            .transform(\n                # Map/harmonise variants to variant annotation dataset:\n                # This function adds columns: variantId, referenceAllele, alternateAllele, chromosome, position\n                lambda df: GWASCatalogCuratedAssociationsParser._map_to_variant_annotation_variants(\n                    df, variant_annotation\n                )\n            )\n            .withColumn(\n                # Perform all quality control checks:\n                \"qualityControls\",\n                GWASCatalogCuratedAssociationsParser._qc_all(\n                    f.array().alias(\"qualityControls\"),\n                    f.col(\"CHR_ID\"),\n                    f.col(\"CHR_POS\").cast(IntegerType()),\n                    f.col(\"referenceAllele\"),\n                    f.col(\"alternateAllele\"),\n                    f.col(\"STRONGEST SNP-RISK ALLELE\"),\n                    *GWASCatalogCuratedAssociationsParser._parse_pvalue(\n                        f.col(\"P-VALUE\")\n                    ),\n                    pvalue_threshold,\n                ),\n            )\n            .select(\n                # INSIDE STUDY-LOCUS SCHEMA:\n                \"studyLocusId\",\n                \"variantId\",\n                # Mapped genomic location of the variant (; separated list)\n                \"chromosome\",\n                \"position\",\n                f.col(\"STUDY ACCESSION\").alias(\"studyId\"),\n                # beta value of the association\n                GWASCatalogCuratedAssociationsParser._harmonise_beta(\n                    GWASCatalogCuratedAssociationsParser._normalise_risk_allele(\n                        f.col(\"STRONGEST SNP-RISK ALLELE\")\n                    ),\n                    f.col(\"referenceAllele\"),\n                    f.col(\"alternateAllele\"),\n                    f.col(\"OR or BETA\"),\n                    f.col(\"95% CI (TEXT)\"),\n                ).alias(\"beta\"),\n                # p-value of the association, string: split into exponent and mantissa.\n                *GWASCatalogCuratedAssociationsParser._parse_pvalue(f.col(\"P-VALUE\")),\n                # Capturing phenotype granularity at the association level\n                GWASCatalogCuratedAssociationsParser._concatenate_substudy_description(\n                    f.col(\"DISEASE/TRAIT\"),\n                    f.col(\"P-VALUE (TEXT)\"),\n                    f.col(\"MAPPED_TRAIT_URI\"),\n                ).alias(\"subStudyDescription\"),\n                # Quality controls (array of strings)\n                \"qualityControls\",\n            ),\n            _schema=StudyLocusGWASCatalog.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/associations/#gentropy.datasource.gwas_catalog.associations.GWASCatalogCuratedAssociationsParser.from_source","title":"<code>from_source(gwas_associations: DataFrame, variant_annotation: VariantAnnotation, pvalue_threshold: float = 5e-08) -&gt; StudyLocusGWASCatalog</code>  <code>classmethod</code>","text":"<p>Read GWASCatalog associations.</p> <p>It reads the GWAS Catalog association dataset, selects and renames columns, casts columns, and applies some pre-defined filters on the data:</p> <p>Parameters:</p> Name Type Description Default <code>gwas_associations</code> <code>DataFrame</code> <p>GWAS Catalog raw associations dataset</p> required <code>variant_annotation</code> <code>VariantAnnotation</code> <p>Variant annotation dataset</p> required <code>pvalue_threshold</code> <code>float</code> <p>P-value threshold for flagging associations</p> <code>5e-08</code> <p>Returns:</p> Name Type Description <code>StudyLocusGWASCatalog</code> <code>StudyLocusGWASCatalog</code> <p>GWASCatalogAssociations dataset</p> Source code in <code>src/gentropy/datasource/gwas_catalog/associations.py</code> <pre><code>@classmethod\ndef from_source(\n    cls: type[GWASCatalogCuratedAssociationsParser],\n    gwas_associations: DataFrame,\n    variant_annotation: VariantAnnotation,\n    pvalue_threshold: float = 5e-8,\n) -&gt; StudyLocusGWASCatalog:\n    \"\"\"Read GWASCatalog associations.\n\n    It reads the GWAS Catalog association dataset, selects and renames columns, casts columns, and\n    applies some pre-defined filters on the data:\n\n    Args:\n        gwas_associations (DataFrame): GWAS Catalog raw associations dataset\n        variant_annotation (VariantAnnotation): Variant annotation dataset\n        pvalue_threshold (float): P-value threshold for flagging associations\n\n    Returns:\n        StudyLocusGWASCatalog: GWASCatalogAssociations dataset\n    \"\"\"\n    return StudyLocusGWASCatalog(\n        _df=gwas_associations.withColumn(\n            \"studyLocusId\", f.monotonically_increasing_id().cast(LongType())\n        )\n        .transform(\n            # Map/harmonise variants to variant annotation dataset:\n            # This function adds columns: variantId, referenceAllele, alternateAllele, chromosome, position\n            lambda df: GWASCatalogCuratedAssociationsParser._map_to_variant_annotation_variants(\n                df, variant_annotation\n            )\n        )\n        .withColumn(\n            # Perform all quality control checks:\n            \"qualityControls\",\n            GWASCatalogCuratedAssociationsParser._qc_all(\n                f.array().alias(\"qualityControls\"),\n                f.col(\"CHR_ID\"),\n                f.col(\"CHR_POS\").cast(IntegerType()),\n                f.col(\"referenceAllele\"),\n                f.col(\"alternateAllele\"),\n                f.col(\"STRONGEST SNP-RISK ALLELE\"),\n                *GWASCatalogCuratedAssociationsParser._parse_pvalue(\n                    f.col(\"P-VALUE\")\n                ),\n                pvalue_threshold,\n            ),\n        )\n        .select(\n            # INSIDE STUDY-LOCUS SCHEMA:\n            \"studyLocusId\",\n            \"variantId\",\n            # Mapped genomic location of the variant (; separated list)\n            \"chromosome\",\n            \"position\",\n            f.col(\"STUDY ACCESSION\").alias(\"studyId\"),\n            # beta value of the association\n            GWASCatalogCuratedAssociationsParser._harmonise_beta(\n                GWASCatalogCuratedAssociationsParser._normalise_risk_allele(\n                    f.col(\"STRONGEST SNP-RISK ALLELE\")\n                ),\n                f.col(\"referenceAllele\"),\n                f.col(\"alternateAllele\"),\n                f.col(\"OR or BETA\"),\n                f.col(\"95% CI (TEXT)\"),\n            ).alias(\"beta\"),\n            # p-value of the association, string: split into exponent and mantissa.\n            *GWASCatalogCuratedAssociationsParser._parse_pvalue(f.col(\"P-VALUE\")),\n            # Capturing phenotype granularity at the association level\n            GWASCatalogCuratedAssociationsParser._concatenate_substudy_description(\n                f.col(\"DISEASE/TRAIT\"),\n                f.col(\"P-VALUE (TEXT)\"),\n                f.col(\"MAPPED_TRAIT_URI\"),\n            ).alias(\"subStudyDescription\"),\n            # Quality controls (array of strings)\n            \"qualityControls\",\n        ),\n        _schema=StudyLocusGWASCatalog.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/associations/#gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog","title":"<code>gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog</code>  <code>dataclass</code>","text":"<p>             Bases: <code>StudyLocus</code></p> <p>Study locus Dataset for GWAS Catalog curated associations.</p> <p>A study index dataset captures all the metadata for all studies including GWAS and Molecular QTL.</p> Source code in <code>src/gentropy/datasource/gwas_catalog/associations.py</code> <pre><code>@dataclass\nclass StudyLocusGWASCatalog(StudyLocus):\n    \"\"\"Study locus Dataset for GWAS Catalog curated associations.\n\n    A study index dataset captures all the metadata for all studies including GWAS and Molecular QTL.\n    \"\"\"\n\n    def update_study_id(\n        self: StudyLocusGWASCatalog, study_annotation: DataFrame\n    ) -&gt; StudyLocusGWASCatalog:\n        \"\"\"Update final studyId and studyLocusId with a dataframe containing study annotation.\n\n        Args:\n            study_annotation (DataFrame): Dataframe containing `updatedStudyId` and key columns `studyId` and `subStudyDescription`.\n\n        Returns:\n            StudyLocusGWASCatalog: Updated study locus with new `studyId` and `studyLocusId`.\n        \"\"\"\n        self.df = (\n            self._df.join(\n                study_annotation, on=[\"studyId\", \"subStudyDescription\"], how=\"left\"\n            )\n            .withColumn(\"studyId\", f.coalesce(\"updatedStudyId\", \"studyId\"))\n            .drop(\"subStudyDescription\", \"updatedStudyId\")\n        ).withColumn(\n            \"studyLocusId\",\n            StudyLocus.assign_study_locus_id(f.col(\"studyId\"), f.col(\"variantId\")),\n        )\n        return self\n\n    def qc_ambiguous_study(self: StudyLocusGWASCatalog) -&gt; StudyLocusGWASCatalog:\n        \"\"\"Flag associations with variants that can not be unambiguously associated with one study.\n\n        Returns:\n            StudyLocusGWASCatalog: Updated study locus.\n        \"\"\"\n        assoc_ambiguity_window = Window.partitionBy(\n            f.col(\"studyId\"), f.col(\"variantId\")\n        )\n\n        self._df.withColumn(\n            \"qualityControls\",\n            StudyLocus.update_quality_flag(\n                f.col(\"qualityControls\"),\n                f.count(f.col(\"variantId\")).over(assoc_ambiguity_window) &gt; 1,\n                StudyLocusQualityCheck.AMBIGUOUS_STUDY,\n            ),\n        )\n        return self\n\n    def apply_inclusion_list(\n        self: StudyLocusGWASCatalog, inclusion_list: DataFrame\n    ) -&gt; StudyLocusGWASCatalog:\n        \"\"\"Restricting GWAS Catalog studies based on a list of accpected study ids.\n\n        Args:\n            inclusion_list (DataFrame): List of accepted GWAS Catalog study identifiers\n\n        Returns:\n            StudyLocusGWASCatalog: Filtered dataset.\n        \"\"\"\n        return StudyLocusGWASCatalog(\n            _df=self.df.join(inclusion_list, on=\"studyId\", how=\"inner\"),\n            _schema=StudyLocusGWASCatalog.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/associations/#gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog.apply_inclusion_list","title":"<code>apply_inclusion_list(inclusion_list: DataFrame) -&gt; StudyLocusGWASCatalog</code>","text":"<p>Restricting GWAS Catalog studies based on a list of accpected study ids.</p> <p>Parameters:</p> Name Type Description Default <code>inclusion_list</code> <code>DataFrame</code> <p>List of accepted GWAS Catalog study identifiers</p> required <p>Returns:</p> Name Type Description <code>StudyLocusGWASCatalog</code> <code>StudyLocusGWASCatalog</code> <p>Filtered dataset.</p> Source code in <code>src/gentropy/datasource/gwas_catalog/associations.py</code> <pre><code>def apply_inclusion_list(\n    self: StudyLocusGWASCatalog, inclusion_list: DataFrame\n) -&gt; StudyLocusGWASCatalog:\n    \"\"\"Restricting GWAS Catalog studies based on a list of accpected study ids.\n\n    Args:\n        inclusion_list (DataFrame): List of accepted GWAS Catalog study identifiers\n\n    Returns:\n        StudyLocusGWASCatalog: Filtered dataset.\n    \"\"\"\n    return StudyLocusGWASCatalog(\n        _df=self.df.join(inclusion_list, on=\"studyId\", how=\"inner\"),\n        _schema=StudyLocusGWASCatalog.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/associations/#gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog.qc_ambiguous_study","title":"<code>qc_ambiguous_study() -&gt; StudyLocusGWASCatalog</code>","text":"<p>Flag associations with variants that can not be unambiguously associated with one study.</p> <p>Returns:</p> Name Type Description <code>StudyLocusGWASCatalog</code> <code>StudyLocusGWASCatalog</code> <p>Updated study locus.</p> Source code in <code>src/gentropy/datasource/gwas_catalog/associations.py</code> <pre><code>def qc_ambiguous_study(self: StudyLocusGWASCatalog) -&gt; StudyLocusGWASCatalog:\n    \"\"\"Flag associations with variants that can not be unambiguously associated with one study.\n\n    Returns:\n        StudyLocusGWASCatalog: Updated study locus.\n    \"\"\"\n    assoc_ambiguity_window = Window.partitionBy(\n        f.col(\"studyId\"), f.col(\"variantId\")\n    )\n\n    self._df.withColumn(\n        \"qualityControls\",\n        StudyLocus.update_quality_flag(\n            f.col(\"qualityControls\"),\n            f.count(f.col(\"variantId\")).over(assoc_ambiguity_window) &gt; 1,\n            StudyLocusQualityCheck.AMBIGUOUS_STUDY,\n        ),\n    )\n    return self\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/associations/#gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog.update_study_id","title":"<code>update_study_id(study_annotation: DataFrame) -&gt; StudyLocusGWASCatalog</code>","text":"<p>Update final studyId and studyLocusId with a dataframe containing study annotation.</p> <p>Parameters:</p> Name Type Description Default <code>study_annotation</code> <code>DataFrame</code> <p>Dataframe containing <code>updatedStudyId</code> and key columns <code>studyId</code> and <code>subStudyDescription</code>.</p> required <p>Returns:</p> Name Type Description <code>StudyLocusGWASCatalog</code> <code>StudyLocusGWASCatalog</code> <p>Updated study locus with new <code>studyId</code> and <code>studyLocusId</code>.</p> Source code in <code>src/gentropy/datasource/gwas_catalog/associations.py</code> <pre><code>def update_study_id(\n    self: StudyLocusGWASCatalog, study_annotation: DataFrame\n) -&gt; StudyLocusGWASCatalog:\n    \"\"\"Update final studyId and studyLocusId with a dataframe containing study annotation.\n\n    Args:\n        study_annotation (DataFrame): Dataframe containing `updatedStudyId` and key columns `studyId` and `subStudyDescription`.\n\n    Returns:\n        StudyLocusGWASCatalog: Updated study locus with new `studyId` and `studyLocusId`.\n    \"\"\"\n    self.df = (\n        self._df.join(\n            study_annotation, on=[\"studyId\", \"subStudyDescription\"], how=\"left\"\n        )\n        .withColumn(\"studyId\", f.coalesce(\"updatedStudyId\", \"studyId\"))\n        .drop(\"subStudyDescription\", \"updatedStudyId\")\n    ).withColumn(\n        \"studyLocusId\",\n        StudyLocus.assign_study_locus_id(f.col(\"studyId\"), f.col(\"variantId\")),\n    )\n    return self\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/study_index/","title":"Study Index","text":""},{"location":"python_api/datasources/gwas_catalog/study_index/#gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalogParser","title":"<code>gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalogParser</code>  <code>dataclass</code>","text":"<p>GWAS Catalog study index parser.</p> <p>The following information is harmonised from the GWAS Catalog:</p> <ul> <li>All publication related information retained.</li> <li>Mapped measured and background traits parsed.</li> <li>Flagged if harmonized summary statistics datasets available.</li> <li>If available, the ftp path to these files presented.</li> <li>Ancestries from the discovery and replication stages are structured with sample counts.</li> <li>Case/control counts extracted.</li> <li>The number of samples with European ancestry extracted.</li> </ul> Source code in <code>src/gentropy/datasource/gwas_catalog/study_index.py</code> <pre><code>@dataclass\nclass StudyIndexGWASCatalogParser:\n    \"\"\"GWAS Catalog study index parser.\n\n    The following information is harmonised from the GWAS Catalog:\n\n    - All publication related information retained.\n    - Mapped measured and background traits parsed.\n    - Flagged if harmonized summary statistics datasets available.\n    - If available, the ftp path to these files presented.\n    - Ancestries from the discovery and replication stages are structured with sample counts.\n    - Case/control counts extracted.\n    - The number of samples with European ancestry extracted.\n\n    \"\"\"\n\n    @staticmethod\n    def _parse_discovery_samples(discovery_samples: Column) -&gt; Column:\n        \"\"\"Parse discovery sample sizes from GWAS Catalog.\n\n        This is a curated field. From publication sometimes it is not clear how the samples were split\n        across the reported ancestries. In such cases we are assuming the ancestries were evenly presented\n        and the total sample size is split:\n\n        [\"European, African\", 100] -&gt; [\"European, 50], [\"African\", 50]\n\n        Args:\n            discovery_samples (Column): Raw discovery sample sizes\n\n        Returns:\n            Column: Parsed and de-duplicated list of discovery ancestries with sample size.\n\n        Examples:\n            &gt;&gt;&gt; data = [('s1', \"European\", 10), ('s1', \"African\", 10), ('s2', \"European, African, Asian\", 100), ('s2', \"European\", 50)]\n            &gt;&gt;&gt; df = (\n            ...    spark.createDataFrame(data, ['studyId', 'ancestry', 'sampleSize'])\n            ...    .groupBy('studyId')\n            ...    .agg(\n            ...        f.collect_set(\n            ...            f.struct('ancestry', 'sampleSize')\n            ...        ).alias('discoverySampleSize')\n            ...    )\n            ...    .orderBy('studyId')\n            ...    .withColumn('discoverySampleSize', StudyIndexGWASCatalogParser._parse_discovery_samples(f.col('discoverySampleSize')))\n            ...    .select('discoverySampleSize')\n            ...    .show(truncate=False)\n            ... )\n            +--------------------------------------------+\n            |discoverySampleSize                         |\n            +--------------------------------------------+\n            |[{African, 10}, {European, 10}]             |\n            |[{European, 83}, {African, 33}, {Asian, 33}]|\n            +--------------------------------------------+\n            &lt;BLANKLINE&gt;\n        \"\"\"\n        # To initialize return objects for aggregate functions, schema has to be defined:\n        schema = t.ArrayType(\n            t.StructType(\n                [\n                    t.StructField(\"ancestry\", t.StringType(), True),\n                    t.StructField(\"sampleSize\", t.IntegerType(), True),\n                ]\n            )\n        )\n\n        # Splitting comma separated ancestries:\n        exploded_ancestries = f.transform(\n            discovery_samples,\n            lambda sample: f.split(sample.ancestry, r\",\\s(?![^()]*\\))\"),\n        )\n\n        # Initialize discoverySample object from unique list of ancestries:\n        unique_ancestries = f.transform(\n            f.aggregate(\n                exploded_ancestries,\n                f.array().cast(t.ArrayType(t.StringType())),\n                lambda x, y: f.array_union(x, y),\n                f.array_distinct,\n            ),\n            lambda ancestry: f.struct(\n                ancestry.alias(\"ancestry\"),\n                f.lit(0).alias(\"sampleSize\"),\n            ),\n        )\n\n        # Computing sample sizes for ancestries when splitting is needed:\n        resolved_sample_count = f.transform(\n            f.arrays_zip(\n                f.transform(exploded_ancestries, lambda pop: f.size(pop)).alias(\n                    \"pop_size\"\n                ),\n                f.transform(discovery_samples, lambda pop: pop.sampleSize).alias(\n                    \"pop_count\"\n                ),\n            ),\n            lambda pop: (pop.pop_count / pop.pop_size).cast(t.IntegerType()),\n        )\n\n        # Flattening out ancestries with sample sizes:\n        parsed_sample_size = f.aggregate(\n            f.transform(\n                f.arrays_zip(\n                    exploded_ancestries.alias(\"ancestries\"),\n                    resolved_sample_count.alias(\"sample_count\"),\n                ),\n                StudyIndexGWASCatalogParser._merge_ancestries_and_counts,\n            ),\n            f.array().cast(schema),\n            lambda x, y: f.array_union(x, y),\n        )\n\n        # Normalize ancestries:\n        return f.aggregate(\n            parsed_sample_size,\n            unique_ancestries,\n            StudyIndexGWASCatalogParser._normalize_ancestries,\n        )\n\n    @staticmethod\n    def _normalize_ancestries(merged: Column, ancestry: Column) -&gt; Column:\n        \"\"\"Normalize ancestries from a list of structs.\n\n        As some ancestry label might be repeated with different sample counts,\n        these counts need to be collected.\n\n        Args:\n            merged (Column): Resulting list of struct with unique ancestries.\n            ancestry (Column): One ancestry object coming from raw.\n\n        Returns:\n            Column: Unique list of ancestries with the sample counts.\n        \"\"\"\n        # Iterating over the list of unique ancestries and adding the sample size if label matches:\n        return f.transform(\n            merged,\n            lambda a: f.when(\n                a.ancestry == ancestry.ancestry,\n                f.struct(\n                    a.ancestry.alias(\"ancestry\"),\n                    (a.sampleSize + ancestry.sampleSize)\n                    .cast(t.IntegerType())\n                    .alias(\"sampleSize\"),\n                ),\n            ).otherwise(a),\n        )\n\n    @staticmethod\n    def _merge_ancestries_and_counts(ancestry_group: Column) -&gt; Column:\n        \"\"\"Merge ancestries with sample sizes.\n\n        After splitting ancestry annotations, all resulting ancestries needs to be assigned\n        with the proper sample size.\n\n        Args:\n            ancestry_group (Column): Each element is a struct with `sample_count` (int) and `ancestries` (list)\n\n        Returns:\n            Column: a list of structs with `ancestry` and `sampleSize` fields.\n\n        Examples:\n            &gt;&gt;&gt; data = [(12, ['African', 'European']),(12, ['African'])]\n            &gt;&gt;&gt; (\n            ...     spark.createDataFrame(data, ['sample_count', 'ancestries'])\n            ...     .select(StudyIndexGWASCatalogParser._merge_ancestries_and_counts(f.struct('sample_count', 'ancestries')).alias('test'))\n            ...     .show(truncate=False)\n            ... )\n            +-------------------------------+\n            |test                           |\n            +-------------------------------+\n            |[{African, 12}, {European, 12}]|\n            |[{African, 12}]                |\n            +-------------------------------+\n            &lt;BLANKLINE&gt;\n        \"\"\"\n        # Extract sample size for the ancestry group:\n        count = ancestry_group.sample_count\n\n        # We need to loop through the ancestries:\n        return f.transform(\n            ancestry_group.ancestries,\n            lambda ancestry: f.struct(\n                ancestry.alias(\"ancestry\"),\n                count.alias(\"sampleSize\"),\n            ),\n        )\n\n    @staticmethod\n    def parse_cohorts(raw_cohort: Column) -&gt; Column:\n        \"\"\"Return a list of unique cohort labels from pipe separated list if provided.\n\n        Args:\n            raw_cohort (Column): Cohort list column, where labels are separated by `|` sign.\n\n        Returns:\n            Column: an array colun with string elements.\n\n        Examples:\n        &gt;&gt;&gt; data = [('BioME|CaPS|Estonia|FHS|UKB|GERA|GERA|GERA',),(None,),]\n        &gt;&gt;&gt; spark.createDataFrame(data, ['cohorts']).select(StudyIndexGWASCatalogParser.parse_cohorts(f.col('cohorts')).alias('parsedCohorts')).show(truncate=False)\n        +--------------------------------------+\n        |parsedCohorts                         |\n        +--------------------------------------+\n        |[BioME, CaPS, Estonia, FHS, UKB, GERA]|\n        |[null]                                |\n        +--------------------------------------+\n        &lt;BLANKLINE&gt;\n        \"\"\"\n        return f.when(\n            (raw_cohort.isNull()) | (raw_cohort == \"\"),\n            f.array(f.lit(None).cast(t.StringType())),\n        ).otherwise(f.array_distinct(f.split(raw_cohort, r\"\\|\")))\n\n    @classmethod\n    def _parse_study_table(\n        cls: type[StudyIndexGWASCatalogParser], catalog_studies: DataFrame\n    ) -&gt; StudyIndexGWASCatalog:\n        \"\"\"Harmonise GWASCatalog study table with `StudyIndex` schema.\n\n        Args:\n            catalog_studies (DataFrame): GWAS Catalog study table\n\n        Returns:\n            StudyIndexGWASCatalog: Parsed and annotated GWAS Catalog study table.\n        \"\"\"\n        return StudyIndexGWASCatalog(\n            _df=catalog_studies.select(\n                f.coalesce(\n                    f.col(\"STUDY ACCESSION\"), f.monotonically_increasing_id()\n                ).alias(\"studyId\"),\n                f.lit(\"GCST\").alias(\"projectId\"),\n                f.lit(\"gwas\").alias(\"studyType\"),\n                f.col(\"PUBMED ID\").alias(\"pubmedId\"),\n                f.col(\"FIRST AUTHOR\").alias(\"publicationFirstAuthor\"),\n                f.col(\"DATE\").alias(\"publicationDate\"),\n                f.col(\"JOURNAL\").alias(\"publicationJournal\"),\n                f.col(\"STUDY\").alias(\"publicationTitle\"),\n                f.coalesce(f.col(\"DISEASE/TRAIT\"), f.lit(\"Unreported\")).alias(\n                    \"traitFromSource\"\n                ),\n                f.col(\"INITIAL SAMPLE SIZE\").alias(\"initialSampleSize\"),\n                parse_efos(f.col(\"MAPPED_TRAIT_URI\")).alias(\"traitFromSourceMappedIds\"),\n                parse_efos(f.col(\"MAPPED BACKGROUND TRAIT URI\")).alias(\n                    \"backgroundTraitFromSourceMappedIds\"\n                ),\n            ),\n            _schema=StudyIndexGWASCatalog.get_schema(),\n        )\n\n    @classmethod\n    def from_source(\n        cls: type[StudyIndexGWASCatalogParser],\n        catalog_studies: DataFrame,\n        ancestry_file: DataFrame,\n        sumstats_lut: DataFrame,\n    ) -&gt; StudyIndexGWASCatalog:\n        \"\"\"Ingests study level metadata from the GWAS Catalog.\n\n        Args:\n            catalog_studies (DataFrame): GWAS Catalog raw study table\n            ancestry_file (DataFrame): GWAS Catalog ancestry table.\n            sumstats_lut (DataFrame): GWAS Catalog summary statistics list.\n\n        Returns:\n            StudyIndexGWASCatalog: Parsed and annotated GWAS Catalog study table.\n        \"\"\"\n        # Read GWAS Catalogue raw data\n        return (\n            cls._parse_study_table(catalog_studies)\n            .annotate_ancestries(ancestry_file)\n            .annotate_sumstats_info(sumstats_lut)\n            .annotate_discovery_sample_sizes()\n        )\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/study_index/#gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalogParser.from_source","title":"<code>from_source(catalog_studies: DataFrame, ancestry_file: DataFrame, sumstats_lut: DataFrame) -&gt; StudyIndexGWASCatalog</code>  <code>classmethod</code>","text":"<p>Ingests study level metadata from the GWAS Catalog.</p> <p>Parameters:</p> Name Type Description Default <code>catalog_studies</code> <code>DataFrame</code> <p>GWAS Catalog raw study table</p> required <code>ancestry_file</code> <code>DataFrame</code> <p>GWAS Catalog ancestry table.</p> required <code>sumstats_lut</code> <code>DataFrame</code> <p>GWAS Catalog summary statistics list.</p> required <p>Returns:</p> Name Type Description <code>StudyIndexGWASCatalog</code> <code>StudyIndexGWASCatalog</code> <p>Parsed and annotated GWAS Catalog study table.</p> Source code in <code>src/gentropy/datasource/gwas_catalog/study_index.py</code> <pre><code>@classmethod\ndef from_source(\n    cls: type[StudyIndexGWASCatalogParser],\n    catalog_studies: DataFrame,\n    ancestry_file: DataFrame,\n    sumstats_lut: DataFrame,\n) -&gt; StudyIndexGWASCatalog:\n    \"\"\"Ingests study level metadata from the GWAS Catalog.\n\n    Args:\n        catalog_studies (DataFrame): GWAS Catalog raw study table\n        ancestry_file (DataFrame): GWAS Catalog ancestry table.\n        sumstats_lut (DataFrame): GWAS Catalog summary statistics list.\n\n    Returns:\n        StudyIndexGWASCatalog: Parsed and annotated GWAS Catalog study table.\n    \"\"\"\n    # Read GWAS Catalogue raw data\n    return (\n        cls._parse_study_table(catalog_studies)\n        .annotate_ancestries(ancestry_file)\n        .annotate_sumstats_info(sumstats_lut)\n        .annotate_discovery_sample_sizes()\n    )\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/study_index/#gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalogParser.parse_cohorts","title":"<code>parse_cohorts(raw_cohort: Column) -&gt; Column</code>  <code>staticmethod</code>","text":"<p>Return a list of unique cohort labels from pipe separated list if provided.</p> <p>Parameters:</p> Name Type Description Default <code>raw_cohort</code> <code>Column</code> <p>Cohort list column, where labels are separated by <code>|</code> sign.</p> required <p>Returns:</p> Name Type Description <code>Column</code> <code>Column</code> <p>an array colun with string elements.</p> <p>Examples:</p> <p>data = [('BioME|CaPS|Estonia|FHS|UKB|GERA|GERA|GERA',),(None,),] spark.createDataFrame(data, ['cohorts']).select(StudyIndexGWASCatalogParser.parse_cohorts(f.col('cohorts')).alias('parsedCohorts')).show(truncate=False) +--------------------------------------+ |parsedCohorts                         | +--------------------------------------+ |[BioME, CaPS, Estonia, FHS, UKB, GERA]| |[null]                                | +--------------------------------------+  Source code in <code>src/gentropy/datasource/gwas_catalog/study_index.py</code> <pre><code>@staticmethod\ndef parse_cohorts(raw_cohort: Column) -&gt; Column:\n    \"\"\"Return a list of unique cohort labels from pipe separated list if provided.\n\n    Args:\n        raw_cohort (Column): Cohort list column, where labels are separated by `|` sign.\n\n    Returns:\n        Column: an array colun with string elements.\n\n    Examples:\n    &gt;&gt;&gt; data = [('BioME|CaPS|Estonia|FHS|UKB|GERA|GERA|GERA',),(None,),]\n    &gt;&gt;&gt; spark.createDataFrame(data, ['cohorts']).select(StudyIndexGWASCatalogParser.parse_cohorts(f.col('cohorts')).alias('parsedCohorts')).show(truncate=False)\n    +--------------------------------------+\n    |parsedCohorts                         |\n    +--------------------------------------+\n    |[BioME, CaPS, Estonia, FHS, UKB, GERA]|\n    |[null]                                |\n    +--------------------------------------+\n    &lt;BLANKLINE&gt;\n    \"\"\"\n    return f.when(\n        (raw_cohort.isNull()) | (raw_cohort == \"\"),\n        f.array(f.lit(None).cast(t.StringType())),\n    ).otherwise(f.array_distinct(f.split(raw_cohort, r\"\\|\")))\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/study_index/#gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog","title":"<code>gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog</code>  <code>dataclass</code>","text":"<p>             Bases: <code>StudyIndex</code></p> <p>Study index dataset from GWAS Catalog.</p> <p>A study index dataset captures all the metadata for all studies including GWAS and Molecular QTL.</p> Source code in <code>src/gentropy/datasource/gwas_catalog/study_index.py</code> <pre><code>@dataclass\nclass StudyIndexGWASCatalog(StudyIndex):\n    \"\"\"Study index dataset from GWAS Catalog.\n\n    A study index dataset captures all the metadata for all studies including GWAS and Molecular QTL.\n    \"\"\"\n\n    def update_study_id(\n        self: StudyIndexGWASCatalog, study_annotation: DataFrame\n    ) -&gt; StudyIndexGWASCatalog:\n        \"\"\"Update studyId with a dataframe containing study.\n\n        Args:\n            study_annotation (DataFrame): Dataframe containing `updatedStudyId`, `traitFromSource`, `traitFromSourceMappedIds` and key column `studyId`.\n\n        Returns:\n            StudyIndexGWASCatalog: Updated study table.\n        \"\"\"\n        self.df = (\n            self._df.join(\n                study_annotation.select(\n                    *[\n                        f.col(c).alias(f\"updated{c}\")\n                        if c not in [\"studyId\", \"updatedStudyId\"]\n                        else f.col(c)\n                        for c in study_annotation.columns\n                    ]\n                ),\n                on=\"studyId\",\n                how=\"left\",\n            )\n            .withColumn(\n                \"studyId\",\n                f.coalesce(f.col(\"updatedStudyId\"), f.col(\"studyId\")),\n            )\n            .withColumn(\n                \"traitFromSource\",\n                f.coalesce(f.col(\"updatedtraitFromSource\"), f.col(\"traitFromSource\")),\n            )\n            .withColumn(\n                \"traitFromSourceMappedIds\",\n                f.coalesce(\n                    f.col(\"updatedtraitFromSourceMappedIds\"),\n                    f.col(\"traitFromSourceMappedIds\"),\n                ),\n            )\n            .select(self._df.columns)\n        )\n\n        return self\n\n    def annotate_from_study_curation(\n        self: StudyIndexGWASCatalog, curation_table: DataFrame | None\n    ) -&gt; StudyIndexGWASCatalog:\n        \"\"\"Annotating study index with curation.\n\n        Args:\n            curation_table (DataFrame | None): Curated GWAS Catalog studies with summary statistics\n\n        Returns:\n            StudyIndexGWASCatalog: Updated study index\n        \"\"\"\n        # Providing curation table is optional. However once this method is called, the quality and studyFlag columns are added.\n        if curation_table is None:\n            return self\n\n        columns = self.df.columns\n\n        # Adding prefix to columns in the curation table:\n        curation_table = curation_table.select(\n            *[\n                f.col(column).alias(f\"curation_{column}\")\n                if column != \"studyId\"\n                else f.col(column)\n                for column in curation_table.columns\n            ]\n        )\n\n        # Create expression how to update/create quality controls dataset:\n        qualityControls_expression = (\n            f.col(\"curation_qualityControls\")\n            if \"qualityControls\" not in columns\n            else f.when(\n                f.col(\"curation_qualityControls\").isNotNull(),\n                f.array_union(\n                    f.col(\"qualityControls\"), f.array(f.col(\"curation_qualityControls\"))\n                ),\n            ).otherwise(f.col(\"qualityControls\"))\n        )\n\n        # Create expression how to update/create analysis flag:\n        analysis_expression = (\n            f.col(\"curation_analysisFlags\")\n            if \"analysisFlags\" not in columns\n            else f.when(\n                f.col(\"curation_analysisFlags\").isNotNull(),\n                f.array_union(\n                    f.col(\"analysisFlags\"), f.array(f.col(\"curation_analysisFlags\"))\n                ),\n            ).otherwise(f.col(\"analysisFlags\"))\n        )\n\n        # Updating columns list. We might or might not list columns twice, but that doesn't matter, unique set will generated:\n        columns = list(set(columns + [\"qualityControls\", \"analysisFlags\"]))\n\n        # Based on the curation table, columns needs to be updated:\n        curated_df = (\n            self.df.join(curation_table, on=\"studyId\", how=\"left\")\n            # Updating study type:\n            .withColumn(\n                \"studyType\", f.coalesce(f.col(\"curation_studyType\"), f.col(\"studyType\"))\n            )\n            # Updating quality controls:\n            .withColumn(\"qualityControls\", qualityControls_expression)\n            # Updating study annotation flags:\n            .withColumn(\"analysisFlags\", analysis_expression)\n            # Dropping columns coming from the curation table:\n            .select(*columns)\n        )\n        return StudyIndexGWASCatalog(\n            _df=curated_df, _schema=StudyIndexGWASCatalog.get_schema()\n        )\n\n    def extract_studies_for_curation(\n        self: StudyIndexGWASCatalog, curation: DataFrame | None\n    ) -&gt; DataFrame:\n        \"\"\"Extract studies for curation.\n\n        Args:\n            curation (DataFrame | None): Dataframe with curation.\n\n        Returns:\n            DataFrame: Updated curation table. New studies are have the `isCurated` False.\n        \"\"\"\n        # If no curation table provided, assume all studies needs curation:\n        if curation is None:\n            return (\n                self.df\n                # Curation only applyed on studies with summary statistics:\n                .filter(f.col(\"hasSumstats\"))\n                # Adding columns expected in the curation table - array columns aready flattened:\n                .withColumn(\"studyType\", f.lit(None).cast(t.StringType()))\n                .withColumn(\"analysisFlag\", f.lit(None).cast(t.StringType()))\n                .withColumn(\"qualityControl\", f.lit(None).cast(t.StringType()))\n                .withColumn(\"isCurated\", f.lit(False).cast(t.StringType()))\n            )\n\n        # Adding prefix to columns in the curation table:\n        curation = curation.select(\n            *[\n                f.col(column).alias(f\"curation_{column}\")\n                if column != \"studyId\"\n                else f.col(column)\n                for column in curation.columns\n            ]\n        )\n\n        return (\n            self.df\n            # Curation only applyed on studies with summary statistics:\n            .filter(f.col(\"hasSumstats\"))\n            .join(curation, on=\"studyId\", how=\"left\")\n            .select(\n                \"studyId\",\n                # Propagate existing curation - array columns are being flattened:\n                f.col(\"curation_studyType\").alias(\"studyType\"),\n                f.array_join(f.col(\"curation_analysisFlags\"), \"|\").alias(\n                    \"analysisFlag\"\n                ),\n                f.array_join(f.col(\"curation_qualityControls\"), \"|\").alias(\n                    \"qualityControl\"\n                ),\n                # This boolean flag needs to be casted to string, because saving to tsv would fail otherwise:\n                f.coalesce(f.col(\"curation_isCurated\"), f.lit(False))\n                .cast(t.StringType())\n                .alias(\"isCurated\"),\n                # The following columns are propagated to make curation easier:\n                \"pubmedId\",\n                \"publicationTitle\",\n                \"traitFromSource\",\n            )\n        )\n\n    def annotate_ancestries(\n        self: StudyIndexGWASCatalog, ancestry_lut: DataFrame\n    ) -&gt; StudyIndexGWASCatalog:\n        \"\"\"Extracting sample sizes and ancestry information.\n\n        This function parses the ancestry data. Also get counts for the europeans in the same\n        discovery stage.\n\n        Args:\n            ancestry_lut (DataFrame): Ancestry table as downloaded from the GWAS Catalog\n\n        Returns:\n            StudyIndexGWASCatalog: Slimmed and cleaned version of the ancestry annotation.\n        \"\"\"\n        from gentropy.datasource.gwas_catalog.study_index import (\n            StudyIndexGWASCatalogParser as GWASCatalogStudyIndexParser,\n        )\n\n        ancestry = (\n            ancestry_lut\n            # Convert column headers to camelcase:\n            .transform(\n                lambda df: df.select(\n                    *[f.expr(column2camel_case(x)) for x in df.columns]\n                )\n            ).withColumnRenamed(\n                \"studyAccession\", \"studyId\"\n            )  # studyId has not been split yet\n        )\n\n        # Parsing cohort information:\n        cohorts = ancestry_lut.select(\n            f.col(\"STUDY ACCESSION\").alias(\"studyId\"),\n            GWASCatalogStudyIndexParser.parse_cohorts(f.col(\"COHORT(S)\")).alias(\n                \"cohorts\"\n            ),\n        ).distinct()\n\n        # Get a high resolution dataset on experimental stage:\n        ancestry_stages = (\n            ancestry.groupBy(\"studyId\")\n            .pivot(\"stage\")\n            .agg(\n                f.collect_set(\n                    f.struct(\n                        f.col(\"broadAncestralCategory\").alias(\"ancestry\"),\n                        f.col(\"numberOfIndividuals\")\n                        .cast(t.IntegerType())\n                        .alias(\"sampleSize\"),\n                    )\n                )\n            )\n            .withColumn(\n                \"discoverySamples\",\n                GWASCatalogStudyIndexParser._parse_discovery_samples(f.col(\"initial\")),\n            )\n            .withColumnRenamed(\"replication\", \"replicationSamples\")\n            # Mapping discovery stage ancestries to LD reference:\n            .withColumn(\n                \"ldPopulationStructure\",\n                self.aggregate_and_map_ancestries(f.col(\"discoverySamples\")),\n            )\n            .drop(\"initial\")\n            .persist()\n        )\n\n        # Generate information on the ancestry composition of the discovery stage, and calculate\n        # the proportion of the Europeans:\n        europeans_deconvoluted = (\n            ancestry\n            # Focus on discovery stage:\n            .filter(f.col(\"stage\") == \"initial\")\n            # Sorting ancestries if European:\n            .withColumn(\n                \"ancestryFlag\",\n                # Excluding finnish:\n                f.when(\n                    f.col(\"initialSampleDescription\").contains(\"Finnish\"),\n                    f.lit(\"other\"),\n                )\n                # Excluding Icelandic population:\n                .when(\n                    f.col(\"initialSampleDescription\").contains(\"Icelandic\"),\n                    f.lit(\"other\"),\n                )\n                # Including European ancestry:\n                .when(f.col(\"broadAncestralCategory\") == \"European\", f.lit(\"european\"))\n                # Exclude all other population:\n                .otherwise(\"other\"),\n            )\n            # Grouping by study accession and initial sample description:\n            .groupBy(\"studyId\")\n            .pivot(\"ancestryFlag\")\n            .agg(\n                # Summarizing sample sizes for all ancestries:\n                f.sum(f.col(\"numberOfIndividuals\"))\n            )\n            # Do arithmetics to make sure we have the right proportion of european in the set:\n            .withColumn(\n                \"initialSampleCountEuropean\",\n                f.when(f.col(\"european\").isNull(), f.lit(0)).otherwise(\n                    f.col(\"european\")\n                ),\n            )\n            .withColumn(\n                \"initialSampleCountOther\",\n                f.when(f.col(\"other\").isNull(), f.lit(0)).otherwise(f.col(\"other\")),\n            )\n            .withColumn(\n                \"initialSampleCount\",\n                f.col(\"initialSampleCountEuropean\") + f.col(\"other\"),\n            )\n            .drop(\n                \"european\",\n                \"other\",\n                \"initialSampleCount\",\n                \"initialSampleCountEuropean\",\n                \"initialSampleCountOther\",\n            )\n        )\n\n        parsed_ancestry_lut = ancestry_stages.join(\n            europeans_deconvoluted, on=\"studyId\", how=\"outer\"\n        ).select(\n            \"studyId\", \"discoverySamples\", \"ldPopulationStructure\", \"replicationSamples\"\n        )\n        self.df = self.df.join(parsed_ancestry_lut, on=\"studyId\", how=\"left\").join(\n            cohorts, on=\"studyId\", how=\"left\"\n        )\n        return self\n\n    def annotate_sumstats_info(\n        self: StudyIndexGWASCatalog, sumstats_lut: DataFrame\n    ) -&gt; StudyIndexGWASCatalog:\n        \"\"\"Annotate summary stat locations.\n\n        Args:\n            sumstats_lut (DataFrame): listing GWAS Catalog summary stats paths\n\n        Returns:\n            StudyIndexGWASCatalog: including `summarystatsLocation` and `hasSumstats` columns\n\n        Raises:\n            ValueError: if the sumstats_lut table doesn't have the right columns\n        \"\"\"\n        gwas_sumstats_base_uri = (\n            \"ftp://ftp.ebi.ac.uk/pub/databases/gwas/summary_statistics/\"\n        )\n\n        if \"_c0\" not in sumstats_lut.columns:\n            raise ValueError(\n                f'Sumstats look-up table needs to have `_c0` column. However it has: {\",\".join(sumstats_lut.columns)}'\n            )\n\n        parsed_sumstats_lut = sumstats_lut.withColumn(\n            \"summarystatsLocation\",\n            f.concat(\n                f.lit(gwas_sumstats_base_uri),\n                f.regexp_replace(f.col(\"_c0\"), r\"^\\.\\/\", \"\"),\n            ),\n        ).select(\n            self._parse_gwas_catalog_study_id(\"summarystatsLocation\").alias(\"studyId\"),\n            \"summarystatsLocation\",\n            f.lit(True).alias(\"hasSumstats\"),\n        )\n        self.df = (\n            self.df.drop(\"hasSumstats\")\n            .join(parsed_sumstats_lut, on=\"studyId\", how=\"left\")\n            .withColumn(\"hasSumstats\", f.coalesce(f.col(\"hasSumstats\"), f.lit(False)))\n        )\n        return self\n\n    def annotate_discovery_sample_sizes(\n        self: StudyIndexGWASCatalog,\n    ) -&gt; StudyIndexGWASCatalog:\n        \"\"\"Extract the sample size of the discovery stage of the study as annotated in the GWAS Catalog.\n\n        For some studies that measure quantitative traits, nCases and nControls can't be extracted. Therefore, we assume these are 0.\n\n        Returns:\n            StudyIndexGWASCatalog: object with columns `nCases`, `nControls`, and `nSamples` per `studyId` correctly extracted.\n        \"\"\"\n        sample_size_lut = (\n            self.df.select(\n                \"studyId\",\n                f.explode_outer(f.split(f.col(\"initialSampleSize\"), r\",\\s+\")).alias(\n                    \"samples\"\n                ),\n            )\n            # Extracting the sample size from the string:\n            .withColumn(\n                \"sampleSize\",\n                f.regexp_extract(\n                    f.regexp_replace(f.col(\"samples\"), \",\", \"\"), r\"[0-9,]+\", 0\n                ).cast(t.IntegerType()),\n            )\n            .select(\n                \"studyId\",\n                \"sampleSize\",\n                f.when(f.col(\"samples\").contains(\"cases\"), f.col(\"sampleSize\"))\n                .otherwise(f.lit(0))\n                .alias(\"nCases\"),\n                f.when(f.col(\"samples\").contains(\"controls\"), f.col(\"sampleSize\"))\n                .otherwise(f.lit(0))\n                .alias(\"nControls\"),\n            )\n            # Aggregating sample sizes for all ancestries:\n            .groupBy(\"studyId\")  # studyId has not been split yet\n            .agg(\n                f.sum(\"nCases\").cast(\"integer\").alias(\"nCases\"),\n                f.sum(\"nControls\").cast(\"integer\").alias(\"nControls\"),\n                f.sum(\"sampleSize\").cast(\"integer\").alias(\"nSamples\"),\n            )\n        )\n        self.df = self.df.join(sample_size_lut, on=\"studyId\", how=\"left\")\n        return self\n\n    def apply_inclusion_list(\n        self: StudyIndexGWASCatalog, inclusion_list: DataFrame\n    ) -&gt; StudyIndexGWASCatalog:\n        \"\"\"Restricting GWAS Catalog studies based on a list of accepted study identifiers.\n\n        Args:\n            inclusion_list (DataFrame): List of accepted GWAS Catalog study identifiers\n\n        Returns:\n            StudyIndexGWASCatalog: Filtered dataset.\n        \"\"\"\n        return StudyIndexGWASCatalog(\n            _df=self.df.join(inclusion_list, on=\"studyId\", how=\"inner\"),\n            _schema=StudyIndexGWASCatalog.get_schema(),\n        )\n\n    @staticmethod\n    def _parse_gwas_catalog_study_id(sumstats_path_column: str) -&gt; Column:\n        \"\"\"Extract GWAS Catalog study accession from the summary statistics path.\n\n        Args:\n            sumstats_path_column (str): column *name* for the summary statistics path\n\n        Returns:\n            Column: GWAS Catalog study accession.\n\n        Examples:\n            &gt;&gt;&gt; data = [\n            ... ('./GCST90086001-GCST90087000/GCST90086758/harmonised/35078996-GCST90086758-EFO_0007937.h.tsv.gz',),\n            ...    ('gs://open-targets-gwas-summary-stats/harmonised/GCST000568.parquet/',),\n            ...    (None,)\n            ... ]\n            &gt;&gt;&gt; spark.createDataFrame(data, ['testColumn']).select(StudyIndexGWASCatalog._parse_gwas_catalog_study_id('testColumn').alias('accessions')).collect()\n            [Row(accessions='GCST90086758'), Row(accessions='GCST000568'), Row(accessions=None)]\n        \"\"\"\n        accesions = f.expr(rf\"regexp_extract_all({sumstats_path_column}, '(GCST\\\\d+)')\")\n        return accesions[f.size(accesions) - 1]\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/study_index/#gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.annotate_ancestries","title":"<code>annotate_ancestries(ancestry_lut: DataFrame) -&gt; StudyIndexGWASCatalog</code>","text":"<p>Extracting sample sizes and ancestry information.</p> <p>This function parses the ancestry data. Also get counts for the europeans in the same discovery stage.</p> <p>Parameters:</p> Name Type Description Default <code>ancestry_lut</code> <code>DataFrame</code> <p>Ancestry table as downloaded from the GWAS Catalog</p> required <p>Returns:</p> Name Type Description <code>StudyIndexGWASCatalog</code> <code>StudyIndexGWASCatalog</code> <p>Slimmed and cleaned version of the ancestry annotation.</p> Source code in <code>src/gentropy/datasource/gwas_catalog/study_index.py</code> <pre><code>def annotate_ancestries(\n    self: StudyIndexGWASCatalog, ancestry_lut: DataFrame\n) -&gt; StudyIndexGWASCatalog:\n    \"\"\"Extracting sample sizes and ancestry information.\n\n    This function parses the ancestry data. Also get counts for the europeans in the same\n    discovery stage.\n\n    Args:\n        ancestry_lut (DataFrame): Ancestry table as downloaded from the GWAS Catalog\n\n    Returns:\n        StudyIndexGWASCatalog: Slimmed and cleaned version of the ancestry annotation.\n    \"\"\"\n    from gentropy.datasource.gwas_catalog.study_index import (\n        StudyIndexGWASCatalogParser as GWASCatalogStudyIndexParser,\n    )\n\n    ancestry = (\n        ancestry_lut\n        # Convert column headers to camelcase:\n        .transform(\n            lambda df: df.select(\n                *[f.expr(column2camel_case(x)) for x in df.columns]\n            )\n        ).withColumnRenamed(\n            \"studyAccession\", \"studyId\"\n        )  # studyId has not been split yet\n    )\n\n    # Parsing cohort information:\n    cohorts = ancestry_lut.select(\n        f.col(\"STUDY ACCESSION\").alias(\"studyId\"),\n        GWASCatalogStudyIndexParser.parse_cohorts(f.col(\"COHORT(S)\")).alias(\n            \"cohorts\"\n        ),\n    ).distinct()\n\n    # Get a high resolution dataset on experimental stage:\n    ancestry_stages = (\n        ancestry.groupBy(\"studyId\")\n        .pivot(\"stage\")\n        .agg(\n            f.collect_set(\n                f.struct(\n                    f.col(\"broadAncestralCategory\").alias(\"ancestry\"),\n                    f.col(\"numberOfIndividuals\")\n                    .cast(t.IntegerType())\n                    .alias(\"sampleSize\"),\n                )\n            )\n        )\n        .withColumn(\n            \"discoverySamples\",\n            GWASCatalogStudyIndexParser._parse_discovery_samples(f.col(\"initial\")),\n        )\n        .withColumnRenamed(\"replication\", \"replicationSamples\")\n        # Mapping discovery stage ancestries to LD reference:\n        .withColumn(\n            \"ldPopulationStructure\",\n            self.aggregate_and_map_ancestries(f.col(\"discoverySamples\")),\n        )\n        .drop(\"initial\")\n        .persist()\n    )\n\n    # Generate information on the ancestry composition of the discovery stage, and calculate\n    # the proportion of the Europeans:\n    europeans_deconvoluted = (\n        ancestry\n        # Focus on discovery stage:\n        .filter(f.col(\"stage\") == \"initial\")\n        # Sorting ancestries if European:\n        .withColumn(\n            \"ancestryFlag\",\n            # Excluding finnish:\n            f.when(\n                f.col(\"initialSampleDescription\").contains(\"Finnish\"),\n                f.lit(\"other\"),\n            )\n            # Excluding Icelandic population:\n            .when(\n                f.col(\"initialSampleDescription\").contains(\"Icelandic\"),\n                f.lit(\"other\"),\n            )\n            # Including European ancestry:\n            .when(f.col(\"broadAncestralCategory\") == \"European\", f.lit(\"european\"))\n            # Exclude all other population:\n            .otherwise(\"other\"),\n        )\n        # Grouping by study accession and initial sample description:\n        .groupBy(\"studyId\")\n        .pivot(\"ancestryFlag\")\n        .agg(\n            # Summarizing sample sizes for all ancestries:\n            f.sum(f.col(\"numberOfIndividuals\"))\n        )\n        # Do arithmetics to make sure we have the right proportion of european in the set:\n        .withColumn(\n            \"initialSampleCountEuropean\",\n            f.when(f.col(\"european\").isNull(), f.lit(0)).otherwise(\n                f.col(\"european\")\n            ),\n        )\n        .withColumn(\n            \"initialSampleCountOther\",\n            f.when(f.col(\"other\").isNull(), f.lit(0)).otherwise(f.col(\"other\")),\n        )\n        .withColumn(\n            \"initialSampleCount\",\n            f.col(\"initialSampleCountEuropean\") + f.col(\"other\"),\n        )\n        .drop(\n            \"european\",\n            \"other\",\n            \"initialSampleCount\",\n            \"initialSampleCountEuropean\",\n            \"initialSampleCountOther\",\n        )\n    )\n\n    parsed_ancestry_lut = ancestry_stages.join(\n        europeans_deconvoluted, on=\"studyId\", how=\"outer\"\n    ).select(\n        \"studyId\", \"discoverySamples\", \"ldPopulationStructure\", \"replicationSamples\"\n    )\n    self.df = self.df.join(parsed_ancestry_lut, on=\"studyId\", how=\"left\").join(\n        cohorts, on=\"studyId\", how=\"left\"\n    )\n    return self\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/study_index/#gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.annotate_discovery_sample_sizes","title":"<code>annotate_discovery_sample_sizes() -&gt; StudyIndexGWASCatalog</code>","text":"<p>Extract the sample size of the discovery stage of the study as annotated in the GWAS Catalog.</p> <p>For some studies that measure quantitative traits, nCases and nControls can't be extracted. Therefore, we assume these are 0.</p> <p>Returns:</p> Name Type Description <code>StudyIndexGWASCatalog</code> <code>StudyIndexGWASCatalog</code> <p>object with columns <code>nCases</code>, <code>nControls</code>, and <code>nSamples</code> per <code>studyId</code> correctly extracted.</p> Source code in <code>src/gentropy/datasource/gwas_catalog/study_index.py</code> <pre><code>def annotate_discovery_sample_sizes(\n    self: StudyIndexGWASCatalog,\n) -&gt; StudyIndexGWASCatalog:\n    \"\"\"Extract the sample size of the discovery stage of the study as annotated in the GWAS Catalog.\n\n    For some studies that measure quantitative traits, nCases and nControls can't be extracted. Therefore, we assume these are 0.\n\n    Returns:\n        StudyIndexGWASCatalog: object with columns `nCases`, `nControls`, and `nSamples` per `studyId` correctly extracted.\n    \"\"\"\n    sample_size_lut = (\n        self.df.select(\n            \"studyId\",\n            f.explode_outer(f.split(f.col(\"initialSampleSize\"), r\",\\s+\")).alias(\n                \"samples\"\n            ),\n        )\n        # Extracting the sample size from the string:\n        .withColumn(\n            \"sampleSize\",\n            f.regexp_extract(\n                f.regexp_replace(f.col(\"samples\"), \",\", \"\"), r\"[0-9,]+\", 0\n            ).cast(t.IntegerType()),\n        )\n        .select(\n            \"studyId\",\n            \"sampleSize\",\n            f.when(f.col(\"samples\").contains(\"cases\"), f.col(\"sampleSize\"))\n            .otherwise(f.lit(0))\n            .alias(\"nCases\"),\n            f.when(f.col(\"samples\").contains(\"controls\"), f.col(\"sampleSize\"))\n            .otherwise(f.lit(0))\n            .alias(\"nControls\"),\n        )\n        # Aggregating sample sizes for all ancestries:\n        .groupBy(\"studyId\")  # studyId has not been split yet\n        .agg(\n            f.sum(\"nCases\").cast(\"integer\").alias(\"nCases\"),\n            f.sum(\"nControls\").cast(\"integer\").alias(\"nControls\"),\n            f.sum(\"sampleSize\").cast(\"integer\").alias(\"nSamples\"),\n        )\n    )\n    self.df = self.df.join(sample_size_lut, on=\"studyId\", how=\"left\")\n    return self\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/study_index/#gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.annotate_from_study_curation","title":"<code>annotate_from_study_curation(curation_table: DataFrame | None) -&gt; StudyIndexGWASCatalog</code>","text":"<p>Annotating study index with curation.</p> <p>Parameters:</p> Name Type Description Default <code>curation_table</code> <code>DataFrame | None</code> <p>Curated GWAS Catalog studies with summary statistics</p> required <p>Returns:</p> Name Type Description <code>StudyIndexGWASCatalog</code> <code>StudyIndexGWASCatalog</code> <p>Updated study index</p> Source code in <code>src/gentropy/datasource/gwas_catalog/study_index.py</code> <pre><code>def annotate_from_study_curation(\n    self: StudyIndexGWASCatalog, curation_table: DataFrame | None\n) -&gt; StudyIndexGWASCatalog:\n    \"\"\"Annotating study index with curation.\n\n    Args:\n        curation_table (DataFrame | None): Curated GWAS Catalog studies with summary statistics\n\n    Returns:\n        StudyIndexGWASCatalog: Updated study index\n    \"\"\"\n    # Providing curation table is optional. However once this method is called, the quality and studyFlag columns are added.\n    if curation_table is None:\n        return self\n\n    columns = self.df.columns\n\n    # Adding prefix to columns in the curation table:\n    curation_table = curation_table.select(\n        *[\n            f.col(column).alias(f\"curation_{column}\")\n            if column != \"studyId\"\n            else f.col(column)\n            for column in curation_table.columns\n        ]\n    )\n\n    # Create expression how to update/create quality controls dataset:\n    qualityControls_expression = (\n        f.col(\"curation_qualityControls\")\n        if \"qualityControls\" not in columns\n        else f.when(\n            f.col(\"curation_qualityControls\").isNotNull(),\n            f.array_union(\n                f.col(\"qualityControls\"), f.array(f.col(\"curation_qualityControls\"))\n            ),\n        ).otherwise(f.col(\"qualityControls\"))\n    )\n\n    # Create expression how to update/create analysis flag:\n    analysis_expression = (\n        f.col(\"curation_analysisFlags\")\n        if \"analysisFlags\" not in columns\n        else f.when(\n            f.col(\"curation_analysisFlags\").isNotNull(),\n            f.array_union(\n                f.col(\"analysisFlags\"), f.array(f.col(\"curation_analysisFlags\"))\n            ),\n        ).otherwise(f.col(\"analysisFlags\"))\n    )\n\n    # Updating columns list. We might or might not list columns twice, but that doesn't matter, unique set will generated:\n    columns = list(set(columns + [\"qualityControls\", \"analysisFlags\"]))\n\n    # Based on the curation table, columns needs to be updated:\n    curated_df = (\n        self.df.join(curation_table, on=\"studyId\", how=\"left\")\n        # Updating study type:\n        .withColumn(\n            \"studyType\", f.coalesce(f.col(\"curation_studyType\"), f.col(\"studyType\"))\n        )\n        # Updating quality controls:\n        .withColumn(\"qualityControls\", qualityControls_expression)\n        # Updating study annotation flags:\n        .withColumn(\"analysisFlags\", analysis_expression)\n        # Dropping columns coming from the curation table:\n        .select(*columns)\n    )\n    return StudyIndexGWASCatalog(\n        _df=curated_df, _schema=StudyIndexGWASCatalog.get_schema()\n    )\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/study_index/#gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.annotate_sumstats_info","title":"<code>annotate_sumstats_info(sumstats_lut: DataFrame) -&gt; StudyIndexGWASCatalog</code>","text":"<p>Annotate summary stat locations.</p> <p>Parameters:</p> Name Type Description Default <code>sumstats_lut</code> <code>DataFrame</code> <p>listing GWAS Catalog summary stats paths</p> required <p>Returns:</p> Name Type Description <code>StudyIndexGWASCatalog</code> <code>StudyIndexGWASCatalog</code> <p>including <code>summarystatsLocation</code> and <code>hasSumstats</code> columns</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>if the sumstats_lut table doesn't have the right columns</p> Source code in <code>src/gentropy/datasource/gwas_catalog/study_index.py</code> <pre><code>def annotate_sumstats_info(\n    self: StudyIndexGWASCatalog, sumstats_lut: DataFrame\n) -&gt; StudyIndexGWASCatalog:\n    \"\"\"Annotate summary stat locations.\n\n    Args:\n        sumstats_lut (DataFrame): listing GWAS Catalog summary stats paths\n\n    Returns:\n        StudyIndexGWASCatalog: including `summarystatsLocation` and `hasSumstats` columns\n\n    Raises:\n        ValueError: if the sumstats_lut table doesn't have the right columns\n    \"\"\"\n    gwas_sumstats_base_uri = (\n        \"ftp://ftp.ebi.ac.uk/pub/databases/gwas/summary_statistics/\"\n    )\n\n    if \"_c0\" not in sumstats_lut.columns:\n        raise ValueError(\n            f'Sumstats look-up table needs to have `_c0` column. However it has: {\",\".join(sumstats_lut.columns)}'\n        )\n\n    parsed_sumstats_lut = sumstats_lut.withColumn(\n        \"summarystatsLocation\",\n        f.concat(\n            f.lit(gwas_sumstats_base_uri),\n            f.regexp_replace(f.col(\"_c0\"), r\"^\\.\\/\", \"\"),\n        ),\n    ).select(\n        self._parse_gwas_catalog_study_id(\"summarystatsLocation\").alias(\"studyId\"),\n        \"summarystatsLocation\",\n        f.lit(True).alias(\"hasSumstats\"),\n    )\n    self.df = (\n        self.df.drop(\"hasSumstats\")\n        .join(parsed_sumstats_lut, on=\"studyId\", how=\"left\")\n        .withColumn(\"hasSumstats\", f.coalesce(f.col(\"hasSumstats\"), f.lit(False)))\n    )\n    return self\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/study_index/#gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.apply_inclusion_list","title":"<code>apply_inclusion_list(inclusion_list: DataFrame) -&gt; StudyIndexGWASCatalog</code>","text":"<p>Restricting GWAS Catalog studies based on a list of accepted study identifiers.</p> <p>Parameters:</p> Name Type Description Default <code>inclusion_list</code> <code>DataFrame</code> <p>List of accepted GWAS Catalog study identifiers</p> required <p>Returns:</p> Name Type Description <code>StudyIndexGWASCatalog</code> <code>StudyIndexGWASCatalog</code> <p>Filtered dataset.</p> Source code in <code>src/gentropy/datasource/gwas_catalog/study_index.py</code> <pre><code>def apply_inclusion_list(\n    self: StudyIndexGWASCatalog, inclusion_list: DataFrame\n) -&gt; StudyIndexGWASCatalog:\n    \"\"\"Restricting GWAS Catalog studies based on a list of accepted study identifiers.\n\n    Args:\n        inclusion_list (DataFrame): List of accepted GWAS Catalog study identifiers\n\n    Returns:\n        StudyIndexGWASCatalog: Filtered dataset.\n    \"\"\"\n    return StudyIndexGWASCatalog(\n        _df=self.df.join(inclusion_list, on=\"studyId\", how=\"inner\"),\n        _schema=StudyIndexGWASCatalog.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/study_index/#gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.extract_studies_for_curation","title":"<code>extract_studies_for_curation(curation: DataFrame | None) -&gt; DataFrame</code>","text":"<p>Extract studies for curation.</p> <p>Parameters:</p> Name Type Description Default <code>curation</code> <code>DataFrame | None</code> <p>Dataframe with curation.</p> required <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>Updated curation table. New studies are have the <code>isCurated</code> False.</p> Source code in <code>src/gentropy/datasource/gwas_catalog/study_index.py</code> <pre><code>def extract_studies_for_curation(\n    self: StudyIndexGWASCatalog, curation: DataFrame | None\n) -&gt; DataFrame:\n    \"\"\"Extract studies for curation.\n\n    Args:\n        curation (DataFrame | None): Dataframe with curation.\n\n    Returns:\n        DataFrame: Updated curation table. New studies are have the `isCurated` False.\n    \"\"\"\n    # If no curation table provided, assume all studies needs curation:\n    if curation is None:\n        return (\n            self.df\n            # Curation only applyed on studies with summary statistics:\n            .filter(f.col(\"hasSumstats\"))\n            # Adding columns expected in the curation table - array columns aready flattened:\n            .withColumn(\"studyType\", f.lit(None).cast(t.StringType()))\n            .withColumn(\"analysisFlag\", f.lit(None).cast(t.StringType()))\n            .withColumn(\"qualityControl\", f.lit(None).cast(t.StringType()))\n            .withColumn(\"isCurated\", f.lit(False).cast(t.StringType()))\n        )\n\n    # Adding prefix to columns in the curation table:\n    curation = curation.select(\n        *[\n            f.col(column).alias(f\"curation_{column}\")\n            if column != \"studyId\"\n            else f.col(column)\n            for column in curation.columns\n        ]\n    )\n\n    return (\n        self.df\n        # Curation only applyed on studies with summary statistics:\n        .filter(f.col(\"hasSumstats\"))\n        .join(curation, on=\"studyId\", how=\"left\")\n        .select(\n            \"studyId\",\n            # Propagate existing curation - array columns are being flattened:\n            f.col(\"curation_studyType\").alias(\"studyType\"),\n            f.array_join(f.col(\"curation_analysisFlags\"), \"|\").alias(\n                \"analysisFlag\"\n            ),\n            f.array_join(f.col(\"curation_qualityControls\"), \"|\").alias(\n                \"qualityControl\"\n            ),\n            # This boolean flag needs to be casted to string, because saving to tsv would fail otherwise:\n            f.coalesce(f.col(\"curation_isCurated\"), f.lit(False))\n            .cast(t.StringType())\n            .alias(\"isCurated\"),\n            # The following columns are propagated to make curation easier:\n            \"pubmedId\",\n            \"publicationTitle\",\n            \"traitFromSource\",\n        )\n    )\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/study_index/#gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.update_study_id","title":"<code>update_study_id(study_annotation: DataFrame) -&gt; StudyIndexGWASCatalog</code>","text":"<p>Update studyId with a dataframe containing study.</p> <p>Parameters:</p> Name Type Description Default <code>study_annotation</code> <code>DataFrame</code> <p>Dataframe containing <code>updatedStudyId</code>, <code>traitFromSource</code>, <code>traitFromSourceMappedIds</code> and key column <code>studyId</code>.</p> required <p>Returns:</p> Name Type Description <code>StudyIndexGWASCatalog</code> <code>StudyIndexGWASCatalog</code> <p>Updated study table.</p> Source code in <code>src/gentropy/datasource/gwas_catalog/study_index.py</code> <pre><code>def update_study_id(\n    self: StudyIndexGWASCatalog, study_annotation: DataFrame\n) -&gt; StudyIndexGWASCatalog:\n    \"\"\"Update studyId with a dataframe containing study.\n\n    Args:\n        study_annotation (DataFrame): Dataframe containing `updatedStudyId`, `traitFromSource`, `traitFromSourceMappedIds` and key column `studyId`.\n\n    Returns:\n        StudyIndexGWASCatalog: Updated study table.\n    \"\"\"\n    self.df = (\n        self._df.join(\n            study_annotation.select(\n                *[\n                    f.col(c).alias(f\"updated{c}\")\n                    if c not in [\"studyId\", \"updatedStudyId\"]\n                    else f.col(c)\n                    for c in study_annotation.columns\n                ]\n            ),\n            on=\"studyId\",\n            how=\"left\",\n        )\n        .withColumn(\n            \"studyId\",\n            f.coalesce(f.col(\"updatedStudyId\"), f.col(\"studyId\")),\n        )\n        .withColumn(\n            \"traitFromSource\",\n            f.coalesce(f.col(\"updatedtraitFromSource\"), f.col(\"traitFromSource\")),\n        )\n        .withColumn(\n            \"traitFromSourceMappedIds\",\n            f.coalesce(\n                f.col(\"updatedtraitFromSourceMappedIds\"),\n                f.col(\"traitFromSourceMappedIds\"),\n            ),\n        )\n        .select(self._df.columns)\n    )\n\n    return self\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/study_splitter/","title":"Study Splitter","text":""},{"location":"python_api/datasources/gwas_catalog/study_splitter/#gentropy.datasource.gwas_catalog.study_splitter.GWASCatalogStudySplitter","title":"<code>gentropy.datasource.gwas_catalog.study_splitter.GWASCatalogStudySplitter</code>","text":"<p>Splitting multi-trait GWAS Catalog studies.</p> Source code in <code>src/gentropy/datasource/gwas_catalog/study_splitter.py</code> <pre><code>class GWASCatalogStudySplitter:\n    \"\"\"Splitting multi-trait GWAS Catalog studies.\"\"\"\n\n    @staticmethod\n    def _resolve_trait(\n        study_trait: Column, association_trait: Column, p_value_text: Column\n    ) -&gt; Column:\n        \"\"\"Resolve trait names by consolidating association-level and study-level trait names.\n\n        Args:\n            study_trait (Column): Study-level trait name.\n            association_trait (Column): Association-level trait name.\n            p_value_text (Column): P-value text.\n\n        Returns:\n            Column: Resolved trait name.\n        \"\"\"\n        return (\n            f.when(\n                (p_value_text.isNotNull()) &amp; (p_value_text != (\"no_pvalue_text\")),\n                f.concat(\n                    association_trait,\n                    f.lit(\" [\"),\n                    p_value_text,\n                    f.lit(\"]\"),\n                ),\n            )\n            .when(\n                association_trait.isNotNull(),\n                association_trait,\n            )\n            .otherwise(study_trait)\n        )\n\n    @staticmethod\n    def _resolve_efo(association_efo: Column, study_efo: Column) -&gt; Column:\n        \"\"\"Resolve EFOs by consolidating association-level and study-level EFOs.\n\n        Args:\n            association_efo (Column): EFO column from the association table.\n            study_efo (Column): EFO column from the study table.\n\n        Returns:\n            Column: Consolidated EFO column.\n        \"\"\"\n        return f.coalesce(f.split(association_efo, r\"\\/\"), study_efo)\n\n    @staticmethod\n    def _resolve_study_id(study_id: Column, sub_study_description: Column) -&gt; Column:\n        \"\"\"Resolve study IDs by exploding association-level information (e.g. pvalue_text, EFO).\n\n        Args:\n            study_id (Column): Study ID column.\n            sub_study_description (Column): Sub-study description column from the association table.\n\n        Returns:\n            Column: Resolved study ID column.\n        \"\"\"\n        split_w = Window.partitionBy(study_id).orderBy(sub_study_description)\n        row_number = f.dense_rank().over(split_w)\n        substudy_count = f.approx_count_distinct(row_number).over(split_w)\n        return f.when(substudy_count == 1, study_id).otherwise(\n            f.concat_ws(\"_\", study_id, row_number)\n        )\n\n    @classmethod\n    def split(\n        cls: type[GWASCatalogStudySplitter],\n        studies: StudyIndexGWASCatalog,\n        associations: StudyLocusGWASCatalog,\n    ) -&gt; Tuple[StudyIndexGWASCatalog, StudyLocusGWASCatalog]:\n        \"\"\"Splitting multi-trait GWAS Catalog studies.\n\n        If assigned disease of the study and the association don't agree, we assume the study needs to be split.\n        Then disease EFOs, trait names and study ID are consolidated\n\n        Args:\n            studies (StudyIndexGWASCatalog): GWAS Catalog studies.\n            associations (StudyLocusGWASCatalog): GWAS Catalog associations.\n\n        Returns:\n            Tuple[StudyIndexGWASCatalog, StudyLocusGWASCatalog]: Split studies and associations.\n        \"\"\"\n        # Composite of studies and associations to resolve scattered information\n        st_ass = (\n            associations.df.join(f.broadcast(studies.df), on=\"studyId\", how=\"inner\")\n            .select(\n                \"studyId\",\n                \"subStudyDescription\",\n                cls._resolve_study_id(\n                    f.col(\"studyId\"), f.col(\"subStudyDescription\")\n                ).alias(\"updatedStudyId\"),\n                cls._resolve_trait(\n                    f.col(\"traitFromSource\"),\n                    f.split(\"subStudyDescription\", r\"\\|\").getItem(0),\n                    f.split(\"subStudyDescription\", r\"\\|\").getItem(1),\n                ).alias(\"traitFromSource\"),\n                cls._resolve_efo(\n                    f.split(\"subStudyDescription\", r\"\\|\").getItem(2),\n                    f.col(\"traitFromSourceMappedIds\"),\n                ).alias(\"traitFromSourceMappedIds\"),\n            )\n            .persist()\n        )\n\n        return (\n            studies.update_study_id(\n                st_ass.select(\n                    \"studyId\",\n                    \"updatedStudyId\",\n                    \"traitFromSource\",\n                    \"traitFromSourceMappedIds\",\n                ).distinct()\n            ),\n            associations.update_study_id(\n                st_ass.select(\n                    \"updatedStudyId\", \"studyId\", \"subStudyDescription\"\n                ).distinct()\n            ).qc_ambiguous_study(),\n        )\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/study_splitter/#gentropy.datasource.gwas_catalog.study_splitter.GWASCatalogStudySplitter.split","title":"<code>split(studies: StudyIndexGWASCatalog, associations: StudyLocusGWASCatalog) -&gt; Tuple[StudyIndexGWASCatalog, StudyLocusGWASCatalog]</code>  <code>classmethod</code>","text":"<p>Splitting multi-trait GWAS Catalog studies.</p> <p>If assigned disease of the study and the association don't agree, we assume the study needs to be split. Then disease EFOs, trait names and study ID are consolidated</p> <p>Parameters:</p> Name Type Description Default <code>studies</code> <code>StudyIndexGWASCatalog</code> <p>GWAS Catalog studies.</p> required <code>associations</code> <code>StudyLocusGWASCatalog</code> <p>GWAS Catalog associations.</p> required <p>Returns:</p> Type Description <code>Tuple[StudyIndexGWASCatalog, StudyLocusGWASCatalog]</code> <p>Tuple[StudyIndexGWASCatalog, StudyLocusGWASCatalog]: Split studies and associations.</p> Source code in <code>src/gentropy/datasource/gwas_catalog/study_splitter.py</code> <pre><code>@classmethod\ndef split(\n    cls: type[GWASCatalogStudySplitter],\n    studies: StudyIndexGWASCatalog,\n    associations: StudyLocusGWASCatalog,\n) -&gt; Tuple[StudyIndexGWASCatalog, StudyLocusGWASCatalog]:\n    \"\"\"Splitting multi-trait GWAS Catalog studies.\n\n    If assigned disease of the study and the association don't agree, we assume the study needs to be split.\n    Then disease EFOs, trait names and study ID are consolidated\n\n    Args:\n        studies (StudyIndexGWASCatalog): GWAS Catalog studies.\n        associations (StudyLocusGWASCatalog): GWAS Catalog associations.\n\n    Returns:\n        Tuple[StudyIndexGWASCatalog, StudyLocusGWASCatalog]: Split studies and associations.\n    \"\"\"\n    # Composite of studies and associations to resolve scattered information\n    st_ass = (\n        associations.df.join(f.broadcast(studies.df), on=\"studyId\", how=\"inner\")\n        .select(\n            \"studyId\",\n            \"subStudyDescription\",\n            cls._resolve_study_id(\n                f.col(\"studyId\"), f.col(\"subStudyDescription\")\n            ).alias(\"updatedStudyId\"),\n            cls._resolve_trait(\n                f.col(\"traitFromSource\"),\n                f.split(\"subStudyDescription\", r\"\\|\").getItem(0),\n                f.split(\"subStudyDescription\", r\"\\|\").getItem(1),\n            ).alias(\"traitFromSource\"),\n            cls._resolve_efo(\n                f.split(\"subStudyDescription\", r\"\\|\").getItem(2),\n                f.col(\"traitFromSourceMappedIds\"),\n            ).alias(\"traitFromSourceMappedIds\"),\n        )\n        .persist()\n    )\n\n    return (\n        studies.update_study_id(\n            st_ass.select(\n                \"studyId\",\n                \"updatedStudyId\",\n                \"traitFromSource\",\n                \"traitFromSourceMappedIds\",\n            ).distinct()\n        ),\n        associations.update_study_id(\n            st_ass.select(\n                \"updatedStudyId\", \"studyId\", \"subStudyDescription\"\n            ).distinct()\n        ).qc_ambiguous_study(),\n    )\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/summary_statistics/","title":"Summary statistics","text":""},{"location":"python_api/datasources/gwas_catalog/summary_statistics/#gentropy.datasource.gwas_catalog.summary_statistics.GWASCatalogSummaryStatistics","title":"<code>gentropy.datasource.gwas_catalog.summary_statistics.GWASCatalogSummaryStatistics</code>  <code>dataclass</code>","text":"<p>             Bases: <code>SummaryStatistics</code></p> <p>GWAS Catalog Summary Statistics reader.</p> Source code in <code>src/gentropy/datasource/gwas_catalog/summary_statistics.py</code> <pre><code>@dataclass\nclass GWASCatalogSummaryStatistics(SummaryStatistics):\n    \"\"\"GWAS Catalog Summary Statistics reader.\"\"\"\n\n    @classmethod\n    def from_gwas_harmonized_summary_stats(\n        cls: type[GWASCatalogSummaryStatistics],\n        spark: SparkSession,\n        sumstats_file: str,\n    ) -&gt; GWASCatalogSummaryStatistics:\n        \"\"\"Create summary statistics object from summary statistics flatfile, harmonized by the GWAS Catalog.\n\n        Things got slightly complicated given the GWAS Catalog harmonization pipelines changed recently so we had to accomodate to\n        both formats.\n\n        Args:\n            spark (SparkSession): spark session\n            sumstats_file (str): list of GWAS Catalog summary stat files, with study ids in them.\n\n        Returns:\n            GWASCatalogSummaryStatistics: Summary statistics object.\n        \"\"\"\n        sumstats_df = spark.read.csv(sumstats_file, sep=\"\\t\", header=True).withColumn(\n            # Parsing GWAS Catalog study identifier from filename:\n            \"studyId\",\n            f.lit(filename_to_study_identifier(sumstats_file)),\n        )\n\n        # Parsing variant id fields:\n        chromosome = (\n            f.col(\"hm_chrom\")\n            if \"hm_chrom\" in sumstats_df.columns\n            else f.col(\"chromosome\")\n        ).cast(t.StringType())\n        position = (\n            f.col(\"hm_pos\")\n            if \"hm_pos\" in sumstats_df.columns\n            else f.col(\"base_pair_location\")\n        ).cast(t.IntegerType())\n        ref_allele = (\n            f.col(\"hm_other_allele\")\n            if \"hm_other_allele\" in sumstats_df.columns\n            else f.col(\"other_allele\")\n        )\n        alt_allele = (\n            f.col(\"hm_effect_allele\")\n            if \"hm_effect_allele\" in sumstats_df.columns\n            else f.col(\"effect_allele\")\n        )\n\n        # Parsing p-value (get a tuple with mantissa and exponent):\n        p_value_expression = (\n            parse_pvalue(f.col(\"p_value\"))\n            if \"p_value\" in sumstats_df.columns\n            else neglog_pvalue_to_mantissa_and_exponent(f.col(\"neg_log_10_p_value\"))\n        )\n\n        # The effect allele frequency is an optional column, we have to test if it is there:\n        allele_frequency = (\n            f.col(\"effect_allele_frequency\")\n            if \"effect_allele_frequency\" in sumstats_df.columns\n            else f.lit(None)\n        ).cast(t.FloatType())\n\n        # Do we have sample size? This expression captures 99.7% of sample size columns.\n        sample_size = (f.col(\"n\") if \"n\" in sumstats_df.columns else f.lit(None)).cast(\n            t.IntegerType()\n        )\n\n        # Depending on the input, we might have beta, but the column might not be there at all also old format calls differently:\n        beta_expression = (\n            f.col(\"hm_beta\")\n            if \"hm_beta\" in sumstats_df.columns\n            else f.col(\"beta\")\n            if \"beta\" in sumstats_df.columns\n            # If no column, create one:\n            else f.lit(None)\n        ).cast(t.DoubleType())\n\n        # We might have odds ratio or hazard ratio, wich are basically the same:\n        odds_ratio_expression = (\n            f.col(\"hm_odds_ratio\")\n            if \"hm_odds_ratio\" in sumstats_df.columns\n            else f.col(\"odds_ratio\")\n            if \"odds_ratio\" in sumstats_df.columns\n            else f.col(\"hazard_ratio\")\n            if \"hazard_ratio\" in sumstats_df.columns\n            # If no column, create one:\n            else f.lit(None)\n        ).cast(t.DoubleType())\n\n        # Does the file have standard error column?\n        standard_error = (\n            f.col(\"standard_error\")\n            if \"standard_error\" in sumstats_df.columns\n            else f.lit(None)\n        ).cast(t.DoubleType())\n\n        # Processing columns of interest:\n        processed_sumstats_df = (\n            sumstats_df\n            # Dropping rows which doesn't have proper position:\n            .select(\n                \"studyId\",\n                # Adding variant identifier:\n                f.concat_ws(\n                    \"_\",\n                    chromosome,\n                    position,\n                    ref_allele,\n                    alt_allele,\n                ).alias(\"variantId\"),\n                chromosome.alias(\"chromosome\"),\n                position.alias(\"position\"),\n                # Parsing p-value mantissa and exponent:\n                *p_value_expression,\n                # Converting/calculating effect and confidence interval:\n                *convert_odds_ratio_to_beta(\n                    beta_expression,\n                    odds_ratio_expression,\n                    standard_error,\n                ),\n                allele_frequency.alias(\"effectAlleleFrequencyFromSource\"),\n                sample_size.alias(\"sampleSize\"),\n            )\n            .filter(\n                # Dropping associations where no harmonized position is available:\n                f.col(\"position\").isNotNull()\n                &amp;\n                # We are not interested in associations with zero effect:\n                (f.col(\"beta\") != 0)\n            )\n            .orderBy(f.col(\"chromosome\"), f.col(\"position\"))\n            # median study size is 200Mb, max is 2.6Gb\n            .repartition(20)\n        )\n\n        # Initializing summary statistics object:\n        return cls(\n            _df=processed_sumstats_df,\n            _schema=cls.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/summary_statistics/#gentropy.datasource.gwas_catalog.summary_statistics.GWASCatalogSummaryStatistics.from_gwas_harmonized_summary_stats","title":"<code>from_gwas_harmonized_summary_stats(spark: SparkSession, sumstats_file: str) -&gt; GWASCatalogSummaryStatistics</code>  <code>classmethod</code>","text":"<p>Create summary statistics object from summary statistics flatfile, harmonized by the GWAS Catalog.</p> <p>Things got slightly complicated given the GWAS Catalog harmonization pipelines changed recently so we had to accomodate to both formats.</p> <p>Parameters:</p> Name Type Description Default <code>spark</code> <code>SparkSession</code> <p>spark session</p> required <code>sumstats_file</code> <code>str</code> <p>list of GWAS Catalog summary stat files, with study ids in them.</p> required <p>Returns:</p> Name Type Description <code>GWASCatalogSummaryStatistics</code> <code>GWASCatalogSummaryStatistics</code> <p>Summary statistics object.</p> Source code in <code>src/gentropy/datasource/gwas_catalog/summary_statistics.py</code> <pre><code>@classmethod\ndef from_gwas_harmonized_summary_stats(\n    cls: type[GWASCatalogSummaryStatistics],\n    spark: SparkSession,\n    sumstats_file: str,\n) -&gt; GWASCatalogSummaryStatistics:\n    \"\"\"Create summary statistics object from summary statistics flatfile, harmonized by the GWAS Catalog.\n\n    Things got slightly complicated given the GWAS Catalog harmonization pipelines changed recently so we had to accomodate to\n    both formats.\n\n    Args:\n        spark (SparkSession): spark session\n        sumstats_file (str): list of GWAS Catalog summary stat files, with study ids in them.\n\n    Returns:\n        GWASCatalogSummaryStatistics: Summary statistics object.\n    \"\"\"\n    sumstats_df = spark.read.csv(sumstats_file, sep=\"\\t\", header=True).withColumn(\n        # Parsing GWAS Catalog study identifier from filename:\n        \"studyId\",\n        f.lit(filename_to_study_identifier(sumstats_file)),\n    )\n\n    # Parsing variant id fields:\n    chromosome = (\n        f.col(\"hm_chrom\")\n        if \"hm_chrom\" in sumstats_df.columns\n        else f.col(\"chromosome\")\n    ).cast(t.StringType())\n    position = (\n        f.col(\"hm_pos\")\n        if \"hm_pos\" in sumstats_df.columns\n        else f.col(\"base_pair_location\")\n    ).cast(t.IntegerType())\n    ref_allele = (\n        f.col(\"hm_other_allele\")\n        if \"hm_other_allele\" in sumstats_df.columns\n        else f.col(\"other_allele\")\n    )\n    alt_allele = (\n        f.col(\"hm_effect_allele\")\n        if \"hm_effect_allele\" in sumstats_df.columns\n        else f.col(\"effect_allele\")\n    )\n\n    # Parsing p-value (get a tuple with mantissa and exponent):\n    p_value_expression = (\n        parse_pvalue(f.col(\"p_value\"))\n        if \"p_value\" in sumstats_df.columns\n        else neglog_pvalue_to_mantissa_and_exponent(f.col(\"neg_log_10_p_value\"))\n    )\n\n    # The effect allele frequency is an optional column, we have to test if it is there:\n    allele_frequency = (\n        f.col(\"effect_allele_frequency\")\n        if \"effect_allele_frequency\" in sumstats_df.columns\n        else f.lit(None)\n    ).cast(t.FloatType())\n\n    # Do we have sample size? This expression captures 99.7% of sample size columns.\n    sample_size = (f.col(\"n\") if \"n\" in sumstats_df.columns else f.lit(None)).cast(\n        t.IntegerType()\n    )\n\n    # Depending on the input, we might have beta, but the column might not be there at all also old format calls differently:\n    beta_expression = (\n        f.col(\"hm_beta\")\n        if \"hm_beta\" in sumstats_df.columns\n        else f.col(\"beta\")\n        if \"beta\" in sumstats_df.columns\n        # If no column, create one:\n        else f.lit(None)\n    ).cast(t.DoubleType())\n\n    # We might have odds ratio or hazard ratio, wich are basically the same:\n    odds_ratio_expression = (\n        f.col(\"hm_odds_ratio\")\n        if \"hm_odds_ratio\" in sumstats_df.columns\n        else f.col(\"odds_ratio\")\n        if \"odds_ratio\" in sumstats_df.columns\n        else f.col(\"hazard_ratio\")\n        if \"hazard_ratio\" in sumstats_df.columns\n        # If no column, create one:\n        else f.lit(None)\n    ).cast(t.DoubleType())\n\n    # Does the file have standard error column?\n    standard_error = (\n        f.col(\"standard_error\")\n        if \"standard_error\" in sumstats_df.columns\n        else f.lit(None)\n    ).cast(t.DoubleType())\n\n    # Processing columns of interest:\n    processed_sumstats_df = (\n        sumstats_df\n        # Dropping rows which doesn't have proper position:\n        .select(\n            \"studyId\",\n            # Adding variant identifier:\n            f.concat_ws(\n                \"_\",\n                chromosome,\n                position,\n                ref_allele,\n                alt_allele,\n            ).alias(\"variantId\"),\n            chromosome.alias(\"chromosome\"),\n            position.alias(\"position\"),\n            # Parsing p-value mantissa and exponent:\n            *p_value_expression,\n            # Converting/calculating effect and confidence interval:\n            *convert_odds_ratio_to_beta(\n                beta_expression,\n                odds_ratio_expression,\n                standard_error,\n            ),\n            allele_frequency.alias(\"effectAlleleFrequencyFromSource\"),\n            sample_size.alias(\"sampleSize\"),\n        )\n        .filter(\n            # Dropping associations where no harmonized position is available:\n            f.col(\"position\").isNotNull()\n            &amp;\n            # We are not interested in associations with zero effect:\n            (f.col(\"beta\") != 0)\n        )\n        .orderBy(f.col(\"chromosome\"), f.col(\"position\"))\n        # median study size is 200Mb, max is 2.6Gb\n        .repartition(20)\n    )\n\n    # Initializing summary statistics object:\n    return cls(\n        _df=processed_sumstats_df,\n        _schema=cls.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasources/intervals/_intervals/","title":"List of Interaction and Interval-based Studies","text":"<p>In this section, we provide a list of studies that focus on interaction and interval-based investigations, shedding light on the intricate relationships between genetic elements and their functional implications.</p> <ol> <li> <p>Promoter Capture Hi-C (Javierre et al., 2016): Title: \"Lineage-Specific Genome Architecture Links Enhancers and Non-coding Disease Variants to Target Gene Promoters\".    This study presents evidence linking genetic variation to genes through the application of Promoter Capture Hi-C across each of the 17 human primary hematopoietic cell types. The method captures interactions between promoters and distal regulatory elements, providing valuable insights into the three-dimensional chromatin architecture. DOI: 10.1016/j.cell.2016.09.037</p> </li> <li> <p>Enhancer-TSS Correlation (Andersson et al., 2014): Title: \"An Atlas of Active Enhancers across Human Cell Types and Tissues\".    This study explores genetic variation's impact on genes by examining the correlation between the transcriptional activity of enhancers and transcription start sites. The findings are documented in the FANTOM5 CAGE expression atlas, offering a comprehensive view of the regulatory landscape. DOI: 10.1038/nature12787</p> </li> <li> <p>DHS-Promoter Correlation (Thurman et al., 2012): Title: \"The accessible chromatin landscape of the human genome\".    Investigating genetic variation's connection to genes, this study employs the correlation of DNase I hypersensitive sites (DHS) and gene promoters. The analysis spans 125 cell and tissue types from the ENCODE project, providing a broad understanding of the regulatory interactions across diverse biological contexts. DOI: 10.1038/nature11232</p> </li> <li> <p>Promoter Capture Hi-C (Jung et al., 2019): Title: \"A compendium of promoter-centered long-range chromatin interactions in the human genome\".    This study compiles a compendium of promoter-centered long-range chromatin interactions in the human genome. By focusing on the three-dimensional organization of chromatin, the research contributes to our understanding of the spatial arrangement of genetic elements and their implications in gene regulation. DOI: 10.1038/s41588-019-0494-8</p> </li> </ol> <p>For in-depth details on each study, you may refer to the respective publications.</p>"},{"location":"python_api/datasources/intervals/andersson/","title":"Andersson et al.","text":""},{"location":"python_api/datasources/intervals/andersson/#gentropy.datasource.intervals.andersson.IntervalsAndersson","title":"<code>gentropy.datasource.intervals.andersson.IntervalsAndersson</code>","text":"<p>Interval dataset from Andersson et al. 2014.</p> Source code in <code>src/gentropy/datasource/intervals/andersson.py</code> <pre><code>class IntervalsAndersson:\n    \"\"\"Interval dataset from Andersson et al. 2014.\"\"\"\n\n    @staticmethod\n    def read(spark: SparkSession, path: str) -&gt; DataFrame:\n        \"\"\"Read andersson2014 dataset.\n\n        Args:\n            spark (SparkSession): Spark session\n            path (str): Path to the dataset\n\n        Returns:\n            DataFrame: Raw Andersson et al. dataframe\n        \"\"\"\n        input_schema = t.StructType.fromJson(\n            json.loads(\n                pkg_resources.read_text(schemas, \"andersson2014.json\", encoding=\"utf-8\")\n            )\n        )\n        return (\n            spark.read.option(\"delimiter\", \"\\t\")\n            .option(\"mode\", \"DROPMALFORMED\")\n            .option(\"header\", \"true\")\n            .schema(input_schema)\n            .csv(path)\n        )\n\n    @classmethod\n    def parse(\n        cls: type[IntervalsAndersson],\n        raw_anderson_df: DataFrame,\n        gene_index: GeneIndex,\n        lift: LiftOverSpark,\n    ) -&gt; Intervals:\n        \"\"\"Parse Andersson et al. 2014 dataset.\n\n        Args:\n            raw_anderson_df (DataFrame): Raw Andersson et al. dataset\n            gene_index (GeneIndex): Gene index\n            lift (LiftOverSpark): LiftOverSpark instance\n\n        Returns:\n            Intervals: Intervals dataset\n        \"\"\"\n        # Constant values:\n        dataset_name = \"andersson2014\"\n        experiment_type = \"fantom5\"\n        pmid = \"24670763\"\n        bio_feature = \"aggregate\"\n        twosided_threshold = 2.45e6  # &lt;-  this needs to phased out. Filter by percentile instead of absolute value.\n\n        # Read the anderson file:\n        parsed_anderson_df = (\n            raw_anderson_df\n            # Parsing score column and casting as float:\n            .withColumn(\"score\", f.col(\"score\").cast(\"float\") / f.lit(1000))\n            # Parsing the 'name' column:\n            .withColumn(\"parsedName\", f.split(f.col(\"name\"), \";\"))\n            .withColumn(\"gene_symbol\", f.col(\"parsedName\")[2])\n            .withColumn(\"location\", f.col(\"parsedName\")[0])\n            .withColumn(\n                \"chrom\",\n                f.regexp_replace(f.split(f.col(\"location\"), \":|-\")[0], \"chr\", \"\"),\n            )\n            .withColumn(\n                \"start\", f.split(f.col(\"location\"), \":|-\")[1].cast(t.IntegerType())\n            )\n            .withColumn(\n                \"end\", f.split(f.col(\"location\"), \":|-\")[2].cast(t.IntegerType())\n            )\n            # Select relevant columns:\n            .select(\"chrom\", \"start\", \"end\", \"gene_symbol\", \"score\")\n            # Drop rows with non-canonical chromosomes:\n            .filter(\n                f.col(\"chrom\").isin([str(x) for x in range(1, 23)] + [\"X\", \"Y\", \"MT\"])\n            )\n            # For each region/gene, keep only one row with the highest score:\n            .groupBy(\"chrom\", \"start\", \"end\", \"gene_symbol\")\n            .agg(f.max(\"score\").alias(\"resourceScore\"))\n            .orderBy(\"chrom\", \"start\")\n        )\n\n        return Intervals(\n            _df=(\n                # Lift over the intervals:\n                lift.convert_intervals(parsed_anderson_df, \"chrom\", \"start\", \"end\")\n                .drop(\"start\", \"end\")\n                .withColumnRenamed(\"mapped_start\", \"start\")\n                .withColumnRenamed(\"mapped_end\", \"end\")\n                .distinct()\n                # Joining with the gene index\n                .alias(\"intervals\")\n                .join(\n                    gene_index.symbols_lut().alias(\"genes\"),\n                    on=[\n                        f.col(\"intervals.gene_symbol\") == f.col(\"genes.geneSymbol\"),\n                        # Drop rows where the TSS is far from the start of the region\n                        f.abs(\n                            (f.col(\"intervals.start\") + f.col(\"intervals.end\")) / 2\n                            - f.col(\"tss\")\n                        )\n                        &lt;= twosided_threshold,\n                    ],\n                    how=\"left\",\n                )\n                # Select relevant columns:\n                .select(\n                    f.col(\"chrom\").alias(\"chromosome\"),\n                    f.col(\"intervals.start\").alias(\"start\"),\n                    f.col(\"intervals.end\").alias(\"end\"),\n                    \"geneId\",\n                    \"resourceScore\",\n                    f.lit(dataset_name).alias(\"datasourceId\"),\n                    f.lit(experiment_type).alias(\"datatypeId\"),\n                    f.lit(pmid).alias(\"pmid\"),\n                    f.lit(bio_feature).alias(\"biofeature\"),\n                )\n            ),\n            _schema=Intervals.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasources/intervals/andersson/#gentropy.datasource.intervals.andersson.IntervalsAndersson.parse","title":"<code>parse(raw_anderson_df: DataFrame, gene_index: GeneIndex, lift: LiftOverSpark) -&gt; Intervals</code>  <code>classmethod</code>","text":"<p>Parse Andersson et al. 2014 dataset.</p> <p>Parameters:</p> Name Type Description Default <code>raw_anderson_df</code> <code>DataFrame</code> <p>Raw Andersson et al. dataset</p> required <code>gene_index</code> <code>GeneIndex</code> <p>Gene index</p> required <code>lift</code> <code>LiftOverSpark</code> <p>LiftOverSpark instance</p> required <p>Returns:</p> Name Type Description <code>Intervals</code> <code>Intervals</code> <p>Intervals dataset</p> Source code in <code>src/gentropy/datasource/intervals/andersson.py</code> <pre><code>@classmethod\ndef parse(\n    cls: type[IntervalsAndersson],\n    raw_anderson_df: DataFrame,\n    gene_index: GeneIndex,\n    lift: LiftOverSpark,\n) -&gt; Intervals:\n    \"\"\"Parse Andersson et al. 2014 dataset.\n\n    Args:\n        raw_anderson_df (DataFrame): Raw Andersson et al. dataset\n        gene_index (GeneIndex): Gene index\n        lift (LiftOverSpark): LiftOverSpark instance\n\n    Returns:\n        Intervals: Intervals dataset\n    \"\"\"\n    # Constant values:\n    dataset_name = \"andersson2014\"\n    experiment_type = \"fantom5\"\n    pmid = \"24670763\"\n    bio_feature = \"aggregate\"\n    twosided_threshold = 2.45e6  # &lt;-  this needs to phased out. Filter by percentile instead of absolute value.\n\n    # Read the anderson file:\n    parsed_anderson_df = (\n        raw_anderson_df\n        # Parsing score column and casting as float:\n        .withColumn(\"score\", f.col(\"score\").cast(\"float\") / f.lit(1000))\n        # Parsing the 'name' column:\n        .withColumn(\"parsedName\", f.split(f.col(\"name\"), \";\"))\n        .withColumn(\"gene_symbol\", f.col(\"parsedName\")[2])\n        .withColumn(\"location\", f.col(\"parsedName\")[0])\n        .withColumn(\n            \"chrom\",\n            f.regexp_replace(f.split(f.col(\"location\"), \":|-\")[0], \"chr\", \"\"),\n        )\n        .withColumn(\n            \"start\", f.split(f.col(\"location\"), \":|-\")[1].cast(t.IntegerType())\n        )\n        .withColumn(\n            \"end\", f.split(f.col(\"location\"), \":|-\")[2].cast(t.IntegerType())\n        )\n        # Select relevant columns:\n        .select(\"chrom\", \"start\", \"end\", \"gene_symbol\", \"score\")\n        # Drop rows with non-canonical chromosomes:\n        .filter(\n            f.col(\"chrom\").isin([str(x) for x in range(1, 23)] + [\"X\", \"Y\", \"MT\"])\n        )\n        # For each region/gene, keep only one row with the highest score:\n        .groupBy(\"chrom\", \"start\", \"end\", \"gene_symbol\")\n        .agg(f.max(\"score\").alias(\"resourceScore\"))\n        .orderBy(\"chrom\", \"start\")\n    )\n\n    return Intervals(\n        _df=(\n            # Lift over the intervals:\n            lift.convert_intervals(parsed_anderson_df, \"chrom\", \"start\", \"end\")\n            .drop(\"start\", \"end\")\n            .withColumnRenamed(\"mapped_start\", \"start\")\n            .withColumnRenamed(\"mapped_end\", \"end\")\n            .distinct()\n            # Joining with the gene index\n            .alias(\"intervals\")\n            .join(\n                gene_index.symbols_lut().alias(\"genes\"),\n                on=[\n                    f.col(\"intervals.gene_symbol\") == f.col(\"genes.geneSymbol\"),\n                    # Drop rows where the TSS is far from the start of the region\n                    f.abs(\n                        (f.col(\"intervals.start\") + f.col(\"intervals.end\")) / 2\n                        - f.col(\"tss\")\n                    )\n                    &lt;= twosided_threshold,\n                ],\n                how=\"left\",\n            )\n            # Select relevant columns:\n            .select(\n                f.col(\"chrom\").alias(\"chromosome\"),\n                f.col(\"intervals.start\").alias(\"start\"),\n                f.col(\"intervals.end\").alias(\"end\"),\n                \"geneId\",\n                \"resourceScore\",\n                f.lit(dataset_name).alias(\"datasourceId\"),\n                f.lit(experiment_type).alias(\"datatypeId\"),\n                f.lit(pmid).alias(\"pmid\"),\n                f.lit(bio_feature).alias(\"biofeature\"),\n            )\n        ),\n        _schema=Intervals.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasources/intervals/andersson/#gentropy.datasource.intervals.andersson.IntervalsAndersson.read","title":"<code>read(spark: SparkSession, path: str) -&gt; DataFrame</code>  <code>staticmethod</code>","text":"<p>Read andersson2014 dataset.</p> <p>Parameters:</p> Name Type Description Default <code>spark</code> <code>SparkSession</code> <p>Spark session</p> required <code>path</code> <code>str</code> <p>Path to the dataset</p> required <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>Raw Andersson et al. dataframe</p> Source code in <code>src/gentropy/datasource/intervals/andersson.py</code> <pre><code>@staticmethod\ndef read(spark: SparkSession, path: str) -&gt; DataFrame:\n    \"\"\"Read andersson2014 dataset.\n\n    Args:\n        spark (SparkSession): Spark session\n        path (str): Path to the dataset\n\n    Returns:\n        DataFrame: Raw Andersson et al. dataframe\n    \"\"\"\n    input_schema = t.StructType.fromJson(\n        json.loads(\n            pkg_resources.read_text(schemas, \"andersson2014.json\", encoding=\"utf-8\")\n        )\n    )\n    return (\n        spark.read.option(\"delimiter\", \"\\t\")\n        .option(\"mode\", \"DROPMALFORMED\")\n        .option(\"header\", \"true\")\n        .schema(input_schema)\n        .csv(path)\n    )\n</code></pre>"},{"location":"python_api/datasources/intervals/javierre/","title":"Javierre et al.","text":""},{"location":"python_api/datasources/intervals/javierre/#gentropy.datasource.intervals.javierre.IntervalsJavierre","title":"<code>gentropy.datasource.intervals.javierre.IntervalsJavierre</code>","text":"<p>Interval dataset from Javierre et al. 2016.</p> Source code in <code>src/gentropy/datasource/intervals/javierre.py</code> <pre><code>class IntervalsJavierre:\n    \"\"\"Interval dataset from Javierre et al. 2016.\"\"\"\n\n    @staticmethod\n    def read(spark: SparkSession, path: str) -&gt; DataFrame:\n        \"\"\"Read Javierre dataset.\n\n        Args:\n            spark (SparkSession): Spark session\n            path (str): Path to dataset\n\n        Returns:\n            DataFrame: Raw Javierre dataset\n        \"\"\"\n        return spark.read.parquet(path)\n\n    @classmethod\n    def parse(\n        cls: type[IntervalsJavierre],\n        javierre_raw: DataFrame,\n        gene_index: GeneIndex,\n        lift: LiftOverSpark,\n    ) -&gt; Intervals:\n        \"\"\"Parse Javierre et al. 2016 dataset.\n\n        Args:\n            javierre_raw (DataFrame): Raw Javierre data\n            gene_index (GeneIndex): Gene index\n            lift (LiftOverSpark): LiftOverSpark instance\n\n        Returns:\n            Intervals: Javierre et al. 2016 interval data\n        \"\"\"\n        # Constant values:\n        dataset_name = \"javierre2016\"\n        experiment_type = \"pchic\"\n        pmid = \"27863249\"\n        twosided_threshold = 2.45e6\n\n        # Read Javierre data:\n        javierre_parsed = (\n            javierre_raw\n            # Splitting name column into chromosome, start, end, and score:\n            .withColumn(\"name_split\", f.split(f.col(\"name\"), r\":|-|,\"))\n            .withColumn(\n                \"name_chr\",\n                f.regexp_replace(f.col(\"name_split\")[0], \"chr\", \"\").cast(\n                    t.StringType()\n                ),\n            )\n            .withColumn(\"name_start\", f.col(\"name_split\")[1].cast(t.IntegerType()))\n            .withColumn(\"name_end\", f.col(\"name_split\")[2].cast(t.IntegerType()))\n            .withColumn(\"name_score\", f.col(\"name_split\")[3].cast(t.FloatType()))\n            # Cleaning up chromosome:\n            .withColumn(\n                \"chrom\",\n                f.regexp_replace(f.col(\"chrom\"), \"chr\", \"\").cast(t.StringType()),\n            )\n            .drop(\"name_split\", \"name\", \"annotation\")\n            # Keep canonical chromosomes and consistent chromosomes with scores:\n            .filter(\n                (f.col(\"name_score\").isNotNull())\n                &amp; (f.col(\"chrom\") == f.col(\"name_chr\"))\n                &amp; f.col(\"name_chr\").isin(\n                    [f\"{x}\" for x in range(1, 23)] + [\"X\", \"Y\", \"MT\"]\n                )\n            )\n        )\n\n        # Lifting over intervals:\n        javierre_remapped = (\n            javierre_parsed\n            # Lifting over to GRCh38 interval 1:\n            .transform(lambda df: lift.convert_intervals(df, \"chrom\", \"start\", \"end\"))\n            .drop(\"start\", \"end\")\n            .withColumnRenamed(\"mapped_chrom\", \"chrom\")\n            .withColumnRenamed(\"mapped_start\", \"start\")\n            .withColumnRenamed(\"mapped_end\", \"end\")\n            # Lifting over interval 2 to GRCh38:\n            .transform(\n                lambda df: lift.convert_intervals(\n                    df, \"name_chr\", \"name_start\", \"name_end\"\n                )\n            )\n            .drop(\"name_start\", \"name_end\")\n            .withColumnRenamed(\"mapped_name_chr\", \"name_chr\")\n            .withColumnRenamed(\"mapped_name_start\", \"name_start\")\n            .withColumnRenamed(\"mapped_name_end\", \"name_end\")\n        )\n\n        # Once the intervals are lifted, extracting the unique intervals:\n        unique_intervals_with_genes = (\n            javierre_remapped.select(\n                f.col(\"chrom\"),\n                f.col(\"start\").cast(t.IntegerType()),\n                f.col(\"end\").cast(t.IntegerType()),\n            )\n            .distinct()\n            .alias(\"intervals\")\n            .join(\n                gene_index.locations_lut().alias(\"genes\"),\n                on=[\n                    f.col(\"intervals.chrom\") == f.col(\"genes.chromosome\"),\n                    (\n                        (f.col(\"intervals.start\") &gt;= f.col(\"genes.start\"))\n                        &amp; (f.col(\"intervals.start\") &lt;= f.col(\"genes.end\"))\n                    )\n                    | (\n                        (f.col(\"intervals.end\") &gt;= f.col(\"genes.start\"))\n                        &amp; (f.col(\"intervals.end\") &lt;= f.col(\"genes.end\"))\n                    ),\n                ],\n                how=\"left\",\n            )\n            .select(\n                f.col(\"intervals.chrom\").alias(\"chrom\"),\n                f.col(\"intervals.start\").alias(\"start\"),\n                f.col(\"intervals.end\").alias(\"end\"),\n                f.col(\"genes.geneId\").alias(\"geneId\"),\n                f.col(\"genes.tss\").alias(\"tss\"),\n            )\n        )\n\n        # Joining back the data:\n        return Intervals(\n            _df=(\n                javierre_remapped.join(\n                    unique_intervals_with_genes,\n                    on=[\"chrom\", \"start\", \"end\"],\n                    how=\"left\",\n                )\n                .filter(\n                    # Drop rows where the TSS is far from the start of the region\n                    f.abs((f.col(\"start\") + f.col(\"end\")) / 2 - f.col(\"tss\"))\n                    &lt;= twosided_threshold\n                )\n                # For each gene, keep only the highest scoring interval:\n                .groupBy(\"name_chr\", \"name_start\", \"name_end\", \"geneId\", \"bio_feature\")\n                .agg(f.max(f.col(\"name_score\")).alias(\"resourceScore\"))\n                # Create the output:\n                .select(\n                    f.col(\"name_chr\").alias(\"chromosome\"),\n                    f.col(\"name_start\").alias(\"start\"),\n                    f.col(\"name_end\").alias(\"end\"),\n                    f.col(\"resourceScore\").cast(t.DoubleType()),\n                    f.col(\"geneId\"),\n                    f.col(\"bio_feature\").alias(\"biofeature\"),\n                    f.lit(dataset_name).alias(\"datasourceId\"),\n                    f.lit(experiment_type).alias(\"datatypeId\"),\n                    f.lit(pmid).alias(\"pmid\"),\n                )\n            ),\n            _schema=Intervals.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasources/intervals/javierre/#gentropy.datasource.intervals.javierre.IntervalsJavierre.parse","title":"<code>parse(javierre_raw: DataFrame, gene_index: GeneIndex, lift: LiftOverSpark) -&gt; Intervals</code>  <code>classmethod</code>","text":"<p>Parse Javierre et al. 2016 dataset.</p> <p>Parameters:</p> Name Type Description Default <code>javierre_raw</code> <code>DataFrame</code> <p>Raw Javierre data</p> required <code>gene_index</code> <code>GeneIndex</code> <p>Gene index</p> required <code>lift</code> <code>LiftOverSpark</code> <p>LiftOverSpark instance</p> required <p>Returns:</p> Name Type Description <code>Intervals</code> <code>Intervals</code> <p>Javierre et al. 2016 interval data</p> Source code in <code>src/gentropy/datasource/intervals/javierre.py</code> <pre><code>@classmethod\ndef parse(\n    cls: type[IntervalsJavierre],\n    javierre_raw: DataFrame,\n    gene_index: GeneIndex,\n    lift: LiftOverSpark,\n) -&gt; Intervals:\n    \"\"\"Parse Javierre et al. 2016 dataset.\n\n    Args:\n        javierre_raw (DataFrame): Raw Javierre data\n        gene_index (GeneIndex): Gene index\n        lift (LiftOverSpark): LiftOverSpark instance\n\n    Returns:\n        Intervals: Javierre et al. 2016 interval data\n    \"\"\"\n    # Constant values:\n    dataset_name = \"javierre2016\"\n    experiment_type = \"pchic\"\n    pmid = \"27863249\"\n    twosided_threshold = 2.45e6\n\n    # Read Javierre data:\n    javierre_parsed = (\n        javierre_raw\n        # Splitting name column into chromosome, start, end, and score:\n        .withColumn(\"name_split\", f.split(f.col(\"name\"), r\":|-|,\"))\n        .withColumn(\n            \"name_chr\",\n            f.regexp_replace(f.col(\"name_split\")[0], \"chr\", \"\").cast(\n                t.StringType()\n            ),\n        )\n        .withColumn(\"name_start\", f.col(\"name_split\")[1].cast(t.IntegerType()))\n        .withColumn(\"name_end\", f.col(\"name_split\")[2].cast(t.IntegerType()))\n        .withColumn(\"name_score\", f.col(\"name_split\")[3].cast(t.FloatType()))\n        # Cleaning up chromosome:\n        .withColumn(\n            \"chrom\",\n            f.regexp_replace(f.col(\"chrom\"), \"chr\", \"\").cast(t.StringType()),\n        )\n        .drop(\"name_split\", \"name\", \"annotation\")\n        # Keep canonical chromosomes and consistent chromosomes with scores:\n        .filter(\n            (f.col(\"name_score\").isNotNull())\n            &amp; (f.col(\"chrom\") == f.col(\"name_chr\"))\n            &amp; f.col(\"name_chr\").isin(\n                [f\"{x}\" for x in range(1, 23)] + [\"X\", \"Y\", \"MT\"]\n            )\n        )\n    )\n\n    # Lifting over intervals:\n    javierre_remapped = (\n        javierre_parsed\n        # Lifting over to GRCh38 interval 1:\n        .transform(lambda df: lift.convert_intervals(df, \"chrom\", \"start\", \"end\"))\n        .drop(\"start\", \"end\")\n        .withColumnRenamed(\"mapped_chrom\", \"chrom\")\n        .withColumnRenamed(\"mapped_start\", \"start\")\n        .withColumnRenamed(\"mapped_end\", \"end\")\n        # Lifting over interval 2 to GRCh38:\n        .transform(\n            lambda df: lift.convert_intervals(\n                df, \"name_chr\", \"name_start\", \"name_end\"\n            )\n        )\n        .drop(\"name_start\", \"name_end\")\n        .withColumnRenamed(\"mapped_name_chr\", \"name_chr\")\n        .withColumnRenamed(\"mapped_name_start\", \"name_start\")\n        .withColumnRenamed(\"mapped_name_end\", \"name_end\")\n    )\n\n    # Once the intervals are lifted, extracting the unique intervals:\n    unique_intervals_with_genes = (\n        javierre_remapped.select(\n            f.col(\"chrom\"),\n            f.col(\"start\").cast(t.IntegerType()),\n            f.col(\"end\").cast(t.IntegerType()),\n        )\n        .distinct()\n        .alias(\"intervals\")\n        .join(\n            gene_index.locations_lut().alias(\"genes\"),\n            on=[\n                f.col(\"intervals.chrom\") == f.col(\"genes.chromosome\"),\n                (\n                    (f.col(\"intervals.start\") &gt;= f.col(\"genes.start\"))\n                    &amp; (f.col(\"intervals.start\") &lt;= f.col(\"genes.end\"))\n                )\n                | (\n                    (f.col(\"intervals.end\") &gt;= f.col(\"genes.start\"))\n                    &amp; (f.col(\"intervals.end\") &lt;= f.col(\"genes.end\"))\n                ),\n            ],\n            how=\"left\",\n        )\n        .select(\n            f.col(\"intervals.chrom\").alias(\"chrom\"),\n            f.col(\"intervals.start\").alias(\"start\"),\n            f.col(\"intervals.end\").alias(\"end\"),\n            f.col(\"genes.geneId\").alias(\"geneId\"),\n            f.col(\"genes.tss\").alias(\"tss\"),\n        )\n    )\n\n    # Joining back the data:\n    return Intervals(\n        _df=(\n            javierre_remapped.join(\n                unique_intervals_with_genes,\n                on=[\"chrom\", \"start\", \"end\"],\n                how=\"left\",\n            )\n            .filter(\n                # Drop rows where the TSS is far from the start of the region\n                f.abs((f.col(\"start\") + f.col(\"end\")) / 2 - f.col(\"tss\"))\n                &lt;= twosided_threshold\n            )\n            # For each gene, keep only the highest scoring interval:\n            .groupBy(\"name_chr\", \"name_start\", \"name_end\", \"geneId\", \"bio_feature\")\n            .agg(f.max(f.col(\"name_score\")).alias(\"resourceScore\"))\n            # Create the output:\n            .select(\n                f.col(\"name_chr\").alias(\"chromosome\"),\n                f.col(\"name_start\").alias(\"start\"),\n                f.col(\"name_end\").alias(\"end\"),\n                f.col(\"resourceScore\").cast(t.DoubleType()),\n                f.col(\"geneId\"),\n                f.col(\"bio_feature\").alias(\"biofeature\"),\n                f.lit(dataset_name).alias(\"datasourceId\"),\n                f.lit(experiment_type).alias(\"datatypeId\"),\n                f.lit(pmid).alias(\"pmid\"),\n            )\n        ),\n        _schema=Intervals.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasources/intervals/javierre/#gentropy.datasource.intervals.javierre.IntervalsJavierre.read","title":"<code>read(spark: SparkSession, path: str) -&gt; DataFrame</code>  <code>staticmethod</code>","text":"<p>Read Javierre dataset.</p> <p>Parameters:</p> Name Type Description Default <code>spark</code> <code>SparkSession</code> <p>Spark session</p> required <code>path</code> <code>str</code> <p>Path to dataset</p> required <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>Raw Javierre dataset</p> Source code in <code>src/gentropy/datasource/intervals/javierre.py</code> <pre><code>@staticmethod\ndef read(spark: SparkSession, path: str) -&gt; DataFrame:\n    \"\"\"Read Javierre dataset.\n\n    Args:\n        spark (SparkSession): Spark session\n        path (str): Path to dataset\n\n    Returns:\n        DataFrame: Raw Javierre dataset\n    \"\"\"\n    return spark.read.parquet(path)\n</code></pre>"},{"location":"python_api/datasources/intervals/jung/","title":"Jung et al.","text":""},{"location":"python_api/datasources/intervals/jung/#gentropy.datasource.intervals.jung.IntervalsJung","title":"<code>gentropy.datasource.intervals.jung.IntervalsJung</code>","text":"<p>Interval dataset from Jung et al. 2019.</p> Source code in <code>src/gentropy/datasource/intervals/jung.py</code> <pre><code>class IntervalsJung:\n    \"\"\"Interval dataset from Jung et al. 2019.\"\"\"\n\n    @staticmethod\n    def read(spark: SparkSession, path: str) -&gt; DataFrame:\n        \"\"\"Read jung dataset.\n\n        Args:\n            spark (SparkSession): Spark session\n            path (str): Path to dataset\n\n        Returns:\n            DataFrame: DataFrame with raw jung data\n        \"\"\"\n        return spark.read.csv(path, sep=\",\", header=True)\n\n    @classmethod\n    def parse(\n        cls: type[IntervalsJung],\n        jung_raw: DataFrame,\n        gene_index: GeneIndex,\n        lift: LiftOverSpark,\n    ) -&gt; Intervals:\n        \"\"\"Parse the Jung et al. 2019 dataset.\n\n        Args:\n            jung_raw (DataFrame): raw Jung et al. 2019 dataset\n            gene_index (GeneIndex): gene index\n            lift (LiftOverSpark): LiftOverSpark instance\n\n        Returns:\n            Intervals: Interval dataset containing Jung et al. 2019 data\n        \"\"\"\n        dataset_name = \"jung2019\"\n        experiment_type = \"pchic\"\n        pmid = \"31501517\"\n\n        # Lifting over the coordinates:\n        return Intervals(\n            _df=(\n                jung_raw.withColumn(\n                    \"interval\", f.split(f.col(\"Interacting_fragment\"), r\"\\.\")\n                )\n                .select(\n                    # Parsing intervals:\n                    f.regexp_replace(f.col(\"interval\")[0], \"chr\", \"\").alias(\"chrom\"),\n                    f.col(\"interval\")[1].cast(t.IntegerType()).alias(\"start\"),\n                    f.col(\"interval\")[2].cast(t.IntegerType()).alias(\"end\"),\n                    # Extract other columns:\n                    f.col(\"Promoter\").alias(\"gene_name\"),\n                    f.col(\"Tissue_type\").alias(\"tissue\"),\n                )\n                # Lifting over to GRCh38 interval 1:\n                .transform(\n                    lambda df: lift.convert_intervals(df, \"chrom\", \"start\", \"end\")\n                )\n                .select(\n                    \"chrom\",\n                    f.col(\"mapped_start\").alias(\"start\"),\n                    f.col(\"mapped_end\").alias(\"end\"),\n                    f.explode(f.split(f.col(\"gene_name\"), \";\")).alias(\"gene_name\"),\n                    \"tissue\",\n                )\n                .alias(\"intervals\")\n                # Joining with genes:\n                .join(\n                    gene_index.symbols_lut().alias(\"genes\"),\n                    on=[f.col(\"intervals.gene_name\") == f.col(\"genes.geneSymbol\")],\n                    how=\"inner\",\n                )\n                # Finalize dataset:\n                .select(\n                    \"chromosome\",\n                    f.col(\"intervals.start\").alias(\"start\"),\n                    f.col(\"intervals.end\").alias(\"end\"),\n                    \"geneId\",\n                    f.col(\"tissue\").alias(\"biofeature\"),\n                    f.lit(1.0).alias(\"score\"),\n                    f.lit(dataset_name).alias(\"datasourceId\"),\n                    f.lit(experiment_type).alias(\"datatypeId\"),\n                    f.lit(pmid).alias(\"pmid\"),\n                )\n                .drop_duplicates()\n            ),\n            _schema=Intervals.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasources/intervals/jung/#gentropy.datasource.intervals.jung.IntervalsJung.parse","title":"<code>parse(jung_raw: DataFrame, gene_index: GeneIndex, lift: LiftOverSpark) -&gt; Intervals</code>  <code>classmethod</code>","text":"<p>Parse the Jung et al. 2019 dataset.</p> <p>Parameters:</p> Name Type Description Default <code>jung_raw</code> <code>DataFrame</code> <p>raw Jung et al. 2019 dataset</p> required <code>gene_index</code> <code>GeneIndex</code> <p>gene index</p> required <code>lift</code> <code>LiftOverSpark</code> <p>LiftOverSpark instance</p> required <p>Returns:</p> Name Type Description <code>Intervals</code> <code>Intervals</code> <p>Interval dataset containing Jung et al. 2019 data</p> Source code in <code>src/gentropy/datasource/intervals/jung.py</code> <pre><code>@classmethod\ndef parse(\n    cls: type[IntervalsJung],\n    jung_raw: DataFrame,\n    gene_index: GeneIndex,\n    lift: LiftOverSpark,\n) -&gt; Intervals:\n    \"\"\"Parse the Jung et al. 2019 dataset.\n\n    Args:\n        jung_raw (DataFrame): raw Jung et al. 2019 dataset\n        gene_index (GeneIndex): gene index\n        lift (LiftOverSpark): LiftOverSpark instance\n\n    Returns:\n        Intervals: Interval dataset containing Jung et al. 2019 data\n    \"\"\"\n    dataset_name = \"jung2019\"\n    experiment_type = \"pchic\"\n    pmid = \"31501517\"\n\n    # Lifting over the coordinates:\n    return Intervals(\n        _df=(\n            jung_raw.withColumn(\n                \"interval\", f.split(f.col(\"Interacting_fragment\"), r\"\\.\")\n            )\n            .select(\n                # Parsing intervals:\n                f.regexp_replace(f.col(\"interval\")[0], \"chr\", \"\").alias(\"chrom\"),\n                f.col(\"interval\")[1].cast(t.IntegerType()).alias(\"start\"),\n                f.col(\"interval\")[2].cast(t.IntegerType()).alias(\"end\"),\n                # Extract other columns:\n                f.col(\"Promoter\").alias(\"gene_name\"),\n                f.col(\"Tissue_type\").alias(\"tissue\"),\n            )\n            # Lifting over to GRCh38 interval 1:\n            .transform(\n                lambda df: lift.convert_intervals(df, \"chrom\", \"start\", \"end\")\n            )\n            .select(\n                \"chrom\",\n                f.col(\"mapped_start\").alias(\"start\"),\n                f.col(\"mapped_end\").alias(\"end\"),\n                f.explode(f.split(f.col(\"gene_name\"), \";\")).alias(\"gene_name\"),\n                \"tissue\",\n            )\n            .alias(\"intervals\")\n            # Joining with genes:\n            .join(\n                gene_index.symbols_lut().alias(\"genes\"),\n                on=[f.col(\"intervals.gene_name\") == f.col(\"genes.geneSymbol\")],\n                how=\"inner\",\n            )\n            # Finalize dataset:\n            .select(\n                \"chromosome\",\n                f.col(\"intervals.start\").alias(\"start\"),\n                f.col(\"intervals.end\").alias(\"end\"),\n                \"geneId\",\n                f.col(\"tissue\").alias(\"biofeature\"),\n                f.lit(1.0).alias(\"score\"),\n                f.lit(dataset_name).alias(\"datasourceId\"),\n                f.lit(experiment_type).alias(\"datatypeId\"),\n                f.lit(pmid).alias(\"pmid\"),\n            )\n            .drop_duplicates()\n        ),\n        _schema=Intervals.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasources/intervals/jung/#gentropy.datasource.intervals.jung.IntervalsJung.read","title":"<code>read(spark: SparkSession, path: str) -&gt; DataFrame</code>  <code>staticmethod</code>","text":"<p>Read jung dataset.</p> <p>Parameters:</p> Name Type Description Default <code>spark</code> <code>SparkSession</code> <p>Spark session</p> required <code>path</code> <code>str</code> <p>Path to dataset</p> required <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>DataFrame with raw jung data</p> Source code in <code>src/gentropy/datasource/intervals/jung.py</code> <pre><code>@staticmethod\ndef read(spark: SparkSession, path: str) -&gt; DataFrame:\n    \"\"\"Read jung dataset.\n\n    Args:\n        spark (SparkSession): Spark session\n        path (str): Path to dataset\n\n    Returns:\n        DataFrame: DataFrame with raw jung data\n    \"\"\"\n    return spark.read.csv(path, sep=\",\", header=True)\n</code></pre>"},{"location":"python_api/datasources/intervals/thurman/","title":"Thurman et al.","text":""},{"location":"python_api/datasources/intervals/thurman/#gentropy.datasource.intervals.thurman.IntervalsThurman","title":"<code>gentropy.datasource.intervals.thurman.IntervalsThurman</code>","text":"<p>Interval dataset from Thurman et al. 2012.</p> Source code in <code>src/gentropy/datasource/intervals/thurman.py</code> <pre><code>class IntervalsThurman:\n    \"\"\"Interval dataset from Thurman et al. 2012.\"\"\"\n\n    @staticmethod\n    def read(spark: SparkSession, path: str) -&gt; DataFrame:\n        \"\"\"Read thurman dataset.\n\n        Args:\n            spark (SparkSession): Spark session\n            path (str): Path to dataset\n\n        Returns:\n            DataFrame: DataFrame with raw thurman data\n        \"\"\"\n        thurman_schema = t.StructType(\n            [\n                t.StructField(\"gene_chr\", t.StringType(), False),\n                t.StructField(\"gene_start\", t.IntegerType(), False),\n                t.StructField(\"gene_end\", t.IntegerType(), False),\n                t.StructField(\"gene_name\", t.StringType(), False),\n                t.StructField(\"chrom\", t.StringType(), False),\n                t.StructField(\"start\", t.IntegerType(), False),\n                t.StructField(\"end\", t.IntegerType(), False),\n                t.StructField(\"score\", t.FloatType(), False),\n            ]\n        )\n        return spark.read.csv(path, sep=\"\\t\", header=True, schema=thurman_schema)\n\n    @classmethod\n    def parse(\n        cls: type[IntervalsThurman],\n        thurman_raw: DataFrame,\n        gene_index: GeneIndex,\n        lift: LiftOverSpark,\n    ) -&gt; Intervals:\n        \"\"\"Parse the Thurman et al. 2012 dataset.\n\n        Args:\n            thurman_raw (DataFrame): raw Thurman et al. 2019 dataset\n            gene_index (GeneIndex): gene index\n            lift (LiftOverSpark): LiftOverSpark instance\n\n        Returns:\n            Intervals: Interval dataset containing Thurman et al. 2012 data\n        \"\"\"\n        dataset_name = \"thurman2012\"\n        experiment_type = \"dhscor\"\n        pmid = \"22955617\"\n\n        return Intervals(\n            _df=(\n                thurman_raw.select(\n                    f.regexp_replace(f.col(\"chrom\"), \"chr\", \"\").alias(\"chrom\"),\n                    \"start\",\n                    \"end\",\n                    \"gene_name\",\n                    \"score\",\n                )\n                # Lift over to the GRCh38 build:\n                .transform(\n                    lambda df: lift.convert_intervals(df, \"chrom\", \"start\", \"end\")\n                )\n                .alias(\"intervals\")\n                # Map gene names to gene IDs:\n                .join(\n                    gene_index.symbols_lut().alias(\"genes\"),\n                    on=[\n                        f.col(\"intervals.gene_name\") == f.col(\"genes.geneSymbol\"),\n                        f.col(\"intervals.chrom\") == f.col(\"genes.chromosome\"),\n                    ],\n                    how=\"inner\",\n                )\n                # Select relevant columns and add constant columns:\n                .select(\n                    f.col(\"chrom\").alias(\"chromosome\"),\n                    f.col(\"mapped_start\").alias(\"start\"),\n                    f.col(\"mapped_end\").alias(\"end\"),\n                    \"geneId\",\n                    f.col(\"score\").cast(t.DoubleType()).alias(\"resourceScore\"),\n                    f.lit(dataset_name).alias(\"datasourceId\"),\n                    f.lit(experiment_type).alias(\"datatypeId\"),\n                    f.lit(pmid).alias(\"pmid\"),\n                )\n                .distinct()\n            ),\n            _schema=Intervals.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasources/intervals/thurman/#gentropy.datasource.intervals.thurman.IntervalsThurman.parse","title":"<code>parse(thurman_raw: DataFrame, gene_index: GeneIndex, lift: LiftOverSpark) -&gt; Intervals</code>  <code>classmethod</code>","text":"<p>Parse the Thurman et al. 2012 dataset.</p> <p>Parameters:</p> Name Type Description Default <code>thurman_raw</code> <code>DataFrame</code> <p>raw Thurman et al. 2019 dataset</p> required <code>gene_index</code> <code>GeneIndex</code> <p>gene index</p> required <code>lift</code> <code>LiftOverSpark</code> <p>LiftOverSpark instance</p> required <p>Returns:</p> Name Type Description <code>Intervals</code> <code>Intervals</code> <p>Interval dataset containing Thurman et al. 2012 data</p> Source code in <code>src/gentropy/datasource/intervals/thurman.py</code> <pre><code>@classmethod\ndef parse(\n    cls: type[IntervalsThurman],\n    thurman_raw: DataFrame,\n    gene_index: GeneIndex,\n    lift: LiftOverSpark,\n) -&gt; Intervals:\n    \"\"\"Parse the Thurman et al. 2012 dataset.\n\n    Args:\n        thurman_raw (DataFrame): raw Thurman et al. 2019 dataset\n        gene_index (GeneIndex): gene index\n        lift (LiftOverSpark): LiftOverSpark instance\n\n    Returns:\n        Intervals: Interval dataset containing Thurman et al. 2012 data\n    \"\"\"\n    dataset_name = \"thurman2012\"\n    experiment_type = \"dhscor\"\n    pmid = \"22955617\"\n\n    return Intervals(\n        _df=(\n            thurman_raw.select(\n                f.regexp_replace(f.col(\"chrom\"), \"chr\", \"\").alias(\"chrom\"),\n                \"start\",\n                \"end\",\n                \"gene_name\",\n                \"score\",\n            )\n            # Lift over to the GRCh38 build:\n            .transform(\n                lambda df: lift.convert_intervals(df, \"chrom\", \"start\", \"end\")\n            )\n            .alias(\"intervals\")\n            # Map gene names to gene IDs:\n            .join(\n                gene_index.symbols_lut().alias(\"genes\"),\n                on=[\n                    f.col(\"intervals.gene_name\") == f.col(\"genes.geneSymbol\"),\n                    f.col(\"intervals.chrom\") == f.col(\"genes.chromosome\"),\n                ],\n                how=\"inner\",\n            )\n            # Select relevant columns and add constant columns:\n            .select(\n                f.col(\"chrom\").alias(\"chromosome\"),\n                f.col(\"mapped_start\").alias(\"start\"),\n                f.col(\"mapped_end\").alias(\"end\"),\n                \"geneId\",\n                f.col(\"score\").cast(t.DoubleType()).alias(\"resourceScore\"),\n                f.lit(dataset_name).alias(\"datasourceId\"),\n                f.lit(experiment_type).alias(\"datatypeId\"),\n                f.lit(pmid).alias(\"pmid\"),\n            )\n            .distinct()\n        ),\n        _schema=Intervals.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasources/intervals/thurman/#gentropy.datasource.intervals.thurman.IntervalsThurman.read","title":"<code>read(spark: SparkSession, path: str) -&gt; DataFrame</code>  <code>staticmethod</code>","text":"<p>Read thurman dataset.</p> <p>Parameters:</p> Name Type Description Default <code>spark</code> <code>SparkSession</code> <p>Spark session</p> required <code>path</code> <code>str</code> <p>Path to dataset</p> required <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>DataFrame with raw thurman data</p> Source code in <code>src/gentropy/datasource/intervals/thurman.py</code> <pre><code>@staticmethod\ndef read(spark: SparkSession, path: str) -&gt; DataFrame:\n    \"\"\"Read thurman dataset.\n\n    Args:\n        spark (SparkSession): Spark session\n        path (str): Path to dataset\n\n    Returns:\n        DataFrame: DataFrame with raw thurman data\n    \"\"\"\n    thurman_schema = t.StructType(\n        [\n            t.StructField(\"gene_chr\", t.StringType(), False),\n            t.StructField(\"gene_start\", t.IntegerType(), False),\n            t.StructField(\"gene_end\", t.IntegerType(), False),\n            t.StructField(\"gene_name\", t.StringType(), False),\n            t.StructField(\"chrom\", t.StringType(), False),\n            t.StructField(\"start\", t.IntegerType(), False),\n            t.StructField(\"end\", t.IntegerType(), False),\n            t.StructField(\"score\", t.FloatType(), False),\n        ]\n    )\n    return spark.read.csv(path, sep=\"\\t\", header=True, schema=thurman_schema)\n</code></pre>"},{"location":"python_api/datasources/open_targets/_open_targets/","title":"Open Targets","text":"<p>The Open Targets Platform is a comprehensive resource that aims to aggregate and harmonize various types of data to facilitate the identification, prioritization, and validation of drug targets. By integrating publicly available datasets, including data generated by the Open Targets consortium, the Platform builds and scores target-disease associations to assist in drug target identification and prioritization. It also integrates relevant annotation information about targets, diseases, phenotypes, and drugs, as well as their most relevant relationships.</p> <p>Within our analyses, we utilize Open Targets to infer two datasets:</p> <ol> <li> <p>The list of targets:    This dataset provides a compilation of targets. In the Open Targets Platform, a target is understood as any naturally-occurring molecule that can be targeted by a medicinal product. The EMBL-EBI Ensembl database serves as the source for human targets in the Platform, with the Ensembl gene ID as the primary identifier. For more details, refer to this link.</p> </li> <li> <p>The list of Gold Standard Positives:    We use this dataset for training the Locus-to-Gene model. The current list contains 496 Gold Standard Positives.</p> </li> </ol>"},{"location":"python_api/datasources/open_targets/l2g_gold_standard/","title":"L2G Gold Standard","text":""},{"location":"python_api/datasources/open_targets/l2g_gold_standard/#gentropy.datasource.open_targets.l2g_gold_standard.OpenTargetsL2GGoldStandard","title":"<code>gentropy.datasource.open_targets.l2g_gold_standard.OpenTargetsL2GGoldStandard</code>","text":"<p>Parser for OTGenetics locus to gene gold standards curation.</p> The curation is processed to generate a dataset with 2 labels <ul> <li>Gold Standard Positive (GSP): When the lead variant is part of a curated list of GWAS loci with known gene-trait associations.</li> <li>Gold Standard Negative (GSN): When the lead variant is not part of a curated list of GWAS loci with known gene-trait associations but is in the vicinity of a gene's TSS.</li> </ul> Source code in <code>src/gentropy/datasource/open_targets/l2g_gold_standard.py</code> <pre><code>class OpenTargetsL2GGoldStandard:\n    \"\"\"Parser for OTGenetics locus to gene gold standards curation.\n\n    The curation is processed to generate a dataset with 2 labels:\n        - Gold Standard Positive (GSP): When the lead variant is part of a curated list of GWAS loci with known gene-trait associations.\n        - Gold Standard Negative (GSN): When the lead variant is not part of a curated list of GWAS loci with known gene-trait associations but is in the vicinity of a gene's TSS.\n    \"\"\"\n\n    LOCUS_TO_GENE_WINDOW = 500_000\n\n    @classmethod\n    def parse_positive_curation(\n        cls: Type[OpenTargetsL2GGoldStandard], gold_standard_curation: DataFrame\n    ) -&gt; DataFrame:\n        \"\"\"Parse positive set from gold standard curation.\n\n        Args:\n            gold_standard_curation (DataFrame): Gold standard curation dataframe\n\n        Returns:\n            DataFrame: Positive set\n        \"\"\"\n        return (\n            gold_standard_curation.filter(\n                f.col(\"gold_standard_info.highest_confidence\").isin([\"High\", \"Medium\"])\n            )\n            .select(\n                f.col(\"association_info.otg_id\").alias(\"studyId\"),\n                f.col(\"gold_standard_info.gene_id\").alias(\"geneId\"),\n                f.concat_ws(\n                    \"_\",\n                    f.col(\"sentinel_variant.locus_GRCh38.chromosome\"),\n                    f.col(\"sentinel_variant.locus_GRCh38.position\"),\n                    f.col(\"sentinel_variant.alleles.reference\"),\n                    f.col(\"sentinel_variant.alleles.alternative\"),\n                ).alias(\"variantId\"),\n                f.col(\"metadata.set_label\").alias(\"source\"),\n            )\n            .withColumn(\n                \"studyLocusId\",\n                StudyLocus.assign_study_locus_id(f.col(\"studyId\"), f.col(\"variantId\")),\n            )\n            .groupBy(\"studyLocusId\", \"studyId\", \"variantId\", \"geneId\")\n            .agg(f.collect_set(\"source\").alias(\"sources\"))\n        )\n\n    @classmethod\n    def expand_gold_standard_with_negatives(\n        cls: Type[OpenTargetsL2GGoldStandard], positive_set: DataFrame, v2g: V2G\n    ) -&gt; DataFrame:\n        \"\"\"Create full set of positive and negative evidence of locus to gene associations.\n\n        Negative evidence consists of all genes within a window of 500kb of the lead variant that are not in the positive set.\n\n        Args:\n            positive_set (DataFrame): Positive set from curation\n            v2g (V2G): Variant to gene dataset to bring distance between a variant and a gene's TSS\n\n        Returns:\n            DataFrame: Full set of positive and negative evidence of locus to gene associations\n        \"\"\"\n        return (\n            positive_set.withColumnRenamed(\"geneId\", \"curated_geneId\")\n            .join(\n                v2g.df.selectExpr(\n                    \"variantId\", \"geneId as non_curated_geneId\", \"distance\"\n                ).filter(f.col(\"distance\") &lt;= cls.LOCUS_TO_GENE_WINDOW),\n                on=\"variantId\",\n                how=\"left\",\n            )\n            .withColumn(\n                \"goldStandardSet\",\n                f.when(\n                    (f.col(\"curated_geneId\") == f.col(\"non_curated_geneId\"))\n                    # to keep the positives that are outside the v2g dataset\n                    | (f.col(\"non_curated_geneId\").isNull()),\n                    f.lit(L2GGoldStandard.GS_POSITIVE_LABEL),\n                ).otherwise(L2GGoldStandard.GS_NEGATIVE_LABEL),\n            )\n            .withColumn(\n                \"geneId\",\n                f.when(\n                    f.col(\"goldStandardSet\") == L2GGoldStandard.GS_POSITIVE_LABEL,\n                    f.col(\"curated_geneId\"),\n                ).otherwise(f.col(\"non_curated_geneId\")),\n            )\n            .drop(\"distance\", \"curated_geneId\", \"non_curated_geneId\")\n        )\n\n    @classmethod\n    def as_l2g_gold_standard(\n        cls: type[OpenTargetsL2GGoldStandard],\n        gold_standard_curation: DataFrame,\n        v2g: V2G,\n    ) -&gt; L2GGoldStandard:\n        \"\"\"Initialise L2GGoldStandard from source dataset.\n\n        Args:\n            gold_standard_curation (DataFrame): Gold standard curation dataframe, extracted from https://github.com/opentargets/genetics-gold-standards\n            v2g (V2G): Variant to gene dataset to bring distance between a variant and a gene's TSS\n\n        Returns:\n            L2GGoldStandard: L2G Gold Standard dataset. False negatives have not yet been removed.\n        \"\"\"\n        return L2GGoldStandard(\n            _df=cls.parse_positive_curation(gold_standard_curation).transform(\n                cls.expand_gold_standard_with_negatives, v2g\n            ),\n            _schema=L2GGoldStandard.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasources/open_targets/l2g_gold_standard/#gentropy.datasource.open_targets.l2g_gold_standard.OpenTargetsL2GGoldStandard.as_l2g_gold_standard","title":"<code>as_l2g_gold_standard(gold_standard_curation: DataFrame, v2g: V2G) -&gt; L2GGoldStandard</code>  <code>classmethod</code>","text":"<p>Initialise L2GGoldStandard from source dataset.</p> <p>Parameters:</p> Name Type Description Default <code>gold_standard_curation</code> <code>DataFrame</code> <p>Gold standard curation dataframe, extracted from https://github.com/opentargets/genetics-gold-standards</p> required <code>v2g</code> <code>V2G</code> <p>Variant to gene dataset to bring distance between a variant and a gene's TSS</p> required <p>Returns:</p> Name Type Description <code>L2GGoldStandard</code> <code>L2GGoldStandard</code> <p>L2G Gold Standard dataset. False negatives have not yet been removed.</p> Source code in <code>src/gentropy/datasource/open_targets/l2g_gold_standard.py</code> <pre><code>@classmethod\ndef as_l2g_gold_standard(\n    cls: type[OpenTargetsL2GGoldStandard],\n    gold_standard_curation: DataFrame,\n    v2g: V2G,\n) -&gt; L2GGoldStandard:\n    \"\"\"Initialise L2GGoldStandard from source dataset.\n\n    Args:\n        gold_standard_curation (DataFrame): Gold standard curation dataframe, extracted from https://github.com/opentargets/genetics-gold-standards\n        v2g (V2G): Variant to gene dataset to bring distance between a variant and a gene's TSS\n\n    Returns:\n        L2GGoldStandard: L2G Gold Standard dataset. False negatives have not yet been removed.\n    \"\"\"\n    return L2GGoldStandard(\n        _df=cls.parse_positive_curation(gold_standard_curation).transform(\n            cls.expand_gold_standard_with_negatives, v2g\n        ),\n        _schema=L2GGoldStandard.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasources/open_targets/l2g_gold_standard/#gentropy.datasource.open_targets.l2g_gold_standard.OpenTargetsL2GGoldStandard.expand_gold_standard_with_negatives","title":"<code>expand_gold_standard_with_negatives(positive_set: DataFrame, v2g: V2G) -&gt; DataFrame</code>  <code>classmethod</code>","text":"<p>Create full set of positive and negative evidence of locus to gene associations.</p> <p>Negative evidence consists of all genes within a window of 500kb of the lead variant that are not in the positive set.</p> <p>Parameters:</p> Name Type Description Default <code>positive_set</code> <code>DataFrame</code> <p>Positive set from curation</p> required <code>v2g</code> <code>V2G</code> <p>Variant to gene dataset to bring distance between a variant and a gene's TSS</p> required <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>Full set of positive and negative evidence of locus to gene associations</p> Source code in <code>src/gentropy/datasource/open_targets/l2g_gold_standard.py</code> <pre><code>@classmethod\ndef expand_gold_standard_with_negatives(\n    cls: Type[OpenTargetsL2GGoldStandard], positive_set: DataFrame, v2g: V2G\n) -&gt; DataFrame:\n    \"\"\"Create full set of positive and negative evidence of locus to gene associations.\n\n    Negative evidence consists of all genes within a window of 500kb of the lead variant that are not in the positive set.\n\n    Args:\n        positive_set (DataFrame): Positive set from curation\n        v2g (V2G): Variant to gene dataset to bring distance between a variant and a gene's TSS\n\n    Returns:\n        DataFrame: Full set of positive and negative evidence of locus to gene associations\n    \"\"\"\n    return (\n        positive_set.withColumnRenamed(\"geneId\", \"curated_geneId\")\n        .join(\n            v2g.df.selectExpr(\n                \"variantId\", \"geneId as non_curated_geneId\", \"distance\"\n            ).filter(f.col(\"distance\") &lt;= cls.LOCUS_TO_GENE_WINDOW),\n            on=\"variantId\",\n            how=\"left\",\n        )\n        .withColumn(\n            \"goldStandardSet\",\n            f.when(\n                (f.col(\"curated_geneId\") == f.col(\"non_curated_geneId\"))\n                # to keep the positives that are outside the v2g dataset\n                | (f.col(\"non_curated_geneId\").isNull()),\n                f.lit(L2GGoldStandard.GS_POSITIVE_LABEL),\n            ).otherwise(L2GGoldStandard.GS_NEGATIVE_LABEL),\n        )\n        .withColumn(\n            \"geneId\",\n            f.when(\n                f.col(\"goldStandardSet\") == L2GGoldStandard.GS_POSITIVE_LABEL,\n                f.col(\"curated_geneId\"),\n            ).otherwise(f.col(\"non_curated_geneId\")),\n        )\n        .drop(\"distance\", \"curated_geneId\", \"non_curated_geneId\")\n    )\n</code></pre>"},{"location":"python_api/datasources/open_targets/l2g_gold_standard/#gentropy.datasource.open_targets.l2g_gold_standard.OpenTargetsL2GGoldStandard.parse_positive_curation","title":"<code>parse_positive_curation(gold_standard_curation: DataFrame) -&gt; DataFrame</code>  <code>classmethod</code>","text":"<p>Parse positive set from gold standard curation.</p> <p>Parameters:</p> Name Type Description Default <code>gold_standard_curation</code> <code>DataFrame</code> <p>Gold standard curation dataframe</p> required <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>Positive set</p> Source code in <code>src/gentropy/datasource/open_targets/l2g_gold_standard.py</code> <pre><code>@classmethod\ndef parse_positive_curation(\n    cls: Type[OpenTargetsL2GGoldStandard], gold_standard_curation: DataFrame\n) -&gt; DataFrame:\n    \"\"\"Parse positive set from gold standard curation.\n\n    Args:\n        gold_standard_curation (DataFrame): Gold standard curation dataframe\n\n    Returns:\n        DataFrame: Positive set\n    \"\"\"\n    return (\n        gold_standard_curation.filter(\n            f.col(\"gold_standard_info.highest_confidence\").isin([\"High\", \"Medium\"])\n        )\n        .select(\n            f.col(\"association_info.otg_id\").alias(\"studyId\"),\n            f.col(\"gold_standard_info.gene_id\").alias(\"geneId\"),\n            f.concat_ws(\n                \"_\",\n                f.col(\"sentinel_variant.locus_GRCh38.chromosome\"),\n                f.col(\"sentinel_variant.locus_GRCh38.position\"),\n                f.col(\"sentinel_variant.alleles.reference\"),\n                f.col(\"sentinel_variant.alleles.alternative\"),\n            ).alias(\"variantId\"),\n            f.col(\"metadata.set_label\").alias(\"source\"),\n        )\n        .withColumn(\n            \"studyLocusId\",\n            StudyLocus.assign_study_locus_id(f.col(\"studyId\"), f.col(\"variantId\")),\n        )\n        .groupBy(\"studyLocusId\", \"studyId\", \"variantId\", \"geneId\")\n        .agg(f.collect_set(\"source\").alias(\"sources\"))\n    )\n</code></pre>"},{"location":"python_api/datasources/open_targets/target/","title":"Target","text":""},{"location":"python_api/datasources/open_targets/target/#gentropy.datasource.open_targets.target.OpenTargetsTarget","title":"<code>gentropy.datasource.open_targets.target.OpenTargetsTarget</code>","text":"<p>Parser for OTPlatform target dataset.</p> <p>Genomic data from Open Targets provides gene identification and genomic coordinates that are integrated into the gene index of our ETL pipeline.</p> <p>The EMBL-EBI Ensembl database is used as a source for human targets in the Platform, with the Ensembl gene ID as the primary identifier. The criteria for target inclusion is: - Genes from all biotypes encoded in canonical chromosomes - Genes in alternative assemblies encoding for a reviewed protein product.</p> Source code in <code>src/gentropy/datasource/open_targets/target.py</code> <pre><code>class OpenTargetsTarget:\n    \"\"\"Parser for OTPlatform target dataset.\n\n    Genomic data from Open Targets provides gene identification and genomic coordinates that are integrated into the gene index of our ETL pipeline.\n\n    The EMBL-EBI Ensembl database is used as a source for human targets in the Platform, with the Ensembl gene ID as the primary identifier. The criteria for target inclusion is:\n    - Genes from all biotypes encoded in canonical chromosomes\n    - Genes in alternative assemblies encoding for a reviewed protein product.\n    \"\"\"\n\n    @staticmethod\n    def _get_gene_tss(strand_col: Column, start_col: Column, end_col: Column) -&gt; Column:\n        \"\"\"Returns the TSS of a gene based on its orientation.\n\n        Args:\n            strand_col (Column): Column containing 1 if the coding strand of the gene is forward, and -1 if it is reverse.\n            start_col (Column): Column containing the start position of the gene.\n            end_col (Column): Column containing the end position of the gene.\n\n        Returns:\n            Column: Column containing the TSS of the gene.\n\n        Examples:\n            &gt;&gt;&gt; df = spark.createDataFrame([{\"strand\": 1, \"start\": 100, \"end\": 200}, {\"strand\": -1, \"start\": 100, \"end\": 200}])\n            &gt;&gt;&gt; df.withColumn(\"tss\", OpenTargetsTarget._get_gene_tss(f.col(\"strand\"), f.col(\"start\"), f.col(\"end\"))).show()\n            +---+-----+------+---+\n            |end|start|strand|tss|\n            +---+-----+------+---+\n            |200|  100|     1|100|\n            |200|  100|    -1|200|\n            +---+-----+------+---+\n            &lt;BLANKLINE&gt;\n\n        \"\"\"\n        return f.when(strand_col == 1, start_col).when(strand_col == -1, end_col)\n\n    @classmethod\n    def as_gene_index(\n        cls: type[OpenTargetsTarget], target_index: DataFrame\n    ) -&gt; GeneIndex:\n        \"\"\"Initialise GeneIndex from source dataset.\n\n        Args:\n            target_index (DataFrame): Target index dataframe\n\n        Returns:\n            GeneIndex: Gene index dataset\n        \"\"\"\n        return GeneIndex(\n            _df=target_index.select(\n                f.coalesce(f.col(\"id\"), f.lit(\"unknown\")).alias(\"geneId\"),\n                \"approvedSymbol\",\n                \"approvedName\",\n                \"biotype\",\n                f.col(\"obsoleteSymbols.label\").alias(\"obsoleteSymbols\"),\n                f.coalesce(f.col(\"genomicLocation.chromosome\"), f.lit(\"unknown\")).alias(\n                    \"chromosome\"\n                ),\n                OpenTargetsTarget._get_gene_tss(\n                    f.col(\"genomicLocation.strand\"),\n                    f.col(\"genomicLocation.start\"),\n                    f.col(\"genomicLocation.end\"),\n                ).alias(\"tss\"),\n                f.col(\"genomicLocation.start\").alias(\"start\"),\n                f.col(\"genomicLocation.end\").alias(\"end\"),\n                f.col(\"genomicLocation.strand\").alias(\"strand\"),\n            ),\n            _schema=GeneIndex.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasources/open_targets/target/#gentropy.datasource.open_targets.target.OpenTargetsTarget.as_gene_index","title":"<code>as_gene_index(target_index: DataFrame) -&gt; GeneIndex</code>  <code>classmethod</code>","text":"<p>Initialise GeneIndex from source dataset.</p> <p>Parameters:</p> Name Type Description Default <code>target_index</code> <code>DataFrame</code> <p>Target index dataframe</p> required <p>Returns:</p> Name Type Description <code>GeneIndex</code> <code>GeneIndex</code> <p>Gene index dataset</p> Source code in <code>src/gentropy/datasource/open_targets/target.py</code> <pre><code>@classmethod\ndef as_gene_index(\n    cls: type[OpenTargetsTarget], target_index: DataFrame\n) -&gt; GeneIndex:\n    \"\"\"Initialise GeneIndex from source dataset.\n\n    Args:\n        target_index (DataFrame): Target index dataframe\n\n    Returns:\n        GeneIndex: Gene index dataset\n    \"\"\"\n    return GeneIndex(\n        _df=target_index.select(\n            f.coalesce(f.col(\"id\"), f.lit(\"unknown\")).alias(\"geneId\"),\n            \"approvedSymbol\",\n            \"approvedName\",\n            \"biotype\",\n            f.col(\"obsoleteSymbols.label\").alias(\"obsoleteSymbols\"),\n            f.coalesce(f.col(\"genomicLocation.chromosome\"), f.lit(\"unknown\")).alias(\n                \"chromosome\"\n            ),\n            OpenTargetsTarget._get_gene_tss(\n                f.col(\"genomicLocation.strand\"),\n                f.col(\"genomicLocation.start\"),\n                f.col(\"genomicLocation.end\"),\n            ).alias(\"tss\"),\n            f.col(\"genomicLocation.start\").alias(\"start\"),\n            f.col(\"genomicLocation.end\").alias(\"end\"),\n            f.col(\"genomicLocation.strand\").alias(\"strand\"),\n        ),\n        _schema=GeneIndex.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/methods/_methods/","title":"Methods","text":"<p>This section consists of all the methods available in the package. It provides detailed explanations and usage examples for each method. Developers can refer to this section to understand how to use the methods effectively in their code. The list of methods is constantly updated.</p>"},{"location":"python_api/methods/carma/","title":"CARMA","text":"<p>CARMA is the method of the fine-mapping and outlier detection, originally implemented in R (CARMA on GitHub).</p> <p>The full repository for the reimplementation of CARMA in Python can be found here.</p> <p>This is a simplified version of CARMA with the following features:</p> <ol> <li>It uses only Spike-slab effect size priors and Poisson model priors.</li> <li>C++ is re-implemented in Python.</li> <li>The way of storing the configuration list is changed. It uses a string with the list of indexes for causal SNPs instead of a sparse matrix.</li> <li>Fixed bugs in PIP calculation.</li> <li>No credible models.</li> <li>No credible sets, only PIPs.</li> <li>No functional annotations.</li> <li>Removed unnecessary parameters.</li> </ol>"},{"location":"python_api/methods/carma/#gentropy.method.carma.CARMA","title":"<code>gentropy.method.carma.CARMA</code>","text":"<p>Implementation of CARMA outlier detection method.</p> Source code in <code>src/gentropy/method/carma.py</code> <pre><code>class CARMA:\n    \"\"\"Implementation of CARMA outlier detection method.\"\"\"\n\n    @staticmethod\n    def time_limited_CARMA_spike_slab_noEM(\n        z: np.ndarray, ld: np.ndarray, sec_threshold: float = 600\n    ) -&gt; dict[str, Any]:\n        \"\"\"The wrapper for the CARMA_spike_slab_noEM function that runs the function in a separate thread and terminates it if it takes too long.\n\n        Args:\n            z (np.ndarray): Numeric vector representing z-scores.\n            ld (np.ndarray): Numeric matrix representing the linkage disequilibrium (LD) matrix.\n            sec_threshold (float): The time threshold in seconds.\n\n        Returns:\n            dict[str, Any]: A dictionary containing the following results:\n                - PIPs: A numeric vector of posterior inclusion probabilities (PIPs) for all SNPs or None.\n                - B_list: A dataframe containing the marginal likelihoods and the corresponding model space or None.\n                - Outliers: A list of outlier SNPs or None.\n        \"\"\"\n        try:\n            # Execute CARMA.CARMA_spike_slab_noEM with a timeout\n            with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:\n                future = executor.submit(CARMA.CARMA_spike_slab_noEM, z=z, ld=ld)\n                result = future.result(timeout=sec_threshold)\n        except concurrent.futures.TimeoutError:\n            # If execution exceeds the timeout, return None\n            result = {\"PIPs\": None, \"B_list\": None, \"Outliers\": None}\n\n        return result\n\n    @staticmethod\n    def CARMA_spike_slab_noEM(\n        z: np.ndarray,\n        ld: np.ndarray,\n        lambda_val: float = 1,\n        Max_Model_Dim: int = 200_000,\n        all_iter: int = 1,\n        all_inner_iter: int = 10,\n        epsilon_threshold: float = 1e-5,\n        num_causal: int = 10,\n        tau: float = 0.04,\n        outlier_switch: bool = True,\n        outlier_BF_index: float = 1 / 3.2,\n    ) -&gt; dict[str, Any]:\n        \"\"\"Perform CARMA analysis using a Spike-and-Slab prior without Expectation-Maximization (EM).\n\n        Args:\n            z (np.ndarray): Numeric vector representing z-scores.\n            ld (np.ndarray): Numeric matrix representing the linkage disequilibrium (LD) matrix.\n            lambda_val (float): Regularization parameter controlling the strength of the L1 penalty.\n            Max_Model_Dim (int): Maximum allowed dimension for the causal models.\n            all_iter (int): The total number of iterations to run the CARMA analysis.\n            all_inner_iter (int): The number of inner iterations in each CARMA iteration.\n            epsilon_threshold (float): Threshold for convergence in CARMA iterations.\n            num_causal (int): Maximal number of causal variants to be selected in the final model.\n            tau (float): Tuning parameter controlling the degree of sparsity in the Spike-and-Slab prior.\n            outlier_switch (bool): Whether to consider outlier detection in the analysis.\n            outlier_BF_index (float): Bayes Factor threshold for identifying outliers.\n\n        Returns:\n            dict[str, Any]: A dictionary containing the following results:\n                - PIPs: A numeric vector of posterior inclusion probabilities (PIPs) for all SNPs.\n                - B_list: A dataframe containing the marginal likelihoods and the corresponding model space.\n                - Outliers: A list of outlier SNPs.\n        \"\"\"\n        p_snp = len(z)\n        epsilon_list = epsilon_threshold * p_snp\n        all_epsilon_threshold = epsilon_threshold * p_snp\n\n        # Zero step\n        all_C_list = CARMA._MCS_modified(\n            z=z,\n            ld_matrix=ld,\n            epsilon=epsilon_list,\n            Max_Model_Dim=Max_Model_Dim,\n            lambda_val=lambda_val,\n            outlier_switch=outlier_switch,\n            tau=tau,\n            num_causal=num_causal,\n            inner_all_iter=all_inner_iter,\n            outlier_BF_index=outlier_BF_index,\n        )\n\n        # Main steps\n        for _ in range(0, all_iter):\n            ac1 = all_C_list[\"B_list\"][\"set_gamma_margin\"]\n            previous_result = np.mean(ac1[0 : round(len(ac1) / 4)])\n\n            all_C_list = CARMA._MCS_modified(\n                z=z,\n                ld_matrix=ld,\n                input_conditional_S_list=all_C_list[\"conditional_S_list\"],\n                Max_Model_Dim=Max_Model_Dim,\n                num_causal=num_causal,\n                epsilon=epsilon_list,\n                outlier_switch=outlier_switch,\n                tau=tau,\n                lambda_val=lambda_val,\n                inner_all_iter=all_inner_iter,\n                outlier_BF_index=outlier_BF_index,\n            )\n\n            ac1 = all_C_list[\"B_list\"][\"set_gamma_margin\"]\n            difference = np.abs(previous_result - np.mean(ac1[0 : round(len(ac1) / 4)]))\n            if difference &lt; all_epsilon_threshold:\n                break\n\n        # Calculate PIPs and Credible Set\n        pip = CARMA._PIP_func(\n            likeli=all_C_list[\"B_list\"][\"set_gamma_margin\"],\n            model_space=all_C_list[\"B_list\"][\"matrix_gamma\"],\n            p=p_snp,\n            num_causal=num_causal,\n        )\n\n        results_list = {\n            \"PIPs\": pip,\n            \"B_list\": all_C_list[\"B_list\"],\n            \"Outliers\": all_C_list[\"conditional_S_list\"],\n        }\n\n        return results_list\n\n    @staticmethod\n    def _ind_normal_sigma_fixed_marginal_fun_indi(\n        zSigmaz_S: np.ndarray, tau: float, p_S: int, det_S: float\n    ) -&gt; float:\n        \"\"\"Internal function for calculating the marginal likelihood of configuration model.\n\n        Args:\n            zSigmaz_S (np.ndarray): The zSigmaz_S value.\n            tau (float): The tau value.\n            p_S (int): The number of SNPs.\n            det_S (float): The det_S value.\n\n        Returns:\n            float: The marginal likelihood of a model.\n\n        Examples:\n            &gt;&gt;&gt; zSigmaz_S = 0.1\n            &gt;&gt;&gt; tau = 1 / 0.05**2\n            &gt;&gt;&gt; p_S = 3\n            &gt;&gt;&gt; det_S = 0.1\n            &gt;&gt;&gt; np.round(CARMA._ind_normal_sigma_fixed_marginal_fun_indi(zSigmaz_S, tau, p_S, det_S),decimals=5)\n            10.18849\n        \"\"\"\n        return p_S / 2.0 * np.log(tau) - 0.5 * np.log(det_S) + zSigmaz_S / 2.0\n\n    @staticmethod\n    def _ind_Normal_fixed_sigma_marginal_external(\n        index_vec_input: np.ndarray,\n        Sigma: np.ndarray,\n        z: np.ndarray,\n        tau: float,\n        p_S: int,\n    ) -&gt; float:\n        \"\"\"Marginal likelihood of configuration model.\n\n        Args:\n            index_vec_input (np.ndarray): The index vector.\n            Sigma (np.ndarray): The Sigma matrix.\n            z (np.ndarray): The z vector.\n            tau (float): The tau value.\n            p_S (int): The number of SNPs.\n\n        Returns:\n            float: The marginal likelihood of a model.\n\n        Examples:\n            &gt;&gt;&gt; index_vec_input = np.array([1, 2])\n            &gt;&gt;&gt; Sigma = np.array([[1, 0.5, 0.2], [0.5, 1, 0.3], [0.2, 0.3, 1]])\n            &gt;&gt;&gt; z = np.array([10, 11, 10])\n            &gt;&gt;&gt; tau = 1\n            &gt;&gt;&gt; p_S = 2\n            &gt;&gt;&gt; np.round(CARMA._ind_Normal_fixed_sigma_marginal_external(index_vec_input, Sigma, z, tau, p_S),decimals=5)\n            43.60579\n        \"\"\"\n        index_vec = index_vec_input - 1\n        Sigma_S = Sigma[np.ix_(index_vec, index_vec)]\n        A = tau * np.eye(p_S)\n\n        det_S = det(Sigma_S + A)\n        Sigma_S_inv = inv(Sigma_S + A)\n\n        sub_z = z[index_vec]\n        zSigmaz_S = np.dot(sub_z.T, np.dot(Sigma_S_inv, sub_z))\n\n        b = CARMA._ind_normal_sigma_fixed_marginal_fun_indi(zSigmaz_S, tau, p_S, det_S)\n\n        results = b\n\n        return results\n\n    @staticmethod\n    def _outlier_ind_Normal_marginal_external(\n        index_vec_input: np.ndarray,\n        Sigma: np.ndarray,\n        z: np.ndarray,\n        tau: float,\n        p_S: int,\n    ) -&gt; float:\n        \"\"\"Likehood of outlier model.\n\n        Args:\n            index_vec_input (np.ndarray): The index vector.\n            Sigma (np.ndarray): The Sigma matrix.\n            z (np.ndarray): The z vector.\n            tau (float): The tau value.\n            p_S (int): The number of SNPs.\n\n        Returns:\n            float: The likelihood of a model.\n\n        Examples:\n            &gt;&gt;&gt; index_vec_input = np.array([1, 2, 3])\n            &gt;&gt;&gt; Sigma = np.array([[1, 0.5, 0.2], [0.5, 1, 0.3], [0.2, 0.3, 1]])\n            &gt;&gt;&gt; z = np.array([0.1, 0.2, 0.3])\n            &gt;&gt;&gt; tau = 1 / 0.05**2\n            &gt;&gt;&gt; p_S = 3\n            &gt;&gt;&gt; np.round(CARMA._outlier_ind_Normal_marginal_external(index_vec_input, Sigma, z, tau, p_S),decimals=5)\n            -8.8497\n        \"\"\"\n        index_vec = index_vec_input - 1\n\n        Sigma_S = Sigma[np.ix_(index_vec, index_vec)]\n        A = tau * np.eye(p_S)\n\n        Sigma_S_I_inv = pinv(Sigma_S + A, rtol=0.00001)\n        Sigma_S_inv = pinv(Sigma_S, rtol=0.00001)\n\n        det_S = np.abs(det(Sigma_S_inv))\n        det_I_S = np.abs(det(Sigma_S_I_inv))\n\n        sub_z = z[index_vec]\n        zSigmaz_S = np.dot(sub_z, np.dot(Sigma_S_inv, sub_z))\n        zSigmaz_I_S = np.dot(sub_z, np.dot(Sigma_S_I_inv, sub_z))\n\n        b = 0.5 * (np.log(det_S) + np.log(det_I_S)) - 0.5 * (zSigmaz_S - zSigmaz_I_S)\n        results = b\n\n        return results\n\n    @staticmethod\n    def _add_function(S_sub: np.ndarray, y: Any) -&gt; np.ndarray:\n        \"\"\"Concatenate two arrays and sort the result.\n\n        Args:\n            S_sub (np.ndarray): The first array.\n            y (Any): The second array.\n\n        Returns:\n            np.ndarray: The concatenated and sorted array.\n\n        Examples:\n            &gt;&gt;&gt; S_sub = np.array([3, 4])\n            &gt;&gt;&gt; y = np.array([1, 2])\n            &gt;&gt;&gt; CARMA._add_function(S_sub, y)\n            array([[1, 2, 3],\n                   [1, 2, 4]])\n        \"\"\"\n        return np.array([np.sort(np.concatenate(([x], y))) for x in S_sub])\n\n    @staticmethod\n    def _set_gamma_func_base(S: Any, p: int) -&gt; dict[int, np.ndarray]:\n        \"\"\"Creates a dictionary of sets of configurations assuming no conditional set.\n\n        Args:\n            S (Any): The input set.\n            p (int): The number of SNPs.\n\n        Returns:\n            dict[int, np.ndarray]: A dictionary of sets of configurations.\n\n        Examples:\n        &gt;&gt;&gt; S = [0,1]\n        &gt;&gt;&gt; p = 4\n        &gt;&gt;&gt; CARMA._set_gamma_func_base(S, p)\n        {0: array([[0],\n               [1]]), 1: array([[0, 1, 2],\n               [0, 1, 3]]), 2: array([[0, 2],\n               [0, 3],\n               [1, 2],\n               [1, 3]])}\n\n        &gt;&gt;&gt; S = [0]\n        &gt;&gt;&gt; p = 2\n        &gt;&gt;&gt; CARMA._set_gamma_func_base(S, p)\n        {0: None, 1: array([[0, 1]]), 2: array([[1]])}\n\n        &gt;&gt;&gt; S = []\n        &gt;&gt;&gt; p = 2\n        &gt;&gt;&gt; CARMA._set_gamma_func_base(S, p)\n        {0: None, 1: array([[0],\n               [1]]), 2: None}\n        \"\"\"\n        set_gamma: dict[int, Any] = {}\n\n        if len(S) == 0:\n            set_gamma[0] = None\n            set_gamma[1] = np.arange(0, p).reshape(-1, 1)\n            set_gamma[2] = None\n\n        if len(S) == 1:\n            S_sub = np.setdiff1d(np.arange(0, p), S)\n            set_gamma[0] = None\n            set_gamma[1] = CARMA._add_function(S_sub, S)\n            set_gamma[2] = S_sub.reshape(-1, 1)\n\n        if len(S) &gt; 1:\n            S_sub = np.setdiff1d(np.arange(0, p), S)\n            S = np.sort(S)\n            set_gamma[0] = np.array(list(combinations(S, len(S) - 1)))\n            set_gamma[1] = CARMA._add_function(S_sub, S)\n            xs = np.vstack([CARMA._add_function(S_sub, row) for row in set_gamma[0]])\n            set_gamma[2] = xs\n\n        return set_gamma\n\n    @staticmethod\n    def _set_gamma_func_conditional(\n        input_S: Any, condition_index: list[int], p: int\n    ) -&gt; dict[int, np.ndarray]:\n        \"\"\"Creates a dictionary of sets of configurations assuming conditional set.\n\n        Args:\n            input_S (Any): The input set.\n            condition_index (list[int]): The conditional set.\n            p (int): The number of SNPs.\n\n        Returns:\n            dict[int, np.ndarray]: A dictionary of sets of configurations.\n\n        Examples:\n        &gt;&gt;&gt; input_S = [0,1,2]\n        &gt;&gt;&gt; condition_index = [2]\n        &gt;&gt;&gt; p = 4\n        &gt;&gt;&gt; CARMA._set_gamma_func_conditional(input_S, condition_index, p)\n        {0: array([[0],\n               [1]]), 1: array([[0, 1, 3]]), 2: array([[0, 3],\n               [1, 3]])}\n        \"\"\"\n        set_gamma: dict[int, Any] = {}\n        S = np.setdiff1d(input_S, condition_index)\n\n        # set of gamma-\n        if len(S) == 0:\n            S_sub = np.setdiff1d(np.arange(0, p), condition_index)\n            set_gamma[0] = None\n            set_gamma[1] = S_sub.reshape(-1, 1)\n            set_gamma[2] = None\n\n        if len(S) == 1:\n            S_sub = np.setdiff1d(np.arange(0, p), input_S)\n            set_gamma[0] = None\n            set_gamma[1] = CARMA._add_function(S_sub, S)\n            set_gamma[2] = S_sub.reshape(-1, 1)\n\n        if len(S) &gt; 1:\n            S_sub = np.setdiff1d(np.arange(0, p), input_S)\n            S = np.sort(S)\n            set_gamma[0] = np.array(list(combinations(S, len(S) - 1)))\n            set_gamma[1] = CARMA._add_function(S_sub, S)\n            xs = np.vstack([CARMA._add_function(S_sub, row) for row in set_gamma[0]])\n            set_gamma[2] = xs\n\n        return set_gamma\n\n    @staticmethod\n    def _set_gamma_func(\n        input_S: Any, p: int, condition_index: list[int] | None = None\n    ) -&gt; dict[int, np.ndarray]:\n        \"\"\"Creates a dictionary of sets of configurations.\n\n        Args:\n            input_S (Any): The input set.\n            p (int): The number of SNPs.\n            condition_index (list[int] | None): The conditional set. Defaults to None.\n\n        Returns:\n            dict[int, np.ndarray]: A dictionary of sets of configurations.\n\n        Examples:\n        &gt;&gt;&gt; input_S = [0,1,2]\n        &gt;&gt;&gt; condition_index=[2]\n        &gt;&gt;&gt; p = 4\n        &gt;&gt;&gt; CARMA._set_gamma_func(input_S, p, condition_index)\n        {0: array([[0],\n               [1]]), 1: array([[0, 1, 3]]), 2: array([[0, 3],\n               [1, 3]])}\n        \"\"\"\n        if condition_index is None:\n            results = CARMA._set_gamma_func_base(input_S, p)\n        else:\n            results = CARMA._set_gamma_func_conditional(input_S, condition_index, p)\n        return results\n\n    @staticmethod\n    def _index_fun_internal(x: np.ndarray) -&gt; str:\n        \"\"\"Convert an array of causal SNP indexes to comma-separated string.\n\n        Args:\n            x (np.ndarray): The input array.\n\n        Returns:\n            str: The comma-separated string.\n\n        Examples:\n        &gt;&gt;&gt; x = np.array([1,2,3])\n        &gt;&gt;&gt; CARMA._index_fun_internal(x)\n        '1,2,3'\n        \"\"\"\n        y = np.sort(x)\n        y = y.astype(str)\n        return \",\".join(y)\n\n    @staticmethod\n    def _index_fun(y: np.ndarray) -&gt; np.ndarray:\n        \"\"\"Convert an array of causal SNP indexes to comma-separated string.\n\n        Args:\n            y (np.ndarray): The input array.\n\n        Returns:\n            np.ndarray: The comma-separated string.\n\n        Examples:\n        &gt;&gt;&gt; y = np.array([[1,2,3],[4,5,6]])\n        &gt;&gt;&gt; CARMA._index_fun(y)\n        array(['1,2,3', '4,5,6'], dtype='&lt;U5')\n        \"\"\"\n        return np.array([CARMA._index_fun_internal(x) for x in y])\n\n    @staticmethod\n    def _ridge_fun(\n        x: float,\n        Sigma: np.ndarray,\n        modi_ld_S: np.ndarray,\n        test_S: np.ndarray,\n        z: np.ndarray,\n        outlier_tau: float,\n        outlier_likelihood: Any,\n    ) -&gt; float:\n        \"\"\"Estimate the matrix shrinkage parameter for outlier detection.\n\n        Args:\n            x (float): The input parameter.\n            Sigma (np.ndarray): The Sigma matrix.\n            modi_ld_S (np.ndarray): The modi_ld_S matrix.\n            test_S (np.ndarray): The test_S matrix.\n            z (np.ndarray): The z vector.\n            outlier_tau (float): The outlier_tau value.\n            outlier_likelihood (Any): The outlier_likelihood function.\n\n        Returns:\n            float: The estimated matrix shrinkage parameter.\n\n        Examples:\n        &gt;&gt;&gt; x = 0.5\n        &gt;&gt;&gt; Sigma = np.array([[1, 0.5, 0.2], [0.5, 1, 0.3], [0.2, 0.3, 1]])\n        &gt;&gt;&gt; modi_ld_S = np.array([[1, 0.5], [0.5, 1]])\n        &gt;&gt;&gt; test_S = np.array([1, 2])\n        &gt;&gt;&gt; z = np.array([0.1, 0.2, 0.3])\n        &gt;&gt;&gt; outlier_tau = 1 / 0.05**2\n        &gt;&gt;&gt; outlier_likelihood = CARMA._outlier_ind_Normal_marginal_external\n        &gt;&gt;&gt; np.round(CARMA._ridge_fun(x, Sigma, modi_ld_S, test_S, z, outlier_tau, outlier_likelihood),decimals=5)\n        6.01486\n        \"\"\"\n        temp_Sigma = Sigma.copy()\n        temp_ld_S = x * modi_ld_S + (1 - x) * np.eye(len(modi_ld_S))\n        temp_Sigma[np.ix_(test_S, test_S)] = temp_ld_S\n        return -outlier_likelihood(\n            index_vec_input=test_S + 1,\n            Sigma=temp_Sigma,\n            z=z,\n            tau=outlier_tau,\n            p_S=len(test_S),\n        )\n\n    @staticmethod\n    def _prior_dist(t: str, lambda_val: float, p: int) -&gt; float:\n        \"\"\"Estimate the priors for the given configurations.\n\n        Args:\n            t (str): The input string for the given configuration.\n            lambda_val (float): The lambda value.\n            p (int): The number of SNPs.\n\n        Returns:\n            float: The estimated prior.\n\n        Examples:\n        &gt;&gt;&gt; t = \"1,2,3\"\n        &gt;&gt;&gt; lambda_val = 1\n        &gt;&gt;&gt; p = 4\n        &gt;&gt;&gt; np.round(CARMA._prior_dist(t, lambda_val, p),decimals=5)\n        -3.17805\n        \"\"\"\n        index_array = t.split(\",\")\n        dim_model = len(index_array)\n        if t == \"\":\n            dim_model = 0\n        return (\n            dim_model * np.log(lambda_val) + lgamma(p - dim_model + 1) - lgamma(p + 1)\n        )\n\n    @staticmethod\n    def _PIP_func(\n        likeli: pd.DataFrame, model_space: pd.DataFrame, p: int, num_causal: int\n    ) -&gt; np.ndarray:\n        \"\"\"Estimates the posterior inclusion probabilities (PIPs) for all SNPs.\n\n        Args:\n            likeli (pd.DataFrame): The marginal likelihoods.\n            model_space (pd.DataFrame): The corresponding model space.\n            p (int): The number of SNPs.\n            num_causal (int): The maximal number of causal SNPs.\n\n        Returns:\n            np.ndarray: The posterior inclusion probabilities (PIPs) for all SNPs.\n\n        Examples:\n        &gt;&gt;&gt; likeli = pd.DataFrame([10, 10, 5,11,0], columns=['likeli']).squeeze()\n        &gt;&gt;&gt; model_space = pd.DataFrame(['0', '1', '2','0,1',''], columns=['config']).squeeze()\n        &gt;&gt;&gt; p = 3\n        &gt;&gt;&gt; num_causal = 2\n        &gt;&gt;&gt; CARMA._PIP_func(likeli, model_space, p, num_causal)\n        array([0.7869271, 0.7869271, 0.001426 ])\n        \"\"\"\n        likeli = likeli.reset_index(drop=True)\n        model_space = model_space.reset_index(drop=True)\n\n        model_space_matrix = np.zeros((len(model_space), p), dtype=int)\n\n        for i in range(len(model_space)):\n            if model_space.iloc[i] != \"\":\n                ind = list(map(int, model_space.iloc[i].split(\",\")))\n                if len(ind) &gt; 0:\n                    model_space_matrix[i, ind] = 1\n\n        infi_index = np.where(np.isinf(likeli))[0]\n        if len(infi_index) != 0:\n            likeli = likeli.drop(infi_index).reset_index(drop=True)\n            model_space_matrix = np.delete(model_space_matrix, infi_index, axis=0)\n\n        na_index = np.where(np.isnan(likeli))[0]\n        if len(na_index) != 0:\n            likeli = likeli.drop(na_index).reset_index(drop=True)\n            model_space_matrix = np.delete(model_space_matrix, na_index, axis=0)\n\n        row_sums = np.sum(model_space_matrix, axis=1)\n        model_space_matrix = model_space_matrix[row_sums &lt;= num_causal]\n        likeli = likeli[row_sums &lt;= num_causal]\n\n        aa = likeli - max(likeli)\n        prob_sum = np.sum(np.exp(aa))\n\n        result_prob = np.zeros(p)\n        for i in range(p):\n            result_prob[i] = (\n                np.sum(np.exp(aa[model_space_matrix[:, i] == 1])) / prob_sum\n            )\n\n        return result_prob\n\n    @staticmethod\n    def _MCS_modified(  # noqa: C901\n        z: np.ndarray,\n        ld_matrix: np.ndarray,\n        Max_Model_Dim: int = 10_000,\n        lambda_val: float = 1,\n        num_causal: int = 10,\n        outlier_switch: bool = True,\n        input_conditional_S_list: list[int] | None = None,\n        tau: float = 1 / 0.05**2,\n        epsilon: float = 1e-3,\n        inner_all_iter: int = 10,\n        outlier_BF_index: float | None = None,\n    ) -&gt; dict[str, Any]:\n        \"\"\"Modified Monte Carlo shotgun sampling (MCS) algorithm.\n\n        Args:\n            z (np.ndarray): Numeric vector representing z-scores.\n            ld_matrix (np.ndarray): Numeric matrix representing the linkage disequilibrium (LD) matrix.\n            Max_Model_Dim (int): Maximum allowed dimension for the causal models.\n            lambda_val (float): Regularization parameter controlling the strength of the L1 penalty.\n            num_causal (int): Maximal number of causal variants to be selected in the final model.\n            outlier_switch (bool): Whether to consider outlier detection in the analysis.\n            input_conditional_S_list (list[int] | None): The conditional set. Defaults to None.\n            tau (float): Tuning parameter controlling the degree of sparsity in the Spike-and-Slab prior.\n            epsilon (float): Threshold for convergence in CARMA iterations.\n            inner_all_iter (int): The number of inner iterations in each CARMA iteration.\n            outlier_BF_index (float | None): Bayes Factor threshold for identifying outliers. Defaults to None.\n\n        Returns:\n            dict[str, Any]: A dictionary containing the following results:\n                - B_list: A dataframe containing the marginal likelihoods and the corresponding model space.\n                - conditional_S_list: A list of outliers.\n\n        Examples:\n        &gt;&gt;&gt; z = np.array([0.1, 0.2, 0.3])\n        &gt;&gt;&gt; ld_matrix = np.array([[1, 0.5, 0.2], [0.5, 1, 0.3], [0.2, 0.3, 1]])\n        &gt;&gt;&gt; Max_Model_Dim = 10_000\n        &gt;&gt;&gt; lambda_val = 1\n\n        &gt;&gt;&gt; num_causal = 10\n        &gt;&gt;&gt; outlier_switch = True\n        \"\"\"\n        p = len(z)\n        marginal_likelihood = CARMA._ind_Normal_fixed_sigma_marginal_external\n        tau_sample = tau\n        if outlier_switch:\n            outlier_likelihood = CARMA._outlier_ind_Normal_marginal_external\n            outlier_tau = tau\n\n        B = Max_Model_Dim\n        stored_bf = 0\n        Sigma = ld_matrix\n\n        S = []\n\n        null_model = \"\"\n        null_margin = CARMA._prior_dist(null_model, lambda_val=lambda_val, p=p)\n\n        B_list = pd.DataFrame({\"set_gamma_margin\": [null_margin], \"matrix_gamma\": [\"\"]})\n\n        if input_conditional_S_list is None:\n            conditional_S = []\n        else:\n            conditional_S = input_conditional_S_list\n            S = conditional_S\n\n        for _i in range(0, inner_all_iter):\n            for _j in range(0, 10):\n                set_gamma = CARMA._set_gamma_func(\n                    input_S=S, p=p, condition_index=conditional_S\n                )\n\n                if conditional_S is None:\n                    working_S = S\n                else:\n                    working_S = np.sort(np.setdiff1d(S, conditional_S)).astype(int)\n\n                set_gamma_margin: list[Any] = [None, None, None]\n                set_gamma_prior: list[Any] = [None, None, None]\n                matrix_gamma: list[Any] = [None, None, None]\n\n                for i in range(0, len(set_gamma)):\n                    if set_gamma[i] is not None:\n                        matrix_gamma[i] = CARMA._index_fun(set_gamma[i])\n                        p_S = set_gamma[i].shape[1]\n                        set_gamma_margin[i] = np.apply_along_axis(\n                            marginal_likelihood,\n                            1,\n                            set_gamma[i] + 1,\n                            Sigma=Sigma,\n                            z=z,\n                            tau=tau_sample,\n                            p_S=p_S,\n                        )\n                        set_gamma_prior[i] = np.array(\n                            [\n                                CARMA._prior_dist(model, lambda_val=lambda_val, p=p)\n                                for model in matrix_gamma[i]\n                            ]\n                        )\n                        set_gamma_margin[i] = set_gamma_prior[i] + set_gamma_margin[i]\n                    else:\n                        set_gamma_margin[i] = np.array(null_margin)\n                        set_gamma_prior[i] = 0\n                        matrix_gamma[i] = np.array(null_model)\n\n                columns = [\"set_gamma_margin\", \"matrix_gamma\"]\n                add_B = pd.DataFrame(columns=columns)\n\n                for i in range(len(set_gamma)):\n                    if isinstance(set_gamma_margin[i].tolist(), list):\n                        new_row = pd.DataFrame(\n                            {\n                                \"set_gamma_margin\": set_gamma_margin[i].tolist(),\n                                \"matrix_gamma\": matrix_gamma[i].tolist(),\n                            }\n                        )\n                        add_B = pd.concat([add_B, new_row], ignore_index=True)\n                    else:\n                        new_row = pd.DataFrame(\n                            {\n                                \"set_gamma_margin\": [set_gamma_margin[i].tolist()],\n                                \"matrix_gamma\": [matrix_gamma[i].tolist()],\n                            }\n                        )\n                        add_B = pd.concat([add_B, new_row], ignore_index=True)\n\n                # Add visited models into the storage space of models\n                B_list = pd.concat([B_list, add_B], ignore_index=True)\n                B_list = B_list.drop_duplicates(\n                    subset=\"matrix_gamma\", ignore_index=True\n                )\n                B_list = B_list.sort_values(\n                    by=\"set_gamma_margin\", ignore_index=True, ascending=False\n                )\n\n                if len(working_S) == 0:\n                    # Create a DataFrame set.star\n                    set_star = pd.DataFrame(\n                        {\n                            \"set_index\": [0, 1, 2],\n                            \"gamma_set_index\": [np.nan, np.nan, np.nan],\n                            \"margin\": [np.nan, np.nan, np.nan],\n                        }\n                    )\n\n                    # Assuming set.gamma.margin and current.log.margin are defined\n                    aa = set_gamma_margin[1]\n                    aa = aa - aa[np.argmax(aa)]\n\n                    min_half_len = min(len(aa), floor(p / 2))\n                    decr_ind = np.argsort(np.exp(aa))[::-1]\n                    decr_half_ind = decr_ind[:min_half_len]\n\n                    probs = np.exp(aa)[decr_half_ind]\n\n                    chosen_index = np.random.choice(\n                        decr_half_ind, 1, p=probs / np.sum(probs)\n                    )\n                    set_star.at[1, \"gamma_set_index\"] = chosen_index[0]\n                    set_star.at[1, \"margin\"] = set_gamma_margin[1][chosen_index[0]]\n\n                    S = set_gamma[1][chosen_index[0]].tolist()\n\n                else:\n                    set_star = pd.DataFrame(\n                        {\n                            \"set_index\": [0, 1, 2],\n                            \"gamma_set_index\": [np.nan, np.nan, np.nan],\n                            \"margin\": [np.nan, np.nan, np.nan],\n                        }\n                    )\n                    for i in range(0, 3):\n                        aa = set_gamma_margin[i]\n                        if np.size(aa) &gt; 1:\n                            aa = aa - aa[np.argmax(aa)]\n                            chosen_index = np.random.choice(\n                                range(0, np.size(set_gamma_margin[i])),\n                                1,\n                                p=np.exp(aa) / np.sum(np.exp(aa)),\n                            )\n                            set_star.at[i, \"gamma_set_index\"] = chosen_index\n                            set_star.at[i, \"margin\"] = set_gamma_margin[i][chosen_index]\n                        else:\n                            set_star.at[i, \"gamma_set_index\"] = 0\n                            set_star.at[i, \"margin\"] = set_gamma_margin[i]\n\n                    if outlier_switch:\n                        for i in range(1, len(set_gamma)):\n                            test_log_BF: float = 100\n                            while True:\n                                aa = set_gamma_margin[i]\n                                aa = aa - aa[np.argmax(aa)]\n                                chosen_index = np.random.choice(\n                                    range(0, np.size(set_gamma_margin[i])),\n                                    1,\n                                    p=np.exp(aa) / np.sum(np.exp(aa)),\n                                )\n                                set_star.at[i, \"gamma_set_index\"] = chosen_index\n                                set_star.at[i, \"margin\"] = set_gamma_margin[i][\n                                    chosen_index\n                                ]\n\n                                test_S = set_gamma[i][int(chosen_index), :]\n\n                                modi_Sigma = Sigma.copy()\n                                if np.size(test_S) &gt; 1:\n                                    modi_ld_S = modi_Sigma[test_S][:, test_S]\n\n                                    result = minimize_scalar(\n                                        CARMA._ridge_fun,\n                                        bounds=(0, 1),\n                                        args=(\n                                            Sigma,\n                                            modi_ld_S,\n                                            test_S,\n                                            z,\n                                            outlier_tau,\n                                            outlier_likelihood,\n                                        ),\n                                        method=\"bounded\",\n                                    )\n                                    modi_ld_S = result.x * modi_ld_S + (\n                                        1 - result.x\n                                    ) * np.eye(len(modi_ld_S))\n\n                                    modi_Sigma[np.ix_(test_S, test_S)] = modi_ld_S\n\n                                    test_log_BF = outlier_likelihood(\n                                        test_S + 1, Sigma, z, outlier_tau, len(test_S)\n                                    ) - outlier_likelihood(\n                                        test_S + 1,\n                                        modi_Sigma,\n                                        z,\n                                        outlier_tau,\n                                        len(test_S),\n                                    )\n                                    test_log_BF = -np.abs(test_log_BF)\n\n                                if np.exp(test_log_BF) &lt; outlier_BF_index:\n                                    set_gamma[i] = np.delete(\n                                        set_gamma[i],\n                                        int(set_star[\"gamma_set_index\"][i]),\n                                        axis=0,\n                                    )\n                                    set_gamma_margin[i] = np.delete(\n                                        set_gamma_margin[i],\n                                        int(set_star[\"gamma_set_index\"][i]),\n                                        axis=0,\n                                    )\n                                    conditional_S = np.concatenate(\n                                        [conditional_S, np.setdiff1d(test_S, working_S)]\n                                    )\n                                    conditional_S = (\n                                        np.unique(conditional_S).astype(int).tolist()\n                                    )\n                                else:\n                                    break\n\n                    if len(working_S) == num_causal:\n                        set_star = set_star.drop(1)\n                        aa = set_star[\"margin\"] - max(set_star[\"margin\"])\n                        sec_sample = np.random.choice(\n                            [0, 2], 1, p=np.exp(aa) / np.sum(np.exp(aa))\n                        )\n                        ind_sec = int(\n                            set_star[\"gamma_set_index\"][\n                                set_star[\"set_index\"] == int(sec_sample)\n                            ]\n                        )\n                        S = set_gamma[sec_sample[0]][ind_sec].tolist()\n                    else:\n                        aa = set_star[\"margin\"] - max(set_star[\"margin\"])\n                        sec_sample = np.random.choice(\n                            range(0, 3), 1, p=np.exp(aa) / np.sum(np.exp(aa))\n                        )\n                        S = set_gamma[sec_sample[0]][\n                            int(set_star[\"gamma_set_index\"][sec_sample[0]])\n                        ].tolist()\n\n                for item in conditional_S:\n                    if item not in S:\n                        S.append(item)\n            # END h_ind loop\n            #\n            if conditional_S is not None:\n                all_c_index = []\n                index_array = [s.split(\",\") for s in B_list[\"matrix_gamma\"]]\n                for tt in conditional_S:\n                    tt_str = str(tt)\n                    ind = [\n                        i for i, sublist in enumerate(index_array) if tt_str in sublist\n                    ]\n                    all_c_index.extend(ind)\n\n                all_c_index = list(set(all_c_index))\n\n                if len(all_c_index) &gt; 0:\n                    temp_B_list = B_list.copy()\n                    temp_B_list = B_list.drop(all_c_index)\n                else:\n                    temp_B_list = B_list.copy()\n            else:\n                temp_B_list = B_list.copy()\n\n            result_B_list = temp_B_list[: min(int(B), len(temp_B_list))]\n\n            rb1 = result_B_list[\"set_gamma_margin\"]\n\n            difference = abs(rb1[: (len(rb1) // 4)].mean() - stored_bf)\n\n            if difference &lt; epsilon:\n                break\n            else:\n                stored_bf = rb1[: (len(rb1) // 4)].mean()\n\n        out = {\"B_list\": result_B_list, \"conditional_S_list\": conditional_S}\n\n        return out\n</code></pre>"},{"location":"python_api/methods/carma/#gentropy.method.carma.CARMA.CARMA_spike_slab_noEM","title":"<code>CARMA_spike_slab_noEM(z: np.ndarray, ld: np.ndarray, lambda_val: float = 1, Max_Model_Dim: int = 200000, all_iter: int = 1, all_inner_iter: int = 10, epsilon_threshold: float = 1e-05, num_causal: int = 10, tau: float = 0.04, outlier_switch: bool = True, outlier_BF_index: float = 1 / 3.2) -&gt; dict[str, Any]</code>  <code>staticmethod</code>","text":"<p>Perform CARMA analysis using a Spike-and-Slab prior without Expectation-Maximization (EM).</p> <p>Parameters:</p> Name Type Description Default <code>z</code> <code>ndarray</code> <p>Numeric vector representing z-scores.</p> required <code>ld</code> <code>ndarray</code> <p>Numeric matrix representing the linkage disequilibrium (LD) matrix.</p> required <code>lambda_val</code> <code>float</code> <p>Regularization parameter controlling the strength of the L1 penalty.</p> <code>1</code> <code>Max_Model_Dim</code> <code>int</code> <p>Maximum allowed dimension for the causal models.</p> <code>200000</code> <code>all_iter</code> <code>int</code> <p>The total number of iterations to run the CARMA analysis.</p> <code>1</code> <code>all_inner_iter</code> <code>int</code> <p>The number of inner iterations in each CARMA iteration.</p> <code>10</code> <code>epsilon_threshold</code> <code>float</code> <p>Threshold for convergence in CARMA iterations.</p> <code>1e-05</code> <code>num_causal</code> <code>int</code> <p>Maximal number of causal variants to be selected in the final model.</p> <code>10</code> <code>tau</code> <code>float</code> <p>Tuning parameter controlling the degree of sparsity in the Spike-and-Slab prior.</p> <code>0.04</code> <code>outlier_switch</code> <code>bool</code> <p>Whether to consider outlier detection in the analysis.</p> <code>True</code> <code>outlier_BF_index</code> <code>float</code> <p>Bayes Factor threshold for identifying outliers.</p> <code>1 / 3.2</code> <p>Returns:</p> Type Description <code>dict[str, Any]</code> <p>dict[str, Any]: A dictionary containing the following results: - PIPs: A numeric vector of posterior inclusion probabilities (PIPs) for all SNPs. - B_list: A dataframe containing the marginal likelihoods and the corresponding model space. - Outliers: A list of outlier SNPs.</p> Source code in <code>src/gentropy/method/carma.py</code> <pre><code>@staticmethod\ndef CARMA_spike_slab_noEM(\n    z: np.ndarray,\n    ld: np.ndarray,\n    lambda_val: float = 1,\n    Max_Model_Dim: int = 200_000,\n    all_iter: int = 1,\n    all_inner_iter: int = 10,\n    epsilon_threshold: float = 1e-5,\n    num_causal: int = 10,\n    tau: float = 0.04,\n    outlier_switch: bool = True,\n    outlier_BF_index: float = 1 / 3.2,\n) -&gt; dict[str, Any]:\n    \"\"\"Perform CARMA analysis using a Spike-and-Slab prior without Expectation-Maximization (EM).\n\n    Args:\n        z (np.ndarray): Numeric vector representing z-scores.\n        ld (np.ndarray): Numeric matrix representing the linkage disequilibrium (LD) matrix.\n        lambda_val (float): Regularization parameter controlling the strength of the L1 penalty.\n        Max_Model_Dim (int): Maximum allowed dimension for the causal models.\n        all_iter (int): The total number of iterations to run the CARMA analysis.\n        all_inner_iter (int): The number of inner iterations in each CARMA iteration.\n        epsilon_threshold (float): Threshold for convergence in CARMA iterations.\n        num_causal (int): Maximal number of causal variants to be selected in the final model.\n        tau (float): Tuning parameter controlling the degree of sparsity in the Spike-and-Slab prior.\n        outlier_switch (bool): Whether to consider outlier detection in the analysis.\n        outlier_BF_index (float): Bayes Factor threshold for identifying outliers.\n\n    Returns:\n        dict[str, Any]: A dictionary containing the following results:\n            - PIPs: A numeric vector of posterior inclusion probabilities (PIPs) for all SNPs.\n            - B_list: A dataframe containing the marginal likelihoods and the corresponding model space.\n            - Outliers: A list of outlier SNPs.\n    \"\"\"\n    p_snp = len(z)\n    epsilon_list = epsilon_threshold * p_snp\n    all_epsilon_threshold = epsilon_threshold * p_snp\n\n    # Zero step\n    all_C_list = CARMA._MCS_modified(\n        z=z,\n        ld_matrix=ld,\n        epsilon=epsilon_list,\n        Max_Model_Dim=Max_Model_Dim,\n        lambda_val=lambda_val,\n        outlier_switch=outlier_switch,\n        tau=tau,\n        num_causal=num_causal,\n        inner_all_iter=all_inner_iter,\n        outlier_BF_index=outlier_BF_index,\n    )\n\n    # Main steps\n    for _ in range(0, all_iter):\n        ac1 = all_C_list[\"B_list\"][\"set_gamma_margin\"]\n        previous_result = np.mean(ac1[0 : round(len(ac1) / 4)])\n\n        all_C_list = CARMA._MCS_modified(\n            z=z,\n            ld_matrix=ld,\n            input_conditional_S_list=all_C_list[\"conditional_S_list\"],\n            Max_Model_Dim=Max_Model_Dim,\n            num_causal=num_causal,\n            epsilon=epsilon_list,\n            outlier_switch=outlier_switch,\n            tau=tau,\n            lambda_val=lambda_val,\n            inner_all_iter=all_inner_iter,\n            outlier_BF_index=outlier_BF_index,\n        )\n\n        ac1 = all_C_list[\"B_list\"][\"set_gamma_margin\"]\n        difference = np.abs(previous_result - np.mean(ac1[0 : round(len(ac1) / 4)]))\n        if difference &lt; all_epsilon_threshold:\n            break\n\n    # Calculate PIPs and Credible Set\n    pip = CARMA._PIP_func(\n        likeli=all_C_list[\"B_list\"][\"set_gamma_margin\"],\n        model_space=all_C_list[\"B_list\"][\"matrix_gamma\"],\n        p=p_snp,\n        num_causal=num_causal,\n    )\n\n    results_list = {\n        \"PIPs\": pip,\n        \"B_list\": all_C_list[\"B_list\"],\n        \"Outliers\": all_C_list[\"conditional_S_list\"],\n    }\n\n    return results_list\n</code></pre>"},{"location":"python_api/methods/carma/#gentropy.method.carma.CARMA.time_limited_CARMA_spike_slab_noEM","title":"<code>time_limited_CARMA_spike_slab_noEM(z: np.ndarray, ld: np.ndarray, sec_threshold: float = 600) -&gt; dict[str, Any]</code>  <code>staticmethod</code>","text":"<p>The wrapper for the CARMA_spike_slab_noEM function that runs the function in a separate thread and terminates it if it takes too long.</p> <p>Parameters:</p> Name Type Description Default <code>z</code> <code>ndarray</code> <p>Numeric vector representing z-scores.</p> required <code>ld</code> <code>ndarray</code> <p>Numeric matrix representing the linkage disequilibrium (LD) matrix.</p> required <code>sec_threshold</code> <code>float</code> <p>The time threshold in seconds.</p> <code>600</code> <p>Returns:</p> Type Description <code>dict[str, Any]</code> <p>dict[str, Any]: A dictionary containing the following results: - PIPs: A numeric vector of posterior inclusion probabilities (PIPs) for all SNPs or None. - B_list: A dataframe containing the marginal likelihoods and the corresponding model space or None. - Outliers: A list of outlier SNPs or None.</p> Source code in <code>src/gentropy/method/carma.py</code> <pre><code>@staticmethod\ndef time_limited_CARMA_spike_slab_noEM(\n    z: np.ndarray, ld: np.ndarray, sec_threshold: float = 600\n) -&gt; dict[str, Any]:\n    \"\"\"The wrapper for the CARMA_spike_slab_noEM function that runs the function in a separate thread and terminates it if it takes too long.\n\n    Args:\n        z (np.ndarray): Numeric vector representing z-scores.\n        ld (np.ndarray): Numeric matrix representing the linkage disequilibrium (LD) matrix.\n        sec_threshold (float): The time threshold in seconds.\n\n    Returns:\n        dict[str, Any]: A dictionary containing the following results:\n            - PIPs: A numeric vector of posterior inclusion probabilities (PIPs) for all SNPs or None.\n            - B_list: A dataframe containing the marginal likelihoods and the corresponding model space or None.\n            - Outliers: A list of outlier SNPs or None.\n    \"\"\"\n    try:\n        # Execute CARMA.CARMA_spike_slab_noEM with a timeout\n        with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:\n            future = executor.submit(CARMA.CARMA_spike_slab_noEM, z=z, ld=ld)\n            result = future.result(timeout=sec_threshold)\n    except concurrent.futures.TimeoutError:\n        # If execution exceeds the timeout, return None\n        result = {\"PIPs\": None, \"B_list\": None, \"Outliers\": None}\n\n    return result\n</code></pre>"},{"location":"python_api/methods/clumping/","title":"Clumping","text":"<p>Clumping is a commonly used post-processing method that allows for the identification of independent association signals from GWAS summary statistics and curated associations. This process is critical because of the complex linkage disequilibrium (LD) structure in human populations, which can result in multiple statistically significant associations within the same genomic region. Clumping methods help reduce redundancy in GWAS results and ensure that each reported association represents an independent signal.</p> <p>We have implemented two clumping methods:</p> <ol> <li>Distance-based clumping: Uses genomic window to clump the significant SNPs into one hit.</li> <li>LD-based clumping: Uses genomic window and LD to clump the significant SNPs into one hit.</li> </ol> <p>The algorithmic logic is similar to classic clumping approaches from PLINK (Reference: PLINK Clump Documentation). See details below:</p>"},{"location":"python_api/methods/clumping/#distance-based-clumping","title":"Distance-based clumping","text":""},{"location":"python_api/methods/clumping/#gentropy.method.window_based_clumping.WindowBasedClumping","title":"<code>gentropy.method.window_based_clumping.WindowBasedClumping</code>","text":"<p>Get semi-lead snps from summary statistics using a window based function.</p> Source code in <code>src/gentropy/method/window_based_clumping.py</code> <pre><code>class WindowBasedClumping:\n    \"\"\"Get semi-lead snps from summary statistics using a window based function.\"\"\"\n\n    @staticmethod\n    def _cluster_peaks(\n        study: Column, chromosome: Column, position: Column, window_length: int\n    ) -&gt; Column:\n        \"\"\"Cluster GWAS significant variants, were clusters are separated by a defined distance.\n\n        !! Important to note that the length of the clusters can be arbitrarily big.\n\n        Args:\n            study (Column): study identifier\n            chromosome (Column): chromosome identifier\n            position (Column): position of the variant\n            window_length (int): window length in basepair\n\n        Returns:\n            Column: containing cluster identifier\n\n        Examples:\n            &gt;&gt;&gt; data = [\n            ...     # Cluster 1:\n            ...     ('s1', 'chr1', 2),\n            ...     ('s1', 'chr1', 4),\n            ...     ('s1', 'chr1', 12),\n            ...     # Cluster 2 - Same chromosome:\n            ...     ('s1', 'chr1', 31),\n            ...     ('s1', 'chr1', 38),\n            ...     ('s1', 'chr1', 42),\n            ...     # Cluster 3 - New chromosome:\n            ...     ('s1', 'chr2', 41),\n            ...     ('s1', 'chr2', 44),\n            ...     ('s1', 'chr2', 50),\n            ...     # Cluster 4 - other study:\n            ...     ('s2', 'chr2', 55),\n            ...     ('s2', 'chr2', 62),\n            ...     ('s2', 'chr2', 70),\n            ... ]\n            &gt;&gt;&gt; window_length = 10\n            &gt;&gt;&gt; (\n            ...     spark.createDataFrame(data, ['studyId', 'chromosome', 'position'])\n            ...     .withColumn(\"cluster_id\",\n            ...         WindowBasedClumping._cluster_peaks(\n            ...             f.col('studyId'),\n            ...             f.col('chromosome'),\n            ...             f.col('position'),\n            ...             window_length\n            ...         )\n            ...     ).show()\n            ... )\n            +-------+----------+--------+----------+\n            |studyId|chromosome|position|cluster_id|\n            +-------+----------+--------+----------+\n            |     s1|      chr1|       2| s1_chr1_2|\n            |     s1|      chr1|       4| s1_chr1_2|\n            |     s1|      chr1|      12| s1_chr1_2|\n            |     s1|      chr1|      31|s1_chr1_31|\n            |     s1|      chr1|      38|s1_chr1_31|\n            |     s1|      chr1|      42|s1_chr1_31|\n            |     s1|      chr2|      41|s1_chr2_41|\n            |     s1|      chr2|      44|s1_chr2_41|\n            |     s1|      chr2|      50|s1_chr2_41|\n            |     s2|      chr2|      55|s2_chr2_55|\n            |     s2|      chr2|      62|s2_chr2_55|\n            |     s2|      chr2|      70|s2_chr2_55|\n            +-------+----------+--------+----------+\n            &lt;BLANKLINE&gt;\n\n        \"\"\"\n        # By adding previous position, the cluster boundary can be identified:\n        previous_position = f.lag(position).over(\n            Window.partitionBy(study, chromosome).orderBy(position)\n        )\n        # We consider a cluster boudary if subsequent snps are further than the defined window:\n        cluster_id = f.when(\n            (previous_position.isNull())\n            | (position - previous_position &gt; window_length),\n            f.concat_ws(\"_\", study, chromosome, position),\n        )\n        # The cluster identifier is propagated across every variant of the cluster:\n        return f.when(\n            cluster_id.isNull(),\n            f.last(cluster_id, ignorenulls=True).over(\n                Window.partitionBy(study, chromosome)\n                .orderBy(position)\n                .rowsBetween(Window.unboundedPreceding, Window.currentRow)\n            ),\n        ).otherwise(cluster_id)\n\n    @staticmethod\n    def _prune_peak(position: NDArray[np.float64], window_size: int) -&gt; DenseVector:\n        \"\"\"Establish lead snps based on their positions listed by p-value.\n\n        The function `find_peak` assigns lead SNPs based on their positions listed by p-value within a specified window size.\n\n        Args:\n            position (NDArray[np.float64]): positions of the SNPs sorted by p-value.\n            window_size (int): the distance in bp within which associations are clumped together around the lead snp.\n\n        Returns:\n            DenseVector: binary vector where 1 indicates a lead SNP and 0 indicates a non-lead SNP.\n\n        Examples:\n            &gt;&gt;&gt; from pyspark.ml import functions as fml\n            &gt;&gt;&gt; from pyspark.ml.linalg import DenseVector\n            &gt;&gt;&gt; WindowBasedClumping._prune_peak(np.array((3, 9, 8, 4, 6)), 2)\n            DenseVector([1.0, 1.0, 0.0, 0.0, 1.0])\n\n        \"\"\"\n        # Initializing the lead list with zeroes:\n        is_lead = np.zeros(len(position))\n\n        # List containing indices of leads:\n        lead_indices: list[int] = []\n\n        # Looping through all positions:\n        for index in range(len(position)):\n            # Looping through leads to find out if they are within a window:\n            for lead_index in lead_indices:\n                # If any of the leads within the window:\n                if abs(position[lead_index] - position[index]) &lt; window_size:\n                    # Skipping further checks:\n                    break\n            else:\n                # None of the leads were within the window:\n                lead_indices.append(index)\n                is_lead[index] = 1\n\n        return DenseVector(is_lead)\n\n    @classmethod\n    def clump(\n        cls: type[WindowBasedClumping],\n        summary_stats: SummaryStatistics,\n        window_length: int,\n        p_value_significance: float = 5e-8,\n    ) -&gt; StudyLocus:\n        \"\"\"Clump summary statistics by distance.\n\n        Args:\n            summary_stats (SummaryStatistics): summary statistics to clump\n            window_length (int): window length in basepair\n            p_value_significance (float): only more significant variants are considered\n\n        Returns:\n            StudyLocus: clumped summary statistics\n        \"\"\"\n        # Create window for locus clusters\n        # - variants where the distance between subsequent variants is below the defined threshold.\n        # - Variants are sorted by descending significance\n        cluster_window = Window.partitionBy(\n            \"studyId\", \"chromosome\", \"cluster_id\"\n        ).orderBy(f.col(\"pValueExponent\").asc(), f.col(\"pValueMantissa\").asc())\n\n        return StudyLocus(\n            _df=(\n                summary_stats\n                # Dropping snps below significance - all subsequent steps are done on significant variants:\n                .pvalue_filter(p_value_significance)\n                .df\n                # Clustering summary variants for efficient windowing (complexity reduction):\n                .withColumn(\n                    \"cluster_id\",\n                    WindowBasedClumping._cluster_peaks(\n                        f.col(\"studyId\"),\n                        f.col(\"chromosome\"),\n                        f.col(\"position\"),\n                        window_length,\n                    ),\n                )\n                # Within each cluster variants are ranked by significance:\n                .withColumn(\"pvRank\", f.row_number().over(cluster_window))\n                # Collect positions in cluster for the most significant variant (complexity reduction):\n                .withColumn(\n                    \"collectedPositions\",\n                    f.when(\n                        f.col(\"pvRank\") == 1,\n                        f.collect_list(f.col(\"position\")).over(\n                            cluster_window.rowsBetween(\n                                Window.currentRow, Window.unboundedFollowing\n                            )\n                        ),\n                    ).otherwise(f.array()),\n                )\n                # Get semi indices only ONCE per cluster:\n                .withColumn(\n                    \"semiIndices\",\n                    f.when(\n                        f.size(f.col(\"collectedPositions\")) &gt; 0,\n                        fml.vector_to_array(\n                            f.udf(WindowBasedClumping._prune_peak, VectorUDT())(\n                                fml.array_to_vector(f.col(\"collectedPositions\")),\n                                f.lit(window_length),\n                            )\n                        ),\n                    ),\n                )\n                # Propagating the result of the above calculation for all rows:\n                .withColumn(\n                    \"semiIndices\",\n                    f.when(\n                        f.col(\"semiIndices\").isNull(),\n                        f.first(f.col(\"semiIndices\"), ignorenulls=True).over(\n                            cluster_window\n                        ),\n                    ).otherwise(f.col(\"semiIndices\")),\n                )\n                # Keeping semi indices only:\n                .filter(f.col(\"semiIndices\")[f.col(\"pvRank\") - 1] &gt; 0)\n                .drop(\"pvRank\", \"collectedPositions\", \"semiIndices\", \"cluster_id\")\n                # Adding study-locus id:\n                .withColumn(\n                    \"studyLocusId\",\n                    StudyLocus.assign_study_locus_id(\n                        f.col(\"studyId\"), f.col(\"variantId\")\n                    ),\n                )\n                # Initialize QC column as array of strings:\n                .withColumn(\n                    \"qualityControls\", f.array().cast(t.ArrayType(t.StringType()))\n                )\n            ),\n            _schema=StudyLocus.get_schema(),\n        )\n\n    @classmethod\n    def clump_with_locus(\n        cls: type[WindowBasedClumping],\n        summary_stats: SummaryStatistics,\n        window_length: int,\n        p_value_significance: float = 5e-8,\n        p_value_baseline: float = 0.05,\n        locus_window_length: int | None = None,\n    ) -&gt; StudyLocus:\n        \"\"\"Clump significant associations while collecting locus around them.\n\n        Args:\n            summary_stats (SummaryStatistics): Input summary statistics dataset\n            window_length (int): Window size in  bp, used for distance based clumping.\n            p_value_significance (float): GWAS significance threshold used to filter peaks. Defaults to 5e-8.\n            p_value_baseline (float): Least significant threshold. Below this, all snps are dropped. Defaults to 0.05.\n            locus_window_length (int | None): The distance for collecting locus around the semi indices. Defaults to None.\n\n        Returns:\n            StudyLocus: StudyLocus after clumping with information about the `locus`\n        \"\"\"\n        # If no locus window provided, using the same value:\n        if locus_window_length is None:\n            locus_window_length = window_length\n\n        # Run distance based clumping on the summary stats:\n        clumped_dataframe = WindowBasedClumping.clump(\n            summary_stats,\n            window_length=window_length,\n            p_value_significance=p_value_significance,\n        ).df.alias(\"clumped\")\n\n        # Get list of columns from clumped dataset for further propagation:\n        clumped_columns = clumped_dataframe.columns\n\n        # Dropping variants not meeting the baseline criteria:\n        sumstats_baseline = summary_stats.pvalue_filter(p_value_baseline).df\n\n        # Renaming columns:\n        sumstats_baseline_renamed = sumstats_baseline.selectExpr(\n            *[f\"{col} as tag_{col}\" for col in sumstats_baseline.columns]\n        ).alias(\"sumstat\")\n\n        study_locus_df = (\n            sumstats_baseline_renamed\n            # Joining the two datasets together:\n            .join(\n                f.broadcast(clumped_dataframe),\n                on=[\n                    (f.col(\"sumstat.tag_studyId\") == f.col(\"clumped.studyId\"))\n                    &amp; (f.col(\"sumstat.tag_chromosome\") == f.col(\"clumped.chromosome\"))\n                    &amp; (\n                        f.col(\"sumstat.tag_position\")\n                        &gt;= (f.col(\"clumped.position\") - locus_window_length)\n                    )\n                    &amp; (\n                        f.col(\"sumstat.tag_position\")\n                        &lt;= (f.col(\"clumped.position\") + locus_window_length)\n                    )\n                ],\n                how=\"right\",\n            )\n            .withColumn(\n                \"locus\",\n                f.struct(\n                    f.col(\"tag_variantId\").alias(\"variantId\"),\n                    f.col(\"tag_beta\").alias(\"beta\"),\n                    f.col(\"tag_pValueMantissa\").alias(\"pValueMantissa\"),\n                    f.col(\"tag_pValueExponent\").alias(\"pValueExponent\"),\n                    f.col(\"tag_standardError\").alias(\"standardError\"),\n                ),\n            )\n            .groupby(\"studyLocusId\")\n            .agg(\n                *[\n                    f.first(col).alias(col)\n                    for col in clumped_columns\n                    if col != \"studyLocusId\"\n                ],\n                f.collect_list(f.col(\"locus\")).alias(\"locus\"),\n            )\n        )\n\n        return StudyLocus(\n            _df=study_locus_df,\n            _schema=StudyLocus.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/methods/clumping/#gentropy.method.window_based_clumping.WindowBasedClumping.clump","title":"<code>clump(summary_stats: SummaryStatistics, window_length: int, p_value_significance: float = 5e-08) -&gt; StudyLocus</code>  <code>classmethod</code>","text":"<p>Clump summary statistics by distance.</p> <p>Parameters:</p> Name Type Description Default <code>summary_stats</code> <code>SummaryStatistics</code> <p>summary statistics to clump</p> required <code>window_length</code> <code>int</code> <p>window length in basepair</p> required <code>p_value_significance</code> <code>float</code> <p>only more significant variants are considered</p> <code>5e-08</code> <p>Returns:</p> Name Type Description <code>StudyLocus</code> <code>StudyLocus</code> <p>clumped summary statistics</p> Source code in <code>src/gentropy/method/window_based_clumping.py</code> <pre><code>@classmethod\ndef clump(\n    cls: type[WindowBasedClumping],\n    summary_stats: SummaryStatistics,\n    window_length: int,\n    p_value_significance: float = 5e-8,\n) -&gt; StudyLocus:\n    \"\"\"Clump summary statistics by distance.\n\n    Args:\n        summary_stats (SummaryStatistics): summary statistics to clump\n        window_length (int): window length in basepair\n        p_value_significance (float): only more significant variants are considered\n\n    Returns:\n        StudyLocus: clumped summary statistics\n    \"\"\"\n    # Create window for locus clusters\n    # - variants where the distance between subsequent variants is below the defined threshold.\n    # - Variants are sorted by descending significance\n    cluster_window = Window.partitionBy(\n        \"studyId\", \"chromosome\", \"cluster_id\"\n    ).orderBy(f.col(\"pValueExponent\").asc(), f.col(\"pValueMantissa\").asc())\n\n    return StudyLocus(\n        _df=(\n            summary_stats\n            # Dropping snps below significance - all subsequent steps are done on significant variants:\n            .pvalue_filter(p_value_significance)\n            .df\n            # Clustering summary variants for efficient windowing (complexity reduction):\n            .withColumn(\n                \"cluster_id\",\n                WindowBasedClumping._cluster_peaks(\n                    f.col(\"studyId\"),\n                    f.col(\"chromosome\"),\n                    f.col(\"position\"),\n                    window_length,\n                ),\n            )\n            # Within each cluster variants are ranked by significance:\n            .withColumn(\"pvRank\", f.row_number().over(cluster_window))\n            # Collect positions in cluster for the most significant variant (complexity reduction):\n            .withColumn(\n                \"collectedPositions\",\n                f.when(\n                    f.col(\"pvRank\") == 1,\n                    f.collect_list(f.col(\"position\")).over(\n                        cluster_window.rowsBetween(\n                            Window.currentRow, Window.unboundedFollowing\n                        )\n                    ),\n                ).otherwise(f.array()),\n            )\n            # Get semi indices only ONCE per cluster:\n            .withColumn(\n                \"semiIndices\",\n                f.when(\n                    f.size(f.col(\"collectedPositions\")) &gt; 0,\n                    fml.vector_to_array(\n                        f.udf(WindowBasedClumping._prune_peak, VectorUDT())(\n                            fml.array_to_vector(f.col(\"collectedPositions\")),\n                            f.lit(window_length),\n                        )\n                    ),\n                ),\n            )\n            # Propagating the result of the above calculation for all rows:\n            .withColumn(\n                \"semiIndices\",\n                f.when(\n                    f.col(\"semiIndices\").isNull(),\n                    f.first(f.col(\"semiIndices\"), ignorenulls=True).over(\n                        cluster_window\n                    ),\n                ).otherwise(f.col(\"semiIndices\")),\n            )\n            # Keeping semi indices only:\n            .filter(f.col(\"semiIndices\")[f.col(\"pvRank\") - 1] &gt; 0)\n            .drop(\"pvRank\", \"collectedPositions\", \"semiIndices\", \"cluster_id\")\n            # Adding study-locus id:\n            .withColumn(\n                \"studyLocusId\",\n                StudyLocus.assign_study_locus_id(\n                    f.col(\"studyId\"), f.col(\"variantId\")\n                ),\n            )\n            # Initialize QC column as array of strings:\n            .withColumn(\n                \"qualityControls\", f.array().cast(t.ArrayType(t.StringType()))\n            )\n        ),\n        _schema=StudyLocus.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/methods/clumping/#gentropy.method.window_based_clumping.WindowBasedClumping.clump_with_locus","title":"<code>clump_with_locus(summary_stats: SummaryStatistics, window_length: int, p_value_significance: float = 5e-08, p_value_baseline: float = 0.05, locus_window_length: int | None = None) -&gt; StudyLocus</code>  <code>classmethod</code>","text":"<p>Clump significant associations while collecting locus around them.</p> <p>Parameters:</p> Name Type Description Default <code>summary_stats</code> <code>SummaryStatistics</code> <p>Input summary statistics dataset</p> required <code>window_length</code> <code>int</code> <p>Window size in  bp, used for distance based clumping.</p> required <code>p_value_significance</code> <code>float</code> <p>GWAS significance threshold used to filter peaks. Defaults to 5e-8.</p> <code>5e-08</code> <code>p_value_baseline</code> <code>float</code> <p>Least significant threshold. Below this, all snps are dropped. Defaults to 0.05.</p> <code>0.05</code> <code>locus_window_length</code> <code>int | None</code> <p>The distance for collecting locus around the semi indices. Defaults to None.</p> <code>None</code> <p>Returns:</p> Name Type Description <code>StudyLocus</code> <code>StudyLocus</code> <p>StudyLocus after clumping with information about the <code>locus</code></p> Source code in <code>src/gentropy/method/window_based_clumping.py</code> <pre><code>@classmethod\ndef clump_with_locus(\n    cls: type[WindowBasedClumping],\n    summary_stats: SummaryStatistics,\n    window_length: int,\n    p_value_significance: float = 5e-8,\n    p_value_baseline: float = 0.05,\n    locus_window_length: int | None = None,\n) -&gt; StudyLocus:\n    \"\"\"Clump significant associations while collecting locus around them.\n\n    Args:\n        summary_stats (SummaryStatistics): Input summary statistics dataset\n        window_length (int): Window size in  bp, used for distance based clumping.\n        p_value_significance (float): GWAS significance threshold used to filter peaks. Defaults to 5e-8.\n        p_value_baseline (float): Least significant threshold. Below this, all snps are dropped. Defaults to 0.05.\n        locus_window_length (int | None): The distance for collecting locus around the semi indices. Defaults to None.\n\n    Returns:\n        StudyLocus: StudyLocus after clumping with information about the `locus`\n    \"\"\"\n    # If no locus window provided, using the same value:\n    if locus_window_length is None:\n        locus_window_length = window_length\n\n    # Run distance based clumping on the summary stats:\n    clumped_dataframe = WindowBasedClumping.clump(\n        summary_stats,\n        window_length=window_length,\n        p_value_significance=p_value_significance,\n    ).df.alias(\"clumped\")\n\n    # Get list of columns from clumped dataset for further propagation:\n    clumped_columns = clumped_dataframe.columns\n\n    # Dropping variants not meeting the baseline criteria:\n    sumstats_baseline = summary_stats.pvalue_filter(p_value_baseline).df\n\n    # Renaming columns:\n    sumstats_baseline_renamed = sumstats_baseline.selectExpr(\n        *[f\"{col} as tag_{col}\" for col in sumstats_baseline.columns]\n    ).alias(\"sumstat\")\n\n    study_locus_df = (\n        sumstats_baseline_renamed\n        # Joining the two datasets together:\n        .join(\n            f.broadcast(clumped_dataframe),\n            on=[\n                (f.col(\"sumstat.tag_studyId\") == f.col(\"clumped.studyId\"))\n                &amp; (f.col(\"sumstat.tag_chromosome\") == f.col(\"clumped.chromosome\"))\n                &amp; (\n                    f.col(\"sumstat.tag_position\")\n                    &gt;= (f.col(\"clumped.position\") - locus_window_length)\n                )\n                &amp; (\n                    f.col(\"sumstat.tag_position\")\n                    &lt;= (f.col(\"clumped.position\") + locus_window_length)\n                )\n            ],\n            how=\"right\",\n        )\n        .withColumn(\n            \"locus\",\n            f.struct(\n                f.col(\"tag_variantId\").alias(\"variantId\"),\n                f.col(\"tag_beta\").alias(\"beta\"),\n                f.col(\"tag_pValueMantissa\").alias(\"pValueMantissa\"),\n                f.col(\"tag_pValueExponent\").alias(\"pValueExponent\"),\n                f.col(\"tag_standardError\").alias(\"standardError\"),\n            ),\n        )\n        .groupby(\"studyLocusId\")\n        .agg(\n            *[\n                f.first(col).alias(col)\n                for col in clumped_columns\n                if col != \"studyLocusId\"\n            ],\n            f.collect_list(f.col(\"locus\")).alias(\"locus\"),\n        )\n    )\n\n    return StudyLocus(\n        _df=study_locus_df,\n        _schema=StudyLocus.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/methods/clumping/#ld-based-clumping","title":"LD-based clumping:","text":""},{"location":"python_api/methods/clumping/#gentropy.method.clump.LDclumping","title":"<code>gentropy.method.clump.LDclumping</code>","text":"<p>LD clumping reports the most significant genetic associations in a region in terms of a smaller number of \u201cclumps\u201d of genetically linked SNPs.</p> Source code in <code>src/gentropy/method/clump.py</code> <pre><code>class LDclumping:\n    \"\"\"LD clumping reports the most significant genetic associations in a region in terms of a smaller number of \u201cclumps\u201d of genetically linked SNPs.\"\"\"\n\n    @staticmethod\n    def _is_lead_linked(\n        study_id: Column,\n        variant_id: Column,\n        p_value_exponent: Column,\n        p_value_mantissa: Column,\n        ld_set: Column,\n    ) -&gt; Column:\n        \"\"\"Evaluates whether a lead variant is linked to a tag (with lowest p-value) in the same studyLocus dataset.\n\n        Args:\n            study_id (Column): studyId\n            variant_id (Column): Lead variant id\n            p_value_exponent (Column): p-value exponent\n            p_value_mantissa (Column): p-value mantissa\n            ld_set (Column): Array of variants in LD with the lead variant\n\n        Returns:\n            Column: Boolean in which True indicates that the lead is linked to another tag in the same dataset.\n        \"\"\"\n        leads_in_study = f.collect_set(variant_id).over(Window.partitionBy(study_id))\n        tags_in_studylocus = f.array_union(\n            # Get all tag variants from the credible set per studyLocusId\n            f.transform(ld_set, lambda x: x.tagVariantId),\n            # And append the lead variant so that the intersection is the same for all studyLocusIds in a study\n            f.array(variant_id),\n        )\n        intersect_lead_tags = f.array_sort(\n            f.array_intersect(leads_in_study, tags_in_studylocus)\n        )\n        return (\n            # If the lead is in the credible set, we rank the peaks by p-value\n            f.when(\n                f.size(intersect_lead_tags) &gt; 0,\n                f.row_number().over(\n                    Window.partitionBy(study_id, intersect_lead_tags).orderBy(\n                        p_value_exponent, p_value_mantissa\n                    )\n                )\n                &gt; 1,\n            )\n            # If the intersection is empty (lead is not in the credible set or cred set is empty), the association is not linked\n            .otherwise(f.lit(False))\n        )\n\n    @classmethod\n    def clump(cls: type[LDclumping], associations: StudyLocus) -&gt; StudyLocus:\n        \"\"\"Perform clumping on studyLocus dataset.\n\n        Args:\n            associations (StudyLocus): StudyLocus dataset\n\n        Returns:\n            StudyLocus: including flag and removing locus information for LD clumped loci.\n        \"\"\"\n        return associations.clump()\n</code></pre>"},{"location":"python_api/methods/clumping/#gentropy.method.clump.LDclumping.clump","title":"<code>clump(associations: StudyLocus) -&gt; StudyLocus</code>  <code>classmethod</code>","text":"<p>Perform clumping on studyLocus dataset.</p> <p>Parameters:</p> Name Type Description Default <code>associations</code> <code>StudyLocus</code> <p>StudyLocus dataset</p> required <p>Returns:</p> Name Type Description <code>StudyLocus</code> <code>StudyLocus</code> <p>including flag and removing locus information for LD clumped loci.</p> Source code in <code>src/gentropy/method/clump.py</code> <pre><code>@classmethod\ndef clump(cls: type[LDclumping], associations: StudyLocus) -&gt; StudyLocus:\n    \"\"\"Perform clumping on studyLocus dataset.\n\n    Args:\n        associations (StudyLocus): StudyLocus dataset\n\n    Returns:\n        StudyLocus: including flag and removing locus information for LD clumped loci.\n    \"\"\"\n    return associations.clump()\n</code></pre>"},{"location":"python_api/methods/coloc/","title":"Coloc","text":""},{"location":"python_api/methods/coloc/#gentropy.method.colocalisation.Coloc","title":"<code>gentropy.method.colocalisation.Coloc</code>","text":"<p>Calculate bayesian colocalisation based on overlapping signals from credible sets.</p> <p>Based on the R COLOC package, which uses the Bayes factors from the credible set to estimate the posterior probability of colocalisation. This method makes the simplifying assumption that only one single causal variant exists for any given trait in any genomic region.</p> Hypothesis Description H<sub>0</sub> no association with either trait in the region H<sub>1</sub> association with trait 1 only H<sub>2</sub> association with trait 2 only H<sub>3</sub> both traits are associated, but have different single causal variants H<sub>4</sub> both traits are associated and share the same single causal variant <p>Bayes factors required</p> <p>Coloc requires the availability of Bayes factors (BF) for each variant in the credible set (<code>logBF</code> column).</p> <p>Attributes:</p> Name Type Description <code>PSEUDOCOUNT</code> <code>float</code> <p>Pseudocount to avoid log(0). Defaults to 1e-10.</p> Source code in <code>src/gentropy/method/colocalisation.py</code> <pre><code>class Coloc:\n    \"\"\"Calculate bayesian colocalisation based on overlapping signals from credible sets.\n\n    Based on the [R COLOC package](https://github.com/chr1swallace/coloc/blob/main/R/claudia.R), which uses the Bayes factors from the credible set to estimate the posterior probability of colocalisation. This method makes the simplifying assumption that **only one single causal variant** exists for any given trait in any genomic region.\n\n    | Hypothesis    | Description                                                           |\n    | ------------- | --------------------------------------------------------------------- |\n    | H&lt;sub&gt;0&lt;/sub&gt; | no association with either trait in the region                        |\n    | H&lt;sub&gt;1&lt;/sub&gt; | association with trait 1 only                                         |\n    | H&lt;sub&gt;2&lt;/sub&gt; | association with trait 2 only                                         |\n    | H&lt;sub&gt;3&lt;/sub&gt; | both traits are associated, but have different single causal variants |\n    | H&lt;sub&gt;4&lt;/sub&gt; | both traits are associated and share the same single causal variant   |\n\n    !!! warning \"Bayes factors required\"\n\n        Coloc requires the availability of Bayes factors (BF) for each variant in the credible set (`logBF` column).\n\n    Attributes:\n        PSEUDOCOUNT (float): Pseudocount to avoid log(0). Defaults to 1e-10.\n    \"\"\"\n\n    PSEUDOCOUNT: float = 1e-10\n\n    @staticmethod\n    def _get_posteriors(all_bfs: NDArray[np.float64]) -&gt; DenseVector:\n        \"\"\"Calculate posterior probabilities for each hypothesis.\n\n        Args:\n            all_bfs (NDArray[np.float64]): h0-h4 bayes factors\n\n        Returns:\n            DenseVector: Posterior\n\n        Example:\n            &gt;&gt;&gt; l = np.array([0.2, 0.1, 0.05, 0])\n            &gt;&gt;&gt; Coloc._get_posteriors(l)\n            DenseVector([0.279, 0.2524, 0.2401, 0.2284])\n        \"\"\"\n        diff = all_bfs - get_logsum(all_bfs)\n        bfs_posteriors = np.exp(diff)\n        return Vectors.dense(bfs_posteriors)\n\n    @classmethod\n    def colocalise(\n        cls: type[Coloc],\n        overlapping_signals: StudyLocusOverlap,\n        priorc1: float = 1e-4,\n        priorc2: float = 1e-4,\n        priorc12: float = 1e-5,\n    ) -&gt; Colocalisation:\n        \"\"\"Calculate bayesian colocalisation based on overlapping signals.\n\n        Args:\n            overlapping_signals (StudyLocusOverlap): overlapping peaks\n\n            priorc1 (float): Prior on variant being causal for trait 1. Defaults to 1e-4.\n            priorc2 (float): Prior on variant being causal for trait 2. Defaults to 1e-4.\n            priorc12 (float): Prior on variant being causal for traits 1 and 2. Defaults to 1e-5.\n\n        Returns:\n            Colocalisation: Colocalisation results\n        \"\"\"\n        # register udfs\n        logsum = f.udf(get_logsum, DoubleType())\n        posteriors = f.udf(Coloc._get_posteriors, VectorUDT())\n        return Colocalisation(\n            _df=(\n                overlapping_signals.df\n                # Before summing log_BF columns nulls need to be filled with 0:\n                .fillna(0, subset=[\"statistics.left_logBF\", \"statistics.right_logBF\"])\n                # Sum of log_BFs for each pair of signals\n                .withColumn(\n                    \"sum_log_bf\",\n                    f.col(\"statistics.left_logBF\") + f.col(\"statistics.right_logBF\"),\n                )\n                # Group by overlapping peak and generating dense vectors of log_BF:\n                .groupBy(\"chromosome\", \"leftStudyLocusId\", \"rightStudyLocusId\")\n                .agg(\n                    f.count(\"*\").alias(\"numberColocalisingVariants\"),\n                    fml.array_to_vector(\n                        f.collect_list(f.col(\"statistics.left_logBF\"))\n                    ).alias(\"left_logBF\"),\n                    fml.array_to_vector(\n                        f.collect_list(f.col(\"statistics.right_logBF\"))\n                    ).alias(\"right_logBF\"),\n                    fml.array_to_vector(f.collect_list(f.col(\"sum_log_bf\"))).alias(\n                        \"sum_log_bf\"\n                    ),\n                )\n                .withColumn(\"logsum1\", logsum(f.col(\"left_logBF\")))\n                .withColumn(\"logsum2\", logsum(f.col(\"right_logBF\")))\n                .withColumn(\"logsum12\", logsum(f.col(\"sum_log_bf\")))\n                .drop(\"left_logBF\", \"right_logBF\", \"sum_log_bf\")\n                # Add priors\n                # priorc1 Prior on variant being causal for trait 1\n                .withColumn(\"priorc1\", f.lit(priorc1))\n                # priorc2 Prior on variant being causal for trait 2\n                .withColumn(\"priorc2\", f.lit(priorc2))\n                # priorc12 Prior on variant being causal for traits 1 and 2\n                .withColumn(\"priorc12\", f.lit(priorc12))\n                # h0-h2\n                .withColumn(\"lH0bf\", f.lit(0))\n                .withColumn(\"lH1bf\", f.log(f.col(\"priorc1\")) + f.col(\"logsum1\"))\n                .withColumn(\"lH2bf\", f.log(f.col(\"priorc2\")) + f.col(\"logsum2\"))\n                # h3\n                .withColumn(\"sumlogsum\", f.col(\"logsum1\") + f.col(\"logsum2\"))\n                .withColumn(\"max\", f.greatest(\"sumlogsum\", \"logsum12\"))\n                .withColumn(\n                    \"logdiff\",\n                    f.when(\n                        f.col(\"sumlogsum\") == f.col(\"logsum12\"), Coloc.PSEUDOCOUNT\n                    ).otherwise(\n                        f.col(\"max\")\n                        + f.log(\n                            f.exp(f.col(\"sumlogsum\") - f.col(\"max\"))\n                            - f.exp(f.col(\"logsum12\") - f.col(\"max\"))\n                        )\n                    ),\n                )\n                .withColumn(\n                    \"lH3bf\",\n                    f.log(f.col(\"priorc1\"))\n                    + f.log(f.col(\"priorc2\"))\n                    + f.col(\"logdiff\"),\n                )\n                .drop(\"right_logsum\", \"left_logsum\", \"sumlogsum\", \"max\", \"logdiff\")\n                # h4\n                .withColumn(\"lH4bf\", f.log(f.col(\"priorc12\")) + f.col(\"logsum12\"))\n                # cleaning\n                .drop(\n                    \"priorc1\", \"priorc2\", \"priorc12\", \"logsum1\", \"logsum2\", \"logsum12\"\n                )\n                # posteriors\n                .withColumn(\n                    \"allBF\",\n                    fml.array_to_vector(\n                        f.array(\n                            f.col(\"lH0bf\"),\n                            f.col(\"lH1bf\"),\n                            f.col(\"lH2bf\"),\n                            f.col(\"lH3bf\"),\n                            f.col(\"lH4bf\"),\n                        )\n                    ),\n                )\n                .withColumn(\n                    \"posteriors\", fml.vector_to_array(posteriors(f.col(\"allBF\")))\n                )\n                .withColumn(\"h0\", f.col(\"posteriors\").getItem(0))\n                .withColumn(\"h1\", f.col(\"posteriors\").getItem(1))\n                .withColumn(\"h2\", f.col(\"posteriors\").getItem(2))\n                .withColumn(\"h3\", f.col(\"posteriors\").getItem(3))\n                .withColumn(\"h4\", f.col(\"posteriors\").getItem(4))\n                .withColumn(\"h4h3\", f.col(\"h4\") / f.col(\"h3\"))\n                .withColumn(\"log2h4h3\", f.log2(f.col(\"h4h3\")))\n                # clean up\n                .drop(\n                    \"posteriors\",\n                    \"allBF\",\n                    \"h4h3\",\n                    \"lH0bf\",\n                    \"lH1bf\",\n                    \"lH2bf\",\n                    \"lH3bf\",\n                    \"lH4bf\",\n                )\n                .withColumn(\"colocalisationMethod\", f.lit(\"COLOC\"))\n            ),\n            _schema=Colocalisation.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/methods/coloc/#gentropy.method.colocalisation.Coloc.colocalise","title":"<code>colocalise(overlapping_signals: StudyLocusOverlap, priorc1: float = 0.0001, priorc2: float = 0.0001, priorc12: float = 1e-05) -&gt; Colocalisation</code>  <code>classmethod</code>","text":"<p>Calculate bayesian colocalisation based on overlapping signals.</p> <p>Parameters:</p> Name Type Description Default <code>overlapping_signals</code> <code>StudyLocusOverlap</code> <p>overlapping peaks</p> required <code>priorc1</code> <code>float</code> <p>Prior on variant being causal for trait 1. Defaults to 1e-4.</p> <code>0.0001</code> <code>priorc2</code> <code>float</code> <p>Prior on variant being causal for trait 2. Defaults to 1e-4.</p> <code>0.0001</code> <code>priorc12</code> <code>float</code> <p>Prior on variant being causal for traits 1 and 2. Defaults to 1e-5.</p> <code>1e-05</code> <p>Returns:</p> Name Type Description <code>Colocalisation</code> <code>Colocalisation</code> <p>Colocalisation results</p> Source code in <code>src/gentropy/method/colocalisation.py</code> <pre><code>@classmethod\ndef colocalise(\n    cls: type[Coloc],\n    overlapping_signals: StudyLocusOverlap,\n    priorc1: float = 1e-4,\n    priorc2: float = 1e-4,\n    priorc12: float = 1e-5,\n) -&gt; Colocalisation:\n    \"\"\"Calculate bayesian colocalisation based on overlapping signals.\n\n    Args:\n        overlapping_signals (StudyLocusOverlap): overlapping peaks\n\n        priorc1 (float): Prior on variant being causal for trait 1. Defaults to 1e-4.\n        priorc2 (float): Prior on variant being causal for trait 2. Defaults to 1e-4.\n        priorc12 (float): Prior on variant being causal for traits 1 and 2. Defaults to 1e-5.\n\n    Returns:\n        Colocalisation: Colocalisation results\n    \"\"\"\n    # register udfs\n    logsum = f.udf(get_logsum, DoubleType())\n    posteriors = f.udf(Coloc._get_posteriors, VectorUDT())\n    return Colocalisation(\n        _df=(\n            overlapping_signals.df\n            # Before summing log_BF columns nulls need to be filled with 0:\n            .fillna(0, subset=[\"statistics.left_logBF\", \"statistics.right_logBF\"])\n            # Sum of log_BFs for each pair of signals\n            .withColumn(\n                \"sum_log_bf\",\n                f.col(\"statistics.left_logBF\") + f.col(\"statistics.right_logBF\"),\n            )\n            # Group by overlapping peak and generating dense vectors of log_BF:\n            .groupBy(\"chromosome\", \"leftStudyLocusId\", \"rightStudyLocusId\")\n            .agg(\n                f.count(\"*\").alias(\"numberColocalisingVariants\"),\n                fml.array_to_vector(\n                    f.collect_list(f.col(\"statistics.left_logBF\"))\n                ).alias(\"left_logBF\"),\n                fml.array_to_vector(\n                    f.collect_list(f.col(\"statistics.right_logBF\"))\n                ).alias(\"right_logBF\"),\n                fml.array_to_vector(f.collect_list(f.col(\"sum_log_bf\"))).alias(\n                    \"sum_log_bf\"\n                ),\n            )\n            .withColumn(\"logsum1\", logsum(f.col(\"left_logBF\")))\n            .withColumn(\"logsum2\", logsum(f.col(\"right_logBF\")))\n            .withColumn(\"logsum12\", logsum(f.col(\"sum_log_bf\")))\n            .drop(\"left_logBF\", \"right_logBF\", \"sum_log_bf\")\n            # Add priors\n            # priorc1 Prior on variant being causal for trait 1\n            .withColumn(\"priorc1\", f.lit(priorc1))\n            # priorc2 Prior on variant being causal for trait 2\n            .withColumn(\"priorc2\", f.lit(priorc2))\n            # priorc12 Prior on variant being causal for traits 1 and 2\n            .withColumn(\"priorc12\", f.lit(priorc12))\n            # h0-h2\n            .withColumn(\"lH0bf\", f.lit(0))\n            .withColumn(\"lH1bf\", f.log(f.col(\"priorc1\")) + f.col(\"logsum1\"))\n            .withColumn(\"lH2bf\", f.log(f.col(\"priorc2\")) + f.col(\"logsum2\"))\n            # h3\n            .withColumn(\"sumlogsum\", f.col(\"logsum1\") + f.col(\"logsum2\"))\n            .withColumn(\"max\", f.greatest(\"sumlogsum\", \"logsum12\"))\n            .withColumn(\n                \"logdiff\",\n                f.when(\n                    f.col(\"sumlogsum\") == f.col(\"logsum12\"), Coloc.PSEUDOCOUNT\n                ).otherwise(\n                    f.col(\"max\")\n                    + f.log(\n                        f.exp(f.col(\"sumlogsum\") - f.col(\"max\"))\n                        - f.exp(f.col(\"logsum12\") - f.col(\"max\"))\n                    )\n                ),\n            )\n            .withColumn(\n                \"lH3bf\",\n                f.log(f.col(\"priorc1\"))\n                + f.log(f.col(\"priorc2\"))\n                + f.col(\"logdiff\"),\n            )\n            .drop(\"right_logsum\", \"left_logsum\", \"sumlogsum\", \"max\", \"logdiff\")\n            # h4\n            .withColumn(\"lH4bf\", f.log(f.col(\"priorc12\")) + f.col(\"logsum12\"))\n            # cleaning\n            .drop(\n                \"priorc1\", \"priorc2\", \"priorc12\", \"logsum1\", \"logsum2\", \"logsum12\"\n            )\n            # posteriors\n            .withColumn(\n                \"allBF\",\n                fml.array_to_vector(\n                    f.array(\n                        f.col(\"lH0bf\"),\n                        f.col(\"lH1bf\"),\n                        f.col(\"lH2bf\"),\n                        f.col(\"lH3bf\"),\n                        f.col(\"lH4bf\"),\n                    )\n                ),\n            )\n            .withColumn(\n                \"posteriors\", fml.vector_to_array(posteriors(f.col(\"allBF\")))\n            )\n            .withColumn(\"h0\", f.col(\"posteriors\").getItem(0))\n            .withColumn(\"h1\", f.col(\"posteriors\").getItem(1))\n            .withColumn(\"h2\", f.col(\"posteriors\").getItem(2))\n            .withColumn(\"h3\", f.col(\"posteriors\").getItem(3))\n            .withColumn(\"h4\", f.col(\"posteriors\").getItem(4))\n            .withColumn(\"h4h3\", f.col(\"h4\") / f.col(\"h3\"))\n            .withColumn(\"log2h4h3\", f.log2(f.col(\"h4h3\")))\n            # clean up\n            .drop(\n                \"posteriors\",\n                \"allBF\",\n                \"h4h3\",\n                \"lH0bf\",\n                \"lH1bf\",\n                \"lH2bf\",\n                \"lH3bf\",\n                \"lH4bf\",\n            )\n            .withColumn(\"colocalisationMethod\", f.lit(\"COLOC\"))\n        ),\n        _schema=Colocalisation.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/methods/ecaviar/","title":"eCAVIAR","text":""},{"location":"python_api/methods/ecaviar/#gentropy.method.colocalisation.ECaviar","title":"<code>gentropy.method.colocalisation.ECaviar</code>","text":"<p>ECaviar-based colocalisation analysis.</p> <p>It extends CAVIAR\u00a0framework to explicitly estimate the posterior probability that the same variant is causal in 2 studies while accounting for the uncertainty of LD. eCAVIAR computes the colocalization posterior probability (CLPP) by utilizing the marginal posterior probabilities. This framework allows for multiple variants to be causal in a single locus.</p> Source code in <code>src/gentropy/method/colocalisation.py</code> <pre><code>class ECaviar:\n    \"\"\"ECaviar-based colocalisation analysis.\n\n    It extends [CAVIAR](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5142122/#bib18)\u00a0framework to explicitly estimate the posterior probability that the same variant is causal in 2 studies while accounting for the uncertainty of LD. eCAVIAR computes the colocalization posterior probability (**CLPP**) by utilizing the marginal posterior probabilities. This framework allows for **multiple variants to be causal** in a single locus.\n    \"\"\"\n\n    @staticmethod\n    def _get_clpp(left_pp: Column, right_pp: Column) -&gt; Column:\n        \"\"\"Calculate the colocalisation posterior probability (CLPP).\n\n        If the fact that the same variant is found causal for two studies are independent events,\n        CLPP is defined as the product of posterior porbabilities that a variant is causal in both studies.\n\n        Args:\n            left_pp (Column): left posterior probability\n            right_pp (Column): right posterior probability\n\n        Returns:\n            Column: CLPP\n\n        Examples:\n            &gt;&gt;&gt; d = [{\"left_pp\": 0.5, \"right_pp\": 0.5}, {\"left_pp\": 0.25, \"right_pp\": 0.75}]\n            &gt;&gt;&gt; df = spark.createDataFrame(d)\n            &gt;&gt;&gt; df.withColumn(\"clpp\", ECaviar._get_clpp(f.col(\"left_pp\"), f.col(\"right_pp\"))).show()\n            +-------+--------+------+\n            |left_pp|right_pp|  clpp|\n            +-------+--------+------+\n            |    0.5|     0.5|  0.25|\n            |   0.25|    0.75|0.1875|\n            +-------+--------+------+\n            &lt;BLANKLINE&gt;\n\n        \"\"\"\n        return left_pp * right_pp\n\n    @classmethod\n    def colocalise(\n        cls: type[ECaviar], overlapping_signals: StudyLocusOverlap\n    ) -&gt; Colocalisation:\n        \"\"\"Calculate bayesian colocalisation based on overlapping signals.\n\n        Args:\n            overlapping_signals (StudyLocusOverlap): overlapping signals.\n\n        Returns:\n            Colocalisation: colocalisation results based on eCAVIAR.\n        \"\"\"\n        return Colocalisation(\n            _df=(\n                overlapping_signals.df.withColumn(\n                    \"clpp\",\n                    ECaviar._get_clpp(\n                        f.col(\"statistics.left_posteriorProbability\"),\n                        f.col(\"statistics.right_posteriorProbability\"),\n                    ),\n                )\n                .groupBy(\"leftStudyLocusId\", \"rightStudyLocusId\", \"chromosome\")\n                .agg(\n                    f.count(\"*\").alias(\"numberColocalisingVariants\"),\n                    f.sum(f.col(\"clpp\")).alias(\"clpp\"),\n                )\n                .withColumn(\"colocalisationMethod\", f.lit(\"eCAVIAR\"))\n            ),\n            _schema=Colocalisation.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/methods/ecaviar/#gentropy.method.colocalisation.ECaviar.colocalise","title":"<code>colocalise(overlapping_signals: StudyLocusOverlap) -&gt; Colocalisation</code>  <code>classmethod</code>","text":"<p>Calculate bayesian colocalisation based on overlapping signals.</p> <p>Parameters:</p> Name Type Description Default <code>overlapping_signals</code> <code>StudyLocusOverlap</code> <p>overlapping signals.</p> required <p>Returns:</p> Name Type Description <code>Colocalisation</code> <code>Colocalisation</code> <p>colocalisation results based on eCAVIAR.</p> Source code in <code>src/gentropy/method/colocalisation.py</code> <pre><code>@classmethod\ndef colocalise(\n    cls: type[ECaviar], overlapping_signals: StudyLocusOverlap\n) -&gt; Colocalisation:\n    \"\"\"Calculate bayesian colocalisation based on overlapping signals.\n\n    Args:\n        overlapping_signals (StudyLocusOverlap): overlapping signals.\n\n    Returns:\n        Colocalisation: colocalisation results based on eCAVIAR.\n    \"\"\"\n    return Colocalisation(\n        _df=(\n            overlapping_signals.df.withColumn(\n                \"clpp\",\n                ECaviar._get_clpp(\n                    f.col(\"statistics.left_posteriorProbability\"),\n                    f.col(\"statistics.right_posteriorProbability\"),\n                ),\n            )\n            .groupBy(\"leftStudyLocusId\", \"rightStudyLocusId\", \"chromosome\")\n            .agg(\n                f.count(\"*\").alias(\"numberColocalisingVariants\"),\n                f.sum(f.col(\"clpp\")).alias(\"clpp\"),\n            )\n            .withColumn(\"colocalisationMethod\", f.lit(\"eCAVIAR\"))\n        ),\n        _schema=Colocalisation.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/methods/ld_annotator/","title":"LDAnnotator","text":""},{"location":"python_api/methods/ld_annotator/#gentropy.method.ld.LDAnnotator","title":"<code>gentropy.method.ld.LDAnnotator</code>","text":"<p>Class to annotate linkage disequilibrium (LD) operations from GnomAD.</p> Source code in <code>src/gentropy/method/ld.py</code> <pre><code>class LDAnnotator:\n    \"\"\"Class to annotate linkage disequilibrium (LD) operations from GnomAD.\"\"\"\n\n    @staticmethod\n    def _calculate_weighted_r_overall(ld_set: Column) -&gt; Column:\n        \"\"\"Aggregation of weighted R information using ancestry proportions.\n\n        Args:\n            ld_set (Column): LD set\n\n        Returns:\n            Column: LD set with added 'r2Overall' field\n        \"\"\"\n        return f.transform(\n            ld_set,\n            lambda x: f.struct(\n                x[\"tagVariantId\"].alias(\"tagVariantId\"),\n                # r2Overall is the accumulated sum of each r2 relative to the population size\n                f.aggregate(\n                    x[\"rValues\"],\n                    f.lit(0.0),\n                    lambda acc, y: acc\n                    + f.coalesce(\n                        f.pow(y[\"r\"], 2) * y[\"relativeSampleSize\"], f.lit(0.0)\n                    ),  # we use coalesce to avoid problems when r/relativeSampleSize is null\n                ).alias(\"r2Overall\"),\n            ),\n        )\n\n    @staticmethod\n    def _add_population_size(ld_set: Column, study_populations: Column) -&gt; Column:\n        \"\"\"Add population size to each rValues entry in the ldSet.\n\n        Args:\n            ld_set (Column): LD set\n            study_populations (Column): Study populations\n\n        Returns:\n            Column: LD set with added 'relativeSampleSize' field\n        \"\"\"\n        # Create a population to relativeSampleSize map from the struct\n        populations_map = f.map_from_arrays(\n            study_populations[\"ldPopulation\"],\n            study_populations[\"relativeSampleSize\"],\n        )\n        return f.transform(\n            ld_set,\n            lambda x: f.struct(\n                x[\"tagVariantId\"].alias(\"tagVariantId\"),\n                f.transform(\n                    x[\"rValues\"],\n                    lambda y: f.struct(\n                        y[\"population\"].alias(\"population\"),\n                        y[\"r\"].alias(\"r\"),\n                        populations_map[y[\"population\"]].alias(\"relativeSampleSize\"),\n                    ),\n                ).alias(\"rValues\"),\n            ),\n        )\n\n    @staticmethod\n    def _qc_unresolved_ld(ld_set: Column, quality_controls: Column) -&gt; Column:\n        \"\"\"Flag associations with unresolved LD.\n\n        Args:\n            ld_set (Column): LD set\n            quality_controls (Column): Quality controls\n\n        Returns:\n            Column: Quality controls with added 'UNRESOLVED_LD' field\n        \"\"\"\n        return StudyLocus.update_quality_flag(\n            quality_controls,\n            ld_set.isNull(),\n            StudyLocusQualityCheck.UNRESOLVED_LD,\n        )\n\n    @staticmethod\n    def _rescue_lead_variant(ld_set: Column, variant_id: Column) -&gt; Column:\n        \"\"\"Rescue lead variant.\n\n        In cases in which no LD information is available but a lead variant is available, we include the lead as the only variant in the ldSet.\n\n        Args:\n            ld_set (Column): LD set\n            variant_id (Column): Variant ID\n\n        Returns:\n            Column: LD set with added 'tagVariantId' field\n        \"\"\"\n        return f.when(\n            ((ld_set.isNull() | (f.size(ld_set) == 0)) &amp; variant_id.isNotNull()),\n            f.array(\n                f.struct(\n                    variant_id.alias(\"tagVariantId\"),\n                    f.lit(1).alias(\"r2Overall\"),\n                )\n            ),\n        ).otherwise(ld_set)\n\n    @classmethod\n    def ld_annotate(\n        cls: type[LDAnnotator],\n        associations: StudyLocus,\n        studies: StudyIndex,\n        ld_index: LDIndex,\n    ) -&gt; StudyLocus:\n        \"\"\"Annotate linkage disequilibrium (LD) information to a set of studyLocus.\n\n        This function:\n            1. Annotates study locus with population structure information from the study index\n            2. Joins the LD index to the StudyLocus\n            3. Adds the population size of the study to each rValues entry in the ldSet\n            4. Calculates the overall R weighted by the ancestry proportions in every given study.\n            5. Flags associations with variants that are not found in the LD reference\n            6. Rescues lead variant when no LD information is available but lead variant is available\n\n        Args:\n            associations (StudyLocus): Dataset to be LD annotated\n            studies (StudyIndex): Dataset with study information\n            ld_index (LDIndex): Dataset with LD information for every variant present in LD matrix\n\n        Returns:\n            StudyLocus: including additional column with LD information.\n        \"\"\"\n        return StudyLocus(\n            _df=(\n                associations.df\n                # Drop ldSet column if already available\n                .select(*[col for col in associations.df.columns if col != \"ldSet\"])\n                # Annotate study locus with population structure from study index\n                .join(\n                    studies.df.select(\"studyId\", \"ldPopulationStructure\"),\n                    on=\"studyId\",\n                    how=\"left\",\n                )\n                # Bring LD information from LD Index\n                .join(\n                    ld_index.df,\n                    on=[\"variantId\", \"chromosome\"],\n                    how=\"left\",\n                )\n                # Add population size to each rValues entry in the ldSet if population structure available:\n                .withColumn(\n                    \"ldSet\",\n                    f.when(\n                        f.col(\"ldPopulationStructure\").isNotNull(),\n                        cls._add_population_size(\n                            f.col(\"ldSet\"), f.col(\"ldPopulationStructure\")\n                        ),\n                    ),\n                )\n                # Aggregate weighted R information using ancestry proportions\n                .withColumn(\n                    \"ldSet\",\n                    f.when(\n                        f.col(\"ldPopulationStructure\").isNotNull(),\n                        cls._calculate_weighted_r_overall(f.col(\"ldSet\")),\n                    ),\n                )\n                .drop(\"ldPopulationStructure\")\n                # QC: Flag associations with variants that are not found in the LD reference\n                .withColumn(\n                    \"qualityControls\",\n                    cls._qc_unresolved_ld(f.col(\"ldSet\"), f.col(\"qualityControls\")),\n                )\n                # Add lead variant to empty ldSet when no LD information is available but lead variant is available\n                .withColumn(\n                    \"ldSet\",\n                    cls._rescue_lead_variant(f.col(\"ldSet\"), f.col(\"variantId\")),\n                )\n            ),\n            _schema=StudyLocus.get_schema(),\n        )._qc_no_population()\n</code></pre>"},{"location":"python_api/methods/ld_annotator/#gentropy.method.ld.LDAnnotator.ld_annotate","title":"<code>ld_annotate(associations: StudyLocus, studies: StudyIndex, ld_index: LDIndex) -&gt; StudyLocus</code>  <code>classmethod</code>","text":"<p>Annotate linkage disequilibrium (LD) information to a set of studyLocus.</p> This function <ol> <li>Annotates study locus with population structure information from the study index</li> <li>Joins the LD index to the StudyLocus</li> <li>Adds the population size of the study to each rValues entry in the ldSet</li> <li>Calculates the overall R weighted by the ancestry proportions in every given study.</li> <li>Flags associations with variants that are not found in the LD reference</li> <li>Rescues lead variant when no LD information is available but lead variant is available</li> </ol> <p>Parameters:</p> Name Type Description Default <code>associations</code> <code>StudyLocus</code> <p>Dataset to be LD annotated</p> required <code>studies</code> <code>StudyIndex</code> <p>Dataset with study information</p> required <code>ld_index</code> <code>LDIndex</code> <p>Dataset with LD information for every variant present in LD matrix</p> required <p>Returns:</p> Name Type Description <code>StudyLocus</code> <code>StudyLocus</code> <p>including additional column with LD information.</p> Source code in <code>src/gentropy/method/ld.py</code> <pre><code>@classmethod\ndef ld_annotate(\n    cls: type[LDAnnotator],\n    associations: StudyLocus,\n    studies: StudyIndex,\n    ld_index: LDIndex,\n) -&gt; StudyLocus:\n    \"\"\"Annotate linkage disequilibrium (LD) information to a set of studyLocus.\n\n    This function:\n        1. Annotates study locus with population structure information from the study index\n        2. Joins the LD index to the StudyLocus\n        3. Adds the population size of the study to each rValues entry in the ldSet\n        4. Calculates the overall R weighted by the ancestry proportions in every given study.\n        5. Flags associations with variants that are not found in the LD reference\n        6. Rescues lead variant when no LD information is available but lead variant is available\n\n    Args:\n        associations (StudyLocus): Dataset to be LD annotated\n        studies (StudyIndex): Dataset with study information\n        ld_index (LDIndex): Dataset with LD information for every variant present in LD matrix\n\n    Returns:\n        StudyLocus: including additional column with LD information.\n    \"\"\"\n    return StudyLocus(\n        _df=(\n            associations.df\n            # Drop ldSet column if already available\n            .select(*[col for col in associations.df.columns if col != \"ldSet\"])\n            # Annotate study locus with population structure from study index\n            .join(\n                studies.df.select(\"studyId\", \"ldPopulationStructure\"),\n                on=\"studyId\",\n                how=\"left\",\n            )\n            # Bring LD information from LD Index\n            .join(\n                ld_index.df,\n                on=[\"variantId\", \"chromosome\"],\n                how=\"left\",\n            )\n            # Add population size to each rValues entry in the ldSet if population structure available:\n            .withColumn(\n                \"ldSet\",\n                f.when(\n                    f.col(\"ldPopulationStructure\").isNotNull(),\n                    cls._add_population_size(\n                        f.col(\"ldSet\"), f.col(\"ldPopulationStructure\")\n                    ),\n                ),\n            )\n            # Aggregate weighted R information using ancestry proportions\n            .withColumn(\n                \"ldSet\",\n                f.when(\n                    f.col(\"ldPopulationStructure\").isNotNull(),\n                    cls._calculate_weighted_r_overall(f.col(\"ldSet\")),\n                ),\n            )\n            .drop(\"ldPopulationStructure\")\n            # QC: Flag associations with variants that are not found in the LD reference\n            .withColumn(\n                \"qualityControls\",\n                cls._qc_unresolved_ld(f.col(\"ldSet\"), f.col(\"qualityControls\")),\n            )\n            # Add lead variant to empty ldSet when no LD information is available but lead variant is available\n            .withColumn(\n                \"ldSet\",\n                cls._rescue_lead_variant(f.col(\"ldSet\"), f.col(\"variantId\")),\n            )\n        ),\n        _schema=StudyLocus.get_schema(),\n    )._qc_no_population()\n</code></pre>"},{"location":"python_api/methods/pics/","title":"PICS","text":"<p>PICS Overview:</p> <p>PICS is a fine-mapping method designed to identify the most likely causal SNPs associated with a trait or disease within a genomic region. It leverages both haplotype information and the observed association patterns from genome-wide association studies (GWAS).</p> <p>Please refer to the original publication for in-depth details: PICS Publication.</p> <p>We use PICS for both GWAS clumping results and GWAS curated studies.</p>"},{"location":"python_api/methods/pics/#gentropy.method.pics.PICS","title":"<code>gentropy.method.pics.PICS</code>","text":"<p>Probabilistic Identification of Causal SNPs (PICS), an algorithm estimating the probability that an individual variant is causal considering the haplotype structure and observed pattern of association at the genetic locus.</p> Source code in <code>src/gentropy/method/pics.py</code> <pre><code>class PICS:\n    \"\"\"Probabilistic Identification of Causal SNPs (PICS), an algorithm estimating the probability that an individual variant is causal considering the haplotype structure and observed pattern of association at the genetic locus.\"\"\"\n\n    @staticmethod\n    def _pics_relative_posterior_probability(\n        neglog_p: float, pics_snp_mu: float, pics_snp_std: float\n    ) -&gt; float:\n        \"\"\"Compute the PICS posterior probability for a given SNP.\n\n        !!! info \"This probability needs to be scaled to take into account the probabilities of the other variants in the locus.\"\n\n        Args:\n            neglog_p (float): Negative log p-value of the lead variant\n            pics_snp_mu (float): Mean P value of the association between a SNP and a trait\n            pics_snp_std (float): Standard deviation for the P value of the association between a SNP and a trait\n\n        Returns:\n            float: Posterior probability of the association between a SNP and a trait\n\n        Examples:\n            &gt;&gt;&gt; rel_prob = PICS._pics_relative_posterior_probability(neglog_p=10.0, pics_snp_mu=1.0, pics_snp_std=10.0)\n            &gt;&gt;&gt; round(rel_prob, 3)\n            0.368\n        \"\"\"\n        return float(norm(pics_snp_mu, pics_snp_std).sf(neglog_p) * 2)\n\n    @staticmethod\n    def _pics_standard_deviation(neglog_p: float, r2: float, k: float) -&gt; float | None:\n        \"\"\"Compute the PICS standard deviation.\n\n        This distribution is obtained after a series of permutation tests described in the PICS method, and it is only\n        valid when the SNP is highly linked with the lead (r2 &gt; 0.5).\n\n        Args:\n            neglog_p (float): Negative log p-value of the lead variant\n            r2 (float): LD score between a given SNP and the lead variant\n            k (float): Empiric constant that can be adjusted to fit the curve, 6.4 recommended.\n\n        Returns:\n            float | None: Standard deviation for the P value of the association between a SNP and a trait\n\n        Examples:\n            &gt;&gt;&gt; PICS._pics_standard_deviation(neglog_p=1.0, r2=1.0, k=6.4)\n            0.0\n            &gt;&gt;&gt; round(PICS._pics_standard_deviation(neglog_p=10.0, r2=0.5, k=6.4), 3)\n            1.493\n            &gt;&gt;&gt; print(PICS._pics_standard_deviation(neglog_p=1.0, r2=0.0, k=6.4))\n            None\n        \"\"\"\n        return (\n            abs(((1 - (r2**0.5) ** k) ** 0.5) * (neglog_p**0.5) / 2)\n            if r2 &gt;= 0.5\n            else None\n        )\n\n    @staticmethod\n    def _pics_mu(neglog_p: float, r2: float) -&gt; float | None:\n        \"\"\"Compute the PICS mu that estimates the probability of association between a given SNP and the trait.\n\n        This distribution is obtained after a series of permutation tests described in the PICS method, and it is only\n        valid when the SNP is highly linked with the lead (r2 &gt; 0.5).\n\n        Args:\n            neglog_p (float): Negative log p-value of the lead variant\n            r2 (float): LD score between a given SNP and the lead variant\n\n        Returns:\n            float | None: Mean P value of the association between a SNP and a trait\n\n        Examples:\n            &gt;&gt;&gt; PICS._pics_mu(neglog_p=1.0, r2=1.0)\n            1.0\n            &gt;&gt;&gt; PICS._pics_mu(neglog_p=10.0, r2=0.5)\n            5.0\n            &gt;&gt;&gt; print(PICS._pics_mu(neglog_p=10.0, r2=0.3))\n            None\n        \"\"\"\n        return neglog_p * r2 if r2 &gt;= 0.5 else None\n\n    @staticmethod\n    def _finemap(\n        ld_set: list[Row], lead_neglog_p: float, k: float\n    ) -&gt; list[dict[str, Any]] | None:\n        \"\"\"Calculates the probability of a variant being causal in a study-locus context by applying the PICS method.\n\n        It is intended to be applied as an UDF in `PICS.finemap`, where each row is a StudyLocus association.\n        The function iterates over every SNP in the `ldSet` array, and it returns an updated locus with\n        its association signal and causality probability as of PICS.\n\n        Args:\n            ld_set (list[Row]): list of tagging variants after expanding the locus\n            lead_neglog_p (float): P value of the association signal between the lead variant and the study in the form of -log10.\n            k (float): Empiric constant that can be adjusted to fit the curve, 6.4 recommended.\n\n        Returns:\n            list[dict[str, Any]] | None: List of tagging variants with an estimation of the association signal and their posterior probability as of PICS.\n\n        Examples:\n            &gt;&gt;&gt; from pyspark.sql import Row\n            &gt;&gt;&gt; ld_set = [\n            ...     Row(variantId=\"var1\", r2Overall=0.8),\n            ...     Row(variantId=\"var2\", r2Overall=1),\n            ... ]\n            &gt;&gt;&gt; PICS._finemap(ld_set, lead_neglog_p=10.0, k=6.4)\n            [{'variantId': 'var1', 'r2Overall': 0.8, 'standardError': 0.07420896512708416, 'posteriorProbability': 0.07116959886882368}, {'variantId': 'var2', 'r2Overall': 1, 'standardError': 0.9977000638225533, 'posteriorProbability': 0.9288304011311763}]\n            &gt;&gt;&gt; empty_ld_set = []\n            &gt;&gt;&gt; PICS._finemap(empty_ld_set, lead_neglog_p=10.0, k=6.4)\n            []\n            &gt;&gt;&gt; ld_set_with_no_r2 = [\n            ...     Row(variantId=\"var1\", r2Overall=None),\n            ...     Row(variantId=\"var2\", r2Overall=None),\n            ... ]\n            &gt;&gt;&gt; PICS._finemap(ld_set_with_no_r2, lead_neglog_p=10.0, k=6.4)\n            []\n        \"\"\"\n        if ld_set is None:\n            return None\n        elif not ld_set:\n            return []\n        tmp_credible_set = []\n        new_credible_set = []\n        # First iteration: calculation of mu, standard deviation, and the relative posterior probability\n        for tag_struct in ld_set:\n            tag_dict = (\n                tag_struct.asDict()\n            )  # tag_struct is of type pyspark.Row, we'll represent it as a dict\n            if (\n                not tag_dict[\"r2Overall\"]\n                or tag_dict[\"r2Overall\"] &lt; 0.5\n                or not lead_neglog_p\n            ):\n                # If PICS cannot be calculated, we drop the variant from the credible set\n                continue\n\n            pics_snp_mu = PICS._pics_mu(lead_neglog_p, tag_dict[\"r2Overall\"])\n            pics_snp_std = PICS._pics_standard_deviation(\n                lead_neglog_p, tag_dict[\"r2Overall\"], k\n            )\n            pics_snp_std = 0.001 if pics_snp_std == 0 else pics_snp_std\n            if pics_snp_mu is not None and pics_snp_std is not None:\n                posterior_probability = PICS._pics_relative_posterior_probability(\n                    lead_neglog_p, pics_snp_mu, pics_snp_std\n                )\n                tag_dict[\"standardError\"] = 10**-pics_snp_std\n                tag_dict[\"relativePosteriorProbability\"] = posterior_probability\n\n                tmp_credible_set.append(tag_dict)\n\n        # Second iteration: calculation of the sum of all the posteriors in each study-locus, so that we scale them between 0-1\n        total_posteriors = sum(\n            tag_dict.get(\"relativePosteriorProbability\", 0)\n            for tag_dict in tmp_credible_set\n        )\n\n        # Third iteration: calculation of the final posteriorProbability\n        for tag_dict in tmp_credible_set:\n            if total_posteriors != 0:\n                tag_dict[\"posteriorProbability\"] = float(\n                    tag_dict.get(\"relativePosteriorProbability\", 0) / total_posteriors\n                )\n            tag_dict.pop(\"relativePosteriorProbability\")\n            new_credible_set.append(tag_dict)\n        return new_credible_set\n\n    @classmethod\n    def finemap(\n        cls: type[PICS], associations: StudyLocus, k: float = 6.4\n    ) -&gt; StudyLocus:\n        \"\"\"Run PICS on a study locus.\n\n        !!! info \"Study locus needs to be LD annotated\"\n\n            The study locus needs to be LD annotated before PICS can be calculated.\n\n        Args:\n            associations (StudyLocus): Study locus to finemap using PICS\n            k (float): Empiric constant that can be adjusted to fit the curve, 6.4 recommended.\n\n        Returns:\n            StudyLocus: Study locus with PICS results\n        \"\"\"\n        # Register UDF by defining the structure of the output locus array of structs\n        # it also renames tagVariantId to variantId\n\n        picsed_ldset_schema = t.ArrayType(\n            t.StructType(\n                [\n                    t.StructField(\"tagVariantId\", t.StringType(), True),\n                    t.StructField(\"r2Overall\", t.DoubleType(), True),\n                    t.StructField(\"posteriorProbability\", t.DoubleType(), True),\n                    t.StructField(\"standardError\", t.DoubleType(), True),\n                ]\n            )\n        )\n        picsed_study_locus_schema = t.ArrayType(\n            t.StructType(\n                [\n                    t.StructField(\"variantId\", t.StringType(), True),\n                    t.StructField(\"r2Overall\", t.DoubleType(), True),\n                    t.StructField(\"posteriorProbability\", t.DoubleType(), True),\n                    t.StructField(\"standardError\", t.DoubleType(), True),\n                ]\n            )\n        )\n        _finemap_udf = f.udf(\n            lambda locus, neglog_p: PICS._finemap(locus, neglog_p, k),\n            picsed_ldset_schema,\n        )\n        non_picsable_expr = (\n            f.size(f.filter(f.col(\"ldSet\"), lambda x: x.r2Overall &gt;= 0.5)) == 0\n        )\n        return StudyLocus(\n            _df=(\n                associations.df\n                # Old locus column will be dropped if available\n                .select(*[col for col in associations.df.columns if col != \"locus\"])\n                # Estimate neglog_pvalue for the lead variant\n                .withColumn(\"neglog_pvalue\", associations.neglog_pvalue())\n                # New locus containing the PICS results\n                .withColumn(\n                    \"locus\",\n                    f.when(\n                        f.col(\"ldSet\").isNotNull(),\n                        _finemap_udf(f.col(\"ldSet\"), f.col(\"neglog_pvalue\")).cast(\n                            picsed_study_locus_schema\n                        ),\n                    ),\n                )\n                .withColumn(\n                    \"qualityControls\",\n                    StudyLocus.update_quality_flag(\n                        f.col(\"qualityControls\"),\n                        non_picsable_expr,\n                        StudyLocusQualityCheck.NOT_QUALIFYING_LD_BLOCK,\n                    ),\n                )\n                .withColumn(\n                    \"finemappingMethod\",\n                    f.coalesce(f.col(\"finemappingMethod\"), f.lit(\"pics\")),\n                )\n                .drop(\"neglog_pvalue\")\n            ),\n            _schema=StudyLocus.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/methods/pics/#gentropy.method.pics.PICS.finemap","title":"<code>finemap(associations: StudyLocus, k: float = 6.4) -&gt; StudyLocus</code>  <code>classmethod</code>","text":"<p>Run PICS on a study locus.</p> <p>Study locus needs to be LD annotated</p> <p>The study locus needs to be LD annotated before PICS can be calculated.</p> <p>Parameters:</p> Name Type Description Default <code>associations</code> <code>StudyLocus</code> <p>Study locus to finemap using PICS</p> required <code>k</code> <code>float</code> <p>Empiric constant that can be adjusted to fit the curve, 6.4 recommended.</p> <code>6.4</code> <p>Returns:</p> Name Type Description <code>StudyLocus</code> <code>StudyLocus</code> <p>Study locus with PICS results</p> Source code in <code>src/gentropy/method/pics.py</code> <pre><code>@classmethod\ndef finemap(\n    cls: type[PICS], associations: StudyLocus, k: float = 6.4\n) -&gt; StudyLocus:\n    \"\"\"Run PICS on a study locus.\n\n    !!! info \"Study locus needs to be LD annotated\"\n\n        The study locus needs to be LD annotated before PICS can be calculated.\n\n    Args:\n        associations (StudyLocus): Study locus to finemap using PICS\n        k (float): Empiric constant that can be adjusted to fit the curve, 6.4 recommended.\n\n    Returns:\n        StudyLocus: Study locus with PICS results\n    \"\"\"\n    # Register UDF by defining the structure of the output locus array of structs\n    # it also renames tagVariantId to variantId\n\n    picsed_ldset_schema = t.ArrayType(\n        t.StructType(\n            [\n                t.StructField(\"tagVariantId\", t.StringType(), True),\n                t.StructField(\"r2Overall\", t.DoubleType(), True),\n                t.StructField(\"posteriorProbability\", t.DoubleType(), True),\n                t.StructField(\"standardError\", t.DoubleType(), True),\n            ]\n        )\n    )\n    picsed_study_locus_schema = t.ArrayType(\n        t.StructType(\n            [\n                t.StructField(\"variantId\", t.StringType(), True),\n                t.StructField(\"r2Overall\", t.DoubleType(), True),\n                t.StructField(\"posteriorProbability\", t.DoubleType(), True),\n                t.StructField(\"standardError\", t.DoubleType(), True),\n            ]\n        )\n    )\n    _finemap_udf = f.udf(\n        lambda locus, neglog_p: PICS._finemap(locus, neglog_p, k),\n        picsed_ldset_schema,\n    )\n    non_picsable_expr = (\n        f.size(f.filter(f.col(\"ldSet\"), lambda x: x.r2Overall &gt;= 0.5)) == 0\n    )\n    return StudyLocus(\n        _df=(\n            associations.df\n            # Old locus column will be dropped if available\n            .select(*[col for col in associations.df.columns if col != \"locus\"])\n            # Estimate neglog_pvalue for the lead variant\n            .withColumn(\"neglog_pvalue\", associations.neglog_pvalue())\n            # New locus containing the PICS results\n            .withColumn(\n                \"locus\",\n                f.when(\n                    f.col(\"ldSet\").isNotNull(),\n                    _finemap_udf(f.col(\"ldSet\"), f.col(\"neglog_pvalue\")).cast(\n                        picsed_study_locus_schema\n                    ),\n                ),\n            )\n            .withColumn(\n                \"qualityControls\",\n                StudyLocus.update_quality_flag(\n                    f.col(\"qualityControls\"),\n                    non_picsable_expr,\n                    StudyLocusQualityCheck.NOT_QUALIFYING_LD_BLOCK,\n                ),\n            )\n            .withColumn(\n                \"finemappingMethod\",\n                f.coalesce(f.col(\"finemappingMethod\"), f.lit(\"pics\")),\n            )\n            .drop(\"neglog_pvalue\")\n        ),\n        _schema=StudyLocus.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/methods/susie_inf/","title":"SuSiE-inf - Fine-mapping with infinitesimal effects v1.1","text":"<p>This is an implementation of the SuSiE-inf method found here: https://github.com/FinucaneLab/fine-mapping-inf https://www.nature.com/articles/s41588-023-01597-3</p> <p>This fine-mapping approach has two approaches for updating estimates of the variance components - Method of Moments and Maximum Likelihood Estimator ('MoM' / 'MLE') The function takes an array of Z-scores and a numpy array matrix of variant LD to perform finemapping.</p>"},{"location":"python_api/methods/susie_inf/#gentropy.method.susie_inf.SUSIE_inf","title":"<code>gentropy.method.susie_inf.SUSIE_inf</code>  <code>dataclass</code>","text":"<p>SuSiE fine-mapping of a study locus from fine-mapping-inf package.</p> <p>Note: code copied from fine-mapping-inf package as a placeholder https://github.com/FinucaneLab/fine-mapping-inf</p> Source code in <code>src/gentropy/method/susie_inf.py</code> <pre><code>@dataclass\nclass SUSIE_inf:\n    \"\"\"SuSiE fine-mapping of a study locus from fine-mapping-inf package.\n\n    Note: code copied from fine-mapping-inf package as a placeholder\n    https://github.com/FinucaneLab/fine-mapping-inf\n    \"\"\"\n\n    @staticmethod\n    def susie_inf(  # noqa: C901\n        z: np.ndarray,\n        meansq: float = 1,\n        n: int = 100000,\n        L: int = 10,\n        LD: np.ndarray | None = None,\n        V: np.ndarray | None = None,\n        Dsq: np.ndarray | None = None,\n        est_ssq: bool = True,\n        ssq: np.ndarray | None = None,\n        ssq_range: tuple[float, float] = (0, 1),\n        pi0: np.ndarray | None = None,\n        est_sigmasq: bool = True,\n        est_tausq: bool = True,\n        sigmasq: float = 1,\n        tausq: float = 0,\n        sigmasq_range: tuple[float, float] | None = None,\n        tausq_range: tuple[float, float] | None = None,\n        PIP: np.ndarray | None = None,\n        mu: np.ndarray | None = None,\n        method: str = \"moments\",\n        maxiter: int = 100,\n        PIP_tol: float = 0.001,\n    ) -&gt; dict[str, Any]:\n        \"\"\"Susie with random effects.\n\n        Args:\n            z (np.ndarray): vector of z-scores (equal to X'y/sqrt(n))\n            meansq (float): average squared magnitude of y (equal to ||y||^2/n)\n            n (int): sample size\n            L (int): number of modeled causal effects\n            LD (np.ndarray | None): LD matrix (equal to X'X/n)\n            V (np.ndarray | None): precomputed p x p matrix of eigenvectors of X'X\n            Dsq (np.ndarray | None): precomputed length-p vector of eigenvalues of X'X\n            est_ssq (bool): estimate prior effect size variances s^2 using MLE\n            ssq (np.ndarray | None): length-L initialization s^2 for each effect\n            ssq_range (tuple[float, float]): lower and upper bounds for each s^2, if estimated\n            pi0 (np.ndarray | None): length-p vector of prior causal probability for each SNP; must sum to 1\n            est_sigmasq (bool): estimate variance sigma^2\n            est_tausq (bool): estimate both variances sigma^2 and tau^2\n            sigmasq (float): initial value for sigma^2\n            tausq (float): initial value for tau^2\n            sigmasq_range (tuple[float, float] | None): lower and upper bounds for sigma^2, if estimated using MLE\n            tausq_range (tuple[float, float] | None): lower and upper bounds for tau^2, if estimated using MLE\n            PIP (np.ndarray | None): p x L initializations of PIPs\n            mu (np.ndarray | None): p x L initializations of mu\n            method (str): one of {'moments','MLE'}\n            maxiter (int): maximum number of SuSiE iterations\n            PIP_tol (float): convergence threshold for PIP difference between iterations\n\n        Returns:\n            dict[str, Any]: Dictionary with keys:\n                PIP -- p x L matrix of PIPs, individually for each effect\n                mu -- p x L matrix of posterior means conditional on causal\n                omega -- p x L matrix of posterior precisions conditional on causal\n                lbf_variable -- p x L matrix of log-Bayes-factors, for each effect\n                ssq -- length-L array of final effect size variances s^2\n                sigmasq -- final value of sigma^2\n                tausq -- final value of tau^2\n                alpha -- length-p array of posterior means of infinitesimal effects\n                lbf -- length-p array of log-Bayes-factors for each CS\n\n        Raises:\n            RuntimeError: if missing LD\n            RuntimeError: if unsupported variance estimation method\n        \"\"\"\n        p = len(z)\n        # Precompute V,D^2 in the SVD X=UDV', and V'X'y and y'y\n        if (V is None or Dsq is None) and LD is None:\n            raise RuntimeError(\"Missing LD\")\n        elif V is None or Dsq is None:\n            eigvals, V = scipy.linalg.eigh(LD)\n            Dsq = np.maximum(n * eigvals, 0)\n        else:\n            Dsq = np.maximum(Dsq, 0)\n        Xty = np.sqrt(n) * z\n        VtXty = V.T.dot(Xty)\n        yty = n * meansq\n        # Initialize diagonal variances, diag(X' Omega X), X' Omega y\n        var = tausq * Dsq + sigmasq\n        diagXtOmegaX = np.sum(V**2 * (Dsq / var), axis=1)\n        XtOmegay = V.dot(VtXty / var)\n        # Initialize s_l^2, PIP_j, mu_j, omega_j\n        if ssq is None:\n            ssq = np.ones(L) * 0.2\n        if PIP is None:\n            PIP = np.ones((p, L)) / p\n        if mu is None:\n            mu = np.zeros((p, L))\n        lbf_variable = np.zeros((p, L))\n        omega = diagXtOmegaX[:, np.newaxis] + 1 / ssq\n        # Initialize prior causal probabilities\n        if pi0 is None:\n            logpi0 = np.ones(p) * np.log(1.0 / p)\n        else:\n            logpi0 = -np.ones(p) * np.inf\n            inds = np.nonzero(pi0 &gt; 0)[0]\n            logpi0[inds] = np.log(pi0[inds])\n\n        ####### Main SuSiE iteration loop ######\n        def f(x: float) -&gt; float:\n            \"\"\"Negative ELBO as function of x = sigma_e^2.\n\n            Args:\n                x (float): sigma_e^2\n\n            Returns:\n                float: negative ELBO as function of x = sigma_e^2\n            \"\"\"\n            return -scipy.special.logsumexp(\n                -0.5 * np.log(1 + x * diagXtOmegaX)\n                + x * XtOmegar**2 / (2 * (1 + x * diagXtOmegaX))\n                + logpi0\n            )\n\n        for it in range(maxiter):\n            PIP_prev = PIP.copy()\n            # Single effect regression for each effect l = 1,...,L\n            for _l in range(L):\n                # Compute X' Omega r_l for residual r_l\n                b = np.sum(mu * PIP, axis=1) - mu[:, _l] * PIP[:, _l]\n                XtOmegaXb = V.dot(V.T.dot(b) * Dsq / var)\n                XtOmegar = XtOmegay - XtOmegaXb\n                if est_ssq:\n                    # Update prior variance ssq[l]\n                    res = minimize_scalar(f, bounds=ssq_range, method=\"bounded\")\n                    if res.success:\n                        ssq[_l] = res.x\n                # Update omega, mu, and PIP\n                omega[:, _l] = diagXtOmegaX + 1 / ssq[_l]\n                mu[:, _l] = XtOmegar / omega[:, _l]\n                lbf_variable[:, _l] = XtOmegar**2 / (2 * omega[:, _l]) - 0.5 * np.log(\n                    omega[:, _l] * ssq[_l]\n                )\n                logPIP = lbf_variable[:, _l] + logpi0\n                PIP[:, _l] = np.exp(logPIP - scipy.special.logsumexp(logPIP))\n            # Update variance components\n            if est_sigmasq or est_tausq:\n                if method == \"moments\":\n                    (sigmasq, tausq) = SUSIE_inf._MoM(\n                        PIP,\n                        mu,\n                        omega,\n                        sigmasq,\n                        tausq,\n                        n,\n                        V,\n                        Dsq,\n                        VtXty,\n                        Xty,\n                        yty,\n                        est_sigmasq,\n                        est_tausq,\n                    )\n                elif method == \"MLE\":\n                    (sigmasq, tausq) = SUSIE_inf._MLE(\n                        PIP,\n                        mu,\n                        omega,\n                        sigmasq,\n                        tausq,\n                        n,\n                        V,\n                        Dsq,\n                        VtXty,\n                        yty,\n                        est_sigmasq,\n                        est_tausq,\n                        it,\n                        sigmasq_range,\n                        tausq_range,\n                    )\n                else:\n                    raise RuntimeError(\"Unsupported variance estimation method\")\n                # Update X' Omega X, X' Omega y\n                var = tausq * Dsq + sigmasq\n                diagXtOmegaX = np.sum(V**2 * (Dsq / var), axis=1)\n                XtOmegay = V.dot(VtXty / var)\n            # Determine convergence from PIP differences\n            PIP_diff = np.max(np.abs(PIP_prev - PIP))\n            if PIP_diff &lt; PIP_tol:\n                break\n        # Compute posterior means of b and alpha\n        b = np.sum(mu * PIP, axis=1)\n        XtOmegaXb = V.dot(V.T.dot(b) * Dsq / var)\n        XtOmegar = XtOmegay - XtOmegaXb\n        alpha = tausq * XtOmegar\n\n        priors = np.log(np.repeat(1 / p, p))\n        lbf_cs = np.apply_along_axis(\n            lambda x: logsumexp(x + priors), axis=0, arr=lbf_variable\n        )\n        return {\n            \"PIP\": PIP,\n            \"mu\": mu,\n            \"omega\": omega,\n            \"lbf_variable\": lbf_variable,\n            \"ssq\": ssq,\n            \"sigmasq\": sigmasq,\n            \"tausq\": tausq,\n            \"alpha\": alpha,\n            \"lbf\": lbf_cs,\n        }\n\n    @staticmethod\n    def _MoM(\n        PIP: np.ndarray,\n        mu: np.ndarray,\n        omega: np.ndarray,\n        sigmasq: float,\n        tausq: float,\n        n: int,\n        V: np.ndarray,\n        Dsq: np.ndarray,\n        VtXty: np.ndarray,\n        Xty: np.ndarray,\n        yty: float,\n        est_sigmasq: bool,\n        est_tausq: bool,\n    ) -&gt; tuple[float, float]:\n        \"\"\"Subroutine to estimate sigma^2, tau^2 using method-of-moments.\n\n        Args:\n            PIP (np.ndarray): p x L matrix of PIPs\n            mu (np.ndarray): p x L matrix of posterior means conditional on causal\n            omega (np.ndarray): p x L matrix of posterior precisions conditional on causal\n            sigmasq (float): initial value for sigma^2\n            tausq (float): initial value for tau^2\n            n (int): sample size\n            V (np.ndarray): precomputed p x p matrix of eigenvectors of X'X\n            Dsq (np.ndarray): precomputed length-p vector of eigenvalues of X'X\n            VtXty (np.ndarray): precomputed length-p vector V'X'y\n            Xty (np.ndarray): precomputed length-p vector X'y\n            yty (float): precomputed y'y\n            est_sigmasq (bool): estimate variance sigma^2\n            est_tausq (bool): estimate both variances sigma^2 and tau^2\n\n        Returns:\n            tuple[float, float]: (sigmasq,tausq) tuple of updated variances\n        \"\"\"\n        (p, L) = mu.shape\n        # Compute A\n        A = np.array([[n, sum(Dsq)], [0, sum(Dsq**2)]])\n        A[1, 0] = A[0, 1]\n        # Compute diag(V'MV)\n        b = np.sum(mu * PIP, axis=1)\n        Vtb = V.T.dot(b)\n        diagVtMV = Vtb**2\n        tmpD = np.zeros(p)\n        for _l in range(L):\n            bl = mu[:, _l] * PIP[:, _l]\n            Vtbl = V.T.dot(bl)\n            diagVtMV -= Vtbl**2\n            tmpD += PIP[:, _l] * (mu[:, _l] ** 2 + 1 / omega[:, _l])\n        diagVtMV += np.sum((V.T) ** 2 * tmpD, axis=1)\n        # Compute x\n        x = np.zeros(2)\n        x[0] = yty - 2 * sum(b * Xty) + sum(Dsq * diagVtMV)\n        x[1] = sum(Xty**2) - 2 * sum(Vtb * VtXty * Dsq) + sum(Dsq**2 * diagVtMV)\n        if est_tausq:\n            sol = scipy.linalg.solve(A, x)\n            if sol[0] &gt; 0 and sol[1] &gt; 0:\n                (sigmasq, tausq) = sol\n            else:\n                (sigmasq, tausq) = (x[0] / n, 0)\n        elif est_sigmasq:\n            sigmasq = (x[0] - A[0, 1] * tausq) / n\n        return sigmasq, tausq\n\n    @staticmethod\n    def _MLE(\n        PIP: np.ndarray,\n        mu: np.ndarray,\n        omega: np.ndarray,\n        sigmasq: float,\n        tausq: float,\n        n: int,\n        V: np.ndarray,\n        Dsq: np.ndarray,\n        VtXty: np.ndarray,\n        yty: float,\n        est_sigmasq: bool,\n        est_tausq: bool,\n        it: int,\n        sigmasq_range: tuple[float, float] | None = None,\n        tausq_range: tuple[float, float] | None = None,\n    ) -&gt; tuple[float, float]:\n        \"\"\"Subroutine to estimate sigma^2, tau^2 using MLE.\n\n        Args:\n            PIP (np.ndarray): p x L matrix of PIPs\n            mu (np.ndarray): p x L matrix of posterior means conditional on causal\n            omega (np.ndarray): p x L matrix of posterior precisions conditional on causal\n            sigmasq (float): initial value for sigma^2\n            tausq (float): initial value for tau^2\n            n (int): sample size\n            V (np.ndarray): precomputed p x p matrix of eigenvectors of X'X\n            Dsq (np.ndarray): precomputed length-p vector of eigenvalues of X'X\n            VtXty (np.ndarray): precomputed length-p vector V'X'y\n            yty (float): precomputed y'y\n            est_sigmasq (bool): estimate variance sigma^2\n            est_tausq (bool): estimate both variances sigma^2 and tau^2\n            it (int): iteration number\n            sigmasq_range (tuple[float, float] | None): lower and upper bounds for sigma^2, if estimated using MLE\n            tausq_range (tuple[float, float] | None): lower and upper bounds for tau^2, if estimated using MLE\n\n        Returns:\n            tuple[float, float]: (sigmasq,tausq) tuple of updated variances\n        \"\"\"\n        (p, L) = mu.shape\n        if sigmasq_range is None:\n            sigmasq_range = (0.2 * yty / n, 1.2 * yty / n)\n        if tausq_range is None:\n            tausq_range = (1e-12, 1.2 * yty / (n * p))\n        # Compute diag(V'MV)\n        b = np.sum(mu * PIP, axis=1)\n        Vtb = V.T.dot(b)\n        diagVtMV = Vtb**2\n        tmpD = np.zeros(p)\n        for _l in range(L):\n            bl = mu[:, _l] * PIP[:, _l]\n            Vtbl = V.T.dot(bl)\n            diagVtMV -= Vtbl**2\n            tmpD += PIP[:, _l] * (mu[:, _l] ** 2 + 1 / omega[:, _l])\n        diagVtMV += np.sum((V.T) ** 2 * tmpD, axis=1)\n\n        # negative ELBO as function of x = (sigma_e^2,sigma_g^2)\n        def f(x: tuple[float, float]) -&gt; float:\n            \"\"\"Negative ELBO as function of x = (sigma_e^2,sigma_g^2).\n\n            Args:\n                x (tuple[float, float]): (sigma_e^2,sigma_g^2)\n\n            Returns:\n                float: negative ELBO as function of x = (sigma_e^2,sigma_g^2)\n            \"\"\"\n            return (\n                0.5 * (n - p) * np.log(x[0])\n                + 0.5 / x[0] * yty\n                + np.sum(\n                    0.5 * np.log(x[1] * Dsq + x[0])\n                    - 0.5 * x[1] / x[0] * VtXty**2 / (x[1] * Dsq + x[0])\n                    - Vtb * VtXty / (x[1] * Dsq + x[0])\n                    + 0.5 * Dsq / (x[1] * Dsq + x[0]) * diagVtMV\n                )\n            )\n\n        if est_tausq:\n            res = minimize(\n                f,\n                (sigmasq, tausq),\n                method=\"L-BFGS-B\",\n                bounds=(sigmasq_range, tausq_range),\n            )\n            if res.success:\n                sigmasq, tausq = res.x\n        elif est_sigmasq:\n\n            def g(x: float) -&gt; float:\n                \"\"\"Negative ELBO as function of x = sigma_e^2.\n\n                Args:\n                    x (float): sigma_e^2\n\n                Returns:\n                    float: negative ELBO as function of x = sigma_e^2\n                \"\"\"\n                return f((x, tausq))\n\n            res = minimize(g, sigmasq, method=\"L-BFGS-B\", bounds=(sigmasq_range,))\n            if res.success:\n                sigmasq = res.x\n        return sigmasq, tausq\n\n    @staticmethod\n    def cred_inf(\n        PIP: np.ndarray,\n        n: int = 100_000,\n        coverage: float = 0.9,\n        purity: float = 0.5,\n        LD: np.ndarray | None = None,\n        V: np.ndarray | None = None,\n        Dsq: np.ndarray | None = None,\n        dedup: bool = True,\n    ) -&gt; list[Any]:\n        \"\"\"Compute credible sets from single-effect PIPs.\n\n        Args:\n            PIP (np.ndarray): p x L matrix of PIPs\n            n (int): sample size\n            coverage (float): coverage of credible sets\n            purity (float): purity of credible sets\n            LD (np.ndarray | None): LD matrix (equal to X'X/n)\n            V (np.ndarray | None): precomputed p x p matrix of eigenvectors of X'X\n            Dsq (np.ndarray | None): precomputed length-p vector of eigenvalues of X'X\n            dedup (bool): whether to deduplicate credible sets\n\n        Returns:\n            list[Any]: list of L lists of SNP indices in each credible set\n\n        Raises:\n            RuntimeError: if missing inputs for purity filtering\n        \"\"\"\n        if (V is None or Dsq is None or n is None) and LD is None:\n            raise RuntimeError(\"Missing inputs for purity filtering\")\n        # Compute credible sets\n        cred = []\n        for i in range(PIP.shape[1]):\n            sortinds = np.argsort(PIP[:, i])[::-1]\n            ind = min(np.nonzero(np.cumsum(PIP[sortinds, i]) &gt;= coverage)[0])\n            credset = sortinds[: (ind + 1)]\n            # Filter by purity\n            if len(credset) == 1:\n                cred.append(list(credset))\n                continue\n            if len(credset) &lt; 100:\n                rows = credset\n            else:\n                np.random.seed(123)\n                rows = np.random.choice(credset, size=100, replace=False)\n            if LD is not None:\n                LDloc = LD[np.ix_(rows, rows)]\n            elif V is not None and Dsq is not None:\n                LDloc = (V[rows, :] * Dsq).dot(V[rows, :].T) / n\n            else:\n                raise ValueError(\"Both LD and V, Dsq cannot be None\")\n            if np.min(np.abs(LDloc)) &gt; purity:\n                cred.append(sorted(credset))\n        if dedup:\n            cred = list(\n                map(\n                    list,\n                    sorted(set(map(tuple, cred)), key=list(map(tuple, cred)).index),\n                )\n            )\n        return cred\n</code></pre>"},{"location":"python_api/methods/susie_inf/#gentropy.method.susie_inf.SUSIE_inf.cred_inf","title":"<code>cred_inf(PIP: np.ndarray, n: int = 100000, coverage: float = 0.9, purity: float = 0.5, LD: np.ndarray | None = None, V: np.ndarray | None = None, Dsq: np.ndarray | None = None, dedup: bool = True) -&gt; list[Any]</code>  <code>staticmethod</code>","text":"<p>Compute credible sets from single-effect PIPs.</p> <p>Parameters:</p> Name Type Description Default <code>PIP</code> <code>ndarray</code> <p>p x L matrix of PIPs</p> required <code>n</code> <code>int</code> <p>sample size</p> <code>100000</code> <code>coverage</code> <code>float</code> <p>coverage of credible sets</p> <code>0.9</code> <code>purity</code> <code>float</code> <p>purity of credible sets</p> <code>0.5</code> <code>LD</code> <code>ndarray | None</code> <p>LD matrix (equal to X'X/n)</p> <code>None</code> <code>V</code> <code>ndarray | None</code> <p>precomputed p x p matrix of eigenvectors of X'X</p> <code>None</code> <code>Dsq</code> <code>ndarray | None</code> <p>precomputed length-p vector of eigenvalues of X'X</p> <code>None</code> <code>dedup</code> <code>bool</code> <p>whether to deduplicate credible sets</p> <code>True</code> <p>Returns:</p> Type Description <code>list[Any]</code> <p>list[Any]: list of L lists of SNP indices in each credible set</p> <p>Raises:</p> Type Description <code>RuntimeError</code> <p>if missing inputs for purity filtering</p> Source code in <code>src/gentropy/method/susie_inf.py</code> <pre><code>@staticmethod\ndef cred_inf(\n    PIP: np.ndarray,\n    n: int = 100_000,\n    coverage: float = 0.9,\n    purity: float = 0.5,\n    LD: np.ndarray | None = None,\n    V: np.ndarray | None = None,\n    Dsq: np.ndarray | None = None,\n    dedup: bool = True,\n) -&gt; list[Any]:\n    \"\"\"Compute credible sets from single-effect PIPs.\n\n    Args:\n        PIP (np.ndarray): p x L matrix of PIPs\n        n (int): sample size\n        coverage (float): coverage of credible sets\n        purity (float): purity of credible sets\n        LD (np.ndarray | None): LD matrix (equal to X'X/n)\n        V (np.ndarray | None): precomputed p x p matrix of eigenvectors of X'X\n        Dsq (np.ndarray | None): precomputed length-p vector of eigenvalues of X'X\n        dedup (bool): whether to deduplicate credible sets\n\n    Returns:\n        list[Any]: list of L lists of SNP indices in each credible set\n\n    Raises:\n        RuntimeError: if missing inputs for purity filtering\n    \"\"\"\n    if (V is None or Dsq is None or n is None) and LD is None:\n        raise RuntimeError(\"Missing inputs for purity filtering\")\n    # Compute credible sets\n    cred = []\n    for i in range(PIP.shape[1]):\n        sortinds = np.argsort(PIP[:, i])[::-1]\n        ind = min(np.nonzero(np.cumsum(PIP[sortinds, i]) &gt;= coverage)[0])\n        credset = sortinds[: (ind + 1)]\n        # Filter by purity\n        if len(credset) == 1:\n            cred.append(list(credset))\n            continue\n        if len(credset) &lt; 100:\n            rows = credset\n        else:\n            np.random.seed(123)\n            rows = np.random.choice(credset, size=100, replace=False)\n        if LD is not None:\n            LDloc = LD[np.ix_(rows, rows)]\n        elif V is not None and Dsq is not None:\n            LDloc = (V[rows, :] * Dsq).dot(V[rows, :].T) / n\n        else:\n            raise ValueError(\"Both LD and V, Dsq cannot be None\")\n        if np.min(np.abs(LDloc)) &gt; purity:\n            cred.append(sorted(credset))\n    if dedup:\n        cred = list(\n            map(\n                list,\n                sorted(set(map(tuple, cred)), key=list(map(tuple, cred)).index),\n            )\n        )\n    return cred\n</code></pre>"},{"location":"python_api/methods/susie_inf/#gentropy.method.susie_inf.SUSIE_inf.susie_inf","title":"<code>susie_inf(z: np.ndarray, meansq: float = 1, n: int = 100000, L: int = 10, LD: np.ndarray | None = None, V: np.ndarray | None = None, Dsq: np.ndarray | None = None, est_ssq: bool = True, ssq: np.ndarray | None = None, ssq_range: tuple[float, float] = (0, 1), pi0: np.ndarray | None = None, est_sigmasq: bool = True, est_tausq: bool = True, sigmasq: float = 1, tausq: float = 0, sigmasq_range: tuple[float, float] | None = None, tausq_range: tuple[float, float] | None = None, PIP: np.ndarray | None = None, mu: np.ndarray | None = None, method: str = 'moments', maxiter: int = 100, PIP_tol: float = 0.001) -&gt; dict[str, Any]</code>  <code>staticmethod</code>","text":"<p>Susie with random effects.</p> <p>Parameters:</p> Name Type Description Default <code>z</code> <code>ndarray</code> <p>vector of z-scores (equal to X'y/sqrt(n))</p> required <code>meansq</code> <code>float</code> <p>average squared magnitude of y (equal to ||y||^2/n)</p> <code>1</code> <code>n</code> <code>int</code> <p>sample size</p> <code>100000</code> <code>L</code> <code>int</code> <p>number of modeled causal effects</p> <code>10</code> <code>LD</code> <code>ndarray | None</code> <p>LD matrix (equal to X'X/n)</p> <code>None</code> <code>V</code> <code>ndarray | None</code> <p>precomputed p x p matrix of eigenvectors of X'X</p> <code>None</code> <code>Dsq</code> <code>ndarray | None</code> <p>precomputed length-p vector of eigenvalues of X'X</p> <code>None</code> <code>est_ssq</code> <code>bool</code> <p>estimate prior effect size variances s^2 using MLE</p> <code>True</code> <code>ssq</code> <code>ndarray | None</code> <p>length-L initialization s^2 for each effect</p> <code>None</code> <code>ssq_range</code> <code>tuple[float, float]</code> <p>lower and upper bounds for each s^2, if estimated</p> <code>(0, 1)</code> <code>pi0</code> <code>ndarray | None</code> <p>length-p vector of prior causal probability for each SNP; must sum to 1</p> <code>None</code> <code>est_sigmasq</code> <code>bool</code> <p>estimate variance sigma^2</p> <code>True</code> <code>est_tausq</code> <code>bool</code> <p>estimate both variances sigma^2 and tau^2</p> <code>True</code> <code>sigmasq</code> <code>float</code> <p>initial value for sigma^2</p> <code>1</code> <code>tausq</code> <code>float</code> <p>initial value for tau^2</p> <code>0</code> <code>sigmasq_range</code> <code>tuple[float, float] | None</code> <p>lower and upper bounds for sigma^2, if estimated using MLE</p> <code>None</code> <code>tausq_range</code> <code>tuple[float, float] | None</code> <p>lower and upper bounds for tau^2, if estimated using MLE</p> <code>None</code> <code>PIP</code> <code>ndarray | None</code> <p>p x L initializations of PIPs</p> <code>None</code> <code>mu</code> <code>ndarray | None</code> <p>p x L initializations of mu</p> <code>None</code> <code>method</code> <code>str</code> <p>one of {'moments','MLE'}</p> <code>'moments'</code> <code>maxiter</code> <code>int</code> <p>maximum number of SuSiE iterations</p> <code>100</code> <code>PIP_tol</code> <code>float</code> <p>convergence threshold for PIP difference between iterations</p> <code>0.001</code> <p>Returns:</p> Type Description <code>dict[str, Any]</code> <p>dict[str, Any]: Dictionary with keys: PIP -- p x L matrix of PIPs, individually for each effect mu -- p x L matrix of posterior means conditional on causal omega -- p x L matrix of posterior precisions conditional on causal lbf_variable -- p x L matrix of log-Bayes-factors, for each effect ssq -- length-L array of final effect size variances s^2 sigmasq -- final value of sigma^2 tausq -- final value of tau^2 alpha -- length-p array of posterior means of infinitesimal effects lbf -- length-p array of log-Bayes-factors for each CS</p> <p>Raises:</p> Type Description <code>RuntimeError</code> <p>if missing LD</p> <code>RuntimeError</code> <p>if unsupported variance estimation method</p> Source code in <code>src/gentropy/method/susie_inf.py</code> <pre><code>@staticmethod\ndef susie_inf(  # noqa: C901\n    z: np.ndarray,\n    meansq: float = 1,\n    n: int = 100000,\n    L: int = 10,\n    LD: np.ndarray | None = None,\n    V: np.ndarray | None = None,\n    Dsq: np.ndarray | None = None,\n    est_ssq: bool = True,\n    ssq: np.ndarray | None = None,\n    ssq_range: tuple[float, float] = (0, 1),\n    pi0: np.ndarray | None = None,\n    est_sigmasq: bool = True,\n    est_tausq: bool = True,\n    sigmasq: float = 1,\n    tausq: float = 0,\n    sigmasq_range: tuple[float, float] | None = None,\n    tausq_range: tuple[float, float] | None = None,\n    PIP: np.ndarray | None = None,\n    mu: np.ndarray | None = None,\n    method: str = \"moments\",\n    maxiter: int = 100,\n    PIP_tol: float = 0.001,\n) -&gt; dict[str, Any]:\n    \"\"\"Susie with random effects.\n\n    Args:\n        z (np.ndarray): vector of z-scores (equal to X'y/sqrt(n))\n        meansq (float): average squared magnitude of y (equal to ||y||^2/n)\n        n (int): sample size\n        L (int): number of modeled causal effects\n        LD (np.ndarray | None): LD matrix (equal to X'X/n)\n        V (np.ndarray | None): precomputed p x p matrix of eigenvectors of X'X\n        Dsq (np.ndarray | None): precomputed length-p vector of eigenvalues of X'X\n        est_ssq (bool): estimate prior effect size variances s^2 using MLE\n        ssq (np.ndarray | None): length-L initialization s^2 for each effect\n        ssq_range (tuple[float, float]): lower and upper bounds for each s^2, if estimated\n        pi0 (np.ndarray | None): length-p vector of prior causal probability for each SNP; must sum to 1\n        est_sigmasq (bool): estimate variance sigma^2\n        est_tausq (bool): estimate both variances sigma^2 and tau^2\n        sigmasq (float): initial value for sigma^2\n        tausq (float): initial value for tau^2\n        sigmasq_range (tuple[float, float] | None): lower and upper bounds for sigma^2, if estimated using MLE\n        tausq_range (tuple[float, float] | None): lower and upper bounds for tau^2, if estimated using MLE\n        PIP (np.ndarray | None): p x L initializations of PIPs\n        mu (np.ndarray | None): p x L initializations of mu\n        method (str): one of {'moments','MLE'}\n        maxiter (int): maximum number of SuSiE iterations\n        PIP_tol (float): convergence threshold for PIP difference between iterations\n\n    Returns:\n        dict[str, Any]: Dictionary with keys:\n            PIP -- p x L matrix of PIPs, individually for each effect\n            mu -- p x L matrix of posterior means conditional on causal\n            omega -- p x L matrix of posterior precisions conditional on causal\n            lbf_variable -- p x L matrix of log-Bayes-factors, for each effect\n            ssq -- length-L array of final effect size variances s^2\n            sigmasq -- final value of sigma^2\n            tausq -- final value of tau^2\n            alpha -- length-p array of posterior means of infinitesimal effects\n            lbf -- length-p array of log-Bayes-factors for each CS\n\n    Raises:\n        RuntimeError: if missing LD\n        RuntimeError: if unsupported variance estimation method\n    \"\"\"\n    p = len(z)\n    # Precompute V,D^2 in the SVD X=UDV', and V'X'y and y'y\n    if (V is None or Dsq is None) and LD is None:\n        raise RuntimeError(\"Missing LD\")\n    elif V is None or Dsq is None:\n        eigvals, V = scipy.linalg.eigh(LD)\n        Dsq = np.maximum(n * eigvals, 0)\n    else:\n        Dsq = np.maximum(Dsq, 0)\n    Xty = np.sqrt(n) * z\n    VtXty = V.T.dot(Xty)\n    yty = n * meansq\n    # Initialize diagonal variances, diag(X' Omega X), X' Omega y\n    var = tausq * Dsq + sigmasq\n    diagXtOmegaX = np.sum(V**2 * (Dsq / var), axis=1)\n    XtOmegay = V.dot(VtXty / var)\n    # Initialize s_l^2, PIP_j, mu_j, omega_j\n    if ssq is None:\n        ssq = np.ones(L) * 0.2\n    if PIP is None:\n        PIP = np.ones((p, L)) / p\n    if mu is None:\n        mu = np.zeros((p, L))\n    lbf_variable = np.zeros((p, L))\n    omega = diagXtOmegaX[:, np.newaxis] + 1 / ssq\n    # Initialize prior causal probabilities\n    if pi0 is None:\n        logpi0 = np.ones(p) * np.log(1.0 / p)\n    else:\n        logpi0 = -np.ones(p) * np.inf\n        inds = np.nonzero(pi0 &gt; 0)[0]\n        logpi0[inds] = np.log(pi0[inds])\n\n    ####### Main SuSiE iteration loop ######\n    def f(x: float) -&gt; float:\n        \"\"\"Negative ELBO as function of x = sigma_e^2.\n\n        Args:\n            x (float): sigma_e^2\n\n        Returns:\n            float: negative ELBO as function of x = sigma_e^2\n        \"\"\"\n        return -scipy.special.logsumexp(\n            -0.5 * np.log(1 + x * diagXtOmegaX)\n            + x * XtOmegar**2 / (2 * (1 + x * diagXtOmegaX))\n            + logpi0\n        )\n\n    for it in range(maxiter):\n        PIP_prev = PIP.copy()\n        # Single effect regression for each effect l = 1,...,L\n        for _l in range(L):\n            # Compute X' Omega r_l for residual r_l\n            b = np.sum(mu * PIP, axis=1) - mu[:, _l] * PIP[:, _l]\n            XtOmegaXb = V.dot(V.T.dot(b) * Dsq / var)\n            XtOmegar = XtOmegay - XtOmegaXb\n            if est_ssq:\n                # Update prior variance ssq[l]\n                res = minimize_scalar(f, bounds=ssq_range, method=\"bounded\")\n                if res.success:\n                    ssq[_l] = res.x\n            # Update omega, mu, and PIP\n            omega[:, _l] = diagXtOmegaX + 1 / ssq[_l]\n            mu[:, _l] = XtOmegar / omega[:, _l]\n            lbf_variable[:, _l] = XtOmegar**2 / (2 * omega[:, _l]) - 0.5 * np.log(\n                omega[:, _l] * ssq[_l]\n            )\n            logPIP = lbf_variable[:, _l] + logpi0\n            PIP[:, _l] = np.exp(logPIP - scipy.special.logsumexp(logPIP))\n        # Update variance components\n        if est_sigmasq or est_tausq:\n            if method == \"moments\":\n                (sigmasq, tausq) = SUSIE_inf._MoM(\n                    PIP,\n                    mu,\n                    omega,\n                    sigmasq,\n                    tausq,\n                    n,\n                    V,\n                    Dsq,\n                    VtXty,\n                    Xty,\n                    yty,\n                    est_sigmasq,\n                    est_tausq,\n                )\n            elif method == \"MLE\":\n                (sigmasq, tausq) = SUSIE_inf._MLE(\n                    PIP,\n                    mu,\n                    omega,\n                    sigmasq,\n                    tausq,\n                    n,\n                    V,\n                    Dsq,\n                    VtXty,\n                    yty,\n                    est_sigmasq,\n                    est_tausq,\n                    it,\n                    sigmasq_range,\n                    tausq_range,\n                )\n            else:\n                raise RuntimeError(\"Unsupported variance estimation method\")\n            # Update X' Omega X, X' Omega y\n            var = tausq * Dsq + sigmasq\n            diagXtOmegaX = np.sum(V**2 * (Dsq / var), axis=1)\n            XtOmegay = V.dot(VtXty / var)\n        # Determine convergence from PIP differences\n        PIP_diff = np.max(np.abs(PIP_prev - PIP))\n        if PIP_diff &lt; PIP_tol:\n            break\n    # Compute posterior means of b and alpha\n    b = np.sum(mu * PIP, axis=1)\n    XtOmegaXb = V.dot(V.T.dot(b) * Dsq / var)\n    XtOmegar = XtOmegay - XtOmegaXb\n    alpha = tausq * XtOmegar\n\n    priors = np.log(np.repeat(1 / p, p))\n    lbf_cs = np.apply_along_axis(\n        lambda x: logsumexp(x + priors), axis=0, arr=lbf_variable\n    )\n    return {\n        \"PIP\": PIP,\n        \"mu\": mu,\n        \"omega\": omega,\n        \"lbf_variable\": lbf_variable,\n        \"ssq\": ssq,\n        \"sigmasq\": sigmasq,\n        \"tausq\": tausq,\n        \"alpha\": alpha,\n        \"lbf\": lbf_cs,\n    }\n</code></pre>"},{"location":"python_api/methods/l2g/_l2g/","title":"Locus to Gene (L2G) model","text":"<p>The \u201clocus-to-gene\u201d (L2G) model derives features to prioritize likely causal genes at each GWAS locus based on genetic and functional genomics features. The main categories of predictive features are:</p> <ul> <li>Distance: (from credible set variants to gene)</li> <li>Molecular QTL Colocalization</li> <li>Chromatin Interaction: (e.g., promoter-capture Hi-C)</li> <li>Variant Pathogenicity: (from VEP)</li> </ul> <p>The L2G model is distinct from the variant-to-gene (V2G) pipeline in that it:</p> <ul> <li>Uses a machine-learning model to learn the weights of each evidence source based on a gold standard of previously identified causal genes.</li> <li>Relies upon fine-mapping and colocalization data.</li> </ul> <p>Some of the predictive features weight variant-to-gene (or genomic region-to-gene) evidence based on the posterior probability that the variant is causal, determined through fine-mapping of the GWAS association.</p> <p>Details of the L2G model are provided in our Nature Genetics publication (ref - Nature Genetics Publication):</p> <ul> <li>Title: An open approach to systematically prioritize causal variants and genes at all published human GWAS trait-associated loci.</li> <li>Authors: Mountjoy, E., Schmidt, E.M., Carmona, M. et al.</li> <li>Journal: Nat Genet 53, 1527\u20131533 (2021).</li> <li>DOI: 10.1038/s41588-021-00945-5</li> </ul>"},{"location":"python_api/methods/l2g/evaluator/","title":"W&B evaluator","text":""},{"location":"python_api/methods/l2g/evaluator/#gentropy.method.l2g.evaluator.WandbEvaluator","title":"<code>gentropy.method.l2g.evaluator.WandbEvaluator</code>","text":"<p>             Bases: <code>Evaluator</code></p> <p>Wrapper for pyspark Evaluators. It is expected that the user will provide an Evaluators, and this wrapper will log metrics from said evaluator to W&amp;B.</p> Source code in <code>src/gentropy/method/l2g/evaluator.py</code> <pre><code>class WandbEvaluator(Evaluator):\n    \"\"\"Wrapper for pyspark Evaluators. It is expected that the user will provide an Evaluators, and this wrapper will log metrics from said evaluator to W&amp;B.\"\"\"\n\n    spark_ml_evaluator: Param[Evaluator] = Param(\n        Params._dummy(), \"spark_ml_evaluator\", \"evaluator from pyspark.ml.evaluation\"\n    )\n\n    wandb_run: Param[Run] = Param(\n        Params._dummy(),\n        \"wandb_run\",\n        \"wandb run.  Expects an already initialized run.  You should set this, or wandb_run_kwargs, NOT BOTH\",\n    )\n\n    wandb_run_kwargs: Param[Any] = Param(\n        Params._dummy(),\n        \"wandb_run_kwargs\",\n        \"kwargs to be passed to wandb.init.  You should set this, or wandb_runId, NOT BOTH.  Setting this is useful when using with WandbCrossValdidator\",\n    )\n\n    wandb_runId: Param[str] = Param(  # noqa: N815\n        Params._dummy(),\n        \"wandb_runId\",\n        \"wandb run id.  if not providing an intialized run to wandb_run, a run with id wandb_runId will be resumed\",\n    )\n\n    wandb_project_name: Param[str] = Param(\n        Params._dummy(),\n        \"wandb_project_name\",\n        \"name of W&amp;B project\",\n        typeConverter=TypeConverters.toString,\n    )\n\n    label_values: Param[list[str]] = Param(\n        Params._dummy(),\n        \"label_values\",\n        \"for classification and multiclass classification, this is a list of values the label can assume\\nIf provided Multiclass or Multilabel evaluator without label_values, we'll figure it out from dataset passed through to evaluate.\",\n    )\n\n    _input_kwargs: Dict[str, Any]\n\n    @keyword_only\n    def __init__(\n        self: WandbEvaluator,\n        label_values: list[str] | None = None,\n        **kwargs: BinaryClassificationEvaluator\n        | MulticlassClassificationEvaluator\n        | Run,\n    ) -&gt; None:\n        \"\"\"Initialize a WandbEvaluator.\n\n        Args:\n            label_values (list[str] | None): List of label values.\n            **kwargs (BinaryClassificationEvaluator | MulticlassClassificationEvaluator | Run): Keyword arguments.\n        \"\"\"\n        if label_values is None:\n            label_values = []\n        super(Evaluator, self).__init__()\n\n        self.metrics = {\n            MulticlassClassificationEvaluator: [\n                \"f1\",\n                \"accuracy\",\n                \"weightedPrecision\",\n                \"weightedRecall\",\n                \"weightedTruePositiveRate\",\n                \"weightedFalsePositiveRate\",\n                \"weightedFMeasure\",\n                \"truePositiveRateByLabel\",\n                \"falsePositiveRateByLabel\",\n                \"precisionByLabel\",\n                \"recallByLabel\",\n                \"fMeasureByLabel\",\n                \"logLoss\",\n                \"hammingLoss\",\n            ],\n            BinaryClassificationEvaluator: [\"areaUnderROC\", \"areaUnderPR\"],\n        }\n\n        self._setDefault(label_values=[])\n        kwargs = self._input_kwargs\n        self._set(**kwargs)\n\n    def setspark_ml_evaluator(self: WandbEvaluator, value: Evaluator) -&gt; None:\n        \"\"\"Set the spark_ml_evaluator parameter.\n\n        Args:\n            value (Evaluator): Spark ML evaluator.\n        \"\"\"\n        self._set(spark_ml_evaluator=value)\n\n    def setlabel_values(self: WandbEvaluator, value: list[str]) -&gt; None:\n        \"\"\"Set the label_values parameter.\n\n        Args:\n            value (list[str]): List of label values.\n        \"\"\"\n        self._set(label_values=value)\n\n    def getspark_ml_evaluator(self: WandbEvaluator) -&gt; Evaluator:\n        \"\"\"Get the spark_ml_evaluator parameter.\n\n        Returns:\n            Evaluator: Spark ML evaluator.\n        \"\"\"\n        return self.getOrDefault(self.spark_ml_evaluator)\n\n    def getwandb_run(self: WandbEvaluator) -&gt; Run:\n        \"\"\"Get the wandb_run parameter.\n\n        Returns:\n            Run: Wandb run object.\n        \"\"\"\n        return self.getOrDefault(self.wandb_run)\n\n    def getwandb_project_name(self: WandbEvaluator) -&gt; Any:\n        \"\"\"Get the wandb_project_name parameter.\n\n        Returns:\n            Any: Name of the W&amp;B project.\n        \"\"\"\n        return self.getOrDefault(self.wandb_project_name)\n\n    def getlabel_values(self: WandbEvaluator) -&gt; list[str]:\n        \"\"\"Get the label_values parameter.\n\n        Returns:\n            list[str]: List of label values.\n        \"\"\"\n        return self.getOrDefault(self.label_values)\n\n    def _evaluate(self: WandbEvaluator, dataset: DataFrame) -&gt; float:\n        \"\"\"Evaluate the model on the given dataset.\n\n        Args:\n            dataset (DataFrame): Dataset to evaluate the model on.\n\n        Returns:\n            float: Metric value.\n        \"\"\"\n        dataset.persist()\n        metric_values: list[tuple[str, Any]] = []\n        label_values = self.getlabel_values()\n        spark_ml_evaluator: BinaryClassificationEvaluator | MulticlassClassificationEvaluator = (\n            self.getspark_ml_evaluator()  # type: ignore[assignment, unused-ignore]\n        )\n        run = self.getwandb_run()\n        evaluator_type = type(spark_ml_evaluator)\n        for metric in self.metrics[evaluator_type]:\n            if \"ByLabel\" in metric and label_values == []:\n                print(  # noqa: T201\n                    \"no label_values for the target have been provided and will be determined by the dataset.  This could take some time\"\n                )\n                label_values = [\n                    r[spark_ml_evaluator.getLabelCol()]\n                    for r in dataset.select(spark_ml_evaluator.getLabelCol())\n                    .distinct()\n                    .collect()\n                ]\n                if isinstance(label_values[0], list):\n                    merged = list(itertools.chain(*label_values))\n                    label_values = list(dict.fromkeys(merged).keys())\n                    self.setlabel_values(label_values)\n            for label in label_values:\n                out = spark_ml_evaluator.evaluate(\n                    dataset,\n                    {\n                        spark_ml_evaluator.metricLabel: label,  # type: ignore[assignment, unused-ignore]\n                        spark_ml_evaluator.metricName: metric,\n                    },\n                )\n                metric_values.append((f\"{metric}:{label}\", out))\n            out = spark_ml_evaluator.evaluate(\n                dataset, {spark_ml_evaluator.metricName: metric}\n            )\n            metric_values.append((f\"{metric}\", out))\n        run.log(dict(metric_values))\n        config = [\n            (f\"{k.parent.split('_')[0]}.{k.name}\", v)\n            for k, v in spark_ml_evaluator.extractParamMap().items()\n            if \"metric\" not in k.name\n        ]\n        run.config.update(dict(config))\n        return_metric = spark_ml_evaluator.evaluate(dataset)\n        dataset.unpersist()\n        return return_metric\n</code></pre>"},{"location":"python_api/methods/l2g/evaluator/#gentropy.method.l2g.evaluator.WandbEvaluator.__init__","title":"<code>__init__(label_values: list[str] | None = None, **kwargs: BinaryClassificationEvaluator | MulticlassClassificationEvaluator | Run) -&gt; None</code>","text":"<p>Initialize a WandbEvaluator.</p> <p>Parameters:</p> Name Type Description Default <code>label_values</code> <code>list[str] | None</code> <p>List of label values.</p> <code>None</code> <code>**kwargs</code> <code>BinaryClassificationEvaluator | MulticlassClassificationEvaluator | Run</code> <p>Keyword arguments.</p> <code>{}</code> Source code in <code>src/gentropy/method/l2g/evaluator.py</code> <pre><code>@keyword_only\ndef __init__(\n    self: WandbEvaluator,\n    label_values: list[str] | None = None,\n    **kwargs: BinaryClassificationEvaluator\n    | MulticlassClassificationEvaluator\n    | Run,\n) -&gt; None:\n    \"\"\"Initialize a WandbEvaluator.\n\n    Args:\n        label_values (list[str] | None): List of label values.\n        **kwargs (BinaryClassificationEvaluator | MulticlassClassificationEvaluator | Run): Keyword arguments.\n    \"\"\"\n    if label_values is None:\n        label_values = []\n    super(Evaluator, self).__init__()\n\n    self.metrics = {\n        MulticlassClassificationEvaluator: [\n            \"f1\",\n            \"accuracy\",\n            \"weightedPrecision\",\n            \"weightedRecall\",\n            \"weightedTruePositiveRate\",\n            \"weightedFalsePositiveRate\",\n            \"weightedFMeasure\",\n            \"truePositiveRateByLabel\",\n            \"falsePositiveRateByLabel\",\n            \"precisionByLabel\",\n            \"recallByLabel\",\n            \"fMeasureByLabel\",\n            \"logLoss\",\n            \"hammingLoss\",\n        ],\n        BinaryClassificationEvaluator: [\"areaUnderROC\", \"areaUnderPR\"],\n    }\n\n    self._setDefault(label_values=[])\n    kwargs = self._input_kwargs\n    self._set(**kwargs)\n</code></pre>"},{"location":"python_api/methods/l2g/evaluator/#gentropy.method.l2g.evaluator.WandbEvaluator.getlabel_values","title":"<code>getlabel_values() -&gt; list[str]</code>","text":"<p>Get the label_values parameter.</p> <p>Returns:</p> Type Description <code>list[str]</code> <p>list[str]: List of label values.</p> Source code in <code>src/gentropy/method/l2g/evaluator.py</code> <pre><code>def getlabel_values(self: WandbEvaluator) -&gt; list[str]:\n    \"\"\"Get the label_values parameter.\n\n    Returns:\n        list[str]: List of label values.\n    \"\"\"\n    return self.getOrDefault(self.label_values)\n</code></pre>"},{"location":"python_api/methods/l2g/evaluator/#gentropy.method.l2g.evaluator.WandbEvaluator.getspark_ml_evaluator","title":"<code>getspark_ml_evaluator() -&gt; Evaluator</code>","text":"<p>Get the spark_ml_evaluator parameter.</p> <p>Returns:</p> Name Type Description <code>Evaluator</code> <code>Evaluator</code> <p>Spark ML evaluator.</p> Source code in <code>src/gentropy/method/l2g/evaluator.py</code> <pre><code>def getspark_ml_evaluator(self: WandbEvaluator) -&gt; Evaluator:\n    \"\"\"Get the spark_ml_evaluator parameter.\n\n    Returns:\n        Evaluator: Spark ML evaluator.\n    \"\"\"\n    return self.getOrDefault(self.spark_ml_evaluator)\n</code></pre>"},{"location":"python_api/methods/l2g/evaluator/#gentropy.method.l2g.evaluator.WandbEvaluator.getwandb_project_name","title":"<code>getwandb_project_name() -&gt; Any</code>","text":"<p>Get the wandb_project_name parameter.</p> <p>Returns:</p> Name Type Description <code>Any</code> <code>Any</code> <p>Name of the W&amp;B project.</p> Source code in <code>src/gentropy/method/l2g/evaluator.py</code> <pre><code>def getwandb_project_name(self: WandbEvaluator) -&gt; Any:\n    \"\"\"Get the wandb_project_name parameter.\n\n    Returns:\n        Any: Name of the W&amp;B project.\n    \"\"\"\n    return self.getOrDefault(self.wandb_project_name)\n</code></pre>"},{"location":"python_api/methods/l2g/evaluator/#gentropy.method.l2g.evaluator.WandbEvaluator.getwandb_run","title":"<code>getwandb_run() -&gt; Run</code>","text":"<p>Get the wandb_run parameter.</p> <p>Returns:</p> Name Type Description <code>Run</code> <code>Run</code> <p>Wandb run object.</p> Source code in <code>src/gentropy/method/l2g/evaluator.py</code> <pre><code>def getwandb_run(self: WandbEvaluator) -&gt; Run:\n    \"\"\"Get the wandb_run parameter.\n\n    Returns:\n        Run: Wandb run object.\n    \"\"\"\n    return self.getOrDefault(self.wandb_run)\n</code></pre>"},{"location":"python_api/methods/l2g/evaluator/#gentropy.method.l2g.evaluator.WandbEvaluator.setlabel_values","title":"<code>setlabel_values(value: list[str]) -&gt; None</code>","text":"<p>Set the label_values parameter.</p> <p>Parameters:</p> Name Type Description Default <code>value</code> <code>list[str]</code> <p>List of label values.</p> required Source code in <code>src/gentropy/method/l2g/evaluator.py</code> <pre><code>def setlabel_values(self: WandbEvaluator, value: list[str]) -&gt; None:\n    \"\"\"Set the label_values parameter.\n\n    Args:\n        value (list[str]): List of label values.\n    \"\"\"\n    self._set(label_values=value)\n</code></pre>"},{"location":"python_api/methods/l2g/evaluator/#gentropy.method.l2g.evaluator.WandbEvaluator.setspark_ml_evaluator","title":"<code>setspark_ml_evaluator(value: Evaluator) -&gt; None</code>","text":"<p>Set the spark_ml_evaluator parameter.</p> <p>Parameters:</p> Name Type Description Default <code>value</code> <code>Evaluator</code> <p>Spark ML evaluator.</p> required Source code in <code>src/gentropy/method/l2g/evaluator.py</code> <pre><code>def setspark_ml_evaluator(self: WandbEvaluator, value: Evaluator) -&gt; None:\n    \"\"\"Set the spark_ml_evaluator parameter.\n\n    Args:\n        value (Evaluator): Spark ML evaluator.\n    \"\"\"\n    self._set(spark_ml_evaluator=value)\n</code></pre>"},{"location":"python_api/methods/l2g/feature_factory/","title":"L2G Feature Factory","text":""},{"location":"python_api/methods/l2g/feature_factory/#gentropy.method.l2g.feature_factory.ColocalisationFactory","title":"<code>gentropy.method.l2g.feature_factory.ColocalisationFactory</code>","text":"<p>Feature extraction in colocalisation.</p> Source code in <code>src/gentropy/method/l2g/feature_factory.py</code> <pre><code>class ColocalisationFactory:\n    \"\"\"Feature extraction in colocalisation.\"\"\"\n\n    @staticmethod\n    def _get_max_coloc_per_study_locus(\n        study_locus: StudyLocus,\n        studies: StudyIndex,\n        colocalisation: Colocalisation,\n        colocalisation_method: str,\n    ) -&gt; L2GFeature:\n        \"\"\"Get the maximum colocalisation posterior probability for each pair of overlapping study-locus per type of colocalisation method and QTL type.\n\n        Args:\n            study_locus (StudyLocus): Study locus dataset\n            studies (StudyIndex): Study index dataset\n            colocalisation (Colocalisation): Colocalisation dataset\n            colocalisation_method (str): Colocalisation method to extract the max from\n\n        Returns:\n            L2GFeature: Stores the features with the max coloc probabilities for each pair of study-locus\n\n        Raises:\n            ValueError: If the colocalisation method is not supported\n        \"\"\"\n        if colocalisation_method not in [\"COLOC\", \"eCAVIAR\"]:\n            raise ValueError(\n                f\"Colocalisation method {colocalisation_method} not supported\"\n            )\n        if colocalisation_method == \"COLOC\":\n            coloc_score_col_name = \"log2h4h3\"\n            coloc_feature_col_template = \"ColocLlrMaximum\"\n\n        elif colocalisation_method == \"eCAVIAR\":\n            coloc_score_col_name = \"clpp\"\n            coloc_feature_col_template = \"ColocClppMaximum\"\n\n        colocalising_study_locus = (\n            study_locus.df.select(\"studyLocusId\", \"studyId\")\n            # annotate studyLoci with overlapping IDs on the left - to just keep GWAS associations\n            .join(\n                colocalisation.df.selectExpr(\n                    \"leftStudyLocusId as studyLocusId\",\n                    \"rightStudyLocusId\",\n                    \"colocalisationMethod\",\n                    f\"{coloc_score_col_name} as coloc_score\",\n                ),\n                on=\"studyLocusId\",\n                how=\"inner\",\n            )\n            # bring study metadata to just keep QTL studies on the right\n            .join(\n                study_locus.df.selectExpr(\n                    \"studyLocusId as rightStudyLocusId\", \"studyId as right_studyId\"\n                ),\n                on=\"rightStudyLocusId\",\n                how=\"inner\",\n            )\n            .join(\n                f.broadcast(\n                    studies.df.selectExpr(\n                        \"studyId as right_studyId\",\n                        \"studyType as right_studyType\",\n                        \"geneId\",\n                    )\n                ),\n                on=\"right_studyId\",\n                how=\"inner\",\n            )\n            .filter(\n                (f.col(\"colocalisationMethod\") == colocalisation_method)\n                &amp; (f.col(\"right_studyType\") != \"gwas\")\n            )\n            .select(\"studyLocusId\", \"right_studyType\", \"geneId\", \"coloc_score\")\n        )\n\n        # Max PP calculation per studyLocus AND type of QTL\n        local_max = get_record_with_maximum_value(\n            colocalising_study_locus,\n            [\"studyLocusId\", \"right_studyType\", \"geneId\"],\n            \"coloc_score\",\n        ).persist()\n\n        intercept = 0.0001\n        neighbourhood_max = (\n            (\n                local_max.selectExpr(\n                    \"studyLocusId\", \"coloc_score as coloc_local_max\", \"geneId\"\n                )\n                .join(\n                    # Add maximum in the neighborhood\n                    get_record_with_maximum_value(\n                        colocalising_study_locus.withColumnRenamed(\n                            \"coloc_score\", \"coloc_neighborhood_max\"\n                        ),\n                        [\"studyLocusId\", \"right_studyType\"],\n                        \"coloc_neighborhood_max\",\n                    ).drop(\"geneId\"),\n                    on=\"studyLocusId\",\n                )\n                .withColumn(\n                    f\"{coloc_feature_col_template}Neighborhood\",\n                    f.log10(\n                        f.abs(\n                            f.col(\"coloc_local_max\")\n                            - f.col(\"coloc_neighborhood_max\")\n                            + f.lit(intercept)\n                        )\n                    ),\n                )\n            )\n            .drop(\"coloc_neighborhood_max\", \"coloc_local_max\")\n            .persist()\n        )\n\n        # Split feature per molQTL\n        local_dfs = []\n        nbh_dfs = []\n        qtl_types: list[str] = (\n            colocalising_study_locus.select(\"right_studyType\")\n            .distinct()\n            .toPandas()[\"right_studyType\"]\n            .tolist()\n        ) or [\"eqtl\", \"pqtl\", \"sqtl\"]\n        for qtl_type in qtl_types:\n            filtered_local_max = (\n                local_max.filter(f.col(\"right_studyType\") == qtl_type)\n                .withColumnRenamed(\n                    \"coloc_score\",\n                    f\"{qtl_type}{coloc_feature_col_template}\",\n                )\n                .drop(\"right_studyType\")\n            )\n            local_dfs.append(filtered_local_max)\n\n            filtered_neighbourhood_max = (\n                neighbourhood_max.filter(f.col(\"right_studyType\") == qtl_type)\n                .withColumnRenamed(\n                    f\"{coloc_feature_col_template}Neighborhood\",\n                    f\"{qtl_type}{coloc_feature_col_template}Neighborhood\",\n                )\n                .drop(\"right_studyType\")\n            )\n            nbh_dfs.append(filtered_neighbourhood_max)\n\n        wide_dfs = reduce(\n            lambda x, y: x.unionByName(y, allowMissingColumns=True),\n            local_dfs + nbh_dfs,\n        )\n\n        return L2GFeature(\n            _df=convert_from_wide_to_long(\n                wide_dfs.groupBy(\"studyLocusId\", \"geneId\").agg(\n                    *(\n                        f.first(f.col(c), ignorenulls=True).alias(c)\n                        for c in wide_dfs.columns\n                        if c\n                        not in [\n                            \"studyLocusId\",\n                            \"geneId\",\n                        ]\n                    )\n                ),\n                id_vars=(\"studyLocusId\", \"geneId\"),\n                var_name=\"featureName\",\n                value_name=\"featureValue\",\n            ),\n            _schema=L2GFeature.get_schema(),\n        )\n\n    @staticmethod\n    def _get_coloc_features(\n        study_locus: StudyLocus, studies: StudyIndex, colocalisation: Colocalisation\n    ) -&gt; L2GFeature:\n        \"\"\"Calls _get_max_coloc_per_study_locus for both methods and concatenates the results.\n\n        !!! note \"Colocalisation features are only available for the eCAVIAR results for now.\"\n\n        Args:\n            study_locus (StudyLocus): Study locus dataset\n            studies (StudyIndex): Study index dataset\n            colocalisation (Colocalisation): Colocalisation dataset\n\n        Returns:\n            L2GFeature: Stores the features with the max coloc probabilities for each pair of study-locus\n        \"\"\"\n        coloc_clpp = ColocalisationFactory._get_max_coloc_per_study_locus(\n            study_locus,\n            studies,\n            colocalisation,\n            \"eCAVIAR\",\n        )\n\n        return L2GFeature(\n            _df=coloc_clpp.df,\n            _schema=L2GFeature.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/methods/l2g/feature_factory/#gentropy.method.l2g.feature_factory.StudyLocusFactory","title":"<code>gentropy.method.l2g.feature_factory.StudyLocusFactory</code>","text":"<p>             Bases: <code>StudyLocus</code></p> <p>Feature extraction in study locus.</p> Source code in <code>src/gentropy/method/l2g/feature_factory.py</code> <pre><code>class StudyLocusFactory(StudyLocus):\n    \"\"\"Feature extraction in study locus.\"\"\"\n\n    @staticmethod\n    def _get_tss_distance_features(\n        study_locus: StudyLocus, distances: V2G\n    ) -&gt; L2GFeature:\n        \"\"\"Joins StudyLocus with the V2G to extract the minimum distance to a gene TSS of all variants in a StudyLocus credible set.\n\n        Args:\n            study_locus (StudyLocus): Study locus dataset\n            distances (V2G): Dataframe containing the distances of all variants to all genes TSS within a region\n\n        Returns:\n            L2GFeature: Stores the features with the minimum distance among all variants in the credible set and a gene TSS.\n\n        \"\"\"\n        wide_df = (\n            study_locus.filter_credible_set(CredibleInterval.IS95)\n            .df.select(\n                \"studyLocusId\",\n                \"variantId\",\n                f.explode(\"locus.variantId\").alias(\"tagVariantId\"),\n            )\n            .join(\n                distances.df.selectExpr(\n                    \"variantId as tagVariantId\", \"geneId\", \"distance\"\n                ),\n                on=\"tagVariantId\",\n                how=\"inner\",\n            )\n            .groupBy(\"studyLocusId\", \"geneId\")\n            .agg(\n                f.min(\"distance\").alias(\"distanceTssMinimum\"),\n                f.mean(\"distance\").alias(\"distanceTssMean\"),\n            )\n        )\n\n        return L2GFeature(\n            _df=convert_from_wide_to_long(\n                wide_df,\n                id_vars=(\"studyLocusId\", \"geneId\"),\n                var_name=\"featureName\",\n                value_name=\"featureValue\",\n            ),\n            _schema=L2GFeature.get_schema(),\n        )\n\n    @staticmethod\n    def _get_vep_features(\n        credible_set: StudyLocus,\n        v2g: V2G,\n    ) -&gt; L2GFeature:\n        \"\"\"Get the maximum VEP score for all variants in a locus's 95% credible set.\n\n        This informs about functional impact of the variants in the locus. For more information on variant consequences, see: https://www.ensembl.org/info/genome/variation/prediction/predicted_data.html\n        Two metrics: max VEP score per study locus and gene, and max VEP score per study locus.\n\n\n        Args:\n            credible_set (StudyLocus): Study locus dataset with the associations to be annotated\n            v2g (V2G): V2G dataset with the variant/gene relationships and their consequences\n\n        Returns:\n            L2GFeature: Stores the features with the max VEP score.\n        \"\"\"\n\n        def _aggregate_vep_feature(\n            df: DataFrame,\n            aggregation_expr: Column,\n            aggregation_cols: list[str],\n            feature_name: str,\n        ) -&gt; DataFrame:\n            \"\"\"Extracts the maximum or average VEP score after grouping by the given columns. Different aggregations return different predictive annotations.\n\n            If the group_cols include \"geneId\", the maximum/mean VEP score per gene is returned.\n            Otherwise, the maximum/mean VEP score for all genes in the neighborhood of the locus is returned.\n\n            Args:\n                df (DataFrame): DataFrame with the VEP scores for each variant in a studyLocus\n                aggregation_expr (Column): Aggregation expression to apply\n                aggregation_cols (list[str]): Columns to group by\n                feature_name (str): Name of the feature to be returned\n\n            Returns:\n                DataFrame: DataFrame with the maximum VEP score per locus or per locus/gene\n            \"\"\"\n            if \"geneId\" in aggregation_cols:\n                return df.groupBy(aggregation_cols).agg(\n                    aggregation_expr.alias(feature_name)\n                )\n            return (\n                df.groupBy(aggregation_cols)\n                .agg(\n                    aggregation_expr.alias(feature_name),\n                    f.collect_set(\"geneId\").alias(\"geneId\"),\n                )\n                .withColumn(\"geneId\", f.explode(\"geneId\"))\n            )\n\n        credible_set_w_variant_consequences = (\n            credible_set.filter_credible_set(CredibleInterval.IS95)\n            .df.withColumn(\"variantInLocusId\", f.explode(f.col(\"locus.variantId\")))\n            .withColumn(\n                \"variantInLocusPosteriorProbability\",\n                f.explode(f.col(\"locus.posteriorProbability\")),\n            )\n            .join(\n                # Join with V2G to get variant consequences\n                v2g.df.filter(\n                    f.col(\"datasourceId\") == \"variantConsequence\"\n                ).withColumnRenamed(\"variantId\", \"variantInLocusId\"),\n                on=\"variantInLocusId\",\n            )\n            .withColumn(\n                \"weightedScore\",\n                f.col(\"score\") * f.col(\"variantInLocusPosteriorProbability\"),\n            )\n            .select(\n                \"studyLocusId\",\n                \"variantId\",\n                \"studyId\",\n                \"geneId\",\n                \"score\",\n                \"weightedScore\",\n            )\n            .distinct()\n            .persist()\n        )\n\n        return L2GFeature(\n            _df=convert_from_wide_to_long(\n                reduce(\n                    lambda x, y: x.unionByName(y, allowMissingColumns=True),\n                    [\n                        # Calculate overall max VEP score for all genes in the vicinity\n                        credible_set_w_variant_consequences.transform(\n                            _aggregate_vep_feature,\n                            f.max(\"score\"),\n                            [\"studyLocusId\"],\n                            \"vepMaximumNeighborhood\",\n                        ),\n                        # Calculate overall max VEP score per gene\n                        credible_set_w_variant_consequences.transform(\n                            _aggregate_vep_feature,\n                            f.max(\"score\"),\n                            [\"studyLocusId\", \"geneId\"],\n                            \"vepMaximum\",\n                        ),\n                        # Calculate mean VEP score for all genes in the vicinity\n                        credible_set_w_variant_consequences.transform(\n                            _aggregate_vep_feature,\n                            f.mean(\"weightedScore\"),\n                            [\"studyLocusId\"],\n                            \"vepMeanNeighborhood\",\n                        ),\n                        # Calculate mean VEP score per gene\n                        credible_set_w_variant_consequences.transform(\n                            _aggregate_vep_feature,\n                            f.mean(\"weightedScore\"),\n                            [\"studyLocusId\", \"geneId\"],\n                            \"vepMean\",\n                        ),\n                    ],\n                ),\n                id_vars=(\"studyLocusId\", \"geneId\"),\n                var_name=\"featureName\",\n                value_name=\"featureValue\",\n            ).filter(f.col(\"featureValue\").isNotNull()),\n            _schema=L2GFeature.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/methods/l2g/model/","title":"L2G Model","text":""},{"location":"python_api/methods/l2g/model/#gentropy.method.l2g.model.LocusToGeneModel","title":"<code>gentropy.method.l2g.model.LocusToGeneModel</code>  <code>dataclass</code>","text":"<p>Wrapper for the Locus to Gene classifier.</p> Source code in <code>src/gentropy/method/l2g/model.py</code> <pre><code>@dataclass\nclass LocusToGeneModel:\n    \"\"\"Wrapper for the Locus to Gene classifier.\"\"\"\n\n    features_list: list[str]\n    estimator: Any = None\n    pipeline: Pipeline = Pipeline(stages=[])\n    model: PipelineModel | None = None\n    wandb_l2g_project_name: str = \"otg_l2g\"\n\n    def __post_init__(self: LocusToGeneModel) -&gt; None:\n        \"\"\"Post init that adds the model to the ML pipeline.\"\"\"\n        label_indexer = StringIndexer(\n            inputCol=\"goldStandardSet\", outputCol=\"label\", handleInvalid=\"keep\"\n        )\n        vector_assembler = LocusToGeneModel.features_vector_assembler(\n            self.features_list\n        )\n\n        self.pipeline = Pipeline(\n            stages=[\n                label_indexer,\n                vector_assembler,\n            ]\n        )\n\n    def save(self: LocusToGeneModel, path: str) -&gt; None:\n        \"\"\"Saves fitted pipeline model to disk.\n\n        Args:\n            path (str): Path to save the model to\n\n        Raises:\n            ValueError: If the model has not been fitted yet\n        \"\"\"\n        if self.model is None:\n            raise ValueError(\"Model has not been fitted yet.\")\n        self.model.write().overwrite().save(path)\n\n    @property\n    def classifier(self: LocusToGeneModel) -&gt; Any:\n        \"\"\"Return the model.\n\n        Returns:\n            Any: An estimator object from Spark ML\n        \"\"\"\n        return self.estimator\n\n    @staticmethod\n    def features_vector_assembler(features_cols: list[str]) -&gt; VectorAssembler:\n        \"\"\"Spark transformer to assemble the feature columns into a vector.\n\n        Args:\n            features_cols (list[str]): List of feature columns to assemble\n\n        Returns:\n            VectorAssembler: Spark transformer to assemble the feature columns into a vector\n\n        Examples:\n            &gt;&gt;&gt; from pyspark.ml.feature import VectorAssembler\n            &gt;&gt;&gt; df = spark.createDataFrame([(5.2, 3.5)], schema=\"feature_1 FLOAT, feature_2 FLOAT\")\n            &gt;&gt;&gt; assembler = LocusToGeneModel.features_vector_assembler([\"feature_1\", \"feature_2\"])\n            &gt;&gt;&gt; assembler.transform(df).show()\n            +---------+---------+--------------------+\n            |feature_1|feature_2|            features|\n            +---------+---------+--------------------+\n            |      5.2|      3.5|[5.19999980926513...|\n            +---------+---------+--------------------+\n            &lt;BLANKLINE&gt;\n        \"\"\"\n        return (\n            VectorAssembler(handleInvalid=\"error\")\n            .setInputCols(features_cols)\n            .setOutputCol(\"features\")\n        )\n\n    def log_to_wandb(\n        self: LocusToGeneModel,\n        results: DataFrame,\n        training_data: L2GFeatureMatrix,\n        evaluators: list[\n            BinaryClassificationEvaluator | MulticlassClassificationEvaluator\n        ],\n        wandb_run: Run,\n    ) -&gt; None:\n        \"\"\"Log evaluation results and feature importance to W&amp;B.\n\n        Args:\n            results (DataFrame): Dataframe containing the predictions\n            training_data (L2GFeatureMatrix): Training data used for the model. If provided, the table and the number of positive and negative labels will be logged to W&amp;B\n            evaluators (list[BinaryClassificationEvaluator | MulticlassClassificationEvaluator]): List of Spark ML evaluators to use for evaluation\n            wandb_run (Run): W&amp;B run to log the results to\n        \"\"\"\n        ## Track evaluation metrics\n        for evaluator in evaluators:\n            wandb_evaluator = WandbEvaluator(\n                spark_ml_evaluator=evaluator, wandb_run=wandb_run\n            )\n            wandb_evaluator.evaluate(results)\n        ## Track feature importance\n        wandb_run.log({\"importances\": self.get_feature_importance()})\n        ## Track training set\n        training_table = Table(dataframe=training_data.df.toPandas())\n        wandb_run.log({\"trainingSet\": training_table})\n        # Count number of positive and negative labels\n        gs_counts_dict = {\n            \"goldStandard\" + row[\"goldStandardSet\"].capitalize(): row[\"count\"]\n            for row in training_data.df.groupBy(\"goldStandardSet\").count().collect()\n        }\n        wandb_run.log(gs_counts_dict)\n        # Missingness rates\n        wandb_run.log(\n            {\"missingnessRates\": training_data.calculate_feature_missingness_rate()}\n        )\n\n    @classmethod\n    def load_from_disk(\n        cls: Type[LocusToGeneModel], path: str, features_list: list[str]\n    ) -&gt; LocusToGeneModel:\n        \"\"\"Load a fitted pipeline model from disk.\n\n        Args:\n            path (str): Path to the model\n            features_list (list[str]): List of features used for the model\n\n        Returns:\n            LocusToGeneModel: L2G model loaded from disk\n        \"\"\"\n        return cls(model=PipelineModel.load(path), features_list=features_list)\n\n    @classifier.setter  # type: ignore\n    def classifier(self: LocusToGeneModel, new_estimator: Any) -&gt; None:\n        \"\"\"Set the model.\n\n        Args:\n            new_estimator (Any): An estimator object from Spark ML\n        \"\"\"\n        self.estimator = new_estimator\n\n    def get_param_grid(self: LocusToGeneModel) -&gt; list[Any]:\n        \"\"\"Return the parameter grid for the model.\n\n        Returns:\n            list[Any]: List of parameter maps to use for cross validation\n        \"\"\"\n        return (\n            ParamGridBuilder()\n            .addGrid(self.estimator.max_depth, [3, 5, 7])\n            .addGrid(self.estimator.learning_rate, [0.01, 0.1, 1.0])\n            .build()\n        )\n\n    def add_pipeline_stage(\n        self: LocusToGeneModel, transformer: Transformer\n    ) -&gt; LocusToGeneModel:\n        \"\"\"Adds a stage to the L2G pipeline.\n\n        Args:\n            transformer (Transformer): Spark transformer to add to the pipeline\n\n        Returns:\n            LocusToGeneModel: L2G model with the new transformer\n\n        Examples:\n            &gt;&gt;&gt; from pyspark.ml.regression import LinearRegression\n            &gt;&gt;&gt; estimator = LinearRegression()\n            &gt;&gt;&gt; test_model = LocusToGeneModel(features_list=[\"a\", \"b\"])\n            &gt;&gt;&gt; print(len(test_model.pipeline.getStages()))\n            2\n            &gt;&gt;&gt; print(len(test_model.add_pipeline_stage(estimator).pipeline.getStages()))\n            3\n        \"\"\"\n        pipeline_stages = self.pipeline.getStages()\n        new_stages = pipeline_stages + [transformer]\n        self.pipeline = Pipeline(stages=new_stages)\n        return self\n\n    def evaluate(\n        self: LocusToGeneModel,\n        results: DataFrame,\n        hyperparameters: dict[str, Any],\n        wandb_run_name: str | None,\n        training_data: L2GFeatureMatrix | None = None,\n    ) -&gt; None:\n        \"\"\"Perform evaluation of the model predictions for the test set and track the results with W&amp;B.\n\n        Args:\n            results (DataFrame): Dataframe containing the predictions\n            hyperparameters (dict[str, Any]): Hyperparameters used for the model\n            wandb_run_name (str | None): Descriptive name for the run to be tracked with W&amp;B\n            training_data (L2GFeatureMatrix | None): Training data used for the model. If provided, the ratio of positive to negative labels will be logged to W&amp;B\n        \"\"\"\n        binary_evaluator = BinaryClassificationEvaluator(\n            rawPredictionCol=\"rawPrediction\", labelCol=\"label\"\n        )\n        multi_evaluator = MulticlassClassificationEvaluator(\n            labelCol=\"label\", predictionCol=\"prediction\"\n        )\n\n        if wandb_run_name and training_data:\n            run = wandb_init(\n                project=self.wandb_l2g_project_name,\n                config=hyperparameters,\n                name=wandb_run_name,\n            )\n            if isinstance(run, Run):\n                self.log_to_wandb(\n                    results, training_data, [binary_evaluator, multi_evaluator], run\n                )\n                run.finish()\n\n    @property\n    def feature_name_map(self: LocusToGeneModel) -&gt; dict[str, str]:\n        \"\"\"Return a dictionary mapping encoded feature names to the original names.\n\n        Returns:\n            dict[str, str]: Feature name map of the model\n\n        Raises:\n            ValueError: If the model has not been fitted yet\n        \"\"\"\n        if not self.model:\n            raise ValueError(\"Model not fitted yet. `fit()` has to be called first.\")\n        elif isinstance(self.model.stages[1], VectorAssembler):\n            feature_names = self.model.stages[1].getInputCols()\n        return {f\"f{i}\": feature_name for i, feature_name in enumerate(feature_names)}\n\n    def get_feature_importance(self: LocusToGeneModel) -&gt; dict[str, float]:\n        \"\"\"Return dictionary with relative importances of every feature in the model. Feature names are encoded and have to be mapped back to their original names.\n\n        Returns:\n            dict[str, float]: Dictionary mapping feature names to their importance\n\n        Raises:\n            ValueError: If the model has not been fitted yet or is not an XGBoost model\n        \"\"\"\n        if not self.model or not isinstance(\n            self.model.stages[-1], SparkXGBClassifierModel\n        ):\n            raise ValueError(\n                f\"Model type {type(self.model)} not supported for feature importance.\"\n            )\n        importance_map = self.model.stages[-1].get_feature_importances()\n        return {self.feature_name_map[k]: v for k, v in importance_map.items()}\n\n    def fit(\n        self: LocusToGeneModel,\n        feature_matrix: L2GFeatureMatrix,\n    ) -&gt; LocusToGeneModel:\n        \"\"\"Fit the pipeline to the feature matrix dataframe.\n\n        Args:\n            feature_matrix (L2GFeatureMatrix): Feature matrix dataframe to fit the model to\n\n        Returns:\n            LocusToGeneModel: Fitted model\n        \"\"\"\n        self.model = self.pipeline.fit(feature_matrix.df)\n        return self\n\n    def predict(\n        self: LocusToGeneModel,\n        feature_matrix: L2GFeatureMatrix,\n    ) -&gt; DataFrame:\n        \"\"\"Apply the model to a given feature matrix dataframe. The feature matrix needs to be preprocessed first.\n\n        Args:\n            feature_matrix (L2GFeatureMatrix): Feature matrix dataframe to apply the model to\n\n        Returns:\n            DataFrame: Dataframe with predictions\n\n        Raises:\n            ValueError: If the model has not been fitted yet\n        \"\"\"\n        if not self.model:\n            raise ValueError(\"Model not fitted yet. `fit()` has to be called first.\")\n        return self.model.transform(feature_matrix.df)\n</code></pre>"},{"location":"python_api/methods/l2g/model/#gentropy.method.l2g.model.LocusToGeneModel.classifier","title":"<code>classifier: Any</code>  <code>property</code> <code>writable</code>","text":"<p>Return the model.</p> <p>Returns:</p> Name Type Description <code>Any</code> <code>Any</code> <p>An estimator object from Spark ML</p>"},{"location":"python_api/methods/l2g/model/#gentropy.method.l2g.model.LocusToGeneModel.feature_name_map","title":"<code>feature_name_map: dict[str, str]</code>  <code>property</code>","text":"<p>Return a dictionary mapping encoded feature names to the original names.</p> <p>Returns:</p> Type Description <code>dict[str, str]</code> <p>dict[str, str]: Feature name map of the model</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>If the model has not been fitted yet</p>"},{"location":"python_api/methods/l2g/model/#gentropy.method.l2g.model.LocusToGeneModel.add_pipeline_stage","title":"<code>add_pipeline_stage(transformer: Transformer) -&gt; LocusToGeneModel</code>","text":"<p>Adds a stage to the L2G pipeline.</p> <p>Parameters:</p> Name Type Description Default <code>transformer</code> <code>Transformer</code> <p>Spark transformer to add to the pipeline</p> required <p>Returns:</p> Name Type Description <code>LocusToGeneModel</code> <code>LocusToGeneModel</code> <p>L2G model with the new transformer</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pyspark.ml.regression import LinearRegression\n&gt;&gt;&gt; estimator = LinearRegression()\n&gt;&gt;&gt; test_model = LocusToGeneModel(features_list=[\"a\", \"b\"])\n&gt;&gt;&gt; print(len(test_model.pipeline.getStages()))\n2\n&gt;&gt;&gt; print(len(test_model.add_pipeline_stage(estimator).pipeline.getStages()))\n3\n</code></pre> Source code in <code>src/gentropy/method/l2g/model.py</code> <pre><code>def add_pipeline_stage(\n    self: LocusToGeneModel, transformer: Transformer\n) -&gt; LocusToGeneModel:\n    \"\"\"Adds a stage to the L2G pipeline.\n\n    Args:\n        transformer (Transformer): Spark transformer to add to the pipeline\n\n    Returns:\n        LocusToGeneModel: L2G model with the new transformer\n\n    Examples:\n        &gt;&gt;&gt; from pyspark.ml.regression import LinearRegression\n        &gt;&gt;&gt; estimator = LinearRegression()\n        &gt;&gt;&gt; test_model = LocusToGeneModel(features_list=[\"a\", \"b\"])\n        &gt;&gt;&gt; print(len(test_model.pipeline.getStages()))\n        2\n        &gt;&gt;&gt; print(len(test_model.add_pipeline_stage(estimator).pipeline.getStages()))\n        3\n    \"\"\"\n    pipeline_stages = self.pipeline.getStages()\n    new_stages = pipeline_stages + [transformer]\n    self.pipeline = Pipeline(stages=new_stages)\n    return self\n</code></pre>"},{"location":"python_api/methods/l2g/model/#gentropy.method.l2g.model.LocusToGeneModel.evaluate","title":"<code>evaluate(results: DataFrame, hyperparameters: dict[str, Any], wandb_run_name: str | None, training_data: L2GFeatureMatrix | None = None) -&gt; None</code>","text":"<p>Perform evaluation of the model predictions for the test set and track the results with W&amp;B.</p> <p>Parameters:</p> Name Type Description Default <code>results</code> <code>DataFrame</code> <p>Dataframe containing the predictions</p> required <code>hyperparameters</code> <code>dict[str, Any]</code> <p>Hyperparameters used for the model</p> required <code>wandb_run_name</code> <code>str | None</code> <p>Descriptive name for the run to be tracked with W&amp;B</p> required <code>training_data</code> <code>L2GFeatureMatrix | None</code> <p>Training data used for the model. If provided, the ratio of positive to negative labels will be logged to W&amp;B</p> <code>None</code> Source code in <code>src/gentropy/method/l2g/model.py</code> <pre><code>def evaluate(\n    self: LocusToGeneModel,\n    results: DataFrame,\n    hyperparameters: dict[str, Any],\n    wandb_run_name: str | None,\n    training_data: L2GFeatureMatrix | None = None,\n) -&gt; None:\n    \"\"\"Perform evaluation of the model predictions for the test set and track the results with W&amp;B.\n\n    Args:\n        results (DataFrame): Dataframe containing the predictions\n        hyperparameters (dict[str, Any]): Hyperparameters used for the model\n        wandb_run_name (str | None): Descriptive name for the run to be tracked with W&amp;B\n        training_data (L2GFeatureMatrix | None): Training data used for the model. If provided, the ratio of positive to negative labels will be logged to W&amp;B\n    \"\"\"\n    binary_evaluator = BinaryClassificationEvaluator(\n        rawPredictionCol=\"rawPrediction\", labelCol=\"label\"\n    )\n    multi_evaluator = MulticlassClassificationEvaluator(\n        labelCol=\"label\", predictionCol=\"prediction\"\n    )\n\n    if wandb_run_name and training_data:\n        run = wandb_init(\n            project=self.wandb_l2g_project_name,\n            config=hyperparameters,\n            name=wandb_run_name,\n        )\n        if isinstance(run, Run):\n            self.log_to_wandb(\n                results, training_data, [binary_evaluator, multi_evaluator], run\n            )\n            run.finish()\n</code></pre>"},{"location":"python_api/methods/l2g/model/#gentropy.method.l2g.model.LocusToGeneModel.features_vector_assembler","title":"<code>features_vector_assembler(features_cols: list[str]) -&gt; VectorAssembler</code>  <code>staticmethod</code>","text":"<p>Spark transformer to assemble the feature columns into a vector.</p> <p>Parameters:</p> Name Type Description Default <code>features_cols</code> <code>list[str]</code> <p>List of feature columns to assemble</p> required <p>Returns:</p> Name Type Description <code>VectorAssembler</code> <code>VectorAssembler</code> <p>Spark transformer to assemble the feature columns into a vector</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pyspark.ml.feature import VectorAssembler\n&gt;&gt;&gt; df = spark.createDataFrame([(5.2, 3.5)], schema=\"feature_1 FLOAT, feature_2 FLOAT\")\n&gt;&gt;&gt; assembler = LocusToGeneModel.features_vector_assembler([\"feature_1\", \"feature_2\"])\n&gt;&gt;&gt; assembler.transform(df).show()\n+---------+---------+--------------------+\n|feature_1|feature_2|            features|\n+---------+---------+--------------------+\n|      5.2|      3.5|[5.19999980926513...|\n+---------+---------+--------------------+\n</code></pre> Source code in <code>src/gentropy/method/l2g/model.py</code> <pre><code>@staticmethod\ndef features_vector_assembler(features_cols: list[str]) -&gt; VectorAssembler:\n    \"\"\"Spark transformer to assemble the feature columns into a vector.\n\n    Args:\n        features_cols (list[str]): List of feature columns to assemble\n\n    Returns:\n        VectorAssembler: Spark transformer to assemble the feature columns into a vector\n\n    Examples:\n        &gt;&gt;&gt; from pyspark.ml.feature import VectorAssembler\n        &gt;&gt;&gt; df = spark.createDataFrame([(5.2, 3.5)], schema=\"feature_1 FLOAT, feature_2 FLOAT\")\n        &gt;&gt;&gt; assembler = LocusToGeneModel.features_vector_assembler([\"feature_1\", \"feature_2\"])\n        &gt;&gt;&gt; assembler.transform(df).show()\n        +---------+---------+--------------------+\n        |feature_1|feature_2|            features|\n        +---------+---------+--------------------+\n        |      5.2|      3.5|[5.19999980926513...|\n        +---------+---------+--------------------+\n        &lt;BLANKLINE&gt;\n    \"\"\"\n    return (\n        VectorAssembler(handleInvalid=\"error\")\n        .setInputCols(features_cols)\n        .setOutputCol(\"features\")\n    )\n</code></pre>"},{"location":"python_api/methods/l2g/model/#gentropy.method.l2g.model.LocusToGeneModel.fit","title":"<code>fit(feature_matrix: L2GFeatureMatrix) -&gt; LocusToGeneModel</code>","text":"<p>Fit the pipeline to the feature matrix dataframe.</p> <p>Parameters:</p> Name Type Description Default <code>feature_matrix</code> <code>L2GFeatureMatrix</code> <p>Feature matrix dataframe to fit the model to</p> required <p>Returns:</p> Name Type Description <code>LocusToGeneModel</code> <code>LocusToGeneModel</code> <p>Fitted model</p> Source code in <code>src/gentropy/method/l2g/model.py</code> <pre><code>def fit(\n    self: LocusToGeneModel,\n    feature_matrix: L2GFeatureMatrix,\n) -&gt; LocusToGeneModel:\n    \"\"\"Fit the pipeline to the feature matrix dataframe.\n\n    Args:\n        feature_matrix (L2GFeatureMatrix): Feature matrix dataframe to fit the model to\n\n    Returns:\n        LocusToGeneModel: Fitted model\n    \"\"\"\n    self.model = self.pipeline.fit(feature_matrix.df)\n    return self\n</code></pre>"},{"location":"python_api/methods/l2g/model/#gentropy.method.l2g.model.LocusToGeneModel.get_feature_importance","title":"<code>get_feature_importance() -&gt; dict[str, float]</code>","text":"<p>Return dictionary with relative importances of every feature in the model. Feature names are encoded and have to be mapped back to their original names.</p> <p>Returns:</p> Type Description <code>dict[str, float]</code> <p>dict[str, float]: Dictionary mapping feature names to their importance</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>If the model has not been fitted yet or is not an XGBoost model</p> Source code in <code>src/gentropy/method/l2g/model.py</code> <pre><code>def get_feature_importance(self: LocusToGeneModel) -&gt; dict[str, float]:\n    \"\"\"Return dictionary with relative importances of every feature in the model. Feature names are encoded and have to be mapped back to their original names.\n\n    Returns:\n        dict[str, float]: Dictionary mapping feature names to their importance\n\n    Raises:\n        ValueError: If the model has not been fitted yet or is not an XGBoost model\n    \"\"\"\n    if not self.model or not isinstance(\n        self.model.stages[-1], SparkXGBClassifierModel\n    ):\n        raise ValueError(\n            f\"Model type {type(self.model)} not supported for feature importance.\"\n        )\n    importance_map = self.model.stages[-1].get_feature_importances()\n    return {self.feature_name_map[k]: v for k, v in importance_map.items()}\n</code></pre>"},{"location":"python_api/methods/l2g/model/#gentropy.method.l2g.model.LocusToGeneModel.get_param_grid","title":"<code>get_param_grid() -&gt; list[Any]</code>","text":"<p>Return the parameter grid for the model.</p> <p>Returns:</p> Type Description <code>list[Any]</code> <p>list[Any]: List of parameter maps to use for cross validation</p> Source code in <code>src/gentropy/method/l2g/model.py</code> <pre><code>def get_param_grid(self: LocusToGeneModel) -&gt; list[Any]:\n    \"\"\"Return the parameter grid for the model.\n\n    Returns:\n        list[Any]: List of parameter maps to use for cross validation\n    \"\"\"\n    return (\n        ParamGridBuilder()\n        .addGrid(self.estimator.max_depth, [3, 5, 7])\n        .addGrid(self.estimator.learning_rate, [0.01, 0.1, 1.0])\n        .build()\n    )\n</code></pre>"},{"location":"python_api/methods/l2g/model/#gentropy.method.l2g.model.LocusToGeneModel.load_from_disk","title":"<code>load_from_disk(path: str, features_list: list[str]) -&gt; LocusToGeneModel</code>  <code>classmethod</code>","text":"<p>Load a fitted pipeline model from disk.</p> <p>Parameters:</p> Name Type Description Default <code>path</code> <code>str</code> <p>Path to the model</p> required <code>features_list</code> <code>list[str]</code> <p>List of features used for the model</p> required <p>Returns:</p> Name Type Description <code>LocusToGeneModel</code> <code>LocusToGeneModel</code> <p>L2G model loaded from disk</p> Source code in <code>src/gentropy/method/l2g/model.py</code> <pre><code>@classmethod\ndef load_from_disk(\n    cls: Type[LocusToGeneModel], path: str, features_list: list[str]\n) -&gt; LocusToGeneModel:\n    \"\"\"Load a fitted pipeline model from disk.\n\n    Args:\n        path (str): Path to the model\n        features_list (list[str]): List of features used for the model\n\n    Returns:\n        LocusToGeneModel: L2G model loaded from disk\n    \"\"\"\n    return cls(model=PipelineModel.load(path), features_list=features_list)\n</code></pre>"},{"location":"python_api/methods/l2g/model/#gentropy.method.l2g.model.LocusToGeneModel.log_to_wandb","title":"<code>log_to_wandb(results: DataFrame, training_data: L2GFeatureMatrix, evaluators: list[BinaryClassificationEvaluator | MulticlassClassificationEvaluator], wandb_run: Run) -&gt; None</code>","text":"<p>Log evaluation results and feature importance to W&amp;B.</p> <p>Parameters:</p> Name Type Description Default <code>results</code> <code>DataFrame</code> <p>Dataframe containing the predictions</p> required <code>training_data</code> <code>L2GFeatureMatrix</code> <p>Training data used for the model. If provided, the table and the number of positive and negative labels will be logged to W&amp;B</p> required <code>evaluators</code> <code>list[BinaryClassificationEvaluator | MulticlassClassificationEvaluator]</code> <p>List of Spark ML evaluators to use for evaluation</p> required <code>wandb_run</code> <code>Run</code> <p>W&amp;B run to log the results to</p> required Source code in <code>src/gentropy/method/l2g/model.py</code> <pre><code>def log_to_wandb(\n    self: LocusToGeneModel,\n    results: DataFrame,\n    training_data: L2GFeatureMatrix,\n    evaluators: list[\n        BinaryClassificationEvaluator | MulticlassClassificationEvaluator\n    ],\n    wandb_run: Run,\n) -&gt; None:\n    \"\"\"Log evaluation results and feature importance to W&amp;B.\n\n    Args:\n        results (DataFrame): Dataframe containing the predictions\n        training_data (L2GFeatureMatrix): Training data used for the model. If provided, the table and the number of positive and negative labels will be logged to W&amp;B\n        evaluators (list[BinaryClassificationEvaluator | MulticlassClassificationEvaluator]): List of Spark ML evaluators to use for evaluation\n        wandb_run (Run): W&amp;B run to log the results to\n    \"\"\"\n    ## Track evaluation metrics\n    for evaluator in evaluators:\n        wandb_evaluator = WandbEvaluator(\n            spark_ml_evaluator=evaluator, wandb_run=wandb_run\n        )\n        wandb_evaluator.evaluate(results)\n    ## Track feature importance\n    wandb_run.log({\"importances\": self.get_feature_importance()})\n    ## Track training set\n    training_table = Table(dataframe=training_data.df.toPandas())\n    wandb_run.log({\"trainingSet\": training_table})\n    # Count number of positive and negative labels\n    gs_counts_dict = {\n        \"goldStandard\" + row[\"goldStandardSet\"].capitalize(): row[\"count\"]\n        for row in training_data.df.groupBy(\"goldStandardSet\").count().collect()\n    }\n    wandb_run.log(gs_counts_dict)\n    # Missingness rates\n    wandb_run.log(\n        {\"missingnessRates\": training_data.calculate_feature_missingness_rate()}\n    )\n</code></pre>"},{"location":"python_api/methods/l2g/model/#gentropy.method.l2g.model.LocusToGeneModel.predict","title":"<code>predict(feature_matrix: L2GFeatureMatrix) -&gt; DataFrame</code>","text":"<p>Apply the model to a given feature matrix dataframe. The feature matrix needs to be preprocessed first.</p> <p>Parameters:</p> Name Type Description Default <code>feature_matrix</code> <code>L2GFeatureMatrix</code> <p>Feature matrix dataframe to apply the model to</p> required <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>Dataframe with predictions</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>If the model has not been fitted yet</p> Source code in <code>src/gentropy/method/l2g/model.py</code> <pre><code>def predict(\n    self: LocusToGeneModel,\n    feature_matrix: L2GFeatureMatrix,\n) -&gt; DataFrame:\n    \"\"\"Apply the model to a given feature matrix dataframe. The feature matrix needs to be preprocessed first.\n\n    Args:\n        feature_matrix (L2GFeatureMatrix): Feature matrix dataframe to apply the model to\n\n    Returns:\n        DataFrame: Dataframe with predictions\n\n    Raises:\n        ValueError: If the model has not been fitted yet\n    \"\"\"\n    if not self.model:\n        raise ValueError(\"Model not fitted yet. `fit()` has to be called first.\")\n    return self.model.transform(feature_matrix.df)\n</code></pre>"},{"location":"python_api/methods/l2g/model/#gentropy.method.l2g.model.LocusToGeneModel.save","title":"<code>save(path: str) -&gt; None</code>","text":"<p>Saves fitted pipeline model to disk.</p> <p>Parameters:</p> Name Type Description Default <code>path</code> <code>str</code> <p>Path to save the model to</p> required <p>Raises:</p> Type Description <code>ValueError</code> <p>If the model has not been fitted yet</p> Source code in <code>src/gentropy/method/l2g/model.py</code> <pre><code>def save(self: LocusToGeneModel, path: str) -&gt; None:\n    \"\"\"Saves fitted pipeline model to disk.\n\n    Args:\n        path (str): Path to save the model to\n\n    Raises:\n        ValueError: If the model has not been fitted yet\n    \"\"\"\n    if self.model is None:\n        raise ValueError(\"Model has not been fitted yet.\")\n    self.model.write().overwrite().save(path)\n</code></pre>"},{"location":"python_api/methods/l2g/trainer/","title":"L2G Trainer","text":""},{"location":"python_api/methods/l2g/trainer/#gentropy.method.l2g.trainer.LocusToGeneTrainer","title":"<code>gentropy.method.l2g.trainer.LocusToGeneTrainer</code>  <code>dataclass</code>","text":"<p>Modelling of what is the most likely causal gene associated with a given locus.</p> Source code in <code>src/gentropy/method/l2g/trainer.py</code> <pre><code>@dataclass\nclass LocusToGeneTrainer:\n    \"\"\"Modelling of what is the most likely causal gene associated with a given locus.\"\"\"\n\n    _model: LocusToGeneModel\n    train_set: L2GFeatureMatrix\n\n    @classmethod\n    def train(\n        cls: type[LocusToGeneTrainer],\n        data: L2GFeatureMatrix,\n        l2g_model: LocusToGeneModel,\n        features_list: list[str],\n        evaluate: bool,\n        wandb_run_name: str | None = None,\n        model_path: str | None = None,\n        **hyperparams: dict[str, Any],\n    ) -&gt; LocusToGeneModel:\n        \"\"\"Train the Locus to Gene model.\n\n        Args:\n            data (L2GFeatureMatrix): Feature matrix containing the data\n            l2g_model (LocusToGeneModel): Model to fit to the data on\n            features_list (list[str]): List of features to use for the model\n            evaluate (bool): Whether to evaluate the model on a test set\n            wandb_run_name (str | None): Descriptive name for the run to be tracked with W&amp;B\n            model_path (str | None): Path to save the model to\n            **hyperparams (dict[str, Any]): Hyperparameters to use for the model\n\n        Returns:\n            LocusToGeneModel: Trained model\n        \"\"\"\n        train, test = data.select_features(features_list).train_test_split(fraction=0.8)\n\n        model = l2g_model.add_pipeline_stage(l2g_model.estimator).fit(train)\n\n        if evaluate:\n            l2g_model.evaluate(\n                results=model.predict(test),\n                hyperparameters=hyperparams,\n                wandb_run_name=wandb_run_name,\n                training_data=train,\n            )\n        if model_path:\n            l2g_model.save(model_path)\n        return l2g_model\n\n    @classmethod\n    def cross_validate(\n        cls: type[LocusToGeneTrainer],\n        l2g_model: LocusToGeneModel,\n        data: L2GFeatureMatrix,\n        num_folds: int,\n        param_grid: Optional[list] = None,  # type: ignore\n    ) -&gt; LocusToGeneModel:\n        \"\"\"Perform k-fold cross validation on the model.\n\n        By providing a model with a parameter grid, this method will perform k-fold cross validation on the model for each\n        combination of parameters and return the best model.\n\n        Args:\n            l2g_model (LocusToGeneModel): Model to fit to the data on\n            data (L2GFeatureMatrix): Data to perform cross validation on\n            num_folds (int): Number of folds to use for cross validation\n            param_grid (Optional[list]): List of parameter maps to use for cross validation\n\n        Returns:\n            LocusToGeneModel: Trained model fitted with the best hyperparameters\n\n        Raises:\n            ValueError: Parameter grid is empty. Cannot perform cross-validation.\n            ValueError: Unable to retrieve the best model.\n        \"\"\"\n        evaluator = MulticlassClassificationEvaluator()\n        params_grid = param_grid or l2g_model.get_param_grid()\n        if not param_grid:\n            raise ValueError(\n                \"Parameter grid is empty. Cannot perform cross-validation.\"\n            )\n        cv = CrossValidator(\n            numFolds=num_folds,\n            estimator=l2g_model.estimator,\n            estimatorParamMaps=params_grid,\n            evaluator=evaluator,\n            parallelism=2,\n            collectSubModels=False,\n            seed=42,\n        )\n\n        l2g_model.add_pipeline_stage(cv)  # type: ignore[assignment, unused-ignore]\n\n        # Integrate the best model from the last stage of the pipeline\n        if (full_pipeline_model := l2g_model.fit(data).model) is None or not hasattr(\n            full_pipeline_model, \"stages\"\n        ):\n            raise ValueError(\"Unable to retrieve the best model.\")\n        l2g_model.model = full_pipeline_model.stages[-1].bestModel  # type: ignore[assignment, unused-ignore]\n        return l2g_model\n</code></pre>"},{"location":"python_api/methods/l2g/trainer/#gentropy.method.l2g.trainer.LocusToGeneTrainer.cross_validate","title":"<code>cross_validate(l2g_model: LocusToGeneModel, data: L2GFeatureMatrix, num_folds: int, param_grid: Optional[list] = None) -&gt; LocusToGeneModel</code>  <code>classmethod</code>","text":"<p>Perform k-fold cross validation on the model.</p> <p>By providing a model with a parameter grid, this method will perform k-fold cross validation on the model for each combination of parameters and return the best model.</p> <p>Parameters:</p> Name Type Description Default <code>l2g_model</code> <code>LocusToGeneModel</code> <p>Model to fit to the data on</p> required <code>data</code> <code>L2GFeatureMatrix</code> <p>Data to perform cross validation on</p> required <code>num_folds</code> <code>int</code> <p>Number of folds to use for cross validation</p> required <code>param_grid</code> <code>Optional[list]</code> <p>List of parameter maps to use for cross validation</p> <code>None</code> <p>Returns:</p> Name Type Description <code>LocusToGeneModel</code> <code>LocusToGeneModel</code> <p>Trained model fitted with the best hyperparameters</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>Parameter grid is empty. Cannot perform cross-validation.</p> <code>ValueError</code> <p>Unable to retrieve the best model.</p> Source code in <code>src/gentropy/method/l2g/trainer.py</code> <pre><code>@classmethod\ndef cross_validate(\n    cls: type[LocusToGeneTrainer],\n    l2g_model: LocusToGeneModel,\n    data: L2GFeatureMatrix,\n    num_folds: int,\n    param_grid: Optional[list] = None,  # type: ignore\n) -&gt; LocusToGeneModel:\n    \"\"\"Perform k-fold cross validation on the model.\n\n    By providing a model with a parameter grid, this method will perform k-fold cross validation on the model for each\n    combination of parameters and return the best model.\n\n    Args:\n        l2g_model (LocusToGeneModel): Model to fit to the data on\n        data (L2GFeatureMatrix): Data to perform cross validation on\n        num_folds (int): Number of folds to use for cross validation\n        param_grid (Optional[list]): List of parameter maps to use for cross validation\n\n    Returns:\n        LocusToGeneModel: Trained model fitted with the best hyperparameters\n\n    Raises:\n        ValueError: Parameter grid is empty. Cannot perform cross-validation.\n        ValueError: Unable to retrieve the best model.\n    \"\"\"\n    evaluator = MulticlassClassificationEvaluator()\n    params_grid = param_grid or l2g_model.get_param_grid()\n    if not param_grid:\n        raise ValueError(\n            \"Parameter grid is empty. Cannot perform cross-validation.\"\n        )\n    cv = CrossValidator(\n        numFolds=num_folds,\n        estimator=l2g_model.estimator,\n        estimatorParamMaps=params_grid,\n        evaluator=evaluator,\n        parallelism=2,\n        collectSubModels=False,\n        seed=42,\n    )\n\n    l2g_model.add_pipeline_stage(cv)  # type: ignore[assignment, unused-ignore]\n\n    # Integrate the best model from the last stage of the pipeline\n    if (full_pipeline_model := l2g_model.fit(data).model) is None or not hasattr(\n        full_pipeline_model, \"stages\"\n    ):\n        raise ValueError(\"Unable to retrieve the best model.\")\n    l2g_model.model = full_pipeline_model.stages[-1].bestModel  # type: ignore[assignment, unused-ignore]\n    return l2g_model\n</code></pre>"},{"location":"python_api/methods/l2g/trainer/#gentropy.method.l2g.trainer.LocusToGeneTrainer.train","title":"<code>train(data: L2GFeatureMatrix, l2g_model: LocusToGeneModel, features_list: list[str], evaluate: bool, wandb_run_name: str | None = None, model_path: str | None = None, **hyperparams: dict[str, Any]) -&gt; LocusToGeneModel</code>  <code>classmethod</code>","text":"<p>Train the Locus to Gene model.</p> <p>Parameters:</p> Name Type Description Default <code>data</code> <code>L2GFeatureMatrix</code> <p>Feature matrix containing the data</p> required <code>l2g_model</code> <code>LocusToGeneModel</code> <p>Model to fit to the data on</p> required <code>features_list</code> <code>list[str]</code> <p>List of features to use for the model</p> required <code>evaluate</code> <code>bool</code> <p>Whether to evaluate the model on a test set</p> required <code>wandb_run_name</code> <code>str | None</code> <p>Descriptive name for the run to be tracked with W&amp;B</p> <code>None</code> <code>model_path</code> <code>str | None</code> <p>Path to save the model to</p> <code>None</code> <code>**hyperparams</code> <code>dict[str, Any]</code> <p>Hyperparameters to use for the model</p> <code>{}</code> <p>Returns:</p> Name Type Description <code>LocusToGeneModel</code> <code>LocusToGeneModel</code> <p>Trained model</p> Source code in <code>src/gentropy/method/l2g/trainer.py</code> <pre><code>@classmethod\ndef train(\n    cls: type[LocusToGeneTrainer],\n    data: L2GFeatureMatrix,\n    l2g_model: LocusToGeneModel,\n    features_list: list[str],\n    evaluate: bool,\n    wandb_run_name: str | None = None,\n    model_path: str | None = None,\n    **hyperparams: dict[str, Any],\n) -&gt; LocusToGeneModel:\n    \"\"\"Train the Locus to Gene model.\n\n    Args:\n        data (L2GFeatureMatrix): Feature matrix containing the data\n        l2g_model (LocusToGeneModel): Model to fit to the data on\n        features_list (list[str]): List of features to use for the model\n        evaluate (bool): Whether to evaluate the model on a test set\n        wandb_run_name (str | None): Descriptive name for the run to be tracked with W&amp;B\n        model_path (str | None): Path to save the model to\n        **hyperparams (dict[str, Any]): Hyperparameters to use for the model\n\n    Returns:\n        LocusToGeneModel: Trained model\n    \"\"\"\n    train, test = data.select_features(features_list).train_test_split(fraction=0.8)\n\n    model = l2g_model.add_pipeline_stage(l2g_model.estimator).fit(train)\n\n    if evaluate:\n        l2g_model.evaluate(\n            results=model.predict(test),\n            hyperparameters=hyperparams,\n            wandb_run_name=wandb_run_name,\n            training_data=train,\n        )\n    if model_path:\n        l2g_model.save(model_path)\n    return l2g_model\n</code></pre>"},{"location":"python_api/steps/_steps/","title":"Step","text":"<p>This section provides description for the <code>Step</code> class. Each <code>Step</code> uses its own set of Methods and Datasets and implements the logic necessary to read a set of inputs, perform the transformation and write the outputs. All steps are available through the command line interface when running the <code>gentropy</code> command.</p>"},{"location":"python_api/steps/colocalisation/","title":"colocalisation","text":""},{"location":"python_api/steps/colocalisation/#gentropy.colocalisation.ColocalisationStep","title":"<code>gentropy.colocalisation.ColocalisationStep</code>","text":"<p>Colocalisation step.</p> <p>This workflow runs colocalisation analyses that assess the degree to which independent signals of the association share the same causal variant in a region of the genome, typically limited by linkage disequilibrium (LD).</p> Source code in <code>src/gentropy/colocalisation.py</code> <pre><code>class ColocalisationStep:\n    \"\"\"Colocalisation step.\n\n    This workflow runs colocalisation analyses that assess the degree to which independent signals of the association share the same causal variant in a region of the genome, typically limited by linkage disequilibrium (LD).\n    \"\"\"\n\n    def __init__(\n        self,\n        session: Session,\n        credible_set_path: str,\n        study_index_path: str,\n        coloc_path: str,\n    ) -&gt; None:\n        \"\"\"Run Colocalisation step.\n\n        Args:\n            session (Session): Session object.\n            credible_set_path (str): Input credible sets path.\n            study_index_path (str): Input study index path.\n            coloc_path (str): Output Colocalisation path.\n        \"\"\"\n        # Extract\n        credible_set = StudyLocus.from_parquet(\n            session, credible_set_path, recursiveFileLookup=True\n        )\n        si = StudyIndex.from_parquet(\n            session, study_index_path, recursiveFileLookup=True\n        )\n\n        # Transform\n        overlaps = credible_set.filter_credible_set(\n            CredibleInterval.IS95\n        ).find_overlaps(si)\n        ecaviar_results = ECaviar.colocalise(overlaps)\n\n        # Load\n        ecaviar_results.df.write.mode(session.write_mode).parquet(coloc_path)\n</code></pre>"},{"location":"python_api/steps/colocalisation/#gentropy.colocalisation.ColocalisationStep.__init__","title":"<code>__init__(session: Session, credible_set_path: str, study_index_path: str, coloc_path: str) -&gt; None</code>","text":"<p>Run Colocalisation step.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Session object.</p> required <code>credible_set_path</code> <code>str</code> <p>Input credible sets path.</p> required <code>study_index_path</code> <code>str</code> <p>Input study index path.</p> required <code>coloc_path</code> <code>str</code> <p>Output Colocalisation path.</p> required Source code in <code>src/gentropy/colocalisation.py</code> <pre><code>def __init__(\n    self,\n    session: Session,\n    credible_set_path: str,\n    study_index_path: str,\n    coloc_path: str,\n) -&gt; None:\n    \"\"\"Run Colocalisation step.\n\n    Args:\n        session (Session): Session object.\n        credible_set_path (str): Input credible sets path.\n        study_index_path (str): Input study index path.\n        coloc_path (str): Output Colocalisation path.\n    \"\"\"\n    # Extract\n    credible_set = StudyLocus.from_parquet(\n        session, credible_set_path, recursiveFileLookup=True\n    )\n    si = StudyIndex.from_parquet(\n        session, study_index_path, recursiveFileLookup=True\n    )\n\n    # Transform\n    overlaps = credible_set.filter_credible_set(\n        CredibleInterval.IS95\n    ).find_overlaps(si)\n    ecaviar_results = ECaviar.colocalise(overlaps)\n\n    # Load\n    ecaviar_results.df.write.mode(session.write_mode).parquet(coloc_path)\n</code></pre>"},{"location":"python_api/steps/eqtl_catalogue/","title":"eQTL Catalogue","text":""},{"location":"python_api/steps/eqtl_catalogue/#gentropy.eqtl_catalogue.EqtlCatalogueStep","title":"<code>gentropy.eqtl_catalogue.EqtlCatalogueStep</code>","text":"<p>eQTL Catalogue ingestion step.</p> <p>From SuSIE fine mapping results (available at their FTP ), we extract credible sets and study index datasets from gene expression QTL studies.</p> Source code in <code>src/gentropy/eqtl_catalogue.py</code> <pre><code>class EqtlCatalogueStep:\n    \"\"\"eQTL Catalogue ingestion step.\n\n    From SuSIE fine mapping results (available at [their FTP](https://ftp.ebi.ac.uk/pub/databases/spot/eQTL/susie/) ), we extract credible sets and study index datasets from gene expression QTL studies.\n    \"\"\"\n\n    def __init__(\n        self,\n        session: Session,\n        mqtl_quantification_methods_blacklist: list[str],\n        eqtl_catalogue_paths_imported: str,\n        eqtl_catalogue_study_index_out: str,\n        eqtl_catalogue_credible_sets_out: str,\n    ) -&gt; None:\n        \"\"\"Run eQTL Catalogue ingestion step.\n\n        Args:\n            session (Session): Session object.\n            mqtl_quantification_methods_blacklist (list[str]): Molecular trait quantification methods that we don't want to ingest. Available options in https://github.com/eQTL-Catalogue/eQTL-Catalogue-resources/blob/master/data_tables/dataset_metadata.tsv\n            eqtl_catalogue_paths_imported (str): Input eQTL Catalogue fine mapping results path.\n            eqtl_catalogue_study_index_out (str): Output eQTL Catalogue study index path.\n            eqtl_catalogue_credible_sets_out (str): Output eQTL Catalogue credible sets path.\n        \"\"\"\n        # Extract\n        studies_metadata = EqtlCatalogueStudyIndex.read_studies_from_source(\n            session, list(mqtl_quantification_methods_blacklist)\n        )\n\n        # Load raw data only for the studies we are interested in ingestion. This makes the proces much lighter.\n        studies_to_ingest = EqtlCatalogueStudyIndex.get_studies_of_interest(\n            studies_metadata\n        )\n        credible_sets_df = EqtlCatalogueFinemapping.read_credible_set_from_source(\n            session,\n            credible_set_path=[\n                f\"{eqtl_catalogue_paths_imported}/{qtd_id}.credible_sets.tsv\"\n                for qtd_id in studies_to_ingest\n            ],\n        )\n        lbf_df = EqtlCatalogueFinemapping.read_lbf_from_source(\n            session,\n            lbf_path=[\n                f\"{eqtl_catalogue_paths_imported}/{qtd_id}.lbf_variable.txt\"\n                for qtd_id in studies_to_ingest\n            ],\n        )\n\n        # Transform\n        processed_susie_df = EqtlCatalogueFinemapping.parse_susie_results(\n            credible_sets_df, lbf_df, studies_metadata\n        )\n        credible_sets = EqtlCatalogueFinemapping.from_susie_results(processed_susie_df)\n        study_index = EqtlCatalogueStudyIndex.from_susie_results(processed_susie_df)\n\n        # Load\n        study_index.df.write.mode(session.write_mode).parquet(\n            eqtl_catalogue_study_index_out\n        )\n        credible_sets.df.write.mode(session.write_mode).parquet(\n            eqtl_catalogue_credible_sets_out\n        )\n</code></pre>"},{"location":"python_api/steps/eqtl_catalogue/#gentropy.eqtl_catalogue.EqtlCatalogueStep.__init__","title":"<code>__init__(session: Session, mqtl_quantification_methods_blacklist: list[str], eqtl_catalogue_paths_imported: str, eqtl_catalogue_study_index_out: str, eqtl_catalogue_credible_sets_out: str) -&gt; None</code>","text":"<p>Run eQTL Catalogue ingestion step.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Session object.</p> required <code>mqtl_quantification_methods_blacklist</code> <code>list[str]</code> <p>Molecular trait quantification methods that we don't want to ingest. Available options in https://github.com/eQTL-Catalogue/eQTL-Catalogue-resources/blob/master/data_tables/dataset_metadata.tsv</p> required <code>eqtl_catalogue_paths_imported</code> <code>str</code> <p>Input eQTL Catalogue fine mapping results path.</p> required <code>eqtl_catalogue_study_index_out</code> <code>str</code> <p>Output eQTL Catalogue study index path.</p> required <code>eqtl_catalogue_credible_sets_out</code> <code>str</code> <p>Output eQTL Catalogue credible sets path.</p> required Source code in <code>src/gentropy/eqtl_catalogue.py</code> <pre><code>def __init__(\n    self,\n    session: Session,\n    mqtl_quantification_methods_blacklist: list[str],\n    eqtl_catalogue_paths_imported: str,\n    eqtl_catalogue_study_index_out: str,\n    eqtl_catalogue_credible_sets_out: str,\n) -&gt; None:\n    \"\"\"Run eQTL Catalogue ingestion step.\n\n    Args:\n        session (Session): Session object.\n        mqtl_quantification_methods_blacklist (list[str]): Molecular trait quantification methods that we don't want to ingest. Available options in https://github.com/eQTL-Catalogue/eQTL-Catalogue-resources/blob/master/data_tables/dataset_metadata.tsv\n        eqtl_catalogue_paths_imported (str): Input eQTL Catalogue fine mapping results path.\n        eqtl_catalogue_study_index_out (str): Output eQTL Catalogue study index path.\n        eqtl_catalogue_credible_sets_out (str): Output eQTL Catalogue credible sets path.\n    \"\"\"\n    # Extract\n    studies_metadata = EqtlCatalogueStudyIndex.read_studies_from_source(\n        session, list(mqtl_quantification_methods_blacklist)\n    )\n\n    # Load raw data only for the studies we are interested in ingestion. This makes the proces much lighter.\n    studies_to_ingest = EqtlCatalogueStudyIndex.get_studies_of_interest(\n        studies_metadata\n    )\n    credible_sets_df = EqtlCatalogueFinemapping.read_credible_set_from_source(\n        session,\n        credible_set_path=[\n            f\"{eqtl_catalogue_paths_imported}/{qtd_id}.credible_sets.tsv\"\n            for qtd_id in studies_to_ingest\n        ],\n    )\n    lbf_df = EqtlCatalogueFinemapping.read_lbf_from_source(\n        session,\n        lbf_path=[\n            f\"{eqtl_catalogue_paths_imported}/{qtd_id}.lbf_variable.txt\"\n            for qtd_id in studies_to_ingest\n        ],\n    )\n\n    # Transform\n    processed_susie_df = EqtlCatalogueFinemapping.parse_susie_results(\n        credible_sets_df, lbf_df, studies_metadata\n    )\n    credible_sets = EqtlCatalogueFinemapping.from_susie_results(processed_susie_df)\n    study_index = EqtlCatalogueStudyIndex.from_susie_results(processed_susie_df)\n\n    # Load\n    study_index.df.write.mode(session.write_mode).parquet(\n        eqtl_catalogue_study_index_out\n    )\n    credible_sets.df.write.mode(session.write_mode).parquet(\n        eqtl_catalogue_credible_sets_out\n    )\n</code></pre>"},{"location":"python_api/steps/finngen_studies/","title":"finngen_studies","text":""},{"location":"python_api/steps/finngen_studies/#gentropy.finngen_studies.FinnGenStudiesStep","title":"<code>gentropy.finngen_studies.FinnGenStudiesStep</code>","text":"<p>FinnGen study index generation step.</p> Source code in <code>src/gentropy/finngen_studies.py</code> <pre><code>class FinnGenStudiesStep:\n    \"\"\"FinnGen study index generation step.\"\"\"\n\n    def __init__(self, session: Session, finngen_study_index_out: str) -&gt; None:\n        \"\"\"Run FinnGen study index generation step.\n\n        Args:\n            session (Session): Session object.\n            finngen_study_index_out (str): Output FinnGen study index path.\n        \"\"\"\n        # Fetch study index.\n        FinnGenStudyIndex.from_source(session.spark).df.write.mode(\n            session.write_mode\n        ).parquet(finngen_study_index_out)\n</code></pre>"},{"location":"python_api/steps/finngen_studies/#gentropy.finngen_studies.FinnGenStudiesStep.__init__","title":"<code>__init__(session: Session, finngen_study_index_out: str) -&gt; None</code>","text":"<p>Run FinnGen study index generation step.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Session object.</p> required <code>finngen_study_index_out</code> <code>str</code> <p>Output FinnGen study index path.</p> required Source code in <code>src/gentropy/finngen_studies.py</code> <pre><code>def __init__(self, session: Session, finngen_study_index_out: str) -&gt; None:\n    \"\"\"Run FinnGen study index generation step.\n\n    Args:\n        session (Session): Session object.\n        finngen_study_index_out (str): Output FinnGen study index path.\n    \"\"\"\n    # Fetch study index.\n    FinnGenStudyIndex.from_source(session.spark).df.write.mode(\n        session.write_mode\n    ).parquet(finngen_study_index_out)\n</code></pre>"},{"location":"python_api/steps/finngen_sumstat_preprocess/","title":"finngen_sumstat_preprocess","text":""},{"location":"python_api/steps/finngen_sumstat_preprocess/#gentropy.finngen_sumstat_preprocess.FinnGenSumstatPreprocessStep","title":"<code>gentropy.finngen_sumstat_preprocess.FinnGenSumstatPreprocessStep</code>","text":"<p>FinnGen sumstats preprocessing.</p> Source code in <code>src/gentropy/finngen_sumstat_preprocess.py</code> <pre><code>class FinnGenSumstatPreprocessStep:\n    \"\"\"FinnGen sumstats preprocessing.\"\"\"\n\n    def __init__(\n        self, session: Session, raw_sumstats_path: str, out_sumstats_path: str\n    ) -&gt; None:\n        \"\"\"Run FinnGen summary stats preprocessing step.\n\n        Args:\n            session (Session): Session object.\n            raw_sumstats_path (str): Input raw summary stats path.\n            out_sumstats_path (str): Output summary stats path.\n        \"\"\"\n        # Process summary stats.\n        (\n            FinnGenSummaryStats.from_source(session.spark, raw_file=raw_sumstats_path)\n            .df.write.mode(session.write_mode)\n            .parquet(out_sumstats_path)\n        )\n</code></pre>"},{"location":"python_api/steps/finngen_sumstat_preprocess/#gentropy.finngen_sumstat_preprocess.FinnGenSumstatPreprocessStep.__init__","title":"<code>__init__(session: Session, raw_sumstats_path: str, out_sumstats_path: str) -&gt; None</code>","text":"<p>Run FinnGen summary stats preprocessing step.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Session object.</p> required <code>raw_sumstats_path</code> <code>str</code> <p>Input raw summary stats path.</p> required <code>out_sumstats_path</code> <code>str</code> <p>Output summary stats path.</p> required Source code in <code>src/gentropy/finngen_sumstat_preprocess.py</code> <pre><code>def __init__(\n    self, session: Session, raw_sumstats_path: str, out_sumstats_path: str\n) -&gt; None:\n    \"\"\"Run FinnGen summary stats preprocessing step.\n\n    Args:\n        session (Session): Session object.\n        raw_sumstats_path (str): Input raw summary stats path.\n        out_sumstats_path (str): Output summary stats path.\n    \"\"\"\n    # Process summary stats.\n    (\n        FinnGenSummaryStats.from_source(session.spark, raw_file=raw_sumstats_path)\n        .df.write.mode(session.write_mode)\n        .parquet(out_sumstats_path)\n    )\n</code></pre>"},{"location":"python_api/steps/gene_index/","title":"gene_index","text":""},{"location":"python_api/steps/gene_index/#gentropy.gene_index.GeneIndexStep","title":"<code>gentropy.gene_index.GeneIndexStep</code>","text":"<p>Gene index step.</p> <p>This step generates a gene index dataset from an Open Targets Platform target dataset.</p> Source code in <code>src/gentropy/gene_index.py</code> <pre><code>class GeneIndexStep:\n    \"\"\"Gene index step.\n\n    This step generates a gene index dataset from an Open Targets Platform target dataset.\n    \"\"\"\n\n    def __init__(\n        self,\n        session: Session,\n        target_path: str,\n        gene_index_path: str,\n    ) -&gt; None:\n        \"\"\"Initialize step.\n\n        Args:\n            session (Session): Session object.\n            target_path (str): Input Open Targets Platform target dataset path.\n            gene_index_path (str): Output gene index dataset path.\n        \"\"\"\n        platform_target = session.spark.read.parquet(target_path)\n        # Transform\n        gene_index = OpenTargetsTarget.as_gene_index(platform_target)\n        # Load\n        gene_index.df.write.mode(session.write_mode).parquet(gene_index_path)\n</code></pre>"},{"location":"python_api/steps/gene_index/#gentropy.gene_index.GeneIndexStep.__init__","title":"<code>__init__(session: Session, target_path: str, gene_index_path: str) -&gt; None</code>","text":"<p>Initialize step.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Session object.</p> required <code>target_path</code> <code>str</code> <p>Input Open Targets Platform target dataset path.</p> required <code>gene_index_path</code> <code>str</code> <p>Output gene index dataset path.</p> required Source code in <code>src/gentropy/gene_index.py</code> <pre><code>def __init__(\n    self,\n    session: Session,\n    target_path: str,\n    gene_index_path: str,\n) -&gt; None:\n    \"\"\"Initialize step.\n\n    Args:\n        session (Session): Session object.\n        target_path (str): Input Open Targets Platform target dataset path.\n        gene_index_path (str): Output gene index dataset path.\n    \"\"\"\n    platform_target = session.spark.read.parquet(target_path)\n    # Transform\n    gene_index = OpenTargetsTarget.as_gene_index(platform_target)\n    # Load\n    gene_index.df.write.mode(session.write_mode).parquet(gene_index_path)\n</code></pre>"},{"location":"python_api/steps/gwas_catalog_curation/","title":"gwas_catalog_study_curation","text":""},{"location":"python_api/steps/gwas_catalog_curation/#gentropy.gwas_catalog_study_curation.GWASCatalogStudyCurationStep","title":"<code>gentropy.gwas_catalog_study_curation.GWASCatalogStudyCurationStep</code>","text":"<p>Annotate GWAS Catalog studies with additional curation and create a curation backlog.</p> Source code in <code>src/gentropy/gwas_catalog_study_curation.py</code> <pre><code>class GWASCatalogStudyCurationStep:\n    \"\"\"Annotate GWAS Catalog studies with additional curation and create a curation backlog.\"\"\"\n\n    def __init__(\n        self,\n        session: Session,\n        catalog_study_files: list[str],\n        catalog_ancestry_files: list[str],\n        catalog_sumstats_lut: str,\n        gwas_catalog_study_curation_out: str,\n        gwas_catalog_study_curation_file: str | None,\n    ) -&gt; None:\n        \"\"\"Run step to annotate and create backlog.\n\n        Args:\n            session (Session): Session object.\n            catalog_study_files (list[str]): List of raw GWAS catalog studies file.\n            catalog_ancestry_files (list[str]): List of raw ancestry annotations files from GWAS Catalog.\n            catalog_sumstats_lut (str): GWAS Catalog summary statistics lookup table.\n            gwas_catalog_study_curation_out (str): Path for the updated curation table.\n            gwas_catalog_study_curation_file (str | None): Path to the original curation table. Optinal\n        \"\"\"\n        catalog_studies = session.spark.read.csv(\n            list(catalog_study_files), sep=\"\\t\", header=True\n        )\n        ancestry_lut = session.spark.read.csv(\n            list(catalog_ancestry_files), sep=\"\\t\", header=True\n        )\n        sumstats_lut = session.spark.read.csv(\n            catalog_sumstats_lut, sep=\"\\t\", header=False\n        )\n        gwas_catalog_study_curation = read_curation_table(\n            gwas_catalog_study_curation_file, session\n        )\n\n        # Process GWAS Catalog studies and get list of studies for curation:\n        (\n            StudyIndexGWASCatalogParser.from_source(\n                catalog_studies, ancestry_lut, sumstats_lut\n            )\n            # Adding existing curation:\n            .annotate_from_study_curation(gwas_catalog_study_curation)\n            # Extract new studies for curation:\n            .extract_studies_for_curation(gwas_catalog_study_curation)\n            # Save table:\n            .toPandas()\n            .to_csv(gwas_catalog_study_curation_out, sep=\"\\t\", index=False)\n        )\n</code></pre>"},{"location":"python_api/steps/gwas_catalog_curation/#gentropy.gwas_catalog_study_curation.GWASCatalogStudyCurationStep.__init__","title":"<code>__init__(session: Session, catalog_study_files: list[str], catalog_ancestry_files: list[str], catalog_sumstats_lut: str, gwas_catalog_study_curation_out: str, gwas_catalog_study_curation_file: str | None) -&gt; None</code>","text":"<p>Run step to annotate and create backlog.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Session object.</p> required <code>catalog_study_files</code> <code>list[str]</code> <p>List of raw GWAS catalog studies file.</p> required <code>catalog_ancestry_files</code> <code>list[str]</code> <p>List of raw ancestry annotations files from GWAS Catalog.</p> required <code>catalog_sumstats_lut</code> <code>str</code> <p>GWAS Catalog summary statistics lookup table.</p> required <code>gwas_catalog_study_curation_out</code> <code>str</code> <p>Path for the updated curation table.</p> required <code>gwas_catalog_study_curation_file</code> <code>str | None</code> <p>Path to the original curation table. Optinal</p> required Source code in <code>src/gentropy/gwas_catalog_study_curation.py</code> <pre><code>def __init__(\n    self,\n    session: Session,\n    catalog_study_files: list[str],\n    catalog_ancestry_files: list[str],\n    catalog_sumstats_lut: str,\n    gwas_catalog_study_curation_out: str,\n    gwas_catalog_study_curation_file: str | None,\n) -&gt; None:\n    \"\"\"Run step to annotate and create backlog.\n\n    Args:\n        session (Session): Session object.\n        catalog_study_files (list[str]): List of raw GWAS catalog studies file.\n        catalog_ancestry_files (list[str]): List of raw ancestry annotations files from GWAS Catalog.\n        catalog_sumstats_lut (str): GWAS Catalog summary statistics lookup table.\n        gwas_catalog_study_curation_out (str): Path for the updated curation table.\n        gwas_catalog_study_curation_file (str | None): Path to the original curation table. Optinal\n    \"\"\"\n    catalog_studies = session.spark.read.csv(\n        list(catalog_study_files), sep=\"\\t\", header=True\n    )\n    ancestry_lut = session.spark.read.csv(\n        list(catalog_ancestry_files), sep=\"\\t\", header=True\n    )\n    sumstats_lut = session.spark.read.csv(\n        catalog_sumstats_lut, sep=\"\\t\", header=False\n    )\n    gwas_catalog_study_curation = read_curation_table(\n        gwas_catalog_study_curation_file, session\n    )\n\n    # Process GWAS Catalog studies and get list of studies for curation:\n    (\n        StudyIndexGWASCatalogParser.from_source(\n            catalog_studies, ancestry_lut, sumstats_lut\n        )\n        # Adding existing curation:\n        .annotate_from_study_curation(gwas_catalog_study_curation)\n        # Extract new studies for curation:\n        .extract_studies_for_curation(gwas_catalog_study_curation)\n        # Save table:\n        .toPandas()\n        .to_csv(gwas_catalog_study_curation_out, sep=\"\\t\", index=False)\n    )\n</code></pre>"},{"location":"python_api/steps/gwas_catalog_inclusion/","title":"gwas_catalog_study_inclusion","text":""},{"location":"python_api/steps/gwas_catalog_inclusion/#gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator","title":"<code>gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator</code>","text":"<p>GWAS Catalog study eligibility for ingestion based on curation and the provided criteria.</p> Source code in <code>src/gentropy/gwas_catalog_study_inclusion.py</code> <pre><code>class GWASCatalogStudyInclusionGenerator:\n    \"\"\"GWAS Catalog study eligibility for ingestion based on curation and the provided criteria.\"\"\"\n\n    @staticmethod\n    def flag_eligible_studies(\n        study_index: StudyIndexGWASCatalog, criteria: str\n    ) -&gt; DataFrame:\n        \"\"\"Apply filter on GWAS Catalog studies based on the provided criteria.\n\n        Args:\n            study_index (StudyIndexGWASCatalog): complete study index to be filtered based on the provided filter set\n            criteria (str): name of the filter set to be applied.\n\n        Raises:\n            ValueError: if the provided filter set is not in the accepted values.\n\n        Returns:\n            DataFrame: filtered dataframe containing only eligible studies.\n        \"\"\"\n        filters: dict[str, Column] = {\n            # Filters applied on studies for ingesting curated associations:\n            \"curation\": (study_index.is_gwas() &amp; study_index.has_mapped_trait()),\n            # Filters applied on studies for ingesting summary statistics:\n            \"summary_stats\": (\n                study_index.is_gwas()\n                &amp; study_index.has_mapped_trait()\n                &amp; (~study_index.is_quality_flagged())\n                &amp; study_index.has_summarystats()\n            ),\n        }\n\n        if criteria not in filters:\n            raise ValueError(\n                f'Wrong value as filter set ({criteria}). Accepted: {\",\".join(filters.keys())}'\n            )\n\n        # Applying the relevant filter to the study:\n        return study_index.df.select(\n            \"studyId\",\n            \"studyType\",\n            \"traitFromSource\",\n            \"traitFromSourceMappedIds\",\n            \"qualityControls\",\n            \"hasSumstats\",\n            filters[criteria].alias(\"isEligible\"),\n        )\n\n    @staticmethod\n    def process_harmonised_list(studies: list[str], session: Session) -&gt; DataFrame:\n        \"\"\"Generate spark dataframe from the provided list.\n\n        Args:\n            studies (list[str]): list of path pointing to harmonised summary statistics.\n            session (Session): session\n\n        Returns:\n            DataFrame: column name is consistent with original implementatin\n        \"\"\"\n        return session.spark.createDataFrame([{\"_c0\": path} for path in studies])\n\n    @staticmethod\n    def get_gwas_catalog_study_index(\n        session: Session,\n        variant_annotation_path: str,\n        catalog_study_files: list[str],\n        catalog_ancestry_files: list[str],\n        harmonised_study_file: str,\n        catalog_associations_file: str,\n        gwas_catalog_study_curation_file: str,\n    ) -&gt; StudyIndexGWASCatalog:\n        \"\"\"Return GWAS Catalog study index.\n\n        Args:\n            session (Session): Session object.\n            variant_annotation_path (str): Input variant annotation path.\n            catalog_study_files (list[str]): List of raw GWAS catalog studies file.\n            catalog_ancestry_files (list[str]): List of raw ancestry annotations files from GWAS Catalog.\n            harmonised_study_file (str): GWAS Catalog summary statistics lookup table.\n            catalog_associations_file (str): Raw GWAS catalog associations file.\n            gwas_catalog_study_curation_file (str): file of the curation table. Optional.\n\n        Returns:\n            StudyIndexGWASCatalog: Completely processed and fully annotated study index.\n        \"\"\"\n        # Extract\n        va = VariantAnnotation.from_parquet(session, variant_annotation_path)\n        catalog_studies = session.spark.read.csv(\n            list(catalog_study_files), sep=\"\\t\", header=True\n        )\n        ancestry_lut = session.spark.read.csv(\n            list(catalog_ancestry_files), sep=\"\\t\", header=True\n        )\n        sumstats_lut = session.spark.read.csv(\n            harmonised_study_file, sep=\"\\t\", header=False\n        )\n        catalog_associations = session.spark.read.csv(\n            catalog_associations_file, sep=\"\\t\", header=True\n        ).persist()\n        gwas_catalog_study_curation = read_curation_table(\n            gwas_catalog_study_curation_file, session\n        )\n\n        # Transform\n        study_index, _ = GWASCatalogStudySplitter.split(\n            StudyIndexGWASCatalogParser.from_source(\n                catalog_studies,\n                ancestry_lut,\n                sumstats_lut,\n            ).annotate_from_study_curation(gwas_catalog_study_curation),\n            GWASCatalogCuratedAssociationsParser.from_source(catalog_associations, va),\n        )\n\n        return study_index\n\n    def __init__(\n        self,\n        session: Session,\n        catalog_study_files: list[str],\n        catalog_ancestry_files: list[str],\n        catalog_associations_file: str,\n        gwas_catalog_study_curation_file: str,\n        variant_annotation_path: str,\n        harmonised_study_file: str,\n        criteria: str,\n        inclusion_list_path: str,\n        exclusion_list_path: str,\n    ) -&gt; None:\n        \"\"\"Run step.\n\n        Args:\n            session (Session): Session object.\n            catalog_study_files (list[str]): List of raw GWAS catalog studies file.\n            catalog_ancestry_files (list[str]): List of raw ancestry annotations files from GWAS Catalog.\n            catalog_associations_file (str): Raw GWAS catalog associations file.\n            gwas_catalog_study_curation_file (str): file of the curation table. Optional.\n            variant_annotation_path (str): Input variant annotation path.\n            harmonised_study_file (str): GWAS Catalog summary statistics lookup table.\n            criteria (str): name of the filter set to be applied.\n            inclusion_list_path (str): Output path for the inclusion list.\n            exclusion_list_path (str): Output path for the exclusion list.\n        \"\"\"\n        # Create study index:\n        study_index = self.get_gwas_catalog_study_index(\n            session,\n            variant_annotation_path,\n            catalog_study_files,\n            catalog_ancestry_files,\n            harmonised_study_file,\n            catalog_associations_file,\n            gwas_catalog_study_curation_file,\n        )\n\n        # Get study indices for inclusion:\n        flagged_studies = self.flag_eligible_studies(study_index, criteria)\n\n        # Output inclusion list:\n        eligible = (\n            flagged_studies.filter(f.col(\"isEligible\")).select(\"studyId\").persist()\n        )\n        eligible.write.mode(session.write_mode).parquet(inclusion_list_path)\n\n        # Output exclusion list:\n        excluded = flagged_studies.filter(~f.col(\"isEligible\")).persist()\n        excluded.write.mode(session.write_mode).parquet(exclusion_list_path)\n</code></pre>"},{"location":"python_api/steps/gwas_catalog_inclusion/#gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator.__init__","title":"<code>__init__(session: Session, catalog_study_files: list[str], catalog_ancestry_files: list[str], catalog_associations_file: str, gwas_catalog_study_curation_file: str, variant_annotation_path: str, harmonised_study_file: str, criteria: str, inclusion_list_path: str, exclusion_list_path: str) -&gt; None</code>","text":"<p>Run step.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Session object.</p> required <code>catalog_study_files</code> <code>list[str]</code> <p>List of raw GWAS catalog studies file.</p> required <code>catalog_ancestry_files</code> <code>list[str]</code> <p>List of raw ancestry annotations files from GWAS Catalog.</p> required <code>catalog_associations_file</code> <code>str</code> <p>Raw GWAS catalog associations file.</p> required <code>gwas_catalog_study_curation_file</code> <code>str</code> <p>file of the curation table. Optional.</p> required <code>variant_annotation_path</code> <code>str</code> <p>Input variant annotation path.</p> required <code>harmonised_study_file</code> <code>str</code> <p>GWAS Catalog summary statistics lookup table.</p> required <code>criteria</code> <code>str</code> <p>name of the filter set to be applied.</p> required <code>inclusion_list_path</code> <code>str</code> <p>Output path for the inclusion list.</p> required <code>exclusion_list_path</code> <code>str</code> <p>Output path for the exclusion list.</p> required Source code in <code>src/gentropy/gwas_catalog_study_inclusion.py</code> <pre><code>def __init__(\n    self,\n    session: Session,\n    catalog_study_files: list[str],\n    catalog_ancestry_files: list[str],\n    catalog_associations_file: str,\n    gwas_catalog_study_curation_file: str,\n    variant_annotation_path: str,\n    harmonised_study_file: str,\n    criteria: str,\n    inclusion_list_path: str,\n    exclusion_list_path: str,\n) -&gt; None:\n    \"\"\"Run step.\n\n    Args:\n        session (Session): Session object.\n        catalog_study_files (list[str]): List of raw GWAS catalog studies file.\n        catalog_ancestry_files (list[str]): List of raw ancestry annotations files from GWAS Catalog.\n        catalog_associations_file (str): Raw GWAS catalog associations file.\n        gwas_catalog_study_curation_file (str): file of the curation table. Optional.\n        variant_annotation_path (str): Input variant annotation path.\n        harmonised_study_file (str): GWAS Catalog summary statistics lookup table.\n        criteria (str): name of the filter set to be applied.\n        inclusion_list_path (str): Output path for the inclusion list.\n        exclusion_list_path (str): Output path for the exclusion list.\n    \"\"\"\n    # Create study index:\n    study_index = self.get_gwas_catalog_study_index(\n        session,\n        variant_annotation_path,\n        catalog_study_files,\n        catalog_ancestry_files,\n        harmonised_study_file,\n        catalog_associations_file,\n        gwas_catalog_study_curation_file,\n    )\n\n    # Get study indices for inclusion:\n    flagged_studies = self.flag_eligible_studies(study_index, criteria)\n\n    # Output inclusion list:\n    eligible = (\n        flagged_studies.filter(f.col(\"isEligible\")).select(\"studyId\").persist()\n    )\n    eligible.write.mode(session.write_mode).parquet(inclusion_list_path)\n\n    # Output exclusion list:\n    excluded = flagged_studies.filter(~f.col(\"isEligible\")).persist()\n    excluded.write.mode(session.write_mode).parquet(exclusion_list_path)\n</code></pre>"},{"location":"python_api/steps/gwas_catalog_inclusion/#gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator.flag_eligible_studies","title":"<code>flag_eligible_studies(study_index: StudyIndexGWASCatalog, criteria: str) -&gt; DataFrame</code>  <code>staticmethod</code>","text":"<p>Apply filter on GWAS Catalog studies based on the provided criteria.</p> <p>Parameters:</p> Name Type Description Default <code>study_index</code> <code>StudyIndexGWASCatalog</code> <p>complete study index to be filtered based on the provided filter set</p> required <code>criteria</code> <code>str</code> <p>name of the filter set to be applied.</p> required <p>Raises:</p> Type Description <code>ValueError</code> <p>if the provided filter set is not in the accepted values.</p> <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>filtered dataframe containing only eligible studies.</p> Source code in <code>src/gentropy/gwas_catalog_study_inclusion.py</code> <pre><code>@staticmethod\ndef flag_eligible_studies(\n    study_index: StudyIndexGWASCatalog, criteria: str\n) -&gt; DataFrame:\n    \"\"\"Apply filter on GWAS Catalog studies based on the provided criteria.\n\n    Args:\n        study_index (StudyIndexGWASCatalog): complete study index to be filtered based on the provided filter set\n        criteria (str): name of the filter set to be applied.\n\n    Raises:\n        ValueError: if the provided filter set is not in the accepted values.\n\n    Returns:\n        DataFrame: filtered dataframe containing only eligible studies.\n    \"\"\"\n    filters: dict[str, Column] = {\n        # Filters applied on studies for ingesting curated associations:\n        \"curation\": (study_index.is_gwas() &amp; study_index.has_mapped_trait()),\n        # Filters applied on studies for ingesting summary statistics:\n        \"summary_stats\": (\n            study_index.is_gwas()\n            &amp; study_index.has_mapped_trait()\n            &amp; (~study_index.is_quality_flagged())\n            &amp; study_index.has_summarystats()\n        ),\n    }\n\n    if criteria not in filters:\n        raise ValueError(\n            f'Wrong value as filter set ({criteria}). Accepted: {\",\".join(filters.keys())}'\n        )\n\n    # Applying the relevant filter to the study:\n    return study_index.df.select(\n        \"studyId\",\n        \"studyType\",\n        \"traitFromSource\",\n        \"traitFromSourceMappedIds\",\n        \"qualityControls\",\n        \"hasSumstats\",\n        filters[criteria].alias(\"isEligible\"),\n    )\n</code></pre>"},{"location":"python_api/steps/gwas_catalog_inclusion/#gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator.get_gwas_catalog_study_index","title":"<code>get_gwas_catalog_study_index(session: Session, variant_annotation_path: str, catalog_study_files: list[str], catalog_ancestry_files: list[str], harmonised_study_file: str, catalog_associations_file: str, gwas_catalog_study_curation_file: str) -&gt; StudyIndexGWASCatalog</code>  <code>staticmethod</code>","text":"<p>Return GWAS Catalog study index.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Session object.</p> required <code>variant_annotation_path</code> <code>str</code> <p>Input variant annotation path.</p> required <code>catalog_study_files</code> <code>list[str]</code> <p>List of raw GWAS catalog studies file.</p> required <code>catalog_ancestry_files</code> <code>list[str]</code> <p>List of raw ancestry annotations files from GWAS Catalog.</p> required <code>harmonised_study_file</code> <code>str</code> <p>GWAS Catalog summary statistics lookup table.</p> required <code>catalog_associations_file</code> <code>str</code> <p>Raw GWAS catalog associations file.</p> required <code>gwas_catalog_study_curation_file</code> <code>str</code> <p>file of the curation table. Optional.</p> required <p>Returns:</p> Name Type Description <code>StudyIndexGWASCatalog</code> <code>StudyIndexGWASCatalog</code> <p>Completely processed and fully annotated study index.</p> Source code in <code>src/gentropy/gwas_catalog_study_inclusion.py</code> <pre><code>@staticmethod\ndef get_gwas_catalog_study_index(\n    session: Session,\n    variant_annotation_path: str,\n    catalog_study_files: list[str],\n    catalog_ancestry_files: list[str],\n    harmonised_study_file: str,\n    catalog_associations_file: str,\n    gwas_catalog_study_curation_file: str,\n) -&gt; StudyIndexGWASCatalog:\n    \"\"\"Return GWAS Catalog study index.\n\n    Args:\n        session (Session): Session object.\n        variant_annotation_path (str): Input variant annotation path.\n        catalog_study_files (list[str]): List of raw GWAS catalog studies file.\n        catalog_ancestry_files (list[str]): List of raw ancestry annotations files from GWAS Catalog.\n        harmonised_study_file (str): GWAS Catalog summary statistics lookup table.\n        catalog_associations_file (str): Raw GWAS catalog associations file.\n        gwas_catalog_study_curation_file (str): file of the curation table. Optional.\n\n    Returns:\n        StudyIndexGWASCatalog: Completely processed and fully annotated study index.\n    \"\"\"\n    # Extract\n    va = VariantAnnotation.from_parquet(session, variant_annotation_path)\n    catalog_studies = session.spark.read.csv(\n        list(catalog_study_files), sep=\"\\t\", header=True\n    )\n    ancestry_lut = session.spark.read.csv(\n        list(catalog_ancestry_files), sep=\"\\t\", header=True\n    )\n    sumstats_lut = session.spark.read.csv(\n        harmonised_study_file, sep=\"\\t\", header=False\n    )\n    catalog_associations = session.spark.read.csv(\n        catalog_associations_file, sep=\"\\t\", header=True\n    ).persist()\n    gwas_catalog_study_curation = read_curation_table(\n        gwas_catalog_study_curation_file, session\n    )\n\n    # Transform\n    study_index, _ = GWASCatalogStudySplitter.split(\n        StudyIndexGWASCatalogParser.from_source(\n            catalog_studies,\n            ancestry_lut,\n            sumstats_lut,\n        ).annotate_from_study_curation(gwas_catalog_study_curation),\n        GWASCatalogCuratedAssociationsParser.from_source(catalog_associations, va),\n    )\n\n    return study_index\n</code></pre>"},{"location":"python_api/steps/gwas_catalog_inclusion/#gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator.process_harmonised_list","title":"<code>process_harmonised_list(studies: list[str], session: Session) -&gt; DataFrame</code>  <code>staticmethod</code>","text":"<p>Generate spark dataframe from the provided list.</p> <p>Parameters:</p> Name Type Description Default <code>studies</code> <code>list[str]</code> <p>list of path pointing to harmonised summary statistics.</p> required <code>session</code> <code>Session</code> <p>session</p> required <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>column name is consistent with original implementatin</p> Source code in <code>src/gentropy/gwas_catalog_study_inclusion.py</code> <pre><code>@staticmethod\ndef process_harmonised_list(studies: list[str], session: Session) -&gt; DataFrame:\n    \"\"\"Generate spark dataframe from the provided list.\n\n    Args:\n        studies (list[str]): list of path pointing to harmonised summary statistics.\n        session (Session): session\n\n    Returns:\n        DataFrame: column name is consistent with original implementatin\n    \"\"\"\n    return session.spark.createDataFrame([{\"_c0\": path} for path in studies])\n</code></pre>"},{"location":"python_api/steps/gwas_catalog_ingestion/","title":"gwas_catalog_ingestion","text":""},{"location":"python_api/steps/gwas_catalog_ingestion/#gentropy.gwas_catalog_ingestion.GWASCatalogIngestionStep","title":"<code>gentropy.gwas_catalog_ingestion.GWASCatalogIngestionStep</code>","text":"<p>GWAS Catalog ingestion step to extract GWASCatalog Study and StudyLocus tables.</p> <p>!!! note This step currently only processes the GWAS Catalog curated list of top hits.</p> Source code in <code>src/gentropy/gwas_catalog_ingestion.py</code> <pre><code>class GWASCatalogIngestionStep:\n    \"\"\"GWAS Catalog ingestion step to extract GWASCatalog Study and StudyLocus tables.\n\n    !!! note This step currently only processes the GWAS Catalog curated list of top hits.\n    \"\"\"\n\n    def __init__(\n        self,\n        session: Session,\n        catalog_study_files: list[str],\n        catalog_ancestry_files: list[str],\n        catalog_sumstats_lut: str,\n        catalog_associations_file: str,\n        variant_annotation_path: str,\n        catalog_studies_out: str,\n        catalog_associations_out: str,\n        gwas_catalog_study_curation_file: str | None = None,\n        inclusion_list_path: str | None = None,\n    ) -&gt; None:\n        \"\"\"Run step.\n\n        Args:\n            session (Session): Session object.\n            catalog_study_files (list[str]): List of raw GWAS catalog studies file.\n            catalog_ancestry_files (list[str]): List of raw ancestry annotations files from GWAS Catalog.\n            catalog_sumstats_lut (str): GWAS Catalog summary statistics lookup table.\n            catalog_associations_file (str): Raw GWAS catalog associations file.\n            variant_annotation_path (str): Input variant annotation path.\n            catalog_studies_out (str): Output GWAS catalog studies path.\n            catalog_associations_out (str): Output GWAS catalog associations path.\n            gwas_catalog_study_curation_file (str | None): file of the curation table. Optional.\n            inclusion_list_path (str | None): optional inclusion list (parquet)\n        \"\"\"\n        # Extract\n        va = VariantAnnotation.from_parquet(session, variant_annotation_path)\n        catalog_studies = session.spark.read.csv(\n            list(catalog_study_files), sep=\"\\t\", header=True\n        )\n        ancestry_lut = session.spark.read.csv(\n            list(catalog_ancestry_files), sep=\"\\t\", header=True\n        )\n        sumstats_lut = session.spark.read.csv(\n            catalog_sumstats_lut, sep=\"\\t\", header=False\n        )\n        catalog_associations = session.spark.read.csv(\n            catalog_associations_file, sep=\"\\t\", header=True\n        ).persist()\n        gwas_catalog_study_curation = read_curation_table(\n            gwas_catalog_study_curation_file, session\n        )\n\n        # Transform\n        study_index, study_locus = GWASCatalogStudySplitter.split(\n            StudyIndexGWASCatalogParser.from_source(\n                catalog_studies, ancestry_lut, sumstats_lut\n            ).annotate_from_study_curation(gwas_catalog_study_curation),\n            GWASCatalogCuratedAssociationsParser.from_source(catalog_associations, va),\n        )\n\n        # if inclusion list is provided apply filter:\n        if inclusion_list_path:\n            inclusion_list = session.spark.read.parquet(\n                inclusion_list_path, sep=\"\\t\", header=True\n            )\n\n            study_index = study_index.apply_inclusion_list(inclusion_list)\n            study_locus = study_locus.apply_inclusion_list(inclusion_list)\n\n        # Load\n        study_index.df.write.mode(session.write_mode).parquet(catalog_studies_out)\n        study_locus.df.write.mode(session.write_mode).parquet(catalog_associations_out)\n</code></pre>"},{"location":"python_api/steps/gwas_catalog_ingestion/#gentropy.gwas_catalog_ingestion.GWASCatalogIngestionStep.__init__","title":"<code>__init__(session: Session, catalog_study_files: list[str], catalog_ancestry_files: list[str], catalog_sumstats_lut: str, catalog_associations_file: str, variant_annotation_path: str, catalog_studies_out: str, catalog_associations_out: str, gwas_catalog_study_curation_file: str | None = None, inclusion_list_path: str | None = None) -&gt; None</code>","text":"<p>Run step.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Session object.</p> required <code>catalog_study_files</code> <code>list[str]</code> <p>List of raw GWAS catalog studies file.</p> required <code>catalog_ancestry_files</code> <code>list[str]</code> <p>List of raw ancestry annotations files from GWAS Catalog.</p> required <code>catalog_sumstats_lut</code> <code>str</code> <p>GWAS Catalog summary statistics lookup table.</p> required <code>catalog_associations_file</code> <code>str</code> <p>Raw GWAS catalog associations file.</p> required <code>variant_annotation_path</code> <code>str</code> <p>Input variant annotation path.</p> required <code>catalog_studies_out</code> <code>str</code> <p>Output GWAS catalog studies path.</p> required <code>catalog_associations_out</code> <code>str</code> <p>Output GWAS catalog associations path.</p> required <code>gwas_catalog_study_curation_file</code> <code>str | None</code> <p>file of the curation table. Optional.</p> <code>None</code> <code>inclusion_list_path</code> <code>str | None</code> <p>optional inclusion list (parquet)</p> <code>None</code> Source code in <code>src/gentropy/gwas_catalog_ingestion.py</code> <pre><code>def __init__(\n    self,\n    session: Session,\n    catalog_study_files: list[str],\n    catalog_ancestry_files: list[str],\n    catalog_sumstats_lut: str,\n    catalog_associations_file: str,\n    variant_annotation_path: str,\n    catalog_studies_out: str,\n    catalog_associations_out: str,\n    gwas_catalog_study_curation_file: str | None = None,\n    inclusion_list_path: str | None = None,\n) -&gt; None:\n    \"\"\"Run step.\n\n    Args:\n        session (Session): Session object.\n        catalog_study_files (list[str]): List of raw GWAS catalog studies file.\n        catalog_ancestry_files (list[str]): List of raw ancestry annotations files from GWAS Catalog.\n        catalog_sumstats_lut (str): GWAS Catalog summary statistics lookup table.\n        catalog_associations_file (str): Raw GWAS catalog associations file.\n        variant_annotation_path (str): Input variant annotation path.\n        catalog_studies_out (str): Output GWAS catalog studies path.\n        catalog_associations_out (str): Output GWAS catalog associations path.\n        gwas_catalog_study_curation_file (str | None): file of the curation table. Optional.\n        inclusion_list_path (str | None): optional inclusion list (parquet)\n    \"\"\"\n    # Extract\n    va = VariantAnnotation.from_parquet(session, variant_annotation_path)\n    catalog_studies = session.spark.read.csv(\n        list(catalog_study_files), sep=\"\\t\", header=True\n    )\n    ancestry_lut = session.spark.read.csv(\n        list(catalog_ancestry_files), sep=\"\\t\", header=True\n    )\n    sumstats_lut = session.spark.read.csv(\n        catalog_sumstats_lut, sep=\"\\t\", header=False\n    )\n    catalog_associations = session.spark.read.csv(\n        catalog_associations_file, sep=\"\\t\", header=True\n    ).persist()\n    gwas_catalog_study_curation = read_curation_table(\n        gwas_catalog_study_curation_file, session\n    )\n\n    # Transform\n    study_index, study_locus = GWASCatalogStudySplitter.split(\n        StudyIndexGWASCatalogParser.from_source(\n            catalog_studies, ancestry_lut, sumstats_lut\n        ).annotate_from_study_curation(gwas_catalog_study_curation),\n        GWASCatalogCuratedAssociationsParser.from_source(catalog_associations, va),\n    )\n\n    # if inclusion list is provided apply filter:\n    if inclusion_list_path:\n        inclusion_list = session.spark.read.parquet(\n            inclusion_list_path, sep=\"\\t\", header=True\n        )\n\n        study_index = study_index.apply_inclusion_list(inclusion_list)\n        study_locus = study_locus.apply_inclusion_list(inclusion_list)\n\n    # Load\n    study_index.df.write.mode(session.write_mode).parquet(catalog_studies_out)\n    study_locus.df.write.mode(session.write_mode).parquet(catalog_associations_out)\n</code></pre>"},{"location":"python_api/steps/gwas_catalog_sumstat_preprocess/","title":"gwas_catalog_sumstat_preprocess","text":""},{"location":"python_api/steps/gwas_catalog_sumstat_preprocess/#gentropy.gwas_catalog_sumstat_preprocess.GWASCatalogSumstatsPreprocessStep","title":"<code>gentropy.gwas_catalog_sumstat_preprocess.GWASCatalogSumstatsPreprocessStep</code>","text":"<p>Step to preprocess GWAS Catalog harmonised summary stats.</p> Source code in <code>src/gentropy/gwas_catalog_sumstat_preprocess.py</code> <pre><code>class GWASCatalogSumstatsPreprocessStep:\n    \"\"\"Step to preprocess GWAS Catalog harmonised summary stats.\"\"\"\n\n    def __init__(\n        self, session: Session, raw_sumstats_path: str, out_sumstats_path: str\n    ) -&gt; None:\n        \"\"\"Run step to preprocess GWAS Catalog harmonised summary stats and produce SummaryStatistics dataset.\n\n        Args:\n            session (Session): Session object.\n            raw_sumstats_path (str): Input GWAS Catalog harmonised summary stats path.\n            out_sumstats_path (str): Output SummaryStatistics dataset path.\n        \"\"\"\n        # Processing dataset:\n        GWASCatalogSummaryStatistics.from_gwas_harmonized_summary_stats(\n            session.spark, raw_sumstats_path\n        ).df.write.mode(session.write_mode).parquet(out_sumstats_path)\n        session.logger.info(\"Processing dataset successfully completed.\")\n</code></pre>"},{"location":"python_api/steps/gwas_catalog_sumstat_preprocess/#gentropy.gwas_catalog_sumstat_preprocess.GWASCatalogSumstatsPreprocessStep.__init__","title":"<code>__init__(session: Session, raw_sumstats_path: str, out_sumstats_path: str) -&gt; None</code>","text":"<p>Run step to preprocess GWAS Catalog harmonised summary stats and produce SummaryStatistics dataset.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Session object.</p> required <code>raw_sumstats_path</code> <code>str</code> <p>Input GWAS Catalog harmonised summary stats path.</p> required <code>out_sumstats_path</code> <code>str</code> <p>Output SummaryStatistics dataset path.</p> required Source code in <code>src/gentropy/gwas_catalog_sumstat_preprocess.py</code> <pre><code>def __init__(\n    self, session: Session, raw_sumstats_path: str, out_sumstats_path: str\n) -&gt; None:\n    \"\"\"Run step to preprocess GWAS Catalog harmonised summary stats and produce SummaryStatistics dataset.\n\n    Args:\n        session (Session): Session object.\n        raw_sumstats_path (str): Input GWAS Catalog harmonised summary stats path.\n        out_sumstats_path (str): Output SummaryStatistics dataset path.\n    \"\"\"\n    # Processing dataset:\n    GWASCatalogSummaryStatistics.from_gwas_harmonized_summary_stats(\n        session.spark, raw_sumstats_path\n    ).df.write.mode(session.write_mode).parquet(out_sumstats_path)\n    session.logger.info(\"Processing dataset successfully completed.\")\n</code></pre>"},{"location":"python_api/steps/l2g/","title":"locus_to_gene","text":""},{"location":"python_api/steps/l2g/#gentropy.l2g.LocusToGeneStep","title":"<code>gentropy.l2g.LocusToGeneStep</code>","text":"<p>Locus to gene step.</p> Source code in <code>src/gentropy/l2g.py</code> <pre><code>class LocusToGeneStep:\n    \"\"\"Locus to gene step.\"\"\"\n\n    def __init__(\n        self,\n        session: Session,\n        run_mode: str,\n        model_path: str,\n        predictions_path: str,\n        credible_set_path: str,\n        variant_gene_path: str,\n        colocalisation_path: str,\n        study_index_path: str,\n        gold_standard_curation_path: str,\n        gene_interactions_path: str,\n        features_list: list[str],\n        hyperparameters: dict[str, Any],\n        wandb_run_name: str | None = None,\n        perform_cross_validation: bool = False,\n    ) -&gt; None:\n        \"\"\"Run step.\n\n        Args:\n            session (Session): Session object.\n            run_mode (str): One of \"train\" or \"predict\".\n            model_path (str): Path to save the model.\n            predictions_path (str): Path to save the predictions.\n            credible_set_path (str): Path to credible set Parquet files.\n            variant_gene_path (str): Path to variant to gene Parquet files.\n            colocalisation_path (str): Path to colocalisation Parquet files.\n            study_index_path (str): Path to study index Parquet files.\n            gold_standard_curation_path (str): Path to gold standard curation JSON files.\n            gene_interactions_path (str): Path to gene interactions Parquet files.\n            features_list (list[str]): List of features to use.\n            hyperparameters (dict[str, Any]): Hyperparameters for the model.\n            wandb_run_name (str | None): Name of the run to be tracked on W&amp;B.\n            perform_cross_validation (bool): Whether to perform cross validation.\n\n        Raises:\n            ValueError: if run_mode is not one of \"train\" or \"predict\".\n        \"\"\"\n        print(\"Sci-kit learn version: \", sklearn.__version__)  # noqa: T201\n        if run_mode not in [\"train\", \"predict\"]:\n            raise ValueError(\n                f\"run_mode must be one of 'train' or 'predict', got {run_mode}\"\n            )\n        # Load common inputs\n        credible_set = StudyLocus.from_parquet(\n            session, credible_set_path, recursiveFileLookup=True\n        )\n        studies = StudyIndex.from_parquet(\n            session, study_index_path, recursiveFileLookup=True\n        )\n        v2g = V2G.from_parquet(session, variant_gene_path)\n        coloc = Colocalisation.from_parquet(session, colocalisation_path)\n\n        if run_mode == \"predict\":\n            if not model_path or not predictions_path:\n                raise ValueError(\n                    \"model_path and predictions_path must be set for predict mode.\"\n                )\n            predictions = L2GPrediction.from_credible_set(\n                model_path, list(features_list), credible_set, studies, v2g, coloc\n            )\n            predictions.df.write.mode(session.write_mode).parquet(predictions_path)\n            session.logger.info(predictions_path)\n        elif (\n            run_mode == \"train\"\n            and gold_standard_curation_path\n            and gene_interactions_path\n        ):\n            # Process gold standard and L2G features\n            gs_curation = session.spark.read.json(gold_standard_curation_path).persist()\n            interactions = session.spark.read.parquet(gene_interactions_path)\n            study_locus_overlap = StudyLocus(\n                # We just extract overlaps of associations in the gold standard. This parsing is a duplication of the one in the gold standard curation,\n                # but we need to do it here to be able to parse gold standards later\n                _df=credible_set.df.join(\n                    f.broadcast(\n                        gs_curation.select(\n                            StudyLocus.assign_study_locus_id(\n                                f.col(\"association_info.otg_id\"),  # studyId\n                                f.concat_ws(  # variantId\n                                    \"_\",\n                                    f.col(\"sentinel_variant.locus_GRCh38.chromosome\"),\n                                    f.col(\"sentinel_variant.locus_GRCh38.position\"),\n                                    f.col(\"sentinel_variant.alleles.reference\"),\n                                    f.col(\"sentinel_variant.alleles.alternative\"),\n                                ),\n                            ).alias(\"studyLocusId\"),\n                        )\n                    ),\n                    \"studyLocusId\",\n                    \"inner\",\n                ),\n                _schema=StudyLocus.get_schema(),\n            ).find_overlaps(studies)\n\n            gold_standards = L2GGoldStandard.from_otg_curation(\n                gold_standard_curation=gs_curation,\n                v2g=v2g,\n                study_locus_overlap=study_locus_overlap,\n                interactions=interactions,\n            )\n\n            fm = L2GFeatureMatrix.generate_features(\n                features_list=features_list,\n                study_locus=credible_set,\n                study_index=studies,\n                variant_gene=v2g,\n                colocalisation=coloc,\n            )\n\n            # Join and fill null values with 0\n            data = L2GFeatureMatrix(\n                _df=fm.df.join(\n                    f.broadcast(\n                        gold_standards.df.drop(\"variantId\", \"studyId\", \"sources\")\n                    ),\n                    on=[\"studyLocusId\", \"geneId\"],\n                    how=\"inner\",\n                ),\n                _schema=L2GFeatureMatrix.get_schema(),\n            ).fill_na()\n\n            # Instantiate classifier\n            estimator = SparkXGBClassifier(\n                eval_metric=\"logloss\",\n                features_col=\"features\",\n                label_col=\"label\",\n                max_depth=5,\n            )\n            l2g_model = LocusToGeneModel(\n                features_list=list(features_list), estimator=estimator\n            )\n            if perform_cross_validation:\n                # Perform cross validation to extract what are the best hyperparameters\n                cv_folds = hyperparameters.get(\"cross_validation_folds\", 5)\n                LocusToGeneTrainer.cross_validate(\n                    l2g_model=l2g_model,\n                    data=data,\n                    num_folds=cv_folds,\n                )\n            else:\n                # Train model\n                LocusToGeneTrainer.train(\n                    data=data,\n                    l2g_model=l2g_model,\n                    features_list=list(features_list),\n                    model_path=model_path,\n                    evaluate=True,\n                    wandb_run_name=wandb_run_name,\n                    **hyperparameters,\n                )\n                session.logger.info(model_path)\n</code></pre>"},{"location":"python_api/steps/l2g/#gentropy.l2g.LocusToGeneStep.__init__","title":"<code>__init__(session: Session, run_mode: str, model_path: str, predictions_path: str, credible_set_path: str, variant_gene_path: str, colocalisation_path: str, study_index_path: str, gold_standard_curation_path: str, gene_interactions_path: str, features_list: list[str], hyperparameters: dict[str, Any], wandb_run_name: str | None = None, perform_cross_validation: bool = False) -&gt; None</code>","text":"<p>Run step.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Session object.</p> required <code>run_mode</code> <code>str</code> <p>One of \"train\" or \"predict\".</p> required <code>model_path</code> <code>str</code> <p>Path to save the model.</p> required <code>predictions_path</code> <code>str</code> <p>Path to save the predictions.</p> required <code>credible_set_path</code> <code>str</code> <p>Path to credible set Parquet files.</p> required <code>variant_gene_path</code> <code>str</code> <p>Path to variant to gene Parquet files.</p> required <code>colocalisation_path</code> <code>str</code> <p>Path to colocalisation Parquet files.</p> required <code>study_index_path</code> <code>str</code> <p>Path to study index Parquet files.</p> required <code>gold_standard_curation_path</code> <code>str</code> <p>Path to gold standard curation JSON files.</p> required <code>gene_interactions_path</code> <code>str</code> <p>Path to gene interactions Parquet files.</p> required <code>features_list</code> <code>list[str]</code> <p>List of features to use.</p> required <code>hyperparameters</code> <code>dict[str, Any]</code> <p>Hyperparameters for the model.</p> required <code>wandb_run_name</code> <code>str | None</code> <p>Name of the run to be tracked on W&amp;B.</p> <code>None</code> <code>perform_cross_validation</code> <code>bool</code> <p>Whether to perform cross validation.</p> <code>False</code> <p>Raises:</p> Type Description <code>ValueError</code> <p>if run_mode is not one of \"train\" or \"predict\".</p> Source code in <code>src/gentropy/l2g.py</code> <pre><code>def __init__(\n    self,\n    session: Session,\n    run_mode: str,\n    model_path: str,\n    predictions_path: str,\n    credible_set_path: str,\n    variant_gene_path: str,\n    colocalisation_path: str,\n    study_index_path: str,\n    gold_standard_curation_path: str,\n    gene_interactions_path: str,\n    features_list: list[str],\n    hyperparameters: dict[str, Any],\n    wandb_run_name: str | None = None,\n    perform_cross_validation: bool = False,\n) -&gt; None:\n    \"\"\"Run step.\n\n    Args:\n        session (Session): Session object.\n        run_mode (str): One of \"train\" or \"predict\".\n        model_path (str): Path to save the model.\n        predictions_path (str): Path to save the predictions.\n        credible_set_path (str): Path to credible set Parquet files.\n        variant_gene_path (str): Path to variant to gene Parquet files.\n        colocalisation_path (str): Path to colocalisation Parquet files.\n        study_index_path (str): Path to study index Parquet files.\n        gold_standard_curation_path (str): Path to gold standard curation JSON files.\n        gene_interactions_path (str): Path to gene interactions Parquet files.\n        features_list (list[str]): List of features to use.\n        hyperparameters (dict[str, Any]): Hyperparameters for the model.\n        wandb_run_name (str | None): Name of the run to be tracked on W&amp;B.\n        perform_cross_validation (bool): Whether to perform cross validation.\n\n    Raises:\n        ValueError: if run_mode is not one of \"train\" or \"predict\".\n    \"\"\"\n    print(\"Sci-kit learn version: \", sklearn.__version__)  # noqa: T201\n    if run_mode not in [\"train\", \"predict\"]:\n        raise ValueError(\n            f\"run_mode must be one of 'train' or 'predict', got {run_mode}\"\n        )\n    # Load common inputs\n    credible_set = StudyLocus.from_parquet(\n        session, credible_set_path, recursiveFileLookup=True\n    )\n    studies = StudyIndex.from_parquet(\n        session, study_index_path, recursiveFileLookup=True\n    )\n    v2g = V2G.from_parquet(session, variant_gene_path)\n    coloc = Colocalisation.from_parquet(session, colocalisation_path)\n\n    if run_mode == \"predict\":\n        if not model_path or not predictions_path:\n            raise ValueError(\n                \"model_path and predictions_path must be set for predict mode.\"\n            )\n        predictions = L2GPrediction.from_credible_set(\n            model_path, list(features_list), credible_set, studies, v2g, coloc\n        )\n        predictions.df.write.mode(session.write_mode).parquet(predictions_path)\n        session.logger.info(predictions_path)\n    elif (\n        run_mode == \"train\"\n        and gold_standard_curation_path\n        and gene_interactions_path\n    ):\n        # Process gold standard and L2G features\n        gs_curation = session.spark.read.json(gold_standard_curation_path).persist()\n        interactions = session.spark.read.parquet(gene_interactions_path)\n        study_locus_overlap = StudyLocus(\n            # We just extract overlaps of associations in the gold standard. This parsing is a duplication of the one in the gold standard curation,\n            # but we need to do it here to be able to parse gold standards later\n            _df=credible_set.df.join(\n                f.broadcast(\n                    gs_curation.select(\n                        StudyLocus.assign_study_locus_id(\n                            f.col(\"association_info.otg_id\"),  # studyId\n                            f.concat_ws(  # variantId\n                                \"_\",\n                                f.col(\"sentinel_variant.locus_GRCh38.chromosome\"),\n                                f.col(\"sentinel_variant.locus_GRCh38.position\"),\n                                f.col(\"sentinel_variant.alleles.reference\"),\n                                f.col(\"sentinel_variant.alleles.alternative\"),\n                            ),\n                        ).alias(\"studyLocusId\"),\n                    )\n                ),\n                \"studyLocusId\",\n                \"inner\",\n            ),\n            _schema=StudyLocus.get_schema(),\n        ).find_overlaps(studies)\n\n        gold_standards = L2GGoldStandard.from_otg_curation(\n            gold_standard_curation=gs_curation,\n            v2g=v2g,\n            study_locus_overlap=study_locus_overlap,\n            interactions=interactions,\n        )\n\n        fm = L2GFeatureMatrix.generate_features(\n            features_list=features_list,\n            study_locus=credible_set,\n            study_index=studies,\n            variant_gene=v2g,\n            colocalisation=coloc,\n        )\n\n        # Join and fill null values with 0\n        data = L2GFeatureMatrix(\n            _df=fm.df.join(\n                f.broadcast(\n                    gold_standards.df.drop(\"variantId\", \"studyId\", \"sources\")\n                ),\n                on=[\"studyLocusId\", \"geneId\"],\n                how=\"inner\",\n            ),\n            _schema=L2GFeatureMatrix.get_schema(),\n        ).fill_na()\n\n        # Instantiate classifier\n        estimator = SparkXGBClassifier(\n            eval_metric=\"logloss\",\n            features_col=\"features\",\n            label_col=\"label\",\n            max_depth=5,\n        )\n        l2g_model = LocusToGeneModel(\n            features_list=list(features_list), estimator=estimator\n        )\n        if perform_cross_validation:\n            # Perform cross validation to extract what are the best hyperparameters\n            cv_folds = hyperparameters.get(\"cross_validation_folds\", 5)\n            LocusToGeneTrainer.cross_validate(\n                l2g_model=l2g_model,\n                data=data,\n                num_folds=cv_folds,\n            )\n        else:\n            # Train model\n            LocusToGeneTrainer.train(\n                data=data,\n                l2g_model=l2g_model,\n                features_list=list(features_list),\n                model_path=model_path,\n                evaluate=True,\n                wandb_run_name=wandb_run_name,\n                **hyperparameters,\n            )\n            session.logger.info(model_path)\n</code></pre>"},{"location":"python_api/steps/ld_clump/","title":"ld_based_clumping","text":""},{"location":"python_api/steps/ld_clump/#gentropy.ld_based_clumping.LDBasedClumpingStep","title":"<code>gentropy.ld_based_clumping.LDBasedClumpingStep</code>","text":"<p>Step to perform LD-based clumping on study locus dataset.</p> <p>As a first step, study locus is enriched with population specific linked-variants. That's why the study index and the ld index is required for this step. Study loci are flaggged in the resulting dataset, which can be explained by a more significant association from the same study.</p> Source code in <code>src/gentropy/ld_based_clumping.py</code> <pre><code>class LDBasedClumpingStep:\n    \"\"\"Step to perform LD-based clumping on study locus dataset.\n\n    As a first step, study locus is enriched with population specific linked-variants.\n    That's why the study index and the ld index is required for this step. Study loci are flaggged\n    in the resulting dataset, which can be explained by a more significant association\n    from the same study.\n    \"\"\"\n\n    def __init__(\n        self,\n        session: Session,\n        study_locus_input_path: str,\n        study_index_path: str,\n        ld_index_path: str,\n        clumped_study_locus_output_path: str,\n    ) -&gt; None:\n        \"\"\"Run LD-based clumping step.\n\n        Args:\n            session (Session): Session object.\n            study_locus_input_path (str): Path to the input study locus.\n            study_index_path (str): Path to the study index.\n            ld_index_path (str): Path to the LD index.\n            clumped_study_locus_output_path (str): path of the resulting, clumped study-locus dataset.\n        \"\"\"\n        study_locus = StudyLocus.from_parquet(session, study_locus_input_path)\n        ld_index = LDIndex.from_parquet(session, ld_index_path)\n        study_index = StudyIndex.from_parquet(session, study_index_path)\n\n        (\n            study_locus\n            # Annotating study locus with LD information:\n            .annotate_ld(study_index, ld_index)\n            .clump()\n            # Save result:\n            .df.write.mode(session.write_mode)\n            .parquet(clumped_study_locus_output_path)\n        )\n</code></pre>"},{"location":"python_api/steps/ld_clump/#gentropy.ld_based_clumping.LDBasedClumpingStep.__init__","title":"<code>__init__(session: Session, study_locus_input_path: str, study_index_path: str, ld_index_path: str, clumped_study_locus_output_path: str) -&gt; None</code>","text":"<p>Run LD-based clumping step.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Session object.</p> required <code>study_locus_input_path</code> <code>str</code> <p>Path to the input study locus.</p> required <code>study_index_path</code> <code>str</code> <p>Path to the study index.</p> required <code>ld_index_path</code> <code>str</code> <p>Path to the LD index.</p> required <code>clumped_study_locus_output_path</code> <code>str</code> <p>path of the resulting, clumped study-locus dataset.</p> required Source code in <code>src/gentropy/ld_based_clumping.py</code> <pre><code>def __init__(\n    self,\n    session: Session,\n    study_locus_input_path: str,\n    study_index_path: str,\n    ld_index_path: str,\n    clumped_study_locus_output_path: str,\n) -&gt; None:\n    \"\"\"Run LD-based clumping step.\n\n    Args:\n        session (Session): Session object.\n        study_locus_input_path (str): Path to the input study locus.\n        study_index_path (str): Path to the study index.\n        ld_index_path (str): Path to the LD index.\n        clumped_study_locus_output_path (str): path of the resulting, clumped study-locus dataset.\n    \"\"\"\n    study_locus = StudyLocus.from_parquet(session, study_locus_input_path)\n    ld_index = LDIndex.from_parquet(session, ld_index_path)\n    study_index = StudyIndex.from_parquet(session, study_index_path)\n\n    (\n        study_locus\n        # Annotating study locus with LD information:\n        .annotate_ld(study_index, ld_index)\n        .clump()\n        # Save result:\n        .df.write.mode(session.write_mode)\n        .parquet(clumped_study_locus_output_path)\n    )\n</code></pre>"},{"location":"python_api/steps/ld_index/","title":"ld_index","text":""},{"location":"python_api/steps/ld_index/#gentropy.ld_index.LDIndexStep","title":"<code>gentropy.ld_index.LDIndexStep</code>","text":"<p>LD index step.</p> <p>This step is resource intensive</p> <p>Suggested params: high memory machine, 5TB of boot disk, no SSDs.</p> Source code in <code>src/gentropy/ld_index.py</code> <pre><code>class LDIndexStep:\n    \"\"\"LD index step.\n\n    !!! warning \"This step is resource intensive\"\n\n        Suggested params: high memory machine, 5TB of boot disk, no SSDs.\n    \"\"\"\n\n    def __init__(self, session: Session, min_r2: float, ld_index_out: str) -&gt; None:\n        \"\"\"Run step.\n\n        Args:\n            session (Session): Session object.\n            min_r2 (float): Minimum r2 to consider when considering variants within a window.\n            ld_index_out (str): Output LD index path.\n        \"\"\"\n        hl.init(sc=session.spark.sparkContext, log=\"/dev/null\")\n        (\n            GnomADLDMatrix()\n            .as_ld_index(min_r2)\n            .df.write.partitionBy(\"chromosome\")\n            .mode(session.write_mode)\n            .parquet(ld_index_out)\n        )\n        session.logger.info(ld_index_out)\n</code></pre>"},{"location":"python_api/steps/ld_index/#gentropy.ld_index.LDIndexStep.__init__","title":"<code>__init__(session: Session, min_r2: float, ld_index_out: str) -&gt; None</code>","text":"<p>Run step.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Session object.</p> required <code>min_r2</code> <code>float</code> <p>Minimum r2 to consider when considering variants within a window.</p> required <code>ld_index_out</code> <code>str</code> <p>Output LD index path.</p> required Source code in <code>src/gentropy/ld_index.py</code> <pre><code>def __init__(self, session: Session, min_r2: float, ld_index_out: str) -&gt; None:\n    \"\"\"Run step.\n\n    Args:\n        session (Session): Session object.\n        min_r2 (float): Minimum r2 to consider when considering variants within a window.\n        ld_index_out (str): Output LD index path.\n    \"\"\"\n    hl.init(sc=session.spark.sparkContext, log=\"/dev/null\")\n    (\n        GnomADLDMatrix()\n        .as_ld_index(min_r2)\n        .df.write.partitionBy(\"chromosome\")\n        .mode(session.write_mode)\n        .parquet(ld_index_out)\n    )\n    session.logger.info(ld_index_out)\n</code></pre>"},{"location":"python_api/steps/pics/","title":"pics","text":""},{"location":"python_api/steps/pics/#gentropy.pics.PICSStep","title":"<code>gentropy.pics.PICSStep</code>","text":"<p>PICS finemapping of LD-annotated StudyLocus.</p> Source code in <code>src/gentropy/pics.py</code> <pre><code>class PICSStep:\n    \"\"\"PICS finemapping of LD-annotated StudyLocus.\"\"\"\n\n    def __init__(\n        self,\n        session: Session,\n        study_locus_ld_annotated_in: str,\n        picsed_study_locus_out: str,\n    ) -&gt; None:\n        \"\"\"Run PICS on LD annotated study-locus.\n\n        Args:\n            session (Session): Session object.\n            study_locus_ld_annotated_in (str): Input LD annotated study-locus path.\n            picsed_study_locus_out (str): Output PICSed study-locus path.\n        \"\"\"\n        # Extract\n        study_locus_ld_annotated = StudyLocus.from_parquet(\n            session, study_locus_ld_annotated_in\n        )\n        # PICS\n        picsed_sl = PICS.finemap(study_locus_ld_annotated).annotate_credible_sets()\n        # Write\n        picsed_sl.df.write.mode(session.write_mode).parquet(picsed_study_locus_out)\n</code></pre>"},{"location":"python_api/steps/pics/#gentropy.pics.PICSStep.__init__","title":"<code>__init__(session: Session, study_locus_ld_annotated_in: str, picsed_study_locus_out: str) -&gt; None</code>","text":"<p>Run PICS on LD annotated study-locus.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Session object.</p> required <code>study_locus_ld_annotated_in</code> <code>str</code> <p>Input LD annotated study-locus path.</p> required <code>picsed_study_locus_out</code> <code>str</code> <p>Output PICSed study-locus path.</p> required Source code in <code>src/gentropy/pics.py</code> <pre><code>def __init__(\n    self,\n    session: Session,\n    study_locus_ld_annotated_in: str,\n    picsed_study_locus_out: str,\n) -&gt; None:\n    \"\"\"Run PICS on LD annotated study-locus.\n\n    Args:\n        session (Session): Session object.\n        study_locus_ld_annotated_in (str): Input LD annotated study-locus path.\n        picsed_study_locus_out (str): Output PICSed study-locus path.\n    \"\"\"\n    # Extract\n    study_locus_ld_annotated = StudyLocus.from_parquet(\n        session, study_locus_ld_annotated_in\n    )\n    # PICS\n    picsed_sl = PICS.finemap(study_locus_ld_annotated).annotate_credible_sets()\n    # Write\n    picsed_sl.df.write.mode(session.write_mode).parquet(picsed_study_locus_out)\n</code></pre>"},{"location":"python_api/steps/variant_annotation_step/","title":"variant_annotation","text":""},{"location":"python_api/steps/variant_annotation_step/#gentropy.variant_annotation.VariantAnnotationStep","title":"<code>gentropy.variant_annotation.VariantAnnotationStep</code>","text":"<p>Variant annotation step.</p> <p>Variant annotation step produces a dataset of the type <code>VariantAnnotation</code> derived from gnomADs <code>gnomad.genomes.vX.X.X.sites.ht</code> Hail's table. This dataset is used to validate variants and as a source of annotation.</p> Source code in <code>src/gentropy/variant_annotation.py</code> <pre><code>class VariantAnnotationStep:\n    \"\"\"Variant annotation step.\n\n    Variant annotation step produces a dataset of the type `VariantAnnotation` derived from gnomADs `gnomad.genomes.vX.X.X.sites.ht` Hail's table. This dataset is used to validate variants and as a source of annotation.\n    \"\"\"\n\n    def __init__(self, session: Session, variant_annotation_path: str) -&gt; None:\n        \"\"\"Run Variant Annotation step.\n\n        Args:\n            session (Session): Session object.\n            variant_annotation_path (str): Variant annotation dataset path.\n        \"\"\"\n        # Initialise hail session.\n        hl.init(sc=session.spark.sparkContext, log=\"/dev/null\")\n        # Run variant annotation.\n        variant_annotation = GnomADVariants().as_variant_annotation()\n        # Write data partitioned by chromosome and position.\n        (\n            variant_annotation.df.write.mode(session.write_mode).parquet(\n                variant_annotation_path\n            )\n        )\n</code></pre>"},{"location":"python_api/steps/variant_annotation_step/#gentropy.variant_annotation.VariantAnnotationStep.__init__","title":"<code>__init__(session: Session, variant_annotation_path: str) -&gt; None</code>","text":"<p>Run Variant Annotation step.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Session object.</p> required <code>variant_annotation_path</code> <code>str</code> <p>Variant annotation dataset path.</p> required Source code in <code>src/gentropy/variant_annotation.py</code> <pre><code>def __init__(self, session: Session, variant_annotation_path: str) -&gt; None:\n    \"\"\"Run Variant Annotation step.\n\n    Args:\n        session (Session): Session object.\n        variant_annotation_path (str): Variant annotation dataset path.\n    \"\"\"\n    # Initialise hail session.\n    hl.init(sc=session.spark.sparkContext, log=\"/dev/null\")\n    # Run variant annotation.\n    variant_annotation = GnomADVariants().as_variant_annotation()\n    # Write data partitioned by chromosome and position.\n    (\n        variant_annotation.df.write.mode(session.write_mode).parquet(\n            variant_annotation_path\n        )\n    )\n</code></pre>"},{"location":"python_api/steps/variant_index_step/","title":"variant_index","text":""},{"location":"python_api/steps/variant_index_step/#gentropy.variant_index.VariantIndexStep","title":"<code>gentropy.variant_index.VariantIndexStep</code>","text":"<p>Run variant index step to only variants in study-locus sets.</p> <p>Using a <code>VariantAnnotation</code> dataset as a reference, this step creates and writes a dataset of the type <code>VariantIndex</code> that includes only variants that have disease-association data with a reduced set of annotations.</p> Source code in <code>src/gentropy/variant_index.py</code> <pre><code>class VariantIndexStep:\n    \"\"\"Run variant index step to only variants in study-locus sets.\n\n    Using a `VariantAnnotation` dataset as a reference, this step creates and writes a dataset of the type `VariantIndex` that includes only variants that have disease-association data with a reduced set of annotations.\n    \"\"\"\n\n    def __init__(\n        self: VariantIndexStep,\n        session: Session,\n        variant_annotation_path: str,\n        credible_set_path: str,\n        variant_index_path: str,\n    ) -&gt; None:\n        \"\"\"Run VariantIndex step.\n\n        Args:\n            session (Session): Session object.\n            variant_annotation_path (str): Variant annotation dataset path.\n            credible_set_path (str): Credible set dataset path.\n            variant_index_path (str): Variant index dataset path.\n        \"\"\"\n        # Extract\n        va = VariantAnnotation.from_parquet(session, variant_annotation_path)\n        credible_set = StudyLocus.from_parquet(\n            session, credible_set_path, recursiveFileLookup=True\n        )\n\n        # Transform\n        vi = VariantIndex.from_variant_annotation(va, credible_set)\n\n        (\n            vi.df.write.partitionBy(\"chromosome\")\n            .mode(session.write_mode)\n            .parquet(variant_index_path)\n        )\n</code></pre>"},{"location":"python_api/steps/variant_index_step/#gentropy.variant_index.VariantIndexStep.__init__","title":"<code>__init__(session: Session, variant_annotation_path: str, credible_set_path: str, variant_index_path: str) -&gt; None</code>","text":"<p>Run VariantIndex step.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Session object.</p> required <code>variant_annotation_path</code> <code>str</code> <p>Variant annotation dataset path.</p> required <code>credible_set_path</code> <code>str</code> <p>Credible set dataset path.</p> required <code>variant_index_path</code> <code>str</code> <p>Variant index dataset path.</p> required Source code in <code>src/gentropy/variant_index.py</code> <pre><code>def __init__(\n    self: VariantIndexStep,\n    session: Session,\n    variant_annotation_path: str,\n    credible_set_path: str,\n    variant_index_path: str,\n) -&gt; None:\n    \"\"\"Run VariantIndex step.\n\n    Args:\n        session (Session): Session object.\n        variant_annotation_path (str): Variant annotation dataset path.\n        credible_set_path (str): Credible set dataset path.\n        variant_index_path (str): Variant index dataset path.\n    \"\"\"\n    # Extract\n    va = VariantAnnotation.from_parquet(session, variant_annotation_path)\n    credible_set = StudyLocus.from_parquet(\n        session, credible_set_path, recursiveFileLookup=True\n    )\n\n    # Transform\n    vi = VariantIndex.from_variant_annotation(va, credible_set)\n\n    (\n        vi.df.write.partitionBy(\"chromosome\")\n        .mode(session.write_mode)\n        .parquet(variant_index_path)\n    )\n</code></pre>"},{"location":"python_api/steps/variant_to_gene_step/","title":"variant_to_gene","text":""},{"location":"python_api/steps/variant_to_gene_step/#gentropy.v2g.V2GStep","title":"<code>gentropy.v2g.V2GStep</code>","text":"<p>Variant-to-gene (V2G) step.</p> <p>This step aims to generate a dataset that contains multiple pieces of evidence supporting the functional association of specific variants with genes. Some of the evidence types include:</p> <ol> <li>Chromatin interaction experiments, e.g. Promoter Capture Hi-C (PCHi-C).</li> <li>In silico functional predictions, e.g. Variant Effect Predictor (VEP) from Ensembl.</li> <li>Distance between the variant and each gene's canonical transcription start site (TSS).</li> </ol> <p>Attributes:</p> Name Type Description <code>session</code> <code>Session</code> <p>Session object.</p> <code>variant_index_path</code> <code>str</code> <p>Input variant index path.</p> <code>variant_annotation_path</code> <code>str</code> <p>Input variant annotation path.</p> <code>gene_index_path</code> <code>str</code> <p>Input gene index path.</p> <code>vep_consequences_path</code> <code>str</code> <p>Input VEP consequences path.</p> <code>liftover_chain_file_path</code> <code>str</code> <p>Path to GRCh37 to GRCh38 chain file.</p> <code>liftover_max_length_difference</code> <code>str</code> <p>Maximum length difference for liftover.</p> <code>max_distance</code> <code>int</code> <p>Maximum distance to consider.</p> <code>approved_biotypes</code> <code>list[str]</code> <p>List of approved biotypes.</p> <code>intervals</code> <code>dict</code> <p>Dictionary of interval sources.</p> <code>v2g_path</code> <code>str</code> <p>Output V2G path.</p> Source code in <code>src/gentropy/v2g.py</code> <pre><code>class V2GStep:\n    \"\"\"Variant-to-gene (V2G) step.\n\n    This step aims to generate a dataset that contains multiple pieces of evidence supporting the functional association of specific variants with genes. Some of the evidence types include:\n\n    1. Chromatin interaction experiments, e.g. Promoter Capture Hi-C (PCHi-C).\n    2. In silico functional predictions, e.g. Variant Effect Predictor (VEP) from Ensembl.\n    3. Distance between the variant and each gene's canonical transcription start site (TSS).\n\n    Attributes:\n        session (Session): Session object.\n        variant_index_path (str): Input variant index path.\n        variant_annotation_path (str): Input variant annotation path.\n        gene_index_path (str): Input gene index path.\n        vep_consequences_path (str): Input VEP consequences path.\n        liftover_chain_file_path (str): Path to GRCh37 to GRCh38 chain file.\n        liftover_max_length_difference: Maximum length difference for liftover.\n        max_distance (int): Maximum distance to consider.\n        approved_biotypes (list[str]): List of approved biotypes.\n        intervals (dict): Dictionary of interval sources.\n        v2g_path (str): Output V2G path.\n    \"\"\"\n\n    def __init__(\n        self,\n        session: Session,\n        variant_index_path: str,\n        variant_annotation_path: str,\n        gene_index_path: str,\n        vep_consequences_path: str,\n        liftover_chain_file_path: str,\n        approved_biotypes: list[str],\n        interval_sources: dict[str, str],\n        v2g_path: str,\n        max_distance: int = 500_000,\n        liftover_max_length_difference: int = 100,\n    ) -&gt; None:\n        \"\"\"Run Variant-to-gene (V2G) step.\n\n        Args:\n            session (Session): Session object.\n            variant_index_path (str): Input variant index path.\n            variant_annotation_path (str): Input variant annotation path.\n            gene_index_path (str): Input gene index path.\n            vep_consequences_path (str): Input VEP consequences path.\n            liftover_chain_file_path (str): Path to GRCh37 to GRCh38 chain file.\n            approved_biotypes (list[str]): List of approved biotypes.\n            interval_sources (dict[str, str]): Dictionary of interval sources.\n            v2g_path (str): Output V2G path.\n            max_distance (int): Maximum distance to consider.\n            liftover_max_length_difference (int): Maximum length difference for liftover.\n        \"\"\"\n        # Read\n        gene_index = GeneIndex.from_parquet(session, gene_index_path)\n        vi = VariantIndex.from_parquet(session, variant_index_path).persist()\n        va = VariantAnnotation.from_parquet(session, variant_annotation_path)\n        vep_consequences = session.spark.read.csv(\n            vep_consequences_path, sep=\"\\t\", header=True\n        ).select(\n            f.element_at(f.split(\"Accession\", r\"/\"), -1).alias(\n                \"variantFunctionalConsequenceId\"\n            ),\n            f.col(\"Term\").alias(\"label\"),\n            f.col(\"v2g_score\").cast(\"double\").alias(\"score\"),\n        )\n\n        # Transform\n        lift = LiftOverSpark(\n            # lift over variants to hg38\n            liftover_chain_file_path,\n            liftover_max_length_difference,\n        )\n        gene_index_filtered = gene_index.filter_by_biotypes(\n            # Filter gene index by approved biotypes to define V2G gene universe\n            list(approved_biotypes)\n        )\n        va_slimmed = va.filter_by_variant_df(\n            # Variant annotation reduced to the variant index to define V2G variant universe\n            vi.df\n        ).persist()\n        intervals = Intervals(\n            _df=reduce(\n                lambda x, y: x.unionByName(y, allowMissingColumns=True),\n                # create interval instances by parsing each source\n                [\n                    Intervals.from_source(\n                        session.spark, source_name, source_path, gene_index, lift\n                    ).df\n                    for source_name, source_path in interval_sources.items()\n                ],\n            ),\n            _schema=Intervals.get_schema(),\n        )\n        v2g_datasets = [\n            va_slimmed.get_distance_to_tss(gene_index_filtered, max_distance),\n            va_slimmed.get_most_severe_vep_v2g(vep_consequences, gene_index_filtered),\n            va_slimmed.get_plof_v2g(gene_index_filtered),\n            intervals.v2g(vi),\n        ]\n        v2g = V2G(\n            _df=reduce(\n                lambda x, y: x.unionByName(y, allowMissingColumns=True),\n                [dataset.df for dataset in v2g_datasets],\n            ).repartition(\"chromosome\"),\n            _schema=V2G.get_schema(),\n        )\n\n        # Load\n        (\n            v2g.df.write.partitionBy(\"chromosome\")\n            .mode(session.write_mode)\n            .parquet(v2g_path)\n        )\n</code></pre>"},{"location":"python_api/steps/variant_to_gene_step/#gentropy.v2g.V2GStep.__init__","title":"<code>__init__(session: Session, variant_index_path: str, variant_annotation_path: str, gene_index_path: str, vep_consequences_path: str, liftover_chain_file_path: str, approved_biotypes: list[str], interval_sources: dict[str, str], v2g_path: str, max_distance: int = 500000, liftover_max_length_difference: int = 100) -&gt; None</code>","text":"<p>Run Variant-to-gene (V2G) step.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Session object.</p> required <code>variant_index_path</code> <code>str</code> <p>Input variant index path.</p> required <code>variant_annotation_path</code> <code>str</code> <p>Input variant annotation path.</p> required <code>gene_index_path</code> <code>str</code> <p>Input gene index path.</p> required <code>vep_consequences_path</code> <code>str</code> <p>Input VEP consequences path.</p> required <code>liftover_chain_file_path</code> <code>str</code> <p>Path to GRCh37 to GRCh38 chain file.</p> required <code>approved_biotypes</code> <code>list[str]</code> <p>List of approved biotypes.</p> required <code>interval_sources</code> <code>dict[str, str]</code> <p>Dictionary of interval sources.</p> required <code>v2g_path</code> <code>str</code> <p>Output V2G path.</p> required <code>max_distance</code> <code>int</code> <p>Maximum distance to consider.</p> <code>500000</code> <code>liftover_max_length_difference</code> <code>int</code> <p>Maximum length difference for liftover.</p> <code>100</code> Source code in <code>src/gentropy/v2g.py</code> <pre><code>def __init__(\n    self,\n    session: Session,\n    variant_index_path: str,\n    variant_annotation_path: str,\n    gene_index_path: str,\n    vep_consequences_path: str,\n    liftover_chain_file_path: str,\n    approved_biotypes: list[str],\n    interval_sources: dict[str, str],\n    v2g_path: str,\n    max_distance: int = 500_000,\n    liftover_max_length_difference: int = 100,\n) -&gt; None:\n    \"\"\"Run Variant-to-gene (V2G) step.\n\n    Args:\n        session (Session): Session object.\n        variant_index_path (str): Input variant index path.\n        variant_annotation_path (str): Input variant annotation path.\n        gene_index_path (str): Input gene index path.\n        vep_consequences_path (str): Input VEP consequences path.\n        liftover_chain_file_path (str): Path to GRCh37 to GRCh38 chain file.\n        approved_biotypes (list[str]): List of approved biotypes.\n        interval_sources (dict[str, str]): Dictionary of interval sources.\n        v2g_path (str): Output V2G path.\n        max_distance (int): Maximum distance to consider.\n        liftover_max_length_difference (int): Maximum length difference for liftover.\n    \"\"\"\n    # Read\n    gene_index = GeneIndex.from_parquet(session, gene_index_path)\n    vi = VariantIndex.from_parquet(session, variant_index_path).persist()\n    va = VariantAnnotation.from_parquet(session, variant_annotation_path)\n    vep_consequences = session.spark.read.csv(\n        vep_consequences_path, sep=\"\\t\", header=True\n    ).select(\n        f.element_at(f.split(\"Accession\", r\"/\"), -1).alias(\n            \"variantFunctionalConsequenceId\"\n        ),\n        f.col(\"Term\").alias(\"label\"),\n        f.col(\"v2g_score\").cast(\"double\").alias(\"score\"),\n    )\n\n    # Transform\n    lift = LiftOverSpark(\n        # lift over variants to hg38\n        liftover_chain_file_path,\n        liftover_max_length_difference,\n    )\n    gene_index_filtered = gene_index.filter_by_biotypes(\n        # Filter gene index by approved biotypes to define V2G gene universe\n        list(approved_biotypes)\n    )\n    va_slimmed = va.filter_by_variant_df(\n        # Variant annotation reduced to the variant index to define V2G variant universe\n        vi.df\n    ).persist()\n    intervals = Intervals(\n        _df=reduce(\n            lambda x, y: x.unionByName(y, allowMissingColumns=True),\n            # create interval instances by parsing each source\n            [\n                Intervals.from_source(\n                    session.spark, source_name, source_path, gene_index, lift\n                ).df\n                for source_name, source_path in interval_sources.items()\n            ],\n        ),\n        _schema=Intervals.get_schema(),\n    )\n    v2g_datasets = [\n        va_slimmed.get_distance_to_tss(gene_index_filtered, max_distance),\n        va_slimmed.get_most_severe_vep_v2g(vep_consequences, gene_index_filtered),\n        va_slimmed.get_plof_v2g(gene_index_filtered),\n        intervals.v2g(vi),\n    ]\n    v2g = V2G(\n        _df=reduce(\n            lambda x, y: x.unionByName(y, allowMissingColumns=True),\n            [dataset.df for dataset in v2g_datasets],\n        ).repartition(\"chromosome\"),\n        _schema=V2G.get_schema(),\n    )\n\n    # Load\n    (\n        v2g.df.write.partitionBy(\"chromosome\")\n        .mode(session.write_mode)\n        .parquet(v2g_path)\n    )\n</code></pre>"},{"location":"python_api/steps/window_based_clumping/","title":"window_based_clumping","text":""},{"location":"python_api/steps/window_based_clumping/#gentropy.window_based_clumping.WindowBasedClumpingStep","title":"<code>gentropy.window_based_clumping.WindowBasedClumpingStep</code>","text":"<p>Apply window based clumping on summary statistics datasets.</p> Source code in <code>src/gentropy/window_based_clumping.py</code> <pre><code>class WindowBasedClumpingStep:\n    \"\"\"Apply window based clumping on summary statistics datasets.\"\"\"\n\n    def __init__(\n        self,\n        session: Session,\n        summary_statistics_input_path: str,\n        study_locus_output_path: str,\n        inclusion_list_path: str | None = None,\n        locus_collect_distance: int | None = None,\n    ) -&gt; None:\n        \"\"\"Run window-based clumping step.\n\n        Args:\n            session (Session): Session object.\n            summary_statistics_input_path (str): Path to the harmonized summary statistics dataset.\n            study_locus_output_path (str): Output path for the resulting study locus dataset.\n            inclusion_list_path (str | None): Path to the inclusion list (list of white-listed study identifier). Optional.\n            locus_collect_distance (int | None): Distance, within which tagging variants are collected around the semi-index. Optional.\n        \"\"\"\n        # If inclusion list path is provided, only these studies will be read:\n        if inclusion_list_path:\n            study_ids_to_ingest = [\n                f'{summary_statistics_input_path}/{row[\"studyId\"]}.parquet'\n                for row in session.spark.read.parquet(inclusion_list_path).collect()\n            ]\n        else:\n            # If no inclusion list is provided, read all summary stats in folder:\n            study_ids_to_ingest = [summary_statistics_input_path]\n\n        (\n            SummaryStatistics.from_parquet(\n                session,\n                study_ids_to_ingest,\n                recursiveFileLookup=True,\n            )\n            .coalesce(4000)\n            # Applying window based clumping:\n            .window_based_clumping(locus_collect_distance=locus_collect_distance)\n            # Save resulting study locus dataset:\n            .df.write.mode(session.write_mode)\n            .parquet(study_locus_output_path)\n        )\n</code></pre>"},{"location":"python_api/steps/window_based_clumping/#gentropy.window_based_clumping.WindowBasedClumpingStep.__init__","title":"<code>__init__(session: Session, summary_statistics_input_path: str, study_locus_output_path: str, inclusion_list_path: str | None = None, locus_collect_distance: int | None = None) -&gt; None</code>","text":"<p>Run window-based clumping step.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Session object.</p> required <code>summary_statistics_input_path</code> <code>str</code> <p>Path to the harmonized summary statistics dataset.</p> required <code>study_locus_output_path</code> <code>str</code> <p>Output path for the resulting study locus dataset.</p> required <code>inclusion_list_path</code> <code>str | None</code> <p>Path to the inclusion list (list of white-listed study identifier). Optional.</p> <code>None</code> <code>locus_collect_distance</code> <code>int | None</code> <p>Distance, within which tagging variants are collected around the semi-index. Optional.</p> <code>None</code> Source code in <code>src/gentropy/window_based_clumping.py</code> <pre><code>def __init__(\n    self,\n    session: Session,\n    summary_statistics_input_path: str,\n    study_locus_output_path: str,\n    inclusion_list_path: str | None = None,\n    locus_collect_distance: int | None = None,\n) -&gt; None:\n    \"\"\"Run window-based clumping step.\n\n    Args:\n        session (Session): Session object.\n        summary_statistics_input_path (str): Path to the harmonized summary statistics dataset.\n        study_locus_output_path (str): Output path for the resulting study locus dataset.\n        inclusion_list_path (str | None): Path to the inclusion list (list of white-listed study identifier). Optional.\n        locus_collect_distance (int | None): Distance, within which tagging variants are collected around the semi-index. Optional.\n    \"\"\"\n    # If inclusion list path is provided, only these studies will be read:\n    if inclusion_list_path:\n        study_ids_to_ingest = [\n            f'{summary_statistics_input_path}/{row[\"studyId\"]}.parquet'\n            for row in session.spark.read.parquet(inclusion_list_path).collect()\n        ]\n    else:\n        # If no inclusion list is provided, read all summary stats in folder:\n        study_ids_to_ingest = [summary_statistics_input_path]\n\n    (\n        SummaryStatistics.from_parquet(\n            session,\n            study_ids_to_ingest,\n            recursiveFileLookup=True,\n        )\n        .coalesce(4000)\n        # Applying window based clumping:\n        .window_based_clumping(locus_collect_distance=locus_collect_distance)\n        # Save resulting study locus dataset:\n        .df.write.mode(session.write_mode)\n        .parquet(study_locus_output_path)\n    )\n</code></pre>"}]}
\ No newline at end of file
+{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Open Targets Gentropy","text":"<p>Open Targets Gentropy is a Python package to facilitate the interpretation and analysis of GWAS and functional genomic studies for target identification. This package contains a toolkit for the harmonisation, statistical analysis and prioritisation of genetic signals to assist drug discovery.</p>"},{"location":"#key-features","title":"Key Features:","text":"<ul> <li>Specialized Datatypes: Introduces essential genetics datatypes like StudyLocus, LocusToGene, and SummaryStatistics.</li> <li>Performance-Oriented: Optimized for large-scale genetic data analysis, including locus-to-gene scoring, fine mapping, and colocalization analysis.</li> <li>User-Friendly: The package is designed to be intuitive, allowing both beginners and experienced researchers to conduct complex genetic with ease.</li> </ul>"},{"location":"#about-open-targets","title":"About Open Targets","text":"<p>Open Targets is a pre-competitive, public-private partnership that uses human genetics and genomics data to systematically identify and prioritise drug targets. Through large-scale genomic experiments and the development of innovative computational techniques, the partnership aims to help researchers select the best targets for the development of new therapies. For more information, visit the Open Targets website.</p>"},{"location":"installation/","title":"Installation","text":"<p>Note</p> <p>In the early stages of development, we are using Python version 3.10. We recommend using pyenv or similar tools to manage your local Python version. We intend to support more Python versions in the future.</p>"},{"location":"installation/#pypi","title":"Pypi","text":"<p>We recommend installing Open Targets Gentropy using Pypi:</p> <pre><code>pip install gentropy\n</code></pre>"},{"location":"installation/#source","title":"Source","text":"<p>Alternatively, you can install Open Targets Gentropy from source. Check the contributing section for more information.</p>"},{"location":"installation/#uninstall","title":"Uninstall","text":"<pre><code>pip uninstall gentropy -y\n</code></pre> <p>For any issues with the installation, check the troubleshooting section.</p>"},{"location":"roadmap/","title":"Roadmap","text":"<p>The Open Targets core team is working on refactoring Open Targets Genetics, aiming to:</p> <ul> <li>Re-focus the product around Target ID</li> <li>Create a gold standard toolkit for post-GWAS analysis</li> <li>Faster/robust addition of new datasets and datatypes</li> <li>Reduce computational and financial cost</li> </ul> <p>See here for a list of open issues for this project.</p> <p>Schematic diagram representing the drafted process:</p> <p></p>"},{"location":"development/_development/","title":"Development","text":"<p>This section contains various technical information on how to develop and run the code.</p>"},{"location":"development/airflow/","title":"Airflow configuration","text":"<p>This section describes how to set up a local Airflow server which will orchestrate running workflows in Google Cloud Platform. This is useful for testing and debugging, but for production use, it is recommended to run Airflow on a dedicated server.</p>"},{"location":"development/airflow/#install-pre-requisites","title":"Install pre-requisites","text":"<ul> <li>Docker</li> <li>Google Cloud SDK</li> </ul> <p>Warning</p> <p>On macOS, the default amount of memory available for Docker might not be enough to get Airflow up and running. Allocate at least 4GB of memory for the Docker Engine (ideally 8GB). More info</p>"},{"location":"development/airflow/#configure-airflow-access-to-google-cloud-platform","title":"Configure Airflow access to Google Cloud Platform","text":"<p>Warning</p> <p>Run the next two command with the appropriate Google Cloud project ID and service account name to ensure the correct Google default application credentials are set up.</p> <p>Authenticate to Google Cloud:</p> <pre><code>gcloud auth application-default login --project=&lt;PROJECT&gt;\n</code></pre> <p>Create the service account key file that will be used by Airflow to access Google Cloud Platform resources:</p> <pre><code>gcloud iam service-accounts keys create ~/.config/gcloud/service_account_credentials.json --iam-account=&lt;PROJECT&gt;@appspot.gserviceaccount.com\n</code></pre>"},{"location":"development/airflow/#set-up-airflow","title":"Set up Airflow","text":"<p>Change the working directory so that all subsequent commands will work:</p> <pre><code>cd src/airflow\n</code></pre>"},{"location":"development/airflow/#build-docker-image","title":"Build Docker image","text":"<p>Note</p> <p>The custom Dockerfile built by the command below extends the official Airflow Docker Compose YAML. We add support for Google Cloud SDK, Google Dataproc operators, and access to GCP credentials.</p> <pre><code>docker build . --tag extending_airflow:latest\n</code></pre>"},{"location":"development/airflow/#set-airflow-user-id","title":"Set Airflow user ID","text":"<p>Note</p> <p>These commands allow Airflow running inside Docker to access the credentials file which was generated earlier.</p> <pre><code># If any user ID is already specified in .env, remove it.\ngrep -v \"AIRFLOW_UID\" .env &gt; .env.tmp\n# Add the correct user ID.\necho \"AIRFLOW_UID=$(id -u)\" &gt;&gt; .env.tmp\n# Move the file.\nmv .env.tmp .env\n</code></pre>"},{"location":"development/airflow/#initialise","title":"Initialise","text":"<p>Before starting Airflow, initialise the database:</p> <pre><code>docker compose up airflow-init\n</code></pre> <p>Now start all services:</p> <pre><code>docker compose up -d\n</code></pre> <p>Airflow UI will now be available at <code>http://localhost:8080/</code>. Default username and password are both <code>airflow</code>.</p> <p>For additional information on how to use Airflow visit the official documentation.</p>"},{"location":"development/airflow/#cleaning-up","title":"Cleaning up","text":"<p>At any time, you can check the status of your containers with:</p> <pre><code>docker ps\n</code></pre> <p>To stop Airflow, run:</p> <pre><code>docker compose down\n</code></pre> <p>To cleanup the Airflow database, run:</p> <pre><code>docker compose down --volumes --remove-orphans\n</code></pre>"},{"location":"development/airflow/#advanced-configuration","title":"Advanced configuration","text":"<p>More information on running Airflow with Docker Compose can be found in the official docs.</p> <ol> <li>Increase Airflow concurrency. Modify the <code>docker-compose.yaml</code> and add the following to the x-airflow-common \u2192 environment section:</li> </ol> <pre><code>AIRFLOW__CORE__PARALLELISM: 32\nAIRFLOW__CORE__MAX_ACTIVE_TASKS_PER_DAG: 32\nAIRFLOW__SCHEDULER__MAX_TIS_PER_QUERY: 16\nAIRFLOW__CORE__MAX_ACTIVE_RUNS_PER_DAG: 1\n# Also add the following line if you are using CeleryExecutor (by default, LocalExecutor is used).\nAIRFLOW__CELERY__WORKER_CONCURRENCY: 32\n</code></pre> <ol> <li>Additional pip packages. They can be added to the <code>requirements.txt</code> file.</li> </ol>"},{"location":"development/airflow/#troubleshooting","title":"Troubleshooting","text":"<p>Note that when you a a new workflow under <code>dags/</code>, Airflow will not pick that up immediately. By default the filesystem is only scanned for new DAGs every 300s. However, once the DAG is added, updates are applied nearly instantaneously.</p> <p>Also, if you edit the DAG while an instance of it is running, it might cause problems with the run, as Airflow will try to update the tasks and their properties in DAG according to the file changes.</p>"},{"location":"development/contributing/","title":"Contributing guidelines","text":""},{"location":"development/contributing/#one-time-configuration","title":"One-time configuration","text":"<p>The steps in this section only ever need to be done once on any particular system.</p> <p>For Google Cloud configuration:</p> <ol> <li> <p>Install Google Cloud SDK: https://cloud.google.com/sdk/docs/install.</p> </li> <li> <p>Log in to your work Google Account: run <code>gcloud auth login</code> and follow instructions.</p> </li> <li> <p>Obtain Google application credentials: run <code>gcloud auth application-default login</code> and follow instructions.</p> </li> </ol> <p>Check that you have the <code>make</code> utility installed, and if not (which is unlikely), install it using your system package manager.</p> <p>Check that you have <code>java</code> installed.</p>"},{"location":"development/contributing/#environment-configuration","title":"Environment configuration","text":"<p>Run <code>make setup-dev</code> to install/update the necessary packages and activate the development environment. You need to do this every time you open a new shell.</p> <p>It is recommended to use VS Code as an IDE for development.</p>"},{"location":"development/contributing/#how-to-run-the-code","title":"How to run the code","text":"<p>All pipelines in this repository are intended to be run in Google Dataproc. Running them locally is not currently supported.</p> <p>In order to run the code:</p> <ol> <li> <p>Manually edit your local <code>src/airflow/dags/*</code> file and comment out the steps you do not want to run.</p> </li> <li> <p>Manually edit your local <code>pyproject.toml</code> file and modify the version of the code.</p> </li> <li> <p>This must be different from the version used by any other people working on the repository to avoid any deployment conflicts, so it's a good idea to use your name, for example: <code>1.2.3+jdoe</code>.</p> </li> <li>You can also add a brief branch description, for example: <code>1.2.3+jdoe.myfeature</code>.</li> <li>Note that the version must comply with PEP440 conventions, otherwise Poetry will not allow it to be deployed.</li> <li> <p>Do not use underscores or hyphens in your version name. When building the WHL file, they will be automatically converted to dots, which means the file name will no longer match the version and the build will fail. Use dots instead.</p> </li> <li> <p>Manually edit your local <code>src/airflow/dags/common_airflow.py</code> and set <code>OTG_VERSION</code> to the same version as you did in the previous step.</p> </li> <li> <p>Run <code>make build</code>.</p> </li> <li> <p>This will create a bundle containing the neccessary code, configuration and dependencies to run the ETL pipeline, and then upload this bundle to Google Cloud.</p> </li> <li>A version specific subpath is used, so uploading the code will not affect any branches but your own.</li> <li> <p>If there was already a code bundle uploaded with the same version number, it will be replaced.</p> </li> <li> <p>Open Airflow UI and run the DAG.</p> </li> </ol>"},{"location":"development/contributing/#contributing-checklist","title":"Contributing checklist","text":"<p>When making changes, and especially when implementing a new module or feature, it's essential to ensure that all relevant sections of the code base are modified.</p> <ul> <li>[ ] Run <code>make check</code>. This will run the linter and formatter to ensure that the code is compliant with the project conventions.</li> <li>[ ] Develop unit tests for your code and run <code>make test</code>. This will run all unit tests in the repository, including the examples appended in the docstrings of some methods.</li> <li>[ ] Update the configuration if necessary.</li> <li>[ ] Update the documentation and check it with <code>make build-documentation</code>. This will start a local server to browse it (URL will be printed, usually <code>http://127.0.0.1:8000/</code>)</li> </ul> <p>For more details on each of these steps, see the sections below.</p>"},{"location":"development/contributing/#documentation","title":"Documentation","text":"<ul> <li>If during development you had a question which wasn't covered in the documentation, and someone explained it to you, add it to the documentation. The same applies if you encountered any instructions in the documentation which were obsolete or incorrect.</li> <li>Documentation autogeneration expressions start with <code>:::</code>. They will automatically generate sections of the documentation based on class and method docstrings. Be sure to update them for:</li> <li>Dataset definitions in <code>docs/python_api/datasource/STEP</code> (example: <code>docs/python_api/datasource/finngen/study_index.md</code>)</li> <li>Step definition in <code>docs/python_api/step/STEP.md</code> (example: <code>docs/python_api/step/finngen.md</code>)</li> </ul>"},{"location":"development/contributing/#configuration","title":"Configuration","text":"<ul> <li>Input and output paths in <code>config/datasets/gcp.yaml</code></li> <li>Step configuration in <code>config/step/STEP.yaml</code> (example: <code>config/step/finngen.yaml</code>)</li> </ul>"},{"location":"development/contributing/#classes","title":"Classes","text":"<ul> <li>Dataset class in <code>src/gentropy/datasource/STEP</code> (example: <code>src/gentropy/datasource/finngen/study_index.py</code> \u2192 <code>FinnGenStudyIndex</code>)</li> <li>Step main running class in <code>src/gentropy/STEP.py</code> (example: <code>src/gentropy/finngen.py</code>)</li> </ul>"},{"location":"development/contributing/#tests","title":"Tests","text":"<ul> <li>Test study fixture in <code>tests/conftest.py</code> (example: <code>mock_study_index_finngen</code> in that module)</li> <li>Test sample data in <code>tests/data_samples</code> (example: <code>tests/gentropy/data_samples/finngen_studies_sample.json</code>)</li> <li>Test definition in <code>tests/</code> (example: <code>tests/dataset/test_study_index.py</code> \u2192 <code>test_study_index_finngen_creation</code>)</li> </ul>"},{"location":"development/troubleshooting/","title":"Troubleshooting","text":""},{"location":"development/troubleshooting/#blaslapack","title":"BLAS/LAPACK","text":"<p>If you see errors related to BLAS/LAPACK libraries, see this StackOverflow post for guidance.</p>"},{"location":"development/troubleshooting/#pyenv-and-poetry","title":"Pyenv and Poetry","text":"<p>If you see various errors thrown by Pyenv or Poetry, they can be hard to specifically diagnose and resolve. In this case, it often helps to remove those tools from the system completely. Follow these steps:</p> <ol> <li>Close your currently activated environment, if any: <code>exit</code></li> <li>Uninstall Poetry: <code>curl -sSL https://install.python-poetry.org | python3 - --uninstall</code></li> <li>Clear Poetry cache: <code>rm -rf ~/.cache/pypoetry</code></li> <li>Clear pre-commit cache: <code>rm -rf ~/.cache/pre-commit</code></li> <li>Switch to system Python shell: <code>pyenv shell system</code></li> <li>Edit <code>~/.bashrc</code> to remove the lines related to Pyenv configuration</li> <li>Remove Pyenv configuration and cache: <code>rm -rf ~/.pyenv</code></li> </ol> <p>After that, open a fresh shell session and run <code>make setup-dev</code> again.</p>"},{"location":"development/troubleshooting/#java","title":"Java","text":"<p>Officially, PySpark requires Java version 8 (a.k.a. 1.8) or above to work. However, if you have a very recent version of Java, you may experience issues, as it may introduce breaking changes that PySpark hasn't had time to integrate. For example, as of May 2023, PySpark did not work with Java 20.</p> <p>If you are encountering problems with initialising a Spark session, try using Java 11.</p>"},{"location":"development/troubleshooting/#pre-commit","title":"Pre-commit","text":"<p>If you see an error message thrown by pre-commit, which looks like this (<code>SyntaxError: Unexpected token '?'</code>), followed by a JavaScript traceback, the issue is likely with your system NodeJS version.</p> <p>One solution which can help in this case is to upgrade your system NodeJS version. However, this may not always be possible. For example, Ubuntu repository is several major versions behind the latest version as of July 2023.</p> <p>Another solution which helps is to remove Node, NodeJS, and npm from your system entirely. In this case, pre-commit will not try to rely on a system version of NodeJS and will install its own, suitable one.</p> <p>On Ubuntu, this can be done using <code>sudo apt remove node nodejs npm</code>, followed by <code>sudo apt autoremove</code>. But in some cases, depending on your existing installation, you may need to also manually remove some files. See this StackOverflow answer for guidance.</p> <p>After running these commands, you are advised to open a fresh shell, and then also reinstall Pyenv and Poetry to make sure they pick up the changes (see relevant section above).</p>"},{"location":"development/workflows/","title":"Pipeline workflows","text":"<p>This page describes the high level components of the pipeline, which are organised as Airflow DAGs (directed acyclic graphs).</p>"},{"location":"development/workflows/#note-on-dags-and-dataproc-clusters","title":"Note on DAGs and Dataproc clusters","text":"<p>Each DAG consists of the following general stages:</p> <ol> <li> <p>Create cluster (if it already exists, this step is skipped)</p> </li> <li> <p>Install dependencies on the cluster</p> </li> <li> <p>Run data processing steps for this DAG</p> </li> <li> <p>Delete the cluster</p> </li> </ol> <p>Within a DAG, all data processing steps run on the same Dataproc cluster as separate jobs.</p> <p>There is no need to configure DAGs or steps depending on the size of the input data. Clusters have autoscaling enabled, which means they will increase or decrease the number of worker VMs to accommodate the load.</p>"},{"location":"development/workflows/#dag-1-preprocess","title":"DAG 1: Preprocess","text":"<p>This DAG contains steps which are only supposed to be run once, or very rarely. They ingest external data and apply bespoke transformations specific for each particular data source. The output is normalised according to the data schemas used by the pipeline.</p>"},{"location":"development/workflows/#dag-2-etl","title":"DAG 2: ETL","text":"<p>The ETL DAG takes the inputs of the previous step and performs the main algorithmic processing. This processing is supposed to be data source agnostic.</p>"},{"location":"howto/_howto/","title":"How-to","text":"<p>This page contains a collection of how-to guides for the project.</p> <ul> <li>Command line interface: Learn how to use the Gentropy CLI.</li> <li>Python API: Learn how to use the Gentropy Python package.</li> </ul> <p>For additional information please visit https://community.opentargets.org/</p>"},{"location":"howto/command_line/_command_line/","title":"Command line interface","text":"<p>Gentropy steps can be run using the command line interface (CLI). This section contains a collection of how-to guides for the CLI.</p>"},{"location":"howto/command_line/run_step_in_cli/","title":"Run step in CLI","text":"<p>To run a step in the command line interface (CLI), you need to know the step's name. To list what steps are avaiable in your current environment, simply run <code>gentropy</code> with no arguments. This will list all the steps:</p> <pre><code>You must specify 'step', e.g, step=&lt;OPTION&gt;\nAvailable options:\n        clump\n        colocalisation\n        eqtl_catalogue\n        finngen_studies\n        finngen_sumstat_preprocess\n        gene_index\n        gwas_catalog_ingestion\n        gwas_catalog_sumstat_preprocess\n        ld_index\n        locus_to_gene\n        overlaps\n        pics\n        ukbiobank\n        variant_annotation\n        variant_index\n        variant_to_gene\n\nSet the environment variable HYDRA_FULL_ERROR=1 for a complete stack trace.\n</code></pre> <p>As indicated, you can run a step by specifying the step's name with the <code>step</code> argument. For example, to run the <code>gene_index</code> step, you can run:</p> <pre><code>gentropy step=gene_index\n</code></pre> <p>In most occassions, some mandatory values will be required to run the step. For example, the <code>gene_index</code> step requires the <code>step.target_path</code> and <code>step.gene_index_path</code> argument to be specified. You can complete the necessary arguments by adding them to the command line:</p> <pre><code>gentropy step=gene_index step.target_path=/path/to/target step.gene_index_path=/path/to/gene_index\n</code></pre> <p>You can find more about the available steps in the documentation.</p>"},{"location":"howto/command_line/run_step_using_config/","title":"Run step using YAML config","text":"<p>It's possible to parametrise the functionality of a step using a YAML configuration file. This is useful when you want to run a step multiple times with different parameters or simply to avoid having to specify the same parameters every time you run a step.</p> <p>Info</p> <p>The package uses Hydra to handle configuration files. For more information, please visit the Hydra documentation.</p> <p>To run a step using a configuration file, you need to create a configuration file in YAML format.</p> <pre><code>config/\n\u251c\u2500 step/\n\u2502  \u2514\u2500 my_gene_index.md\n\u2514\u2500 my_config.yml\n</code></pre> <p>The configuration file should contain the parameters you want to use to run the step. For example, to run the <code>gene_index</code> step, you need to specify the <code>step.target_path</code> and <code>step.gene_index_path</code> parameters. The configuration file should look like this:</p> my_config.yamlstep/my_gene_index.md <pre><code>defaults:\n    - config\n    - _self_\n</code></pre> <p>This config file will specify that your configuration file will inherit the default configuration (<code>config</code>) and everything provided (<code>_self_</code>) will overwrite the default configuration.</p> <pre><code>defaults:\n    - gene_index\n\ntarget_path: /path/to/target\ngene_index_path: /path/to/gene_index\n</code></pre> <p>This config file will inherit the default configuration for the <code>gene_index</code> step and overwrite the <code>target_path</code> and <code>gene_index_path</code> parameters.</p> <p>Once you have created the configuration file, you can run your own new <code>my_gene_index</code>:</p> <pre><code>gentropy step=my_gene_index --config-dir=config --config-name=my_config\n</code></pre>"},{"location":"howto/python_api/_python_api/","title":"Python API","text":"<p>This section explains how to use gentropy in a Python environment providing a foundational understanding on how to perform genetics analyses using the package. This section can be useful for users wishing to use Gentropy in their own projects.</p>"},{"location":"howto/python_api/a_creating_spark_session/","title":"Creating a Spark Session","text":"<p>In this section, we'll guide you through creating a Spark session using Gentropy's Session class. Gentropy uses Apache PySpark as the underlying framework for distributed computing. The Session class provides a convenient way to initialize a Spark session with pre-configured settings.</p>"},{"location":"howto/python_api/a_creating_spark_session/#creating-a-default-session","title":"Creating a Default Session","text":"<p>To begin your journey with Gentropy, start by creating a default Spark session. This is the simplest way to initialize your environment.</p> <pre><code>from gentropy.common.session import Session\n\n# Create a default Spark Session\nsession = Session()\n</code></pre> <p>The above code snippet sets up a default Spark session with pre-configured settings. This is ideal for getting started quickly without needing to tweak any configurations.</p>"},{"location":"howto/python_api/a_creating_spark_session/#customizing-your-spark-session","title":"Customizing Your Spark Session","text":"<p>Gentropy allows you to customize the Spark session to suit your specific needs. You can modify various parameters such as memory allocation, number of executors, and more. This flexibility is particularly useful for optimizing performance in steps that are more computationally intensive.</p>"},{"location":"howto/python_api/a_creating_spark_session/#example-increasing-driver-memory","title":"Example: Increasing Driver Memory","text":"<p>If you require more memory for the Spark driver, you can easily adjust this setting:</p> <pre><code>from gentropy.common.session import Session\n\n# Create a Spark session with increased driver memory\nsession = Session(extended_spark_conf={\"spark.driver.memory\": \"4g\"})\n</code></pre> <p>This code snippet demonstrates how to increase the memory allocated to the Spark driver to 16 gigabytes. You can customize other Spark settings similarly, according to your project's requirements.</p>"},{"location":"howto/python_api/a_creating_spark_session/#whats-next","title":"What's next?","text":"<p>Now that you've created a Spark session, you're ready to start using Gentropy. In the next section, we'll show you how to process a large dataset using Gentropy's powerful SummaryStatistics datatype.</p>"},{"location":"howto/python_api/b_create_dataset/","title":"Create a dataset","text":"<p>Gentropy provides a collection of <code>Dataset</code>s that encapsulate key concepts in the field of genetics. For example, to represent summary statistics, you'll use the <code>SummaryStatistics</code> class. This datatype comes with a set of useful operations to disentangle the genetic architecture of a trait or disease.</p> <p>The full list of <code>Dataset</code>s is available in the Python API documentation.</p> <p>Any instance of Dataset will have 2 common attributes</p> <ul> <li>df: the Spark DataFrame that contains the data</li> <li>schema: the definition of the data structure in Spark format</li> </ul> <p>In this section you'll learn the different ways of how to create a <code>Dataset</code> instances.</p>"},{"location":"howto/python_api/b_create_dataset/#creating-a-dataset-from-parquet","title":"Creating a dataset from parquet","text":"<p>All the <code>Dataset</code>s have a <code>from_parquet</code> method that allows you to create any <code>Dataset</code> instance from a parquet file or directory.</p> <pre><code># Create a SummaryStatistics object by loading data from the specified path\nfrom gentropy.dataset.summary_statistics import SummaryStatistics\n\npath = \"path/to/summary/stats\"\nsummary_stats = SummaryStatistics.from_parquet(session, path)\n</code></pre> <p>Parquet files</p> <p>Parquet is a columnar storage format that is widely used in the Spark ecosystem. It is the recommended format for storing large datasets. For more information about parquet, please visit https://parquet.apache.org/.</p>"},{"location":"howto/python_api/b_create_dataset/#creating-a-dataset-from-a-data-source","title":"Creating a dataset from a data source","text":"<p>Alternatively, <code>Dataset</code>s can be created using a data source harmonisation method. For example, to create a <code>SummaryStatistics</code> object from Finngen's raw summary statistics, you can use the <code>FinnGen</code> data source.</p> <pre><code># Create a SummaryStatistics object by loading raw data from Finngen\nfrom gentropy.datasource.finngen.summary_stats import FinnGenSummaryStats\n\npath = \"path/to/finngen/summary/stats\"\nsummary_stats = FinnGenSummaryStats.from_source(session.spark, path)\n</code></pre>"},{"location":"howto/python_api/b_create_dataset/#creating-a-dataset-from-a-pandas-dataframe","title":"Creating a dataset from a pandas DataFrame","text":"<p>If none of our data sources fit your needs, you can create a <code>Dataset</code> object from your own data. To do so, you need to transform your data to fit the <code>Dataset</code> schema.</p> <p>The schema of a Dataset is defined in Spark format</p> <p>The Dataset schemas can be found in the documentation of each Dataset. For example, the schema of the <code>SummaryStatistics</code> dataset can be found here.</p> <p>You can also create a <code>Dataset</code> from a pandas DataFrame. This is useful when you want to create a <code>Dataset</code> from a small dataset that fits in memory.</p> <pre><code>import pyspark.pandas as ps\nfrom gentropy.dataset.summary_statistics import SummaryStatistics\n\n\n# Load your transformed data into a pandas DataFrame\npath = \"path/to/your/data\"\ncustom_summary_stats_pandas_df = pd.read_csv(path)\n\n# Create a SummaryStatistics object specifying the data and schema\ncustom_summary_stats_df = custom_summary_stats_pandas_df.to_spark()\ncustom_summary_stats = SummaryStatistics(\n    _df=custom_summary_stats_df, _schema=SummaryStatistics.get_schema()\n)\n</code></pre>"},{"location":"howto/python_api/b_create_dataset/#whats-next","title":"What's next?","text":"<p>In the next section, we will explore how to apply well-established algorithms that transform and analyse genetic data within the Gentropy framework.</p>"},{"location":"howto/python_api/c_applying_methods/","title":"Applying methods","text":"<p>The available methods implement well established algorithms that transform and analyse data. Methods usually take as input predefined <code>Dataset</code>(s) and produce one or several <code>Dataset</code>(s) as output. This section explains how to apply methods to your data.</p> <p>The full list of available methods can be found in the Python API documentation.</p>"},{"location":"howto/python_api/c_applying_methods/#apply-a-class-method","title":"Apply a class method","text":"<p>Some methods are implemented as class methods. For example, the <code>finemap</code> method is a class method of the <code>PICS</code> class. This method performs fine-mapping using the PICS algorithm. These methods usually take as input one or several <code>Dataset</code>(s) and produce one or several <code>Dataset</code>(s) as output.</p> <pre><code>from gentropy.method.pics import PICS\n\nfinemapped_study_locus = PICS.finemap(\n    study_locus_ld_annotated\n).annotate_credible_sets()\n</code></pre>"},{"location":"howto/python_api/c_applying_methods/#apply-a-dataset-instance-method","title":"Apply a <code>Dataset</code> instance method","text":"<p>Some methods are implemented as instance methods of the <code>Dataset</code> class. For example, the <code>window_based_clumping</code> method is an instance method of the <code>SummaryStatistics</code> class. This method performs window-based clumping on summary statistics.</p> <pre><code># Perform window-based clumping on summary statistics\n# By default, the method uses a 1Mb window and a p-value threshold of 5e-8\nclumped_summary_statistics = summary_stats.window_based_clumping()\n</code></pre> <p>The <code>window_based_clumping</code> method is also available as a class method</p> <p>The <code>window_based_clumping</code> method is also available as a class method of the <code>WindowBasedClumping</code> class. This method performs window-based clumping on summary statistics.</p> <pre><code># Perform window-based clumping on summary statistics\nfrom gentropy.method.window_based_clumping import WindowBasedClumping\n\nclumped_summary_statistics = WindowBasedClumping.clump(\n    summary_stats, distance=250_000\n)\n</code></pre>"},{"location":"howto/python_api/c_applying_methods/#whats-next","title":"What's next?","text":"<p>Up next, we'll show you how to inspect your data to ensure its integrity and the success of your transformations.</p>"},{"location":"howto/python_api/d_inspect_dataset/","title":"Inspect a dataset","text":"<p>We have seen how to create and transform a <code>Dataset</code> instance. This section guides you through inspecting your data to ensure its integrity and the success of your transformations.</p>"},{"location":"howto/python_api/d_inspect_dataset/#inspect-data-in-a-dataset","title":"Inspect data in a <code>Dataset</code>","text":"<p>The <code>df</code> attribute of a Dataset instance is key to interacting with and inspecting the stored data.</p> <p>By accessing the df attribute, you can apply any method that you would typically use on a PySpark DataFrame. See the PySpark documentation for more information.</p>"},{"location":"howto/python_api/d_inspect_dataset/#view-data-samples","title":"View data samples","text":"<pre><code># Inspect the first 10 rows of the data\nsummary_stats.df.show(10)\n</code></pre> <p>This method displays the first 10 rows of your dataset, giving you a snapshot of your data's structure and content.</p>"},{"location":"howto/python_api/d_inspect_dataset/#filter-data","title":"Filter data","text":"<pre><code>import pyspark.sql.functions as f\n\n# Filter summary statistics to only include associations in chromosome 22\nfiltered = summary_stats.filter(condition=f.col(\"chromosome\") == \"22\")\n</code></pre> <p>This method allows you to filter your data based on specific conditions, such as the value of a column. The application of any filter will create a new instance of the <code>Dataset</code> with the filtered data.</p>"},{"location":"howto/python_api/d_inspect_dataset/#understand-the-schema","title":"Understand the schema","text":"<pre><code># Get the Spark schema of any `Dataset` as a `StructType` object\nschema = summary_stats.get_schema()\n\n# Inspect the first 10 rows of the data\nsummary_stats.df.show(10)\n</code></pre>"},{"location":"howto/python_api/d_inspect_dataset/#write-a-dataset-to-disk","title":"Write a <code>Dataset</code> to disk","text":"<pre><code># Write the data to disk in parquet format\nsummary_stats.df.write.parquet(\"path/to/summary/stats\")\n\n# Write the data to disk in csv format\nsummary_stats.df.write.csv(\"path/to/summary/stats\")\n</code></pre> <p>Consider the format's compatibility with your tools, and the partitioning strategy for large datasets to optimize performance.</p>"},{"location":"python_api/_python_api/","title":"Python API","text":"<p>Open Targets Gentropy is a Python package to facilitate the interpretation and analysis of GWAS and functional genomic studies for target identification. The package contains a toolkit for the harmonisation, statistical analysis and prioritisation of genetic signals to assist drug discovery.</p> <p>The overall architecture of the package distinguishes between:</p> <ul> <li>Data Sources: data sources harmonisation tools</li> <li>Datasets: data model</li> <li>Methods: statistical analysis tools</li> <li>Steps: pipeline steps</li> </ul>"},{"location":"python_api/datasets/_datasets/","title":"Datasets","text":"<p>The Dataset classes define the data model behind Open Targets Gentropy. Every class inherits from the <code>Dataset</code> class and contains a dataframe with a predefined schema that can be found in the respective classes.</p>"},{"location":"python_api/datasets/_datasets/#gentropy.dataset.dataset.Dataset","title":"<code>gentropy.dataset.dataset.Dataset</code>  <code>dataclass</code>","text":"<p>             Bases: <code>ABC</code></p> <p>Open Targets Gentropy Dataset.</p> <p><code>Dataset</code> is a wrapper around a Spark DataFrame with a predefined schema. Schemas for each child dataset are described in the <code>schemas</code> module.</p> Source code in <code>src/gentropy/dataset/dataset.py</code> <pre><code>@dataclass\nclass Dataset(ABC):\n    \"\"\"Open Targets Gentropy Dataset.\n\n    `Dataset` is a wrapper around a Spark DataFrame with a predefined schema. Schemas for each child dataset are described in the `schemas` module.\n    \"\"\"\n\n    _df: DataFrame\n    _schema: StructType\n\n    def __post_init__(self: Dataset) -&gt; None:\n        \"\"\"Post init.\"\"\"\n        self.validate_schema()\n\n    @property\n    def df(self: Dataset) -&gt; DataFrame:\n        \"\"\"Dataframe included in the Dataset.\n\n        Returns:\n            DataFrame: Dataframe included in the Dataset\n        \"\"\"\n        return self._df\n\n    @df.setter\n    def df(self: Dataset, new_df: DataFrame) -&gt; None:  # noqa: CCE001\n        \"\"\"Dataframe setter.\n\n        Args:\n            new_df (DataFrame): New dataframe to be included in the Dataset\n        \"\"\"\n        self._df: DataFrame = new_df\n        self.validate_schema()\n\n    @property\n    def schema(self: Dataset) -&gt; StructType:\n        \"\"\"Dataframe expected schema.\n\n        Returns:\n            StructType: Dataframe expected schema\n        \"\"\"\n        return self._schema\n\n    @classmethod\n    @abstractmethod\n    def get_schema(cls: type[Self]) -&gt; StructType:\n        \"\"\"Abstract method to get the schema. Must be implemented by child classes.\n\n        Returns:\n            StructType: Schema for the Dataset\n        \"\"\"\n        pass\n\n    @classmethod\n    def from_parquet(\n        cls: type[Self],\n        session: Session,\n        path: str | list[str],\n        **kwargs: bool | float | int | str | None,\n    ) -&gt; Self:\n        \"\"\"Reads parquet into a Dataset with a given schema.\n\n        Args:\n            session (Session): Spark session\n            path (str | list[str]): Path to the parquet dataset\n            **kwargs (bool | float | int | str | None): Additional arguments to pass to spark.read.parquet\n\n        Returns:\n            Self: Dataset with the parquet file contents\n\n        Raises:\n            ValueError: Parquet file is empty\n        \"\"\"\n        schema = cls.get_schema()\n        df = session.read_parquet(path, schema=schema, **kwargs)\n        if df.isEmpty():\n            raise ValueError(f\"Parquet file is empty: {path}\")\n        return cls(_df=df, _schema=schema)\n\n    def filter(self: Self, condition: Column) -&gt; Self:\n        \"\"\"Creates a new instance of a Dataset with the DataFrame filtered by the condition.\n\n        Args:\n            condition (Column): Condition to filter the DataFrame\n\n        Returns:\n            Self: Filtered Dataset\n        \"\"\"\n        df = self._df.filter(condition)\n        class_constructor = self.__class__\n        return class_constructor(_df=df, _schema=class_constructor.get_schema())\n\n    def validate_schema(self: Dataset) -&gt; None:\n        \"\"\"Validate DataFrame schema against expected class schema.\n\n        Raises:\n            ValueError: DataFrame schema is not valid\n        \"\"\"\n        expected_schema = self._schema\n        expected_fields = flatten_schema(expected_schema)\n        observed_schema = self._df.schema\n        observed_fields = flatten_schema(observed_schema)\n\n        # Unexpected fields in dataset\n        if unexpected_field_names := [\n            x.name\n            for x in observed_fields\n            if x.name not in [y.name for y in expected_fields]\n        ]:\n            raise ValueError(\n                f\"The {unexpected_field_names} fields are not included in DataFrame schema: {expected_fields}\"\n            )\n\n        # Required fields not in dataset\n        required_fields = [x.name for x in expected_schema if not x.nullable]\n        if missing_required_fields := [\n            req\n            for req in required_fields\n            if not any(field.name == req for field in observed_fields)\n        ]:\n            raise ValueError(\n                f\"The {missing_required_fields} fields are required but missing: {required_fields}\"\n            )\n\n        # Fields with duplicated names\n        if duplicated_fields := [\n            x for x in set(observed_fields) if observed_fields.count(x) &gt; 1\n        ]:\n            raise ValueError(\n                f\"The following fields are duplicated in DataFrame schema: {duplicated_fields}\"\n            )\n\n        # Fields with different datatype\n        observed_field_types = {\n            field.name: type(field.dataType) for field in observed_fields\n        }\n        expected_field_types = {\n            field.name: type(field.dataType) for field in expected_fields\n        }\n        if fields_with_different_observed_datatype := [\n            name\n            for name, observed_type in observed_field_types.items()\n            if name in expected_field_types\n            and observed_type != expected_field_types[name]\n        ]:\n            raise ValueError(\n                f\"The following fields present differences in their datatypes: {fields_with_different_observed_datatype}.\"\n            )\n\n    def persist(self: Self) -&gt; Self:\n        \"\"\"Persist in memory the DataFrame included in the Dataset.\n\n        Returns:\n            Self: Persisted Dataset\n        \"\"\"\n        self.df = self._df.persist()\n        return self\n\n    def unpersist(self: Self) -&gt; Self:\n        \"\"\"Remove the persisted DataFrame from memory.\n\n        Returns:\n            Self: Unpersisted Dataset\n        \"\"\"\n        self.df = self._df.unpersist()\n        return self\n\n    def coalesce(self: Self, num_partitions: int, **kwargs: Any) -&gt; Self:\n        \"\"\"Coalesce the DataFrame included in the Dataset.\n\n        Coalescing is efficient for decreasing the number of partitions because it avoids a full shuffle of the data.\n\n        Args:\n            num_partitions (int): Number of partitions to coalesce to\n            **kwargs (Any): Arguments to pass to the coalesce method\n\n        Returns:\n            Self: Coalesced Dataset\n        \"\"\"\n        self.df = self._df.coalesce(num_partitions, **kwargs)\n        return self\n\n    def repartition(self: Self, num_partitions: int, **kwargs: Any) -&gt; Self:\n        \"\"\"Repartition the DataFrame included in the Dataset.\n\n        Repartitioning creates new partitions with data that is distributed evenly.\n\n        Args:\n            num_partitions (int): Number of partitions to repartition to\n            **kwargs (Any): Arguments to pass to the repartition method\n\n        Returns:\n            Self: Repartitioned Dataset\n        \"\"\"\n        self.df = self._df.repartition(num_partitions, **kwargs)\n        return self\n</code></pre>"},{"location":"python_api/datasets/_datasets/#gentropy.dataset.dataset.Dataset.df","title":"<code>df: DataFrame</code>  <code>property</code> <code>writable</code>","text":"<p>Dataframe included in the Dataset.</p> <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>Dataframe included in the Dataset</p>"},{"location":"python_api/datasets/_datasets/#gentropy.dataset.dataset.Dataset.schema","title":"<code>schema: StructType</code>  <code>property</code>","text":"<p>Dataframe expected schema.</p> <p>Returns:</p> Name Type Description <code>StructType</code> <code>StructType</code> <p>Dataframe expected schema</p>"},{"location":"python_api/datasets/_datasets/#gentropy.dataset.dataset.Dataset.coalesce","title":"<code>coalesce(num_partitions: int, **kwargs: Any) -&gt; Self</code>","text":"<p>Coalesce the DataFrame included in the Dataset.</p> <p>Coalescing is efficient for decreasing the number of partitions because it avoids a full shuffle of the data.</p> <p>Parameters:</p> Name Type Description Default <code>num_partitions</code> <code>int</code> <p>Number of partitions to coalesce to</p> required <code>**kwargs</code> <code>Any</code> <p>Arguments to pass to the coalesce method</p> <code>{}</code> <p>Returns:</p> Name Type Description <code>Self</code> <code>Self</code> <p>Coalesced Dataset</p> Source code in <code>src/gentropy/dataset/dataset.py</code> <pre><code>def coalesce(self: Self, num_partitions: int, **kwargs: Any) -&gt; Self:\n    \"\"\"Coalesce the DataFrame included in the Dataset.\n\n    Coalescing is efficient for decreasing the number of partitions because it avoids a full shuffle of the data.\n\n    Args:\n        num_partitions (int): Number of partitions to coalesce to\n        **kwargs (Any): Arguments to pass to the coalesce method\n\n    Returns:\n        Self: Coalesced Dataset\n    \"\"\"\n    self.df = self._df.coalesce(num_partitions, **kwargs)\n    return self\n</code></pre>"},{"location":"python_api/datasets/_datasets/#gentropy.dataset.dataset.Dataset.filter","title":"<code>filter(condition: Column) -&gt; Self</code>","text":"<p>Creates a new instance of a Dataset with the DataFrame filtered by the condition.</p> <p>Parameters:</p> Name Type Description Default <code>condition</code> <code>Column</code> <p>Condition to filter the DataFrame</p> required <p>Returns:</p> Name Type Description <code>Self</code> <code>Self</code> <p>Filtered Dataset</p> Source code in <code>src/gentropy/dataset/dataset.py</code> <pre><code>def filter(self: Self, condition: Column) -&gt; Self:\n    \"\"\"Creates a new instance of a Dataset with the DataFrame filtered by the condition.\n\n    Args:\n        condition (Column): Condition to filter the DataFrame\n\n    Returns:\n        Self: Filtered Dataset\n    \"\"\"\n    df = self._df.filter(condition)\n    class_constructor = self.__class__\n    return class_constructor(_df=df, _schema=class_constructor.get_schema())\n</code></pre>"},{"location":"python_api/datasets/_datasets/#gentropy.dataset.dataset.Dataset.from_parquet","title":"<code>from_parquet(session: Session, path: str | list[str], **kwargs: bool | float | int | str | None) -&gt; Self</code>  <code>classmethod</code>","text":"<p>Reads parquet into a Dataset with a given schema.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Spark session</p> required <code>path</code> <code>str | list[str]</code> <p>Path to the parquet dataset</p> required <code>**kwargs</code> <code>bool | float | int | str | None</code> <p>Additional arguments to pass to spark.read.parquet</p> <code>{}</code> <p>Returns:</p> Name Type Description <code>Self</code> <code>Self</code> <p>Dataset with the parquet file contents</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>Parquet file is empty</p> Source code in <code>src/gentropy/dataset/dataset.py</code> <pre><code>@classmethod\ndef from_parquet(\n    cls: type[Self],\n    session: Session,\n    path: str | list[str],\n    **kwargs: bool | float | int | str | None,\n) -&gt; Self:\n    \"\"\"Reads parquet into a Dataset with a given schema.\n\n    Args:\n        session (Session): Spark session\n        path (str | list[str]): Path to the parquet dataset\n        **kwargs (bool | float | int | str | None): Additional arguments to pass to spark.read.parquet\n\n    Returns:\n        Self: Dataset with the parquet file contents\n\n    Raises:\n        ValueError: Parquet file is empty\n    \"\"\"\n    schema = cls.get_schema()\n    df = session.read_parquet(path, schema=schema, **kwargs)\n    if df.isEmpty():\n        raise ValueError(f\"Parquet file is empty: {path}\")\n    return cls(_df=df, _schema=schema)\n</code></pre>"},{"location":"python_api/datasets/_datasets/#gentropy.dataset.dataset.Dataset.get_schema","title":"<code>get_schema() -&gt; StructType</code>  <code>abstractmethod</code> <code>classmethod</code>","text":"<p>Abstract method to get the schema. Must be implemented by child classes.</p> <p>Returns:</p> Name Type Description <code>StructType</code> <code>StructType</code> <p>Schema for the Dataset</p> Source code in <code>src/gentropy/dataset/dataset.py</code> <pre><code>@classmethod\n@abstractmethod\ndef get_schema(cls: type[Self]) -&gt; StructType:\n    \"\"\"Abstract method to get the schema. Must be implemented by child classes.\n\n    Returns:\n        StructType: Schema for the Dataset\n    \"\"\"\n    pass\n</code></pre>"},{"location":"python_api/datasets/_datasets/#gentropy.dataset.dataset.Dataset.persist","title":"<code>persist() -&gt; Self</code>","text":"<p>Persist in memory the DataFrame included in the Dataset.</p> <p>Returns:</p> Name Type Description <code>Self</code> <code>Self</code> <p>Persisted Dataset</p> Source code in <code>src/gentropy/dataset/dataset.py</code> <pre><code>def persist(self: Self) -&gt; Self:\n    \"\"\"Persist in memory the DataFrame included in the Dataset.\n\n    Returns:\n        Self: Persisted Dataset\n    \"\"\"\n    self.df = self._df.persist()\n    return self\n</code></pre>"},{"location":"python_api/datasets/_datasets/#gentropy.dataset.dataset.Dataset.repartition","title":"<code>repartition(num_partitions: int, **kwargs: Any) -&gt; Self</code>","text":"<p>Repartition the DataFrame included in the Dataset.</p> <p>Repartitioning creates new partitions with data that is distributed evenly.</p> <p>Parameters:</p> Name Type Description Default <code>num_partitions</code> <code>int</code> <p>Number of partitions to repartition to</p> required <code>**kwargs</code> <code>Any</code> <p>Arguments to pass to the repartition method</p> <code>{}</code> <p>Returns:</p> Name Type Description <code>Self</code> <code>Self</code> <p>Repartitioned Dataset</p> Source code in <code>src/gentropy/dataset/dataset.py</code> <pre><code>def repartition(self: Self, num_partitions: int, **kwargs: Any) -&gt; Self:\n    \"\"\"Repartition the DataFrame included in the Dataset.\n\n    Repartitioning creates new partitions with data that is distributed evenly.\n\n    Args:\n        num_partitions (int): Number of partitions to repartition to\n        **kwargs (Any): Arguments to pass to the repartition method\n\n    Returns:\n        Self: Repartitioned Dataset\n    \"\"\"\n    self.df = self._df.repartition(num_partitions, **kwargs)\n    return self\n</code></pre>"},{"location":"python_api/datasets/_datasets/#gentropy.dataset.dataset.Dataset.unpersist","title":"<code>unpersist() -&gt; Self</code>","text":"<p>Remove the persisted DataFrame from memory.</p> <p>Returns:</p> Name Type Description <code>Self</code> <code>Self</code> <p>Unpersisted Dataset</p> Source code in <code>src/gentropy/dataset/dataset.py</code> <pre><code>def unpersist(self: Self) -&gt; Self:\n    \"\"\"Remove the persisted DataFrame from memory.\n\n    Returns:\n        Self: Unpersisted Dataset\n    \"\"\"\n    self.df = self._df.unpersist()\n    return self\n</code></pre>"},{"location":"python_api/datasets/_datasets/#gentropy.dataset.dataset.Dataset.validate_schema","title":"<code>validate_schema() -&gt; None</code>","text":"<p>Validate DataFrame schema against expected class schema.</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>DataFrame schema is not valid</p> Source code in <code>src/gentropy/dataset/dataset.py</code> <pre><code>def validate_schema(self: Dataset) -&gt; None:\n    \"\"\"Validate DataFrame schema against expected class schema.\n\n    Raises:\n        ValueError: DataFrame schema is not valid\n    \"\"\"\n    expected_schema = self._schema\n    expected_fields = flatten_schema(expected_schema)\n    observed_schema = self._df.schema\n    observed_fields = flatten_schema(observed_schema)\n\n    # Unexpected fields in dataset\n    if unexpected_field_names := [\n        x.name\n        for x in observed_fields\n        if x.name not in [y.name for y in expected_fields]\n    ]:\n        raise ValueError(\n            f\"The {unexpected_field_names} fields are not included in DataFrame schema: {expected_fields}\"\n        )\n\n    # Required fields not in dataset\n    required_fields = [x.name for x in expected_schema if not x.nullable]\n    if missing_required_fields := [\n        req\n        for req in required_fields\n        if not any(field.name == req for field in observed_fields)\n    ]:\n        raise ValueError(\n            f\"The {missing_required_fields} fields are required but missing: {required_fields}\"\n        )\n\n    # Fields with duplicated names\n    if duplicated_fields := [\n        x for x in set(observed_fields) if observed_fields.count(x) &gt; 1\n    ]:\n        raise ValueError(\n            f\"The following fields are duplicated in DataFrame schema: {duplicated_fields}\"\n        )\n\n    # Fields with different datatype\n    observed_field_types = {\n        field.name: type(field.dataType) for field in observed_fields\n    }\n    expected_field_types = {\n        field.name: type(field.dataType) for field in expected_fields\n    }\n    if fields_with_different_observed_datatype := [\n        name\n        for name, observed_type in observed_field_types.items()\n        if name in expected_field_types\n        and observed_type != expected_field_types[name]\n    ]:\n        raise ValueError(\n            f\"The following fields present differences in their datatypes: {fields_with_different_observed_datatype}.\"\n        )\n</code></pre>"},{"location":"python_api/datasets/colocalisation/","title":"Colocalisation","text":""},{"location":"python_api/datasets/colocalisation/#gentropy.dataset.colocalisation.Colocalisation","title":"<code>gentropy.dataset.colocalisation.Colocalisation</code>  <code>dataclass</code>","text":"<p>             Bases: <code>Dataset</code></p> <p>Colocalisation results for pairs of overlapping study-locus.</p> Source code in <code>src/gentropy/dataset/colocalisation.py</code> <pre><code>@dataclass\nclass Colocalisation(Dataset):\n    \"\"\"Colocalisation results for pairs of overlapping study-locus.\"\"\"\n\n    @classmethod\n    def get_schema(cls: type[Colocalisation]) -&gt; StructType:\n        \"\"\"Provides the schema for the Colocalisation dataset.\n\n        Returns:\n            StructType: Schema for the Colocalisation dataset\n        \"\"\"\n        return parse_spark_schema(\"colocalisation.json\")\n</code></pre>"},{"location":"python_api/datasets/colocalisation/#gentropy.dataset.colocalisation.Colocalisation.get_schema","title":"<code>get_schema() -&gt; StructType</code>  <code>classmethod</code>","text":"<p>Provides the schema for the Colocalisation dataset.</p> <p>Returns:</p> Name Type Description <code>StructType</code> <code>StructType</code> <p>Schema for the Colocalisation dataset</p> Source code in <code>src/gentropy/dataset/colocalisation.py</code> <pre><code>@classmethod\ndef get_schema(cls: type[Colocalisation]) -&gt; StructType:\n    \"\"\"Provides the schema for the Colocalisation dataset.\n\n    Returns:\n        StructType: Schema for the Colocalisation dataset\n    \"\"\"\n    return parse_spark_schema(\"colocalisation.json\")\n</code></pre>"},{"location":"python_api/datasets/colocalisation/#schema","title":"Schema","text":"<pre><code>root\n |-- leftStudyLocusId: long (nullable = false)\n |-- rightStudyLocusId: long (nullable = false)\n |-- chromosome: string (nullable = false)\n |-- colocalisationMethod: string (nullable = false)\n |-- numberColocalisingVariants: long (nullable = false)\n |-- h0: double (nullable = true)\n |-- h1: double (nullable = true)\n |-- h2: double (nullable = true)\n |-- h3: double (nullable = true)\n |-- h4: double (nullable = true)\n |-- log2h4h3: double (nullable = true)\n |-- clpp: double (nullable = true)\n</code></pre>"},{"location":"python_api/datasets/gene_index/","title":"Gene Index","text":""},{"location":"python_api/datasets/gene_index/#gentropy.dataset.gene_index.GeneIndex","title":"<code>gentropy.dataset.gene_index.GeneIndex</code>  <code>dataclass</code>","text":"<p>             Bases: <code>Dataset</code></p> <p>Gene index dataset.</p> <p>Gene-based annotation.</p> Source code in <code>src/gentropy/dataset/gene_index.py</code> <pre><code>@dataclass\nclass GeneIndex(Dataset):\n    \"\"\"Gene index dataset.\n\n    Gene-based annotation.\n    \"\"\"\n\n    @classmethod\n    def get_schema(cls: type[GeneIndex]) -&gt; StructType:\n        \"\"\"Provides the schema for the GeneIndex dataset.\n\n        Returns:\n            StructType: Schema for the GeneIndex dataset\n        \"\"\"\n        return parse_spark_schema(\"gene_index.json\")\n\n    def filter_by_biotypes(self: GeneIndex, biotypes: list[str]) -&gt; GeneIndex:\n        \"\"\"Filter by approved biotypes.\n\n        Args:\n            biotypes (list[str]): List of Ensembl biotypes to keep.\n\n        Returns:\n            GeneIndex: Gene index dataset filtered by biotypes.\n        \"\"\"\n        self.df = self._df.filter(f.col(\"biotype\").isin(biotypes))\n        return self\n\n    def locations_lut(self: GeneIndex) -&gt; DataFrame:\n        \"\"\"Gene location information.\n\n        Returns:\n            DataFrame: Gene LUT including genomic location information.\n        \"\"\"\n        return self.df.select(\n            \"geneId\",\n            \"chromosome\",\n            \"start\",\n            \"end\",\n            \"strand\",\n            \"tss\",\n        )\n\n    def symbols_lut(self: GeneIndex) -&gt; DataFrame:\n        \"\"\"Gene symbol lookup table.\n\n        Pre-processess gene/target dataset to create lookup table of gene symbols, including\n        obsoleted gene symbols.\n\n        Returns:\n            DataFrame: Gene LUT for symbol mapping containing `geneId` and `geneSymbol` columns.\n        \"\"\"\n        return self.df.select(\n            f.explode(\n                f.array_union(f.array(\"approvedSymbol\"), f.col(\"obsoleteSymbols\"))\n            ).alias(\"geneSymbol\"),\n            \"*\",\n        )\n</code></pre>"},{"location":"python_api/datasets/gene_index/#gentropy.dataset.gene_index.GeneIndex.filter_by_biotypes","title":"<code>filter_by_biotypes(biotypes: list[str]) -&gt; GeneIndex</code>","text":"<p>Filter by approved biotypes.</p> <p>Parameters:</p> Name Type Description Default <code>biotypes</code> <code>list[str]</code> <p>List of Ensembl biotypes to keep.</p> required <p>Returns:</p> Name Type Description <code>GeneIndex</code> <code>GeneIndex</code> <p>Gene index dataset filtered by biotypes.</p> Source code in <code>src/gentropy/dataset/gene_index.py</code> <pre><code>def filter_by_biotypes(self: GeneIndex, biotypes: list[str]) -&gt; GeneIndex:\n    \"\"\"Filter by approved biotypes.\n\n    Args:\n        biotypes (list[str]): List of Ensembl biotypes to keep.\n\n    Returns:\n        GeneIndex: Gene index dataset filtered by biotypes.\n    \"\"\"\n    self.df = self._df.filter(f.col(\"biotype\").isin(biotypes))\n    return self\n</code></pre>"},{"location":"python_api/datasets/gene_index/#gentropy.dataset.gene_index.GeneIndex.get_schema","title":"<code>get_schema() -&gt; StructType</code>  <code>classmethod</code>","text":"<p>Provides the schema for the GeneIndex dataset.</p> <p>Returns:</p> Name Type Description <code>StructType</code> <code>StructType</code> <p>Schema for the GeneIndex dataset</p> Source code in <code>src/gentropy/dataset/gene_index.py</code> <pre><code>@classmethod\ndef get_schema(cls: type[GeneIndex]) -&gt; StructType:\n    \"\"\"Provides the schema for the GeneIndex dataset.\n\n    Returns:\n        StructType: Schema for the GeneIndex dataset\n    \"\"\"\n    return parse_spark_schema(\"gene_index.json\")\n</code></pre>"},{"location":"python_api/datasets/gene_index/#gentropy.dataset.gene_index.GeneIndex.locations_lut","title":"<code>locations_lut() -&gt; DataFrame</code>","text":"<p>Gene location information.</p> <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>Gene LUT including genomic location information.</p> Source code in <code>src/gentropy/dataset/gene_index.py</code> <pre><code>def locations_lut(self: GeneIndex) -&gt; DataFrame:\n    \"\"\"Gene location information.\n\n    Returns:\n        DataFrame: Gene LUT including genomic location information.\n    \"\"\"\n    return self.df.select(\n        \"geneId\",\n        \"chromosome\",\n        \"start\",\n        \"end\",\n        \"strand\",\n        \"tss\",\n    )\n</code></pre>"},{"location":"python_api/datasets/gene_index/#gentropy.dataset.gene_index.GeneIndex.symbols_lut","title":"<code>symbols_lut() -&gt; DataFrame</code>","text":"<p>Gene symbol lookup table.</p> <p>Pre-processess gene/target dataset to create lookup table of gene symbols, including obsoleted gene symbols.</p> <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>Gene LUT for symbol mapping containing <code>geneId</code> and <code>geneSymbol</code> columns.</p> Source code in <code>src/gentropy/dataset/gene_index.py</code> <pre><code>def symbols_lut(self: GeneIndex) -&gt; DataFrame:\n    \"\"\"Gene symbol lookup table.\n\n    Pre-processess gene/target dataset to create lookup table of gene symbols, including\n    obsoleted gene symbols.\n\n    Returns:\n        DataFrame: Gene LUT for symbol mapping containing `geneId` and `geneSymbol` columns.\n    \"\"\"\n    return self.df.select(\n        f.explode(\n            f.array_union(f.array(\"approvedSymbol\"), f.col(\"obsoleteSymbols\"))\n        ).alias(\"geneSymbol\"),\n        \"*\",\n    )\n</code></pre>"},{"location":"python_api/datasets/gene_index/#schema","title":"Schema","text":"<pre><code>root\n |-- geneId: string (nullable = false)\n |-- chromosome: string (nullable = false)\n |-- approvedSymbol: string (nullable = true)\n |-- biotype: string (nullable = true)\n |-- approvedName: string (nullable = true)\n |-- obsoleteSymbols: array (nullable = true)\n |    |-- element: string (containsNull = true)\n |-- tss: long (nullable = true)\n |-- start: long (nullable = true)\n |-- end: long (nullable = true)\n |-- strand: integer (nullable = true)\n</code></pre>"},{"location":"python_api/datasets/intervals/","title":"Intervals","text":""},{"location":"python_api/datasets/intervals/#gentropy.dataset.intervals.Intervals","title":"<code>gentropy.dataset.intervals.Intervals</code>  <code>dataclass</code>","text":"<p>             Bases: <code>Dataset</code></p> <p>Intervals dataset links genes to genomic regions based on genome interaction studies.</p> Source code in <code>src/gentropy/dataset/intervals.py</code> <pre><code>@dataclass\nclass Intervals(Dataset):\n    \"\"\"Intervals dataset links genes to genomic regions based on genome interaction studies.\"\"\"\n\n    @classmethod\n    def get_schema(cls: type[Intervals]) -&gt; StructType:\n        \"\"\"Provides the schema for the Intervals dataset.\n\n        Returns:\n            StructType: Schema for the Intervals dataset\n        \"\"\"\n        return parse_spark_schema(\"intervals.json\")\n\n    @classmethod\n    def from_source(\n        cls: type[Intervals],\n        spark: SparkSession,\n        source_name: str,\n        source_path: str,\n        gene_index: GeneIndex,\n        lift: LiftOverSpark,\n    ) -&gt; Intervals:\n        \"\"\"Collect interval data for a particular source.\n\n        Args:\n            spark (SparkSession): Spark session\n            source_name (str): Name of the interval source\n            source_path (str): Path to the interval source file\n            gene_index (GeneIndex): Gene index\n            lift (LiftOverSpark): LiftOverSpark instance to convert coordinats from hg37 to hg38\n\n        Returns:\n            Intervals: Intervals dataset\n\n        Raises:\n            ValueError: If the source name is not recognised\n        \"\"\"\n        from gentropy.datasource.intervals.andersson import IntervalsAndersson\n        from gentropy.datasource.intervals.javierre import IntervalsJavierre\n        from gentropy.datasource.intervals.jung import IntervalsJung\n        from gentropy.datasource.intervals.thurman import IntervalsThurman\n\n        source_to_class = {\n            \"andersson\": IntervalsAndersson,\n            \"javierre\": IntervalsJavierre,\n            \"jung\": IntervalsJung,\n            \"thurman\": IntervalsThurman,\n        }\n\n        if source_name not in source_to_class:\n            raise ValueError(f\"Unknown interval source: {source_name}\")\n\n        source_class = source_to_class[source_name]\n        data = source_class.read(spark, source_path)  # type: ignore\n        return source_class.parse(data, gene_index, lift)  # type: ignore\n\n    def v2g(self: Intervals, variant_index: VariantIndex) -&gt; V2G:\n        \"\"\"Convert intervals into V2G by intersecting with a variant index.\n\n        Args:\n            variant_index (VariantIndex): Variant index dataset\n\n        Returns:\n            V2G: Variant-to-gene evidence dataset\n        \"\"\"\n        return V2G(\n            _df=(\n                self.df.alias(\"interval\")\n                .join(\n                    variant_index.df.selectExpr(\n                        \"chromosome as vi_chromosome\", \"variantId\", \"position\"\n                    ).alias(\"vi\"),\n                    on=[\n                        f.col(\"vi.vi_chromosome\") == f.col(\"interval.chromosome\"),\n                        f.col(\"vi.position\").between(\n                            f.col(\"interval.start\"), f.col(\"interval.end\")\n                        ),\n                    ],\n                    how=\"inner\",\n                )\n                .drop(\"start\", \"end\", \"vi_chromosome\", \"position\")\n            ),\n            _schema=V2G.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasets/intervals/#gentropy.dataset.intervals.Intervals.from_source","title":"<code>from_source(spark: SparkSession, source_name: str, source_path: str, gene_index: GeneIndex, lift: LiftOverSpark) -&gt; Intervals</code>  <code>classmethod</code>","text":"<p>Collect interval data for a particular source.</p> <p>Parameters:</p> Name Type Description Default <code>spark</code> <code>SparkSession</code> <p>Spark session</p> required <code>source_name</code> <code>str</code> <p>Name of the interval source</p> required <code>source_path</code> <code>str</code> <p>Path to the interval source file</p> required <code>gene_index</code> <code>GeneIndex</code> <p>Gene index</p> required <code>lift</code> <code>LiftOverSpark</code> <p>LiftOverSpark instance to convert coordinats from hg37 to hg38</p> required <p>Returns:</p> Name Type Description <code>Intervals</code> <code>Intervals</code> <p>Intervals dataset</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>If the source name is not recognised</p> Source code in <code>src/gentropy/dataset/intervals.py</code> <pre><code>@classmethod\ndef from_source(\n    cls: type[Intervals],\n    spark: SparkSession,\n    source_name: str,\n    source_path: str,\n    gene_index: GeneIndex,\n    lift: LiftOverSpark,\n) -&gt; Intervals:\n    \"\"\"Collect interval data for a particular source.\n\n    Args:\n        spark (SparkSession): Spark session\n        source_name (str): Name of the interval source\n        source_path (str): Path to the interval source file\n        gene_index (GeneIndex): Gene index\n        lift (LiftOverSpark): LiftOverSpark instance to convert coordinats from hg37 to hg38\n\n    Returns:\n        Intervals: Intervals dataset\n\n    Raises:\n        ValueError: If the source name is not recognised\n    \"\"\"\n    from gentropy.datasource.intervals.andersson import IntervalsAndersson\n    from gentropy.datasource.intervals.javierre import IntervalsJavierre\n    from gentropy.datasource.intervals.jung import IntervalsJung\n    from gentropy.datasource.intervals.thurman import IntervalsThurman\n\n    source_to_class = {\n        \"andersson\": IntervalsAndersson,\n        \"javierre\": IntervalsJavierre,\n        \"jung\": IntervalsJung,\n        \"thurman\": IntervalsThurman,\n    }\n\n    if source_name not in source_to_class:\n        raise ValueError(f\"Unknown interval source: {source_name}\")\n\n    source_class = source_to_class[source_name]\n    data = source_class.read(spark, source_path)  # type: ignore\n    return source_class.parse(data, gene_index, lift)  # type: ignore\n</code></pre>"},{"location":"python_api/datasets/intervals/#gentropy.dataset.intervals.Intervals.get_schema","title":"<code>get_schema() -&gt; StructType</code>  <code>classmethod</code>","text":"<p>Provides the schema for the Intervals dataset.</p> <p>Returns:</p> Name Type Description <code>StructType</code> <code>StructType</code> <p>Schema for the Intervals dataset</p> Source code in <code>src/gentropy/dataset/intervals.py</code> <pre><code>@classmethod\ndef get_schema(cls: type[Intervals]) -&gt; StructType:\n    \"\"\"Provides the schema for the Intervals dataset.\n\n    Returns:\n        StructType: Schema for the Intervals dataset\n    \"\"\"\n    return parse_spark_schema(\"intervals.json\")\n</code></pre>"},{"location":"python_api/datasets/intervals/#gentropy.dataset.intervals.Intervals.v2g","title":"<code>v2g(variant_index: VariantIndex) -&gt; V2G</code>","text":"<p>Convert intervals into V2G by intersecting with a variant index.</p> <p>Parameters:</p> Name Type Description Default <code>variant_index</code> <code>VariantIndex</code> <p>Variant index dataset</p> required <p>Returns:</p> Name Type Description <code>V2G</code> <code>V2G</code> <p>Variant-to-gene evidence dataset</p> Source code in <code>src/gentropy/dataset/intervals.py</code> <pre><code>def v2g(self: Intervals, variant_index: VariantIndex) -&gt; V2G:\n    \"\"\"Convert intervals into V2G by intersecting with a variant index.\n\n    Args:\n        variant_index (VariantIndex): Variant index dataset\n\n    Returns:\n        V2G: Variant-to-gene evidence dataset\n    \"\"\"\n    return V2G(\n        _df=(\n            self.df.alias(\"interval\")\n            .join(\n                variant_index.df.selectExpr(\n                    \"chromosome as vi_chromosome\", \"variantId\", \"position\"\n                ).alias(\"vi\"),\n                on=[\n                    f.col(\"vi.vi_chromosome\") == f.col(\"interval.chromosome\"),\n                    f.col(\"vi.position\").between(\n                        f.col(\"interval.start\"), f.col(\"interval.end\")\n                    ),\n                ],\n                how=\"inner\",\n            )\n            .drop(\"start\", \"end\", \"vi_chromosome\", \"position\")\n        ),\n        _schema=V2G.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasets/intervals/#schema","title":"Schema","text":"<pre><code>root\n |-- chromosome: string (nullable = false)\n |-- start: string (nullable = false)\n |-- end: string (nullable = false)\n |-- geneId: string (nullable = false)\n |-- resourceScore: double (nullable = true)\n |-- score: double (nullable = true)\n |-- datasourceId: string (nullable = false)\n |-- datatypeId: string (nullable = false)\n |-- pmid: string (nullable = true)\n |-- biofeature: string (nullable = true)\n</code></pre>"},{"location":"python_api/datasets/l2g_feature/","title":"L2G Feature","text":""},{"location":"python_api/datasets/l2g_feature/#gentropy.method.l2g.feature_factory.L2GFeature","title":"<code>gentropy.method.l2g.feature_factory.L2GFeature</code>  <code>dataclass</code>","text":"<p>             Bases: <code>Dataset</code></p> <p>Locus-to-gene feature dataset.</p> Source code in <code>src/gentropy/dataset/l2g_feature.py</code> <pre><code>@dataclass\nclass L2GFeature(Dataset):\n    \"\"\"Locus-to-gene feature dataset.\"\"\"\n\n    @classmethod\n    def get_schema(cls: type[L2GFeature]) -&gt; StructType:\n        \"\"\"Provides the schema for the L2GFeature dataset.\n\n        Returns:\n            StructType: Schema for the L2GFeature dataset\n        \"\"\"\n        return parse_spark_schema(\"l2g_feature.json\")\n</code></pre>"},{"location":"python_api/datasets/l2g_feature/#gentropy.method.l2g.feature_factory.L2GFeature.get_schema","title":"<code>get_schema() -&gt; StructType</code>  <code>classmethod</code>","text":"<p>Provides the schema for the L2GFeature dataset.</p> <p>Returns:</p> Name Type Description <code>StructType</code> <code>StructType</code> <p>Schema for the L2GFeature dataset</p> Source code in <code>src/gentropy/dataset/l2g_feature.py</code> <pre><code>@classmethod\ndef get_schema(cls: type[L2GFeature]) -&gt; StructType:\n    \"\"\"Provides the schema for the L2GFeature dataset.\n\n    Returns:\n        StructType: Schema for the L2GFeature dataset\n    \"\"\"\n    return parse_spark_schema(\"l2g_feature.json\")\n</code></pre>"},{"location":"python_api/datasets/l2g_feature/#schema","title":"Schema","text":"<pre><code>root\n |-- studyLocusId: long (nullable = false)\n |-- geneId: string (nullable = false)\n |-- featureName: string (nullable = false)\n |-- featureValue: float (nullable = false)\n</code></pre>"},{"location":"python_api/datasets/l2g_feature_matrix/","title":"L2G Feature Matrix","text":""},{"location":"python_api/datasets/l2g_feature_matrix/#gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix","title":"<code>gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix</code>  <code>dataclass</code>","text":"<p>             Bases: <code>Dataset</code></p> <p>Dataset with features for Locus to Gene prediction.</p> <p>Attributes:</p> Name Type Description <code>features_list</code> <code>list[str] | None</code> <p>List of features to use. If None, all possible features are used.</p> Source code in <code>src/gentropy/dataset/l2g_feature_matrix.py</code> <pre><code>@dataclass\nclass L2GFeatureMatrix(Dataset):\n    \"\"\"Dataset with features for Locus to Gene prediction.\n\n    Attributes:\n        features_list (list[str] | None): List of features to use. If None, all possible features are used.\n    \"\"\"\n\n    features_list: list[str] | None = None\n\n    def __post_init__(self: L2GFeatureMatrix) -&gt; None:\n        \"\"\"Post-initialisation to set the features list. If not provided, all columns except the fixed ones are used.\"\"\"\n        fixed_cols = [\"studyLocusId\", \"geneId\", \"goldStandardSet\"]\n        self.features_list = self.features_list or [\n            col for col in self._df.columns if col not in fixed_cols\n        ]\n\n    @classmethod\n    def generate_features(\n        cls: Type[L2GFeatureMatrix],\n        features_list: list[str],\n        credible_set: StudyLocus,\n        study_index: StudyIndex,\n        variant_gene: V2G,\n        colocalisation: Colocalisation,\n    ) -&gt; L2GFeatureMatrix:\n        \"\"\"Generate features from the gentropy datasets.\n\n        Args:\n            features_list (list[str]): List of features to generate\n            credible_set (StudyLocus): Credible set dataset\n            study_index (StudyIndex): Study index dataset\n            variant_gene (V2G): Variant to gene dataset\n            colocalisation (Colocalisation): Colocalisation dataset\n\n        Returns:\n            L2GFeatureMatrix: L2G feature matrix dataset\n\n        Raises:\n            ValueError: If the feature matrix is empty\n        \"\"\"\n        if features_dfs := [\n            # Extract features\n            ColocalisationFactory._get_max_coloc_per_credible_set(\n                colocalisation,\n                credible_set,\n                study_index,\n            ).df,\n            StudyLocusFactory._get_tss_distance_features(credible_set, variant_gene).df,\n            StudyLocusFactory._get_vep_features(credible_set, variant_gene).df,\n        ]:\n            fm = reduce(\n                lambda x, y: x.unionByName(y),\n                features_dfs,\n            )\n        else:\n            raise ValueError(\"No features found\")\n\n        # raise error if the feature matrix is empty\n        return cls(\n            _df=convert_from_long_to_wide(\n                fm, [\"studyLocusId\", \"geneId\"], \"featureName\", \"featureValue\"\n            ),\n            _schema=cls.get_schema(),\n            features_list=features_list,\n        )\n\n    @classmethod\n    def get_schema(cls: type[L2GFeatureMatrix]) -&gt; StructType:\n        \"\"\"Provides the schema for the L2gFeatureMatrix dataset.\n\n        Returns:\n            StructType: Schema for the L2gFeatureMatrix dataset\n        \"\"\"\n        return parse_spark_schema(\"l2g_feature_matrix.json\")\n\n    def calculate_feature_missingness_rate(\n        self: L2GFeatureMatrix,\n    ) -&gt; dict[str, float]:\n        \"\"\"Calculate the proportion of missing values in each feature.\n\n        Returns:\n            dict[str, float]: Dictionary of feature names and their missingness rate.\n\n        Raises:\n            ValueError: If no features are found.\n        \"\"\"\n        total_count = self._df.count()\n        if not self.features_list:\n            raise ValueError(\"No features found\")\n\n        return {\n            feature: (\n                self._df.filter(\n                    (self._df[feature].isNull()) | (self._df[feature] == 0)\n                ).count()\n                / total_count\n            )\n            for feature in self.features_list\n        }\n\n    def fill_na(\n        self: L2GFeatureMatrix, value: float = 0.0, subset: list[str] | None = None\n    ) -&gt; L2GFeatureMatrix:\n        \"\"\"Fill missing values in a column with a given value.\n\n        Args:\n            value (float): Value to replace missing values with. Defaults to 0.0.\n            subset (list[str] | None): Subset of columns to consider. Defaults to None.\n\n        Returns:\n            L2GFeatureMatrix: L2G feature matrix dataset\n        \"\"\"\n        self.df = self._df.fillna(value, subset=subset)\n        return self\n\n    def select_features(\n        self: L2GFeatureMatrix, features_list: list[str] | None\n    ) -&gt; L2GFeatureMatrix:\n        \"\"\"Select a subset of features from the feature matrix.\n\n        Args:\n            features_list (list[str] | None): List of features to select\n\n        Returns:\n            L2GFeatureMatrix: L2G feature matrix dataset\n        \"\"\"\n        features_list = features_list or self.features_list\n        fixed_cols = [\"studyLocusId\", \"geneId\", \"goldStandardSet\"]\n        self.df = self._df.select(fixed_cols + features_list)  # type: ignore\n        return self\n\n    def train_test_split(\n        self: L2GFeatureMatrix, fraction: float\n    ) -&gt; tuple[L2GFeatureMatrix, L2GFeatureMatrix]:\n        \"\"\"Split the dataset into training and test sets.\n\n        Args:\n            fraction (float): Fraction of the dataset to use for training\n\n        Returns:\n            tuple[L2GFeatureMatrix, L2GFeatureMatrix]: Training and test datasets\n        \"\"\"\n        train, test = self._df.randomSplit([fraction, 1 - fraction], seed=42)\n        return (\n            L2GFeatureMatrix(_df=train, _schema=L2GFeatureMatrix.get_schema()),\n            L2GFeatureMatrix(_df=test, _schema=L2GFeatureMatrix.get_schema()),\n        )\n</code></pre>"},{"location":"python_api/datasets/l2g_feature_matrix/#gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.calculate_feature_missingness_rate","title":"<code>calculate_feature_missingness_rate() -&gt; dict[str, float]</code>","text":"<p>Calculate the proportion of missing values in each feature.</p> <p>Returns:</p> Type Description <code>dict[str, float]</code> <p>dict[str, float]: Dictionary of feature names and their missingness rate.</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>If no features are found.</p> Source code in <code>src/gentropy/dataset/l2g_feature_matrix.py</code> <pre><code>def calculate_feature_missingness_rate(\n    self: L2GFeatureMatrix,\n) -&gt; dict[str, float]:\n    \"\"\"Calculate the proportion of missing values in each feature.\n\n    Returns:\n        dict[str, float]: Dictionary of feature names and their missingness rate.\n\n    Raises:\n        ValueError: If no features are found.\n    \"\"\"\n    total_count = self._df.count()\n    if not self.features_list:\n        raise ValueError(\"No features found\")\n\n    return {\n        feature: (\n            self._df.filter(\n                (self._df[feature].isNull()) | (self._df[feature] == 0)\n            ).count()\n            / total_count\n        )\n        for feature in self.features_list\n    }\n</code></pre>"},{"location":"python_api/datasets/l2g_feature_matrix/#gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.fill_na","title":"<code>fill_na(value: float = 0.0, subset: list[str] | None = None) -&gt; L2GFeatureMatrix</code>","text":"<p>Fill missing values in a column with a given value.</p> <p>Parameters:</p> Name Type Description Default <code>value</code> <code>float</code> <p>Value to replace missing values with. Defaults to 0.0.</p> <code>0.0</code> <code>subset</code> <code>list[str] | None</code> <p>Subset of columns to consider. Defaults to None.</p> <code>None</code> <p>Returns:</p> Name Type Description <code>L2GFeatureMatrix</code> <code>L2GFeatureMatrix</code> <p>L2G feature matrix dataset</p> Source code in <code>src/gentropy/dataset/l2g_feature_matrix.py</code> <pre><code>def fill_na(\n    self: L2GFeatureMatrix, value: float = 0.0, subset: list[str] | None = None\n) -&gt; L2GFeatureMatrix:\n    \"\"\"Fill missing values in a column with a given value.\n\n    Args:\n        value (float): Value to replace missing values with. Defaults to 0.0.\n        subset (list[str] | None): Subset of columns to consider. Defaults to None.\n\n    Returns:\n        L2GFeatureMatrix: L2G feature matrix dataset\n    \"\"\"\n    self.df = self._df.fillna(value, subset=subset)\n    return self\n</code></pre>"},{"location":"python_api/datasets/l2g_feature_matrix/#gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.generate_features","title":"<code>generate_features(features_list: list[str], credible_set: StudyLocus, study_index: StudyIndex, variant_gene: V2G, colocalisation: Colocalisation) -&gt; L2GFeatureMatrix</code>  <code>classmethod</code>","text":"<p>Generate features from the gentropy datasets.</p> <p>Parameters:</p> Name Type Description Default <code>features_list</code> <code>list[str]</code> <p>List of features to generate</p> required <code>credible_set</code> <code>StudyLocus</code> <p>Credible set dataset</p> required <code>study_index</code> <code>StudyIndex</code> <p>Study index dataset</p> required <code>variant_gene</code> <code>V2G</code> <p>Variant to gene dataset</p> required <code>colocalisation</code> <code>Colocalisation</code> <p>Colocalisation dataset</p> required <p>Returns:</p> Name Type Description <code>L2GFeatureMatrix</code> <code>L2GFeatureMatrix</code> <p>L2G feature matrix dataset</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>If the feature matrix is empty</p> Source code in <code>src/gentropy/dataset/l2g_feature_matrix.py</code> <pre><code>@classmethod\ndef generate_features(\n    cls: Type[L2GFeatureMatrix],\n    features_list: list[str],\n    credible_set: StudyLocus,\n    study_index: StudyIndex,\n    variant_gene: V2G,\n    colocalisation: Colocalisation,\n) -&gt; L2GFeatureMatrix:\n    \"\"\"Generate features from the gentropy datasets.\n\n    Args:\n        features_list (list[str]): List of features to generate\n        credible_set (StudyLocus): Credible set dataset\n        study_index (StudyIndex): Study index dataset\n        variant_gene (V2G): Variant to gene dataset\n        colocalisation (Colocalisation): Colocalisation dataset\n\n    Returns:\n        L2GFeatureMatrix: L2G feature matrix dataset\n\n    Raises:\n        ValueError: If the feature matrix is empty\n    \"\"\"\n    if features_dfs := [\n        # Extract features\n        ColocalisationFactory._get_max_coloc_per_credible_set(\n            colocalisation,\n            credible_set,\n            study_index,\n        ).df,\n        StudyLocusFactory._get_tss_distance_features(credible_set, variant_gene).df,\n        StudyLocusFactory._get_vep_features(credible_set, variant_gene).df,\n    ]:\n        fm = reduce(\n            lambda x, y: x.unionByName(y),\n            features_dfs,\n        )\n    else:\n        raise ValueError(\"No features found\")\n\n    # raise error if the feature matrix is empty\n    return cls(\n        _df=convert_from_long_to_wide(\n            fm, [\"studyLocusId\", \"geneId\"], \"featureName\", \"featureValue\"\n        ),\n        _schema=cls.get_schema(),\n        features_list=features_list,\n    )\n</code></pre>"},{"location":"python_api/datasets/l2g_feature_matrix/#gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.get_schema","title":"<code>get_schema() -&gt; StructType</code>  <code>classmethod</code>","text":"<p>Provides the schema for the L2gFeatureMatrix dataset.</p> <p>Returns:</p> Name Type Description <code>StructType</code> <code>StructType</code> <p>Schema for the L2gFeatureMatrix dataset</p> Source code in <code>src/gentropy/dataset/l2g_feature_matrix.py</code> <pre><code>@classmethod\ndef get_schema(cls: type[L2GFeatureMatrix]) -&gt; StructType:\n    \"\"\"Provides the schema for the L2gFeatureMatrix dataset.\n\n    Returns:\n        StructType: Schema for the L2gFeatureMatrix dataset\n    \"\"\"\n    return parse_spark_schema(\"l2g_feature_matrix.json\")\n</code></pre>"},{"location":"python_api/datasets/l2g_feature_matrix/#gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.select_features","title":"<code>select_features(features_list: list[str] | None) -&gt; L2GFeatureMatrix</code>","text":"<p>Select a subset of features from the feature matrix.</p> <p>Parameters:</p> Name Type Description Default <code>features_list</code> <code>list[str] | None</code> <p>List of features to select</p> required <p>Returns:</p> Name Type Description <code>L2GFeatureMatrix</code> <code>L2GFeatureMatrix</code> <p>L2G feature matrix dataset</p> Source code in <code>src/gentropy/dataset/l2g_feature_matrix.py</code> <pre><code>def select_features(\n    self: L2GFeatureMatrix, features_list: list[str] | None\n) -&gt; L2GFeatureMatrix:\n    \"\"\"Select a subset of features from the feature matrix.\n\n    Args:\n        features_list (list[str] | None): List of features to select\n\n    Returns:\n        L2GFeatureMatrix: L2G feature matrix dataset\n    \"\"\"\n    features_list = features_list or self.features_list\n    fixed_cols = [\"studyLocusId\", \"geneId\", \"goldStandardSet\"]\n    self.df = self._df.select(fixed_cols + features_list)  # type: ignore\n    return self\n</code></pre>"},{"location":"python_api/datasets/l2g_feature_matrix/#gentropy.dataset.l2g_feature_matrix.L2GFeatureMatrix.train_test_split","title":"<code>train_test_split(fraction: float) -&gt; tuple[L2GFeatureMatrix, L2GFeatureMatrix]</code>","text":"<p>Split the dataset into training and test sets.</p> <p>Parameters:</p> Name Type Description Default <code>fraction</code> <code>float</code> <p>Fraction of the dataset to use for training</p> required <p>Returns:</p> Type Description <code>tuple[L2GFeatureMatrix, L2GFeatureMatrix]</code> <p>tuple[L2GFeatureMatrix, L2GFeatureMatrix]: Training and test datasets</p> Source code in <code>src/gentropy/dataset/l2g_feature_matrix.py</code> <pre><code>def train_test_split(\n    self: L2GFeatureMatrix, fraction: float\n) -&gt; tuple[L2GFeatureMatrix, L2GFeatureMatrix]:\n    \"\"\"Split the dataset into training and test sets.\n\n    Args:\n        fraction (float): Fraction of the dataset to use for training\n\n    Returns:\n        tuple[L2GFeatureMatrix, L2GFeatureMatrix]: Training and test datasets\n    \"\"\"\n    train, test = self._df.randomSplit([fraction, 1 - fraction], seed=42)\n    return (\n        L2GFeatureMatrix(_df=train, _schema=L2GFeatureMatrix.get_schema()),\n        L2GFeatureMatrix(_df=test, _schema=L2GFeatureMatrix.get_schema()),\n    )\n</code></pre>"},{"location":"python_api/datasets/l2g_feature_matrix/#schema","title":"Schema","text":"<pre><code>root\n |-- studyLocusId: long (nullable = false)\n |-- geneId: string (nullable = false)\n |-- goldStandardSet: string (nullable = true)\n |-- distanceTssMean: float (nullable = true)\n |-- distanceTssMinimum: float (nullable = true)\n |-- vepMaximumNeighborhood: float (nullable = true)\n |-- vepMaximum: float (nullable = true)\n |-- vepMeanNeighborhood: float (nullable = true)\n |-- vepMean: float (nullable = true)\n |-- eqtlColocClppMaximum: float (nullable = true)\n |-- eqtlColocClppMaximumNeighborhood: float (nullable = true)\n |-- eqtlColocLlrMaximum: float (nullable = true)\n |-- eqtlColocLlrMaximumNeighborhood: float (nullable = true)\n |-- pqtlColocClppMaximum: float (nullable = true)\n |-- pqtlColocClppMaximumNeighborhood: float (nullable = true)\n |-- pqtlColocLlrMaximum: float (nullable = true)\n |-- pqtlColocLlrMaximumNeighborhood: float (nullable = true)\n |-- sqtlColocClppMaximum: float (nullable = true)\n |-- sqtlColocClppMaximumNeighborhood: float (nullable = true)\n |-- sqtlColocLlrMaximum: float (nullable = true)\n |-- sqtlColocLlrMaximumNeighborhood: float (nullable = true)\n |-- tuqtlColocClppMaximum: float (nullable = true)\n |-- tuqtlColocClppMaximumNeighborhood: float (nullable = true)\n |-- tuqtlColocLlrMaximum: float (nullable = true)\n |-- tuqtlColocLlrMaximumNeighborhood: float (nullable = true)\n</code></pre>"},{"location":"python_api/datasets/l2g_gold_standard/","title":"L2G Gold Standard","text":""},{"location":"python_api/datasets/l2g_gold_standard/#gentropy.dataset.l2g_gold_standard.L2GGoldStandard","title":"<code>gentropy.dataset.l2g_gold_standard.L2GGoldStandard</code>  <code>dataclass</code>","text":"<p>             Bases: <code>Dataset</code></p> <p>L2G gold standard dataset.</p> Source code in <code>src/gentropy/dataset/l2g_gold_standard.py</code> <pre><code>@dataclass\nclass L2GGoldStandard(Dataset):\n    \"\"\"L2G gold standard dataset.\"\"\"\n\n    INTERACTION_THRESHOLD = 0.7\n    GS_POSITIVE_LABEL = \"positive\"\n    GS_NEGATIVE_LABEL = \"negative\"\n\n    @classmethod\n    def from_otg_curation(\n        cls: type[L2GGoldStandard],\n        gold_standard_curation: DataFrame,\n        v2g: V2G,\n        study_locus_overlap: StudyLocusOverlap,\n        interactions: DataFrame,\n    ) -&gt; L2GGoldStandard:\n        \"\"\"Initialise L2GGoldStandard from source dataset.\n\n        Args:\n            gold_standard_curation (DataFrame): Gold standard curation dataframe, extracted from\n            v2g (V2G): Variant to gene dataset to bring distance between a variant and a gene's TSS\n            study_locus_overlap (StudyLocusOverlap): Study locus overlap dataset to remove duplicated loci\n            interactions (DataFrame): Gene-gene interactions dataset to remove negative cases where the gene interacts with a positive gene\n\n        Returns:\n            L2GGoldStandard: L2G Gold Standard dataset\n        \"\"\"\n        from gentropy.datasource.open_targets.l2g_gold_standard import (\n            OpenTargetsL2GGoldStandard,\n        )\n\n        interactions_df = cls.process_gene_interactions(interactions)\n\n        return (\n            OpenTargetsL2GGoldStandard.as_l2g_gold_standard(gold_standard_curation, v2g)\n            # .filter_unique_associations(study_locus_overlap)\n            .remove_false_negatives(interactions_df)\n        )\n\n    @classmethod\n    def get_schema(cls: type[L2GGoldStandard]) -&gt; StructType:\n        \"\"\"Provides the schema for the L2GGoldStandard dataset.\n\n        Returns:\n            StructType: Spark schema for the L2GGoldStandard dataset\n        \"\"\"\n        return parse_spark_schema(\"l2g_gold_standard.json\")\n\n    @classmethod\n    def process_gene_interactions(\n        cls: Type[L2GGoldStandard], interactions: DataFrame\n    ) -&gt; DataFrame:\n        \"\"\"Extract top scoring gene-gene interaction from the interactions dataset of the Platform.\n\n        Args:\n            interactions (DataFrame): Gene-gene interactions dataset from the Open Targets Platform\n\n        Returns:\n            DataFrame: Top scoring gene-gene interaction per pair of genes\n\n        Examples:\n            &gt;&gt;&gt; interactions = spark.createDataFrame([(\"gene1\", \"gene2\", 0.8), (\"gene1\", \"gene2\", 0.5), (\"gene2\", \"gene3\", 0.7)], [\"targetA\", \"targetB\", \"scoring\"])\n            &gt;&gt;&gt; L2GGoldStandard.process_gene_interactions(interactions).show()\n            +-------+-------+-----+\n            |geneIdA|geneIdB|score|\n            +-------+-------+-----+\n            |  gene1|  gene2|  0.8|\n            |  gene2|  gene3|  0.7|\n            +-------+-------+-----+\n            &lt;BLANKLINE&gt;\n        \"\"\"\n        return get_record_with_maximum_value(\n            interactions,\n            [\"targetA\", \"targetB\"],\n            \"scoring\",\n        ).selectExpr(\n            \"targetA as geneIdA\",\n            \"targetB as geneIdB\",\n            \"scoring as score\",\n        )\n\n    def filter_unique_associations(\n        self: L2GGoldStandard,\n        study_locus_overlap: StudyLocusOverlap,\n    ) -&gt; L2GGoldStandard:\n        \"\"\"Refines the gold standard to filter out loci that are not independent.\n\n        Rules:\n        - If two loci point to the same gene, one positive and one negative, and have overlapping variants, we keep the positive one.\n        - If two loci point to the same gene, both positive or negative, and have overlapping variants, we drop one.\n        - If two loci point to different genes, and have overlapping variants, we keep both.\n\n        Args:\n            study_locus_overlap (StudyLocusOverlap): A dataset detailing variants that overlap between StudyLocus.\n\n        Returns:\n            L2GGoldStandard: L2GGoldStandard updated to exclude false negatives and redundant positives.\n        \"\"\"\n        squared_overlaps = study_locus_overlap._convert_to_square_matrix()\n        unique_associations = (\n            self.df.alias(\"left\")\n            # identify all the study loci that point to the same gene\n            .withColumn(\n                \"sl_same_gene\",\n                f.collect_set(\"studyLocusId\").over(Window.partitionBy(\"geneId\")),\n            )\n            # identify all the study loci that have an overlapping variant\n            .join(\n                squared_overlaps.df.alias(\"right\"),\n                (f.col(\"left.studyLocusId\") == f.col(\"right.leftStudyLocusId\"))\n                &amp; (f.col(\"left.variantId\") == f.col(\"right.tagVariantId\")),\n                \"left\",\n            )\n            .withColumn(\n                \"overlaps\",\n                f.when(f.col(\"right.tagVariantId\").isNotNull(), f.lit(True)).otherwise(\n                    f.lit(False)\n                ),\n            )\n            # drop redundant rows: where the variantid overlaps and the gene is \"explained\" by more than one study locus\n            .filter(~((f.size(\"sl_same_gene\") &gt; 1) &amp; (f.col(\"overlaps\") == 1)))\n            .select(*self.df.columns)\n        )\n        return L2GGoldStandard(_df=unique_associations, _schema=self.get_schema())\n\n    def remove_false_negatives(\n        self: L2GGoldStandard,\n        interactions_df: DataFrame,\n    ) -&gt; L2GGoldStandard:\n        \"\"\"Refines the gold standard to remove negative gold standard instances where the gene interacts with a positive gene.\n\n        Args:\n            interactions_df (DataFrame): Top scoring gene-gene interaction per pair of genes\n\n        Returns:\n            L2GGoldStandard: A refined set of locus-to-gene associations with increased reliability, having excluded loci that were likely false negatives due to gene-gene interaction confounding.\n        \"\"\"\n        squared_interactions = interactions_df.unionByName(\n            interactions_df.selectExpr(\n                \"geneIdB as geneIdA\", \"geneIdA as geneIdB\", \"score\"\n            )\n        ).filter(f.col(\"score\") &gt; self.INTERACTION_THRESHOLD)\n        df = (\n            self.df.alias(\"left\")\n            .join(\n                # bring gene partners\n                squared_interactions.alias(\"right\"),\n                f.col(\"left.geneId\") == f.col(\"right.geneIdA\"),\n                \"left\",\n            )\n            .withColumnRenamed(\"geneIdB\", \"interactorGeneId\")\n            .join(\n                # bring gold standard status for gene partners\n                self.df.selectExpr(\n                    \"geneId as interactorGeneId\",\n                    \"goldStandardSet as interactorGeneIdGoldStandardSet\",\n                ),\n                \"interactorGeneId\",\n                \"left\",\n            )\n            # remove self-interactions\n            .filter(\n                (f.col(\"geneId\") != f.col(\"interactorGeneId\"))\n                | (f.col(\"interactorGeneId\").isNull())\n            )\n            # remove false negatives\n            .filter(\n                # drop rows where the GS gene is negative but the interactor is a GS positive\n                ~(f.col(\"goldStandardSet\") == \"negative\")\n                &amp; (f.col(\"interactorGeneIdGoldStandardSet\") == \"positive\")\n                |\n                # keep rows where the gene does not interact\n                (f.col(\"interactorGeneId\").isNull())\n            )\n            .select(*self.df.columns)\n            .distinct()\n        )\n        return L2GGoldStandard(_df=df, _schema=self.get_schema())\n</code></pre>"},{"location":"python_api/datasets/l2g_gold_standard/#gentropy.dataset.l2g_gold_standard.L2GGoldStandard.filter_unique_associations","title":"<code>filter_unique_associations(study_locus_overlap: StudyLocusOverlap) -&gt; L2GGoldStandard</code>","text":"<p>Refines the gold standard to filter out loci that are not independent.</p> <p>Rules: - If two loci point to the same gene, one positive and one negative, and have overlapping variants, we keep the positive one. - If two loci point to the same gene, both positive or negative, and have overlapping variants, we drop one. - If two loci point to different genes, and have overlapping variants, we keep both.</p> <p>Parameters:</p> Name Type Description Default <code>study_locus_overlap</code> <code>StudyLocusOverlap</code> <p>A dataset detailing variants that overlap between StudyLocus.</p> required <p>Returns:</p> Name Type Description <code>L2GGoldStandard</code> <code>L2GGoldStandard</code> <p>L2GGoldStandard updated to exclude false negatives and redundant positives.</p> Source code in <code>src/gentropy/dataset/l2g_gold_standard.py</code> <pre><code>def filter_unique_associations(\n    self: L2GGoldStandard,\n    study_locus_overlap: StudyLocusOverlap,\n) -&gt; L2GGoldStandard:\n    \"\"\"Refines the gold standard to filter out loci that are not independent.\n\n    Rules:\n    - If two loci point to the same gene, one positive and one negative, and have overlapping variants, we keep the positive one.\n    - If two loci point to the same gene, both positive or negative, and have overlapping variants, we drop one.\n    - If two loci point to different genes, and have overlapping variants, we keep both.\n\n    Args:\n        study_locus_overlap (StudyLocusOverlap): A dataset detailing variants that overlap between StudyLocus.\n\n    Returns:\n        L2GGoldStandard: L2GGoldStandard updated to exclude false negatives and redundant positives.\n    \"\"\"\n    squared_overlaps = study_locus_overlap._convert_to_square_matrix()\n    unique_associations = (\n        self.df.alias(\"left\")\n        # identify all the study loci that point to the same gene\n        .withColumn(\n            \"sl_same_gene\",\n            f.collect_set(\"studyLocusId\").over(Window.partitionBy(\"geneId\")),\n        )\n        # identify all the study loci that have an overlapping variant\n        .join(\n            squared_overlaps.df.alias(\"right\"),\n            (f.col(\"left.studyLocusId\") == f.col(\"right.leftStudyLocusId\"))\n            &amp; (f.col(\"left.variantId\") == f.col(\"right.tagVariantId\")),\n            \"left\",\n        )\n        .withColumn(\n            \"overlaps\",\n            f.when(f.col(\"right.tagVariantId\").isNotNull(), f.lit(True)).otherwise(\n                f.lit(False)\n            ),\n        )\n        # drop redundant rows: where the variantid overlaps and the gene is \"explained\" by more than one study locus\n        .filter(~((f.size(\"sl_same_gene\") &gt; 1) &amp; (f.col(\"overlaps\") == 1)))\n        .select(*self.df.columns)\n    )\n    return L2GGoldStandard(_df=unique_associations, _schema=self.get_schema())\n</code></pre>"},{"location":"python_api/datasets/l2g_gold_standard/#gentropy.dataset.l2g_gold_standard.L2GGoldStandard.from_otg_curation","title":"<code>from_otg_curation(gold_standard_curation: DataFrame, v2g: V2G, study_locus_overlap: StudyLocusOverlap, interactions: DataFrame) -&gt; L2GGoldStandard</code>  <code>classmethod</code>","text":"<p>Initialise L2GGoldStandard from source dataset.</p> <p>Parameters:</p> Name Type Description Default <code>gold_standard_curation</code> <code>DataFrame</code> <p>Gold standard curation dataframe, extracted from</p> required <code>v2g</code> <code>V2G</code> <p>Variant to gene dataset to bring distance between a variant and a gene's TSS</p> required <code>study_locus_overlap</code> <code>StudyLocusOverlap</code> <p>Study locus overlap dataset to remove duplicated loci</p> required <code>interactions</code> <code>DataFrame</code> <p>Gene-gene interactions dataset to remove negative cases where the gene interacts with a positive gene</p> required <p>Returns:</p> Name Type Description <code>L2GGoldStandard</code> <code>L2GGoldStandard</code> <p>L2G Gold Standard dataset</p> Source code in <code>src/gentropy/dataset/l2g_gold_standard.py</code> <pre><code>@classmethod\ndef from_otg_curation(\n    cls: type[L2GGoldStandard],\n    gold_standard_curation: DataFrame,\n    v2g: V2G,\n    study_locus_overlap: StudyLocusOverlap,\n    interactions: DataFrame,\n) -&gt; L2GGoldStandard:\n    \"\"\"Initialise L2GGoldStandard from source dataset.\n\n    Args:\n        gold_standard_curation (DataFrame): Gold standard curation dataframe, extracted from\n        v2g (V2G): Variant to gene dataset to bring distance between a variant and a gene's TSS\n        study_locus_overlap (StudyLocusOverlap): Study locus overlap dataset to remove duplicated loci\n        interactions (DataFrame): Gene-gene interactions dataset to remove negative cases where the gene interacts with a positive gene\n\n    Returns:\n        L2GGoldStandard: L2G Gold Standard dataset\n    \"\"\"\n    from gentropy.datasource.open_targets.l2g_gold_standard import (\n        OpenTargetsL2GGoldStandard,\n    )\n\n    interactions_df = cls.process_gene_interactions(interactions)\n\n    return (\n        OpenTargetsL2GGoldStandard.as_l2g_gold_standard(gold_standard_curation, v2g)\n        # .filter_unique_associations(study_locus_overlap)\n        .remove_false_negatives(interactions_df)\n    )\n</code></pre>"},{"location":"python_api/datasets/l2g_gold_standard/#gentropy.dataset.l2g_gold_standard.L2GGoldStandard.get_schema","title":"<code>get_schema() -&gt; StructType</code>  <code>classmethod</code>","text":"<p>Provides the schema for the L2GGoldStandard dataset.</p> <p>Returns:</p> Name Type Description <code>StructType</code> <code>StructType</code> <p>Spark schema for the L2GGoldStandard dataset</p> Source code in <code>src/gentropy/dataset/l2g_gold_standard.py</code> <pre><code>@classmethod\ndef get_schema(cls: type[L2GGoldStandard]) -&gt; StructType:\n    \"\"\"Provides the schema for the L2GGoldStandard dataset.\n\n    Returns:\n        StructType: Spark schema for the L2GGoldStandard dataset\n    \"\"\"\n    return parse_spark_schema(\"l2g_gold_standard.json\")\n</code></pre>"},{"location":"python_api/datasets/l2g_gold_standard/#gentropy.dataset.l2g_gold_standard.L2GGoldStandard.process_gene_interactions","title":"<code>process_gene_interactions(interactions: DataFrame) -&gt; DataFrame</code>  <code>classmethod</code>","text":"<p>Extract top scoring gene-gene interaction from the interactions dataset of the Platform.</p> <p>Parameters:</p> Name Type Description Default <code>interactions</code> <code>DataFrame</code> <p>Gene-gene interactions dataset from the Open Targets Platform</p> required <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>Top scoring gene-gene interaction per pair of genes</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; interactions = spark.createDataFrame([(\"gene1\", \"gene2\", 0.8), (\"gene1\", \"gene2\", 0.5), (\"gene2\", \"gene3\", 0.7)], [\"targetA\", \"targetB\", \"scoring\"])\n&gt;&gt;&gt; L2GGoldStandard.process_gene_interactions(interactions).show()\n+-------+-------+-----+\n|geneIdA|geneIdB|score|\n+-------+-------+-----+\n|  gene1|  gene2|  0.8|\n|  gene2|  gene3|  0.7|\n+-------+-------+-----+\n</code></pre> Source code in <code>src/gentropy/dataset/l2g_gold_standard.py</code> <pre><code>@classmethod\ndef process_gene_interactions(\n    cls: Type[L2GGoldStandard], interactions: DataFrame\n) -&gt; DataFrame:\n    \"\"\"Extract top scoring gene-gene interaction from the interactions dataset of the Platform.\n\n    Args:\n        interactions (DataFrame): Gene-gene interactions dataset from the Open Targets Platform\n\n    Returns:\n        DataFrame: Top scoring gene-gene interaction per pair of genes\n\n    Examples:\n        &gt;&gt;&gt; interactions = spark.createDataFrame([(\"gene1\", \"gene2\", 0.8), (\"gene1\", \"gene2\", 0.5), (\"gene2\", \"gene3\", 0.7)], [\"targetA\", \"targetB\", \"scoring\"])\n        &gt;&gt;&gt; L2GGoldStandard.process_gene_interactions(interactions).show()\n        +-------+-------+-----+\n        |geneIdA|geneIdB|score|\n        +-------+-------+-----+\n        |  gene1|  gene2|  0.8|\n        |  gene2|  gene3|  0.7|\n        +-------+-------+-----+\n        &lt;BLANKLINE&gt;\n    \"\"\"\n    return get_record_with_maximum_value(\n        interactions,\n        [\"targetA\", \"targetB\"],\n        \"scoring\",\n    ).selectExpr(\n        \"targetA as geneIdA\",\n        \"targetB as geneIdB\",\n        \"scoring as score\",\n    )\n</code></pre>"},{"location":"python_api/datasets/l2g_gold_standard/#gentropy.dataset.l2g_gold_standard.L2GGoldStandard.remove_false_negatives","title":"<code>remove_false_negatives(interactions_df: DataFrame) -&gt; L2GGoldStandard</code>","text":"<p>Refines the gold standard to remove negative gold standard instances where the gene interacts with a positive gene.</p> <p>Parameters:</p> Name Type Description Default <code>interactions_df</code> <code>DataFrame</code> <p>Top scoring gene-gene interaction per pair of genes</p> required <p>Returns:</p> Name Type Description <code>L2GGoldStandard</code> <code>L2GGoldStandard</code> <p>A refined set of locus-to-gene associations with increased reliability, having excluded loci that were likely false negatives due to gene-gene interaction confounding.</p> Source code in <code>src/gentropy/dataset/l2g_gold_standard.py</code> <pre><code>def remove_false_negatives(\n    self: L2GGoldStandard,\n    interactions_df: DataFrame,\n) -&gt; L2GGoldStandard:\n    \"\"\"Refines the gold standard to remove negative gold standard instances where the gene interacts with a positive gene.\n\n    Args:\n        interactions_df (DataFrame): Top scoring gene-gene interaction per pair of genes\n\n    Returns:\n        L2GGoldStandard: A refined set of locus-to-gene associations with increased reliability, having excluded loci that were likely false negatives due to gene-gene interaction confounding.\n    \"\"\"\n    squared_interactions = interactions_df.unionByName(\n        interactions_df.selectExpr(\n            \"geneIdB as geneIdA\", \"geneIdA as geneIdB\", \"score\"\n        )\n    ).filter(f.col(\"score\") &gt; self.INTERACTION_THRESHOLD)\n    df = (\n        self.df.alias(\"left\")\n        .join(\n            # bring gene partners\n            squared_interactions.alias(\"right\"),\n            f.col(\"left.geneId\") == f.col(\"right.geneIdA\"),\n            \"left\",\n        )\n        .withColumnRenamed(\"geneIdB\", \"interactorGeneId\")\n        .join(\n            # bring gold standard status for gene partners\n            self.df.selectExpr(\n                \"geneId as interactorGeneId\",\n                \"goldStandardSet as interactorGeneIdGoldStandardSet\",\n            ),\n            \"interactorGeneId\",\n            \"left\",\n        )\n        # remove self-interactions\n        .filter(\n            (f.col(\"geneId\") != f.col(\"interactorGeneId\"))\n            | (f.col(\"interactorGeneId\").isNull())\n        )\n        # remove false negatives\n        .filter(\n            # drop rows where the GS gene is negative but the interactor is a GS positive\n            ~(f.col(\"goldStandardSet\") == \"negative\")\n            &amp; (f.col(\"interactorGeneIdGoldStandardSet\") == \"positive\")\n            |\n            # keep rows where the gene does not interact\n            (f.col(\"interactorGeneId\").isNull())\n        )\n        .select(*self.df.columns)\n        .distinct()\n    )\n    return L2GGoldStandard(_df=df, _schema=self.get_schema())\n</code></pre>"},{"location":"python_api/datasets/l2g_gold_standard/#schema","title":"Schema","text":"<pre><code>root\n |-- studyLocusId: long (nullable = false)\n |-- variantId: string (nullable = false)\n |-- studyId: string (nullable = false)\n |-- geneId: string (nullable = false)\n |-- goldStandardSet: string (nullable = false)\n |-- sources: array (nullable = true)\n |    |-- element: string (containsNull = true)\n</code></pre>"},{"location":"python_api/datasets/l2g_prediction/","title":"L2G Prediction","text":""},{"location":"python_api/datasets/l2g_prediction/#gentropy.dataset.l2g_prediction.L2GPrediction","title":"<code>gentropy.dataset.l2g_prediction.L2GPrediction</code>  <code>dataclass</code>","text":"<p>             Bases: <code>Dataset</code></p> <p>Dataset that contains the Locus to Gene predictions.</p> <p>It is the result of applying the L2G model on a feature matrix, which contains all the study/locus pairs and their functional annotations. The score column informs the confidence of the prediction that a gene is causal to an association.</p> Source code in <code>src/gentropy/dataset/l2g_prediction.py</code> <pre><code>@dataclass\nclass L2GPrediction(Dataset):\n    \"\"\"Dataset that contains the Locus to Gene predictions.\n\n    It is the result of applying the L2G model on a feature matrix, which contains all\n    the study/locus pairs and their functional annotations. The score column informs the\n    confidence of the prediction that a gene is causal to an association.\n    \"\"\"\n\n    @classmethod\n    def get_schema(cls: type[L2GPrediction]) -&gt; StructType:\n        \"\"\"Provides the schema for the L2GPrediction dataset.\n\n        Returns:\n            StructType: Schema for the L2GPrediction dataset\n        \"\"\"\n        return parse_spark_schema(\"l2g_predictions.json\")\n\n    @classmethod\n    def from_credible_set(\n        cls: Type[L2GPrediction],\n        model_path: str,\n        features_list: list[str],\n        credible_set: StudyLocus,\n        study_index: StudyIndex,\n        v2g: V2G,\n        coloc: Colocalisation,\n    ) -&gt; L2GPrediction:\n        \"\"\"Extract L2G predictions for a set of credible sets derived from GWAS.\n\n        Args:\n            model_path (str): Path to the fitted model\n            features_list (list[str]): List of features to use for the model\n            credible_set (StudyLocus): Credible set dataset\n            study_index (StudyIndex): Study index dataset\n            v2g (V2G): Variant to gene dataset\n            coloc (Colocalisation): Colocalisation dataset\n\n        Returns:\n            L2GPrediction: L2G dataset\n        \"\"\"\n        fm = L2GFeatureMatrix.generate_features(\n            features_list=features_list,\n            credible_set=credible_set,\n            study_index=study_index,\n            variant_gene=v2g,\n            colocalisation=coloc,\n        ).fill_na()\n\n        gwas_fm = L2GFeatureMatrix(\n            _df=(\n                fm.df.join(\n                    credible_set.filter_by_study_type(\"gwas\", study_index).df,\n                    on=\"studyLocusId\",\n                )\n            ),\n            _schema=cls.get_schema(),\n        )\n        return L2GPrediction(\n            # Load and apply fitted model\n            _df=(\n                LocusToGeneModel.load_from_disk(\n                    model_path,\n                    features_list=features_list,\n                )\n                .predict(gwas_fm)\n                # the probability of the positive class is the second element inside the probability array\n                # - this is selected as the L2G probability\n                .select(\n                    \"studyLocusId\",\n                    \"geneId\",\n                    vector_to_array(f.col(\"probability\"))[1].alias(\"score\"),\n                )\n            ),\n            _schema=cls.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasets/l2g_prediction/#gentropy.dataset.l2g_prediction.L2GPrediction.from_credible_set","title":"<code>from_credible_set(model_path: str, features_list: list[str], credible_set: StudyLocus, study_index: StudyIndex, v2g: V2G, coloc: Colocalisation) -&gt; L2GPrediction</code>  <code>classmethod</code>","text":"<p>Extract L2G predictions for a set of credible sets derived from GWAS.</p> <p>Parameters:</p> Name Type Description Default <code>model_path</code> <code>str</code> <p>Path to the fitted model</p> required <code>features_list</code> <code>list[str]</code> <p>List of features to use for the model</p> required <code>credible_set</code> <code>StudyLocus</code> <p>Credible set dataset</p> required <code>study_index</code> <code>StudyIndex</code> <p>Study index dataset</p> required <code>v2g</code> <code>V2G</code> <p>Variant to gene dataset</p> required <code>coloc</code> <code>Colocalisation</code> <p>Colocalisation dataset</p> required <p>Returns:</p> Name Type Description <code>L2GPrediction</code> <code>L2GPrediction</code> <p>L2G dataset</p> Source code in <code>src/gentropy/dataset/l2g_prediction.py</code> <pre><code>@classmethod\ndef from_credible_set(\n    cls: Type[L2GPrediction],\n    model_path: str,\n    features_list: list[str],\n    credible_set: StudyLocus,\n    study_index: StudyIndex,\n    v2g: V2G,\n    coloc: Colocalisation,\n) -&gt; L2GPrediction:\n    \"\"\"Extract L2G predictions for a set of credible sets derived from GWAS.\n\n    Args:\n        model_path (str): Path to the fitted model\n        features_list (list[str]): List of features to use for the model\n        credible_set (StudyLocus): Credible set dataset\n        study_index (StudyIndex): Study index dataset\n        v2g (V2G): Variant to gene dataset\n        coloc (Colocalisation): Colocalisation dataset\n\n    Returns:\n        L2GPrediction: L2G dataset\n    \"\"\"\n    fm = L2GFeatureMatrix.generate_features(\n        features_list=features_list,\n        credible_set=credible_set,\n        study_index=study_index,\n        variant_gene=v2g,\n        colocalisation=coloc,\n    ).fill_na()\n\n    gwas_fm = L2GFeatureMatrix(\n        _df=(\n            fm.df.join(\n                credible_set.filter_by_study_type(\"gwas\", study_index).df,\n                on=\"studyLocusId\",\n            )\n        ),\n        _schema=cls.get_schema(),\n    )\n    return L2GPrediction(\n        # Load and apply fitted model\n        _df=(\n            LocusToGeneModel.load_from_disk(\n                model_path,\n                features_list=features_list,\n            )\n            .predict(gwas_fm)\n            # the probability of the positive class is the second element inside the probability array\n            # - this is selected as the L2G probability\n            .select(\n                \"studyLocusId\",\n                \"geneId\",\n                vector_to_array(f.col(\"probability\"))[1].alias(\"score\"),\n            )\n        ),\n        _schema=cls.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasets/l2g_prediction/#gentropy.dataset.l2g_prediction.L2GPrediction.get_schema","title":"<code>get_schema() -&gt; StructType</code>  <code>classmethod</code>","text":"<p>Provides the schema for the L2GPrediction dataset.</p> <p>Returns:</p> Name Type Description <code>StructType</code> <code>StructType</code> <p>Schema for the L2GPrediction dataset</p> Source code in <code>src/gentropy/dataset/l2g_prediction.py</code> <pre><code>@classmethod\ndef get_schema(cls: type[L2GPrediction]) -&gt; StructType:\n    \"\"\"Provides the schema for the L2GPrediction dataset.\n\n    Returns:\n        StructType: Schema for the L2GPrediction dataset\n    \"\"\"\n    return parse_spark_schema(\"l2g_predictions.json\")\n</code></pre>"},{"location":"python_api/datasets/l2g_prediction/#schema","title":"Schema","text":""},{"location":"python_api/datasets/ld_index/","title":"LD Index","text":""},{"location":"python_api/datasets/ld_index/#gentropy.dataset.ld_index.LDIndex","title":"<code>gentropy.dataset.ld_index.LDIndex</code>  <code>dataclass</code>","text":"<p>             Bases: <code>Dataset</code></p> <p>Dataset containing linkage desequilibrium information between variants.</p> Source code in <code>src/gentropy/dataset/ld_index.py</code> <pre><code>@dataclass\nclass LDIndex(Dataset):\n    \"\"\"Dataset containing linkage desequilibrium information between variants.\"\"\"\n\n    @classmethod\n    def get_schema(cls: type[LDIndex]) -&gt; StructType:\n        \"\"\"Provides the schema for the LDIndex dataset.\n\n        Returns:\n            StructType: Schema for the LDIndex dataset\n        \"\"\"\n        return parse_spark_schema(\"ld_index.json\")\n</code></pre>"},{"location":"python_api/datasets/ld_index/#gentropy.dataset.ld_index.LDIndex.get_schema","title":"<code>get_schema() -&gt; StructType</code>  <code>classmethod</code>","text":"<p>Provides the schema for the LDIndex dataset.</p> <p>Returns:</p> Name Type Description <code>StructType</code> <code>StructType</code> <p>Schema for the LDIndex dataset</p> Source code in <code>src/gentropy/dataset/ld_index.py</code> <pre><code>@classmethod\ndef get_schema(cls: type[LDIndex]) -&gt; StructType:\n    \"\"\"Provides the schema for the LDIndex dataset.\n\n    Returns:\n        StructType: Schema for the LDIndex dataset\n    \"\"\"\n    return parse_spark_schema(\"ld_index.json\")\n</code></pre>"},{"location":"python_api/datasets/ld_index/#schema","title":"Schema","text":"<pre><code>root\n |-- variantId: string (nullable = false)\n |-- chromosome: string (nullable = false)\n |-- ldSet: array (nullable = false)\n |    |-- element: struct (containsNull = false)\n |    |    |-- tagVariantId: string (nullable = false)\n |    |    |-- rValues: array (nullable = false)\n |    |    |    |-- element: struct (containsNull = false)\n |    |    |    |    |-- population: string (nullable = false)\n |    |    |    |    |-- r: double (nullable = false)\n</code></pre>"},{"location":"python_api/datasets/study_index/","title":"Study Index","text":""},{"location":"python_api/datasets/study_index/#gentropy.dataset.study_index.StudyIndex","title":"<code>gentropy.dataset.study_index.StudyIndex</code>  <code>dataclass</code>","text":"<p>             Bases: <code>Dataset</code></p> <p>Study index dataset.</p> <p>A study index dataset captures all the metadata for all studies including GWAS and Molecular QTL.</p> Source code in <code>src/gentropy/dataset/study_index.py</code> <pre><code>@dataclass\nclass StudyIndex(Dataset):\n    \"\"\"Study index dataset.\n\n    A study index dataset captures all the metadata for all studies including GWAS and Molecular QTL.\n    \"\"\"\n\n    @staticmethod\n    def _aggregate_samples_by_ancestry(merged: Column, ancestry: Column) -&gt; Column:\n        \"\"\"Aggregate sample counts by ancestry in a list of struct colmns.\n\n        Args:\n            merged (Column): A column representing merged data (list of structs).\n            ancestry (Column): The `ancestry` parameter is a column that represents the ancestry of each\n                sample. (a struct)\n\n        Returns:\n            Column: the modified \"merged\" column after aggregating the samples by ancestry.\n        \"\"\"\n        # Iterating over the list of ancestries and adding the sample size if label matches:\n        return f.transform(\n            merged,\n            lambda a: f.when(\n                a.ancestry == ancestry.ancestry,\n                f.struct(\n                    a.ancestry.alias(\"ancestry\"),\n                    (a.sampleSize + ancestry.sampleSize).alias(\"sampleSize\"),\n                ),\n            ).otherwise(a),\n        )\n\n    @staticmethod\n    def _map_ancestries_to_ld_population(gwas_ancestry_label: Column) -&gt; Column:\n        \"\"\"Normalise ancestry column from GWAS studies into reference LD panel based on a pre-defined map.\n\n        This function assumes all possible ancestry categories have a corresponding\n        LD panel in the LD index. It is very important to have the ancestry labels\n        moved to the LD panel map.\n\n        Args:\n            gwas_ancestry_label (Column): A struct column with ancestry label like Finnish,\n                European, African etc. and the corresponding sample size.\n\n        Returns:\n            Column: Struct column with the mapped LD population label and the sample size.\n        \"\"\"\n        # Loading ancestry label to LD population label:\n        json_dict = json.loads(\n            pkg_resources.read_text(\n                data, \"gwas_population_2_LD_panel_map.json\", encoding=\"utf-8\"\n            )\n        )\n        map_expr = f.create_map(*[f.lit(x) for x in chain(*json_dict.items())])\n\n        return f.struct(\n            map_expr[gwas_ancestry_label.ancestry].alias(\"ancestry\"),\n            gwas_ancestry_label.sampleSize.alias(\"sampleSize\"),\n        )\n\n    @classmethod\n    def get_schema(cls: type[StudyIndex]) -&gt; StructType:\n        \"\"\"Provide the schema for the StudyIndex dataset.\n\n        Returns:\n            StructType: The schema of the StudyIndex dataset.\n        \"\"\"\n        return parse_spark_schema(\"study_index.json\")\n\n    @classmethod\n    def aggregate_and_map_ancestries(\n        cls: type[StudyIndex], discovery_samples: Column\n    ) -&gt; Column:\n        \"\"\"Map ancestries to populations in the LD reference and calculate relative sample size.\n\n        Args:\n            discovery_samples (Column): A list of struct column. Has an `ancestry` column and a `sampleSize` columns\n\n        Returns:\n            Column: A list of struct with mapped LD population and their relative sample size.\n        \"\"\"\n        # Map ancestry categories to population labels of the LD index:\n        mapped_ancestries = f.transform(\n            discovery_samples, cls._map_ancestries_to_ld_population\n        )\n\n        # Aggregate sample sizes belonging to the same LD population:\n        aggregated_counts = f.aggregate(\n            mapped_ancestries,\n            f.array_distinct(\n                f.transform(\n                    mapped_ancestries,\n                    lambda x: f.struct(\n                        x.ancestry.alias(\"ancestry\"), f.lit(0.0).alias(\"sampleSize\")\n                    ),\n                )\n            ),\n            cls._aggregate_samples_by_ancestry,\n        )\n        # Getting total sample count:\n        total_sample_count = f.aggregate(\n            aggregated_counts, f.lit(0.0), lambda total, pop: total + pop.sampleSize\n        ).alias(\"sampleSize\")\n\n        # Calculating relative sample size for each LD population:\n        return f.transform(\n            aggregated_counts,\n            lambda ld_population: f.struct(\n                ld_population.ancestry.alias(\"ldPopulation\"),\n                (ld_population.sampleSize / total_sample_count).alias(\n                    \"relativeSampleSize\"\n                ),\n            ),\n        )\n\n    def study_type_lut(self: StudyIndex) -&gt; DataFrame:\n        \"\"\"Return a lookup table of study type.\n\n        Returns:\n            DataFrame: A dataframe containing `studyId` and `studyType` columns.\n        \"\"\"\n        return self.df.select(\"studyId\", \"studyType\")\n\n    def is_qtl(self: StudyIndex) -&gt; Column:\n        \"\"\"Return a boolean column with true values for QTL studies.\n\n        Returns:\n            Column: True if the study is a QTL study.\n        \"\"\"\n        return self.df.studyType.endswith(\"qtl\")\n\n    def is_gwas(self: StudyIndex) -&gt; Column:\n        \"\"\"Return a boolean column with true values for GWAS studies.\n\n        Returns:\n            Column: True if the study is a GWAS study.\n        \"\"\"\n        return self.df.studyType == \"gwas\"\n\n    def has_mapped_trait(self: StudyIndex) -&gt; Column:\n        \"\"\"Return a boolean column indicating if a study has mapped disease.\n\n        Returns:\n            Column: True if the study has mapped disease.\n        \"\"\"\n        return f.size(self.df.traitFromSourceMappedIds) &gt; 0\n\n    def is_quality_flagged(self: StudyIndex) -&gt; Column:\n        \"\"\"Return a boolean column indicating if a study is flagged due to quality issues.\n\n        Returns:\n            Column: True if the study is flagged.\n        \"\"\"\n        # Testing for the presence of the qualityControls column:\n        if \"qualityControls\" not in self.df.columns:\n            return f.lit(False)\n        else:\n            return f.size(self.df.qualityControls) != 0\n\n    def has_summarystats(self: StudyIndex) -&gt; Column:\n        \"\"\"Return a boolean column indicating if a study has harmonized summary statistics.\n\n        Returns:\n            Column: True if the study has harmonized summary statistics.\n        \"\"\"\n        return self.df.hasSumstats\n</code></pre>"},{"location":"python_api/datasets/study_index/#gentropy.dataset.study_index.StudyIndex.aggregate_and_map_ancestries","title":"<code>aggregate_and_map_ancestries(discovery_samples: Column) -&gt; Column</code>  <code>classmethod</code>","text":"<p>Map ancestries to populations in the LD reference and calculate relative sample size.</p> <p>Parameters:</p> Name Type Description Default <code>discovery_samples</code> <code>Column</code> <p>A list of struct column. Has an <code>ancestry</code> column and a <code>sampleSize</code> columns</p> required <p>Returns:</p> Name Type Description <code>Column</code> <code>Column</code> <p>A list of struct with mapped LD population and their relative sample size.</p> Source code in <code>src/gentropy/dataset/study_index.py</code> <pre><code>@classmethod\ndef aggregate_and_map_ancestries(\n    cls: type[StudyIndex], discovery_samples: Column\n) -&gt; Column:\n    \"\"\"Map ancestries to populations in the LD reference and calculate relative sample size.\n\n    Args:\n        discovery_samples (Column): A list of struct column. Has an `ancestry` column and a `sampleSize` columns\n\n    Returns:\n        Column: A list of struct with mapped LD population and their relative sample size.\n    \"\"\"\n    # Map ancestry categories to population labels of the LD index:\n    mapped_ancestries = f.transform(\n        discovery_samples, cls._map_ancestries_to_ld_population\n    )\n\n    # Aggregate sample sizes belonging to the same LD population:\n    aggregated_counts = f.aggregate(\n        mapped_ancestries,\n        f.array_distinct(\n            f.transform(\n                mapped_ancestries,\n                lambda x: f.struct(\n                    x.ancestry.alias(\"ancestry\"), f.lit(0.0).alias(\"sampleSize\")\n                ),\n            )\n        ),\n        cls._aggregate_samples_by_ancestry,\n    )\n    # Getting total sample count:\n    total_sample_count = f.aggregate(\n        aggregated_counts, f.lit(0.0), lambda total, pop: total + pop.sampleSize\n    ).alias(\"sampleSize\")\n\n    # Calculating relative sample size for each LD population:\n    return f.transform(\n        aggregated_counts,\n        lambda ld_population: f.struct(\n            ld_population.ancestry.alias(\"ldPopulation\"),\n            (ld_population.sampleSize / total_sample_count).alias(\n                \"relativeSampleSize\"\n            ),\n        ),\n    )\n</code></pre>"},{"location":"python_api/datasets/study_index/#gentropy.dataset.study_index.StudyIndex.get_schema","title":"<code>get_schema() -&gt; StructType</code>  <code>classmethod</code>","text":"<p>Provide the schema for the StudyIndex dataset.</p> <p>Returns:</p> Name Type Description <code>StructType</code> <code>StructType</code> <p>The schema of the StudyIndex dataset.</p> Source code in <code>src/gentropy/dataset/study_index.py</code> <pre><code>@classmethod\ndef get_schema(cls: type[StudyIndex]) -&gt; StructType:\n    \"\"\"Provide the schema for the StudyIndex dataset.\n\n    Returns:\n        StructType: The schema of the StudyIndex dataset.\n    \"\"\"\n    return parse_spark_schema(\"study_index.json\")\n</code></pre>"},{"location":"python_api/datasets/study_index/#gentropy.dataset.study_index.StudyIndex.has_mapped_trait","title":"<code>has_mapped_trait() -&gt; Column</code>","text":"<p>Return a boolean column indicating if a study has mapped disease.</p> <p>Returns:</p> Name Type Description <code>Column</code> <code>Column</code> <p>True if the study has mapped disease.</p> Source code in <code>src/gentropy/dataset/study_index.py</code> <pre><code>def has_mapped_trait(self: StudyIndex) -&gt; Column:\n    \"\"\"Return a boolean column indicating if a study has mapped disease.\n\n    Returns:\n        Column: True if the study has mapped disease.\n    \"\"\"\n    return f.size(self.df.traitFromSourceMappedIds) &gt; 0\n</code></pre>"},{"location":"python_api/datasets/study_index/#gentropy.dataset.study_index.StudyIndex.has_summarystats","title":"<code>has_summarystats() -&gt; Column</code>","text":"<p>Return a boolean column indicating if a study has harmonized summary statistics.</p> <p>Returns:</p> Name Type Description <code>Column</code> <code>Column</code> <p>True if the study has harmonized summary statistics.</p> Source code in <code>src/gentropy/dataset/study_index.py</code> <pre><code>def has_summarystats(self: StudyIndex) -&gt; Column:\n    \"\"\"Return a boolean column indicating if a study has harmonized summary statistics.\n\n    Returns:\n        Column: True if the study has harmonized summary statistics.\n    \"\"\"\n    return self.df.hasSumstats\n</code></pre>"},{"location":"python_api/datasets/study_index/#gentropy.dataset.study_index.StudyIndex.is_gwas","title":"<code>is_gwas() -&gt; Column</code>","text":"<p>Return a boolean column with true values for GWAS studies.</p> <p>Returns:</p> Name Type Description <code>Column</code> <code>Column</code> <p>True if the study is a GWAS study.</p> Source code in <code>src/gentropy/dataset/study_index.py</code> <pre><code>def is_gwas(self: StudyIndex) -&gt; Column:\n    \"\"\"Return a boolean column with true values for GWAS studies.\n\n    Returns:\n        Column: True if the study is a GWAS study.\n    \"\"\"\n    return self.df.studyType == \"gwas\"\n</code></pre>"},{"location":"python_api/datasets/study_index/#gentropy.dataset.study_index.StudyIndex.is_qtl","title":"<code>is_qtl() -&gt; Column</code>","text":"<p>Return a boolean column with true values for QTL studies.</p> <p>Returns:</p> Name Type Description <code>Column</code> <code>Column</code> <p>True if the study is a QTL study.</p> Source code in <code>src/gentropy/dataset/study_index.py</code> <pre><code>def is_qtl(self: StudyIndex) -&gt; Column:\n    \"\"\"Return a boolean column with true values for QTL studies.\n\n    Returns:\n        Column: True if the study is a QTL study.\n    \"\"\"\n    return self.df.studyType.endswith(\"qtl\")\n</code></pre>"},{"location":"python_api/datasets/study_index/#gentropy.dataset.study_index.StudyIndex.is_quality_flagged","title":"<code>is_quality_flagged() -&gt; Column</code>","text":"<p>Return a boolean column indicating if a study is flagged due to quality issues.</p> <p>Returns:</p> Name Type Description <code>Column</code> <code>Column</code> <p>True if the study is flagged.</p> Source code in <code>src/gentropy/dataset/study_index.py</code> <pre><code>def is_quality_flagged(self: StudyIndex) -&gt; Column:\n    \"\"\"Return a boolean column indicating if a study is flagged due to quality issues.\n\n    Returns:\n        Column: True if the study is flagged.\n    \"\"\"\n    # Testing for the presence of the qualityControls column:\n    if \"qualityControls\" not in self.df.columns:\n        return f.lit(False)\n    else:\n        return f.size(self.df.qualityControls) != 0\n</code></pre>"},{"location":"python_api/datasets/study_index/#gentropy.dataset.study_index.StudyIndex.study_type_lut","title":"<code>study_type_lut() -&gt; DataFrame</code>","text":"<p>Return a lookup table of study type.</p> <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>A dataframe containing <code>studyId</code> and <code>studyType</code> columns.</p> Source code in <code>src/gentropy/dataset/study_index.py</code> <pre><code>def study_type_lut(self: StudyIndex) -&gt; DataFrame:\n    \"\"\"Return a lookup table of study type.\n\n    Returns:\n        DataFrame: A dataframe containing `studyId` and `studyType` columns.\n    \"\"\"\n    return self.df.select(\"studyId\", \"studyType\")\n</code></pre>"},{"location":"python_api/datasets/study_index/#schema","title":"Schema","text":"<pre><code>root\n |-- studyId: string (nullable = false)\n |-- projectId: string (nullable = false)\n |-- studyType: string (nullable = false)\n |-- traitFromSource: string (nullable = true)\n |-- traitFromSourceMappedIds: array (nullable = true)\n |    |-- element: string (containsNull = true)\n |-- geneId: string (nullable = true)\n |-- tissueFromSourceId: string (nullable = true)\n |-- pubmedId: string (nullable = true)\n |-- publicationTitle: string (nullable = true)\n |-- publicationFirstAuthor: string (nullable = true)\n |-- publicationDate: string (nullable = true)\n |-- publicationJournal: string (nullable = true)\n |-- backgroundTraitFromSourceMappedIds: array (nullable = true)\n |    |-- element: string (containsNull = true)\n |-- initialSampleSize: string (nullable = true)\n |-- nCases: integer (nullable = true)\n |-- nControls: integer (nullable = true)\n |-- nSamples: integer (nullable = true)\n |-- cohorts: array (nullable = true)\n |    |-- element: string (containsNull = true)\n |-- ldPopulationStructure: array (nullable = true)\n |    |-- element: struct (containsNull = false)\n |    |    |-- ldPopulation: string (nullable = true)\n |    |    |-- relativeSampleSize: double (nullable = true)\n |-- discoverySamples: array (nullable = true)\n |    |-- element: struct (containsNull = false)\n |    |    |-- sampleSize: integer (nullable = true)\n |    |    |-- ancestry: string (nullable = true)\n |-- replicationSamples: array (nullable = true)\n |    |-- element: struct (containsNull = false)\n |    |    |-- sampleSize: integer (nullable = true)\n |    |    |-- ancestry: string (nullable = true)\n |-- qualityControls: array (nullable = true)\n |    |-- element: string (containsNull = true)\n |-- analysisFlags: array (nullable = true)\n |    |-- element: string (containsNull = true)\n |-- summarystatsLocation: string (nullable = true)\n |-- hasSumstats: boolean (nullable = true)\n</code></pre>"},{"location":"python_api/datasets/study_locus/","title":"Study Locus","text":""},{"location":"python_api/datasets/study_locus/#gentropy.dataset.study_locus.StudyLocus","title":"<code>gentropy.dataset.study_locus.StudyLocus</code>  <code>dataclass</code>","text":"<p>             Bases: <code>Dataset</code></p> <p>Study-Locus dataset.</p> <p>This dataset captures associations between study/traits and a genetic loci as provided by finemapping methods.</p> Source code in <code>src/gentropy/dataset/study_locus.py</code> <pre><code>@dataclass\nclass StudyLocus(Dataset):\n    \"\"\"Study-Locus dataset.\n\n    This dataset captures associations between study/traits and a genetic loci as provided by finemapping methods.\n    \"\"\"\n\n    @staticmethod\n    def _overlapping_peaks(\n        credset_to_overlap: DataFrame, intra_study_overlap: bool = False\n    ) -&gt; DataFrame:\n        \"\"\"Calculate overlapping signals (study-locus) between GWAS-GWAS and GWAS-Molecular trait.\n\n        Args:\n            credset_to_overlap (DataFrame): DataFrame containing at least `studyLocusId`, `studyType`, `chromosome` and `tagVariantId` columns.\n            intra_study_overlap (bool): When True, finds intra-study overlaps for credible set deduplication. Default is False.\n\n        Returns:\n            DataFrame: containing `leftStudyLocusId`, `rightStudyLocusId` and `chromosome` columns.\n        \"\"\"\n        # Reduce columns to the minimum to reduce the size of the dataframe\n        credset_to_overlap = credset_to_overlap.select(\n            \"studyLocusId\",\n            \"studyId\",\n            \"studyType\",\n            \"chromosome\",\n            \"region\",\n            \"tagVariantId\",\n        )\n        # Define join condition - if intra_study_overlap is True, finds overlaps within the same study. Otherwise finds gwas vs everything overlaps for coloc.\n        join_condition = (\n            [\n                f.col(\"left.studyId\") == f.col(\"right.studyId\"),\n                f.col(\"left.chromosome\") == f.col(\"right.chromosome\"),\n                f.col(\"left.tagVariantId\") == f.col(\"right.tagVariantId\"),\n                f.col(\"left.studyLocusId\") &gt; f.col(\"right.studyLocusId\"),\n                f.col(\"left.region\") != f.col(\"right.region\"),\n            ]\n            if intra_study_overlap\n            else [\n                f.col(\"left.chromosome\") == f.col(\"right.chromosome\"),\n                f.col(\"left.tagVariantId\") == f.col(\"right.tagVariantId\"),\n                (f.col(\"right.studyType\") != \"gwas\")\n                | (f.col(\"left.studyLocusId\") &gt; f.col(\"right.studyLocusId\")),\n                f.col(\"left.studyType\") == f.lit(\"gwas\"),\n            ]\n        )\n\n        return (\n            credset_to_overlap.alias(\"left\")\n            # Self join with complex condition.\n            .join(\n                credset_to_overlap.alias(\"right\"),\n                on=join_condition,\n                how=\"inner\",\n            )\n            .select(\n                f.col(\"left.studyLocusId\").alias(\"leftStudyLocusId\"),\n                f.col(\"right.studyLocusId\").alias(\"rightStudyLocusId\"),\n                f.col(\"left.chromosome\").alias(\"chromosome\"),\n            )\n            .distinct()\n            .repartition(\"chromosome\")\n            .persist()\n        )\n\n    @staticmethod\n    def _align_overlapping_tags(\n        loci_to_overlap: DataFrame, peak_overlaps: DataFrame\n    ) -&gt; StudyLocusOverlap:\n        \"\"\"Align overlapping tags in pairs of overlapping study-locus, keeping all tags in both loci.\n\n        Args:\n            loci_to_overlap (DataFrame): containing `studyLocusId`, `studyType`, `chromosome`, `tagVariantId`, `logBF` and `posteriorProbability` columns.\n            peak_overlaps (DataFrame): containing `leftStudyLocusId`, `rightStudyLocusId` and `chromosome` columns.\n\n        Returns:\n            StudyLocusOverlap: Pairs of overlapping study-locus with aligned tags.\n        \"\"\"\n        # Complete information about all tags in the left study-locus of the overlap\n        stats_cols = [\n            \"logBF\",\n            \"posteriorProbability\",\n            \"beta\",\n            \"pValueMantissa\",\n            \"pValueExponent\",\n        ]\n        overlapping_left = loci_to_overlap.select(\n            f.col(\"chromosome\"),\n            f.col(\"tagVariantId\"),\n            f.col(\"studyLocusId\").alias(\"leftStudyLocusId\"),\n            *[f.col(col).alias(f\"left_{col}\") for col in stats_cols],\n        ).join(peak_overlaps, on=[\"chromosome\", \"leftStudyLocusId\"], how=\"inner\")\n\n        # Complete information about all tags in the right study-locus of the overlap\n        overlapping_right = loci_to_overlap.select(\n            f.col(\"chromosome\"),\n            f.col(\"tagVariantId\"),\n            f.col(\"studyLocusId\").alias(\"rightStudyLocusId\"),\n            *[f.col(col).alias(f\"right_{col}\") for col in stats_cols],\n        ).join(peak_overlaps, on=[\"chromosome\", \"rightStudyLocusId\"], how=\"inner\")\n\n        # Include information about all tag variants in both study-locus aligned by tag variant id\n        overlaps = overlapping_left.join(\n            overlapping_right,\n            on=[\n                \"chromosome\",\n                \"rightStudyLocusId\",\n                \"leftStudyLocusId\",\n                \"tagVariantId\",\n            ],\n            how=\"outer\",\n        ).select(\n            \"leftStudyLocusId\",\n            \"rightStudyLocusId\",\n            \"chromosome\",\n            \"tagVariantId\",\n            f.struct(\n                *[f\"left_{e}\" for e in stats_cols] + [f\"right_{e}\" for e in stats_cols]\n            ).alias(\"statistics\"),\n        )\n        return StudyLocusOverlap(\n            _df=overlaps,\n            _schema=StudyLocusOverlap.get_schema(),\n        )\n\n    @staticmethod\n    def update_quality_flag(\n        qc: Column, flag_condition: Column, flag_text: StudyLocusQualityCheck\n    ) -&gt; Column:\n        \"\"\"Update the provided quality control list with a new flag if condition is met.\n\n        Args:\n            qc (Column): Array column with the current list of qc flags.\n            flag_condition (Column): This is a column of booleans, signing which row should be flagged\n            flag_text (StudyLocusQualityCheck): Text for the new quality control flag\n\n        Returns:\n            Column: Array column with the updated list of qc flags.\n        \"\"\"\n        qc = f.when(qc.isNull(), f.array()).otherwise(qc)\n        return f.when(\n            flag_condition,\n            f.array_union(qc, f.array(f.lit(flag_text.value))),\n        ).otherwise(qc)\n\n    @staticmethod\n    def assign_study_locus_id(study_id_col: Column, variant_id_col: Column) -&gt; Column:\n        \"\"\"Hashes a column with a variant ID and a study ID to extract a consistent studyLocusId.\n\n        Args:\n            study_id_col (Column): column name with a study ID\n            variant_id_col (Column): column name with a variant ID\n\n        Returns:\n            Column: column with a study locus ID\n\n        Examples:\n            &gt;&gt;&gt; df = spark.createDataFrame([(\"GCST000001\", \"1_1000_A_C\"), (\"GCST000002\", \"1_1000_A_C\")]).toDF(\"studyId\", \"variantId\")\n            &gt;&gt;&gt; df.withColumn(\"study_locus_id\", StudyLocus.assign_study_locus_id(f.col(\"studyId\"), f.col(\"variantId\"))).show()\n            +----------+----------+-------------------+\n            |   studyId| variantId|     study_locus_id|\n            +----------+----------+-------------------+\n            |GCST000001|1_1000_A_C|1553357789130151995|\n            |GCST000002|1_1000_A_C|-415050894682709184|\n            +----------+----------+-------------------+\n            &lt;BLANKLINE&gt;\n        \"\"\"\n        variant_id_col = f.coalesce(variant_id_col, f.rand().cast(\"string\"))\n        return f.xxhash64(study_id_col, variant_id_col).alias(\"studyLocusId\")\n\n    @classmethod\n    def calculate_credible_set_log10bf(cls: type[StudyLocus], logbfs: Column) -&gt; Column:\n        \"\"\"Calculate Bayes factor for the entire credible set. The Bayes factor is calculated as the logsumexp of the logBF values of the variants in the locus.\n\n        Args:\n            logbfs (Column): Array column with the logBF values of the variants in the locus.\n\n        Returns:\n            Column: log10 Bayes factor for the entire credible set.\n\n        Examples:\n            &gt;&gt;&gt; spark.createDataFrame([([0.2, 0.1, 0.05, 0.0],)]).toDF(\"logBF\").select(f.round(StudyLocus.calculate_credible_set_log10bf(f.col(\"logBF\")), 7).alias(\"credibleSetlog10BF\")).show()\n            +------------------+\n            |credibleSetlog10BF|\n            +------------------+\n            |         1.4765565|\n            +------------------+\n            &lt;BLANKLINE&gt;\n        \"\"\"\n        logsumexp_udf = f.udf(lambda x: get_logsum(x), FloatType())\n        return logsumexp_udf(logbfs).cast(\"double\").alias(\"credibleSetlog10BF\")\n\n    @classmethod\n    def get_schema(cls: type[StudyLocus]) -&gt; StructType:\n        \"\"\"Provides the schema for the StudyLocus dataset.\n\n        Returns:\n            StructType: schema for the StudyLocus dataset.\n        \"\"\"\n        return parse_spark_schema(\"study_locus.json\")\n\n    def filter_by_study_type(\n        self: StudyLocus, study_type: str, study_index: StudyIndex\n    ) -&gt; StudyLocus:\n        \"\"\"Creates a new StudyLocus dataset filtered by study type.\n\n        Args:\n            study_type (str): Study type to filter for. Can be one of `gwas`, `eqtl`, `pqtl`, `eqtl`.\n            study_index (StudyIndex): Study index to resolve study types.\n\n        Returns:\n            StudyLocus: Filtered study-locus dataset.\n\n        Raises:\n            ValueError: If study type is not supported.\n        \"\"\"\n        if study_type not in [\"gwas\", \"eqtl\", \"pqtl\", \"sqtl\"]:\n            raise ValueError(\n                f\"Study type {study_type} not supported. Supported types are: gwas, eqtl, pqtl, sqtl.\"\n            )\n        new_df = (\n            self.df.join(study_index.study_type_lut(), on=\"studyId\", how=\"inner\")\n            .filter(f.col(\"studyType\") == study_type)\n            .drop(\"studyType\")\n        )\n        return StudyLocus(\n            _df=new_df,\n            _schema=self._schema,\n        )\n\n    def filter_credible_set(\n        self: StudyLocus,\n        credible_interval: CredibleInterval,\n    ) -&gt; StudyLocus:\n        \"\"\"Filter study-locus tag variants based on given credible interval.\n\n        Args:\n            credible_interval (CredibleInterval): Credible interval to filter for.\n\n        Returns:\n            StudyLocus: Filtered study-locus dataset.\n        \"\"\"\n        self.df = self._df.withColumn(\n            \"locus\",\n            f.filter(\n                f.col(\"locus\"),\n                lambda tag: (tag[credible_interval.value]),\n            ),\n        )\n        return self\n\n    def find_overlaps(\n        self: StudyLocus, study_index: StudyIndex, intra_study_overlap: bool = False\n    ) -&gt; StudyLocusOverlap:\n        \"\"\"Calculate overlapping study-locus.\n\n        Find overlapping study-locus that share at least one tagging variant. All GWAS-GWAS and all GWAS-Molecular traits are computed with the Molecular traits always\n        appearing on the right side.\n\n        Args:\n            study_index (StudyIndex): Study index to resolve study types.\n            intra_study_overlap (bool): If True, finds intra-study overlaps for credible set deduplication. Default is False.\n\n        Returns:\n            StudyLocusOverlap: Pairs of overlapping study-locus with aligned tags.\n        \"\"\"\n        loci_to_overlap = (\n            self.df.join(study_index.study_type_lut(), on=\"studyId\", how=\"inner\")\n            .withColumn(\"locus\", f.explode(\"locus\"))\n            .select(\n                \"studyLocusId\",\n                \"studyId\",\n                \"studyType\",\n                \"chromosome\",\n                \"region\",\n                f.col(\"locus.variantId\").alias(\"tagVariantId\"),\n                f.col(\"locus.logBF\").alias(\"logBF\"),\n                f.col(\"locus.posteriorProbability\").alias(\"posteriorProbability\"),\n                f.col(\"locus.pValueMantissa\").alias(\"pValueMantissa\"),\n                f.col(\"locus.pValueExponent\").alias(\"pValueExponent\"),\n                f.col(\"locus.beta\").alias(\"beta\"),\n            )\n            .persist()\n        )\n\n        # overlapping study-locus\n        peak_overlaps = self._overlapping_peaks(loci_to_overlap, intra_study_overlap)\n\n        # study-locus overlap by aligning overlapping variants\n        return self._align_overlapping_tags(loci_to_overlap, peak_overlaps)\n\n    def unique_variants_in_locus(self: StudyLocus) -&gt; DataFrame:\n        \"\"\"All unique variants collected in a `StudyLocus` dataframe.\n\n        Returns:\n            DataFrame: A dataframe containing `variantId` and `chromosome` columns.\n        \"\"\"\n        return (\n            self.df.withColumn(\n                \"variantId\",\n                # Joint array of variants in that studylocus. Locus can be null\n                f.explode(\n                    f.array_union(\n                        f.array(f.col(\"variantId\")),\n                        f.coalesce(f.col(\"locus.variantId\"), f.array()),\n                    )\n                ),\n            )\n            .select(\n                \"variantId\", f.split(f.col(\"variantId\"), \"_\")[0].alias(\"chromosome\")\n            )\n            .distinct()\n        )\n\n    def neglog_pvalue(self: StudyLocus) -&gt; Column:\n        \"\"\"Returns the negative log p-value.\n\n        Returns:\n            Column: Negative log p-value\n        \"\"\"\n        return calculate_neglog_pvalue(\n            self.df.pValueMantissa,\n            self.df.pValueExponent,\n        )\n\n    def annotate_credible_sets(self: StudyLocus) -&gt; StudyLocus:\n        \"\"\"Annotate study-locus dataset with credible set flags.\n\n        Sorts the array in the `locus` column elements by their `posteriorProbability` values in descending order and adds\n        `is95CredibleSet` and `is99CredibleSet` fields to the elements, indicating which are the tagging variants whose cumulative sum\n        of their `posteriorProbability` values is below 0.95 and 0.99, respectively.\n\n        Returns:\n            StudyLocus: including annotation on `is95CredibleSet` and `is99CredibleSet`.\n\n        Raises:\n            ValueError: If `locus` column is not available.\n        \"\"\"\n        if \"locus\" not in self.df.columns:\n            raise ValueError(\"Locus column not available.\")\n\n        self.df = self.df.withColumn(\n            # Sort credible set by posterior probability in descending order\n            \"locus\",\n            f.when(\n                f.col(\"locus\").isNotNull() &amp; (f.size(f.col(\"locus\")) &gt; 0),\n                order_array_of_structs_by_field(\"locus\", \"posteriorProbability\"),\n            ),\n        ).withColumn(\n            # Calculate array of cumulative sums of posterior probabilities to determine which variants are in the 95% and 99% credible sets\n            # and zip the cumulative sums array with the credible set array to add the flags\n            \"locus\",\n            f.when(\n                f.col(\"locus\").isNotNull() &amp; (f.size(f.col(\"locus\")) &gt; 0),\n                f.zip_with(\n                    f.col(\"locus\"),\n                    f.transform(\n                        f.sequence(f.lit(1), f.size(f.col(\"locus\"))),\n                        lambda index: f.aggregate(\n                            f.slice(\n                                # By using `index - 1` we introduce a value of `0.0` in the cumulative sums array. to ensure that the last variant\n                                # that exceeds the 0.95 threshold is included in the cumulative sum, as its probability is necessary to satisfy the threshold.\n                                f.col(\"locus.posteriorProbability\"),\n                                1,\n                                index - 1,\n                            ),\n                            f.lit(0.0),\n                            lambda acc, el: acc + el,\n                        ),\n                    ),\n                    lambda struct_e, acc: struct_e.withField(\n                        CredibleInterval.IS95.value, (acc &lt; 0.95) &amp; acc.isNotNull()\n                    ).withField(\n                        CredibleInterval.IS99.value, (acc &lt; 0.99) &amp; acc.isNotNull()\n                    ),\n                ),\n            ),\n        )\n        return self\n\n    def annotate_locus_statistics(\n        self: StudyLocus,\n        summary_statistics: SummaryStatistics,\n        collect_locus_distance: int,\n    ) -&gt; StudyLocus:\n        \"\"\"Annotates study locus with summary statistics in the specified distance around the position.\n\n        Args:\n            summary_statistics (SummaryStatistics): Summary statistics to be used for annotation.\n            collect_locus_distance (int): distance from variant defining window for inclusion of variants in locus.\n\n        Returns:\n            StudyLocus: Study locus annotated with summary statistics in `locus` column. If no statistics are found, the `locus` column will be empty.\n        \"\"\"\n        # The clumps will be used several times (persisting)\n        self.df.persist()\n        # Renaming columns:\n        sumstats_renamed = summary_statistics.df.selectExpr(\n            *[f\"{col} as tag_{col}\" for col in summary_statistics.df.columns]\n        ).alias(\"sumstat\")\n\n        locus_df = (\n            sumstats_renamed\n            # Joining the two datasets together:\n            .join(\n                f.broadcast(\n                    self.df.alias(\"clumped\").select(\n                        \"position\", \"chromosome\", \"studyId\", \"studyLocusId\"\n                    )\n                ),\n                on=[\n                    (f.col(\"sumstat.tag_studyId\") == f.col(\"clumped.studyId\"))\n                    &amp; (f.col(\"sumstat.tag_chromosome\") == f.col(\"clumped.chromosome\"))\n                    &amp; (\n                        f.col(\"sumstat.tag_position\")\n                        &gt;= (f.col(\"clumped.position\") - collect_locus_distance)\n                    )\n                    &amp; (\n                        f.col(\"sumstat.tag_position\")\n                        &lt;= (f.col(\"clumped.position\") + collect_locus_distance)\n                    )\n                ],\n                how=\"inner\",\n            )\n            .withColumn(\n                \"locus\",\n                f.struct(\n                    f.col(\"tag_variantId\").alias(\"variantId\"),\n                    f.col(\"tag_beta\").alias(\"beta\"),\n                    f.col(\"tag_pValueMantissa\").alias(\"pValueMantissa\"),\n                    f.col(\"tag_pValueExponent\").alias(\"pValueExponent\"),\n                    f.col(\"tag_standardError\").alias(\"standardError\"),\n                ),\n            )\n            .groupBy(\"studyLocusId\")\n            .agg(\n                f.collect_list(f.col(\"locus\")).alias(\"locus\"),\n            )\n        )\n\n        self.df = self.df.drop(\"locus\").join(\n            locus_df,\n            on=\"studyLocusId\",\n            how=\"left\",\n        )\n\n        return self\n\n    def annotate_ld(\n        self: StudyLocus, study_index: StudyIndex, ld_index: LDIndex\n    ) -&gt; StudyLocus:\n        \"\"\"Annotate LD information to study-locus.\n\n        Args:\n            study_index (StudyIndex): Study index to resolve ancestries.\n            ld_index (LDIndex): LD index to resolve LD information.\n\n        Returns:\n            StudyLocus: Study locus annotated with ld information from LD index.\n        \"\"\"\n        from gentropy.method.ld import LDAnnotator\n\n        return LDAnnotator.ld_annotate(self, study_index, ld_index)\n\n    def clump(self: StudyLocus) -&gt; StudyLocus:\n        \"\"\"Perform LD clumping of the studyLocus.\n\n        Evaluates whether a lead variant is linked to a tag (with lowest p-value) in the same studyLocus dataset.\n\n        Returns:\n            StudyLocus: with empty credible sets for linked variants and QC flag.\n        \"\"\"\n        self.df = (\n            self.df.withColumn(\n                \"is_lead_linked\",\n                LDclumping._is_lead_linked(\n                    self.df.studyId,\n                    self.df.variantId,\n                    self.df.pValueExponent,\n                    self.df.pValueMantissa,\n                    self.df.ldSet,\n                ),\n            )\n            .withColumn(\n                \"ldSet\",\n                f.when(f.col(\"is_lead_linked\"), f.array()).otherwise(f.col(\"ldSet\")),\n            )\n            .withColumn(\n                \"qualityControls\",\n                StudyLocus.update_quality_flag(\n                    f.col(\"qualityControls\"),\n                    f.col(\"is_lead_linked\"),\n                    StudyLocusQualityCheck.LD_CLUMPED,\n                ),\n            )\n            .drop(\"is_lead_linked\")\n        )\n        return self\n\n    def _qc_no_population(self: StudyLocus) -&gt; StudyLocus:\n        \"\"\"Flag associations where the study doesn't have population information to resolve LD.\n\n        Returns:\n            StudyLocus: Updated study locus.\n        \"\"\"\n        # If the tested column is not present, return self unchanged:\n        if \"ldPopulationStructure\" not in self.df.columns:\n            return self\n\n        self.df = self.df.withColumn(\n            \"qualityControls\",\n            self.update_quality_flag(\n                f.col(\"qualityControls\"),\n                f.col(\"ldPopulationStructure\").isNull(),\n                StudyLocusQualityCheck.NO_POPULATION,\n            ),\n        )\n        return self\n</code></pre>"},{"location":"python_api/datasets/study_locus/#gentropy.dataset.study_locus.StudyLocus.annotate_credible_sets","title":"<code>annotate_credible_sets() -&gt; StudyLocus</code>","text":"<p>Annotate study-locus dataset with credible set flags.</p> <p>Sorts the array in the <code>locus</code> column elements by their <code>posteriorProbability</code> values in descending order and adds <code>is95CredibleSet</code> and <code>is99CredibleSet</code> fields to the elements, indicating which are the tagging variants whose cumulative sum of their <code>posteriorProbability</code> values is below 0.95 and 0.99, respectively.</p> <p>Returns:</p> Name Type Description <code>StudyLocus</code> <code>StudyLocus</code> <p>including annotation on <code>is95CredibleSet</code> and <code>is99CredibleSet</code>.</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>If <code>locus</code> column is not available.</p> Source code in <code>src/gentropy/dataset/study_locus.py</code> <pre><code>def annotate_credible_sets(self: StudyLocus) -&gt; StudyLocus:\n    \"\"\"Annotate study-locus dataset with credible set flags.\n\n    Sorts the array in the `locus` column elements by their `posteriorProbability` values in descending order and adds\n    `is95CredibleSet` and `is99CredibleSet` fields to the elements, indicating which are the tagging variants whose cumulative sum\n    of their `posteriorProbability` values is below 0.95 and 0.99, respectively.\n\n    Returns:\n        StudyLocus: including annotation on `is95CredibleSet` and `is99CredibleSet`.\n\n    Raises:\n        ValueError: If `locus` column is not available.\n    \"\"\"\n    if \"locus\" not in self.df.columns:\n        raise ValueError(\"Locus column not available.\")\n\n    self.df = self.df.withColumn(\n        # Sort credible set by posterior probability in descending order\n        \"locus\",\n        f.when(\n            f.col(\"locus\").isNotNull() &amp; (f.size(f.col(\"locus\")) &gt; 0),\n            order_array_of_structs_by_field(\"locus\", \"posteriorProbability\"),\n        ),\n    ).withColumn(\n        # Calculate array of cumulative sums of posterior probabilities to determine which variants are in the 95% and 99% credible sets\n        # and zip the cumulative sums array with the credible set array to add the flags\n        \"locus\",\n        f.when(\n            f.col(\"locus\").isNotNull() &amp; (f.size(f.col(\"locus\")) &gt; 0),\n            f.zip_with(\n                f.col(\"locus\"),\n                f.transform(\n                    f.sequence(f.lit(1), f.size(f.col(\"locus\"))),\n                    lambda index: f.aggregate(\n                        f.slice(\n                            # By using `index - 1` we introduce a value of `0.0` in the cumulative sums array. to ensure that the last variant\n                            # that exceeds the 0.95 threshold is included in the cumulative sum, as its probability is necessary to satisfy the threshold.\n                            f.col(\"locus.posteriorProbability\"),\n                            1,\n                            index - 1,\n                        ),\n                        f.lit(0.0),\n                        lambda acc, el: acc + el,\n                    ),\n                ),\n                lambda struct_e, acc: struct_e.withField(\n                    CredibleInterval.IS95.value, (acc &lt; 0.95) &amp; acc.isNotNull()\n                ).withField(\n                    CredibleInterval.IS99.value, (acc &lt; 0.99) &amp; acc.isNotNull()\n                ),\n            ),\n        ),\n    )\n    return self\n</code></pre>"},{"location":"python_api/datasets/study_locus/#gentropy.dataset.study_locus.StudyLocus.annotate_ld","title":"<code>annotate_ld(study_index: StudyIndex, ld_index: LDIndex) -&gt; StudyLocus</code>","text":"<p>Annotate LD information to study-locus.</p> <p>Parameters:</p> Name Type Description Default <code>study_index</code> <code>StudyIndex</code> <p>Study index to resolve ancestries.</p> required <code>ld_index</code> <code>LDIndex</code> <p>LD index to resolve LD information.</p> required <p>Returns:</p> Name Type Description <code>StudyLocus</code> <code>StudyLocus</code> <p>Study locus annotated with ld information from LD index.</p> Source code in <code>src/gentropy/dataset/study_locus.py</code> <pre><code>def annotate_ld(\n    self: StudyLocus, study_index: StudyIndex, ld_index: LDIndex\n) -&gt; StudyLocus:\n    \"\"\"Annotate LD information to study-locus.\n\n    Args:\n        study_index (StudyIndex): Study index to resolve ancestries.\n        ld_index (LDIndex): LD index to resolve LD information.\n\n    Returns:\n        StudyLocus: Study locus annotated with ld information from LD index.\n    \"\"\"\n    from gentropy.method.ld import LDAnnotator\n\n    return LDAnnotator.ld_annotate(self, study_index, ld_index)\n</code></pre>"},{"location":"python_api/datasets/study_locus/#gentropy.dataset.study_locus.StudyLocus.annotate_locus_statistics","title":"<code>annotate_locus_statistics(summary_statistics: SummaryStatistics, collect_locus_distance: int) -&gt; StudyLocus</code>","text":"<p>Annotates study locus with summary statistics in the specified distance around the position.</p> <p>Parameters:</p> Name Type Description Default <code>summary_statistics</code> <code>SummaryStatistics</code> <p>Summary statistics to be used for annotation.</p> required <code>collect_locus_distance</code> <code>int</code> <p>distance from variant defining window for inclusion of variants in locus.</p> required <p>Returns:</p> Name Type Description <code>StudyLocus</code> <code>StudyLocus</code> <p>Study locus annotated with summary statistics in <code>locus</code> column. If no statistics are found, the <code>locus</code> column will be empty.</p> Source code in <code>src/gentropy/dataset/study_locus.py</code> <pre><code>def annotate_locus_statistics(\n    self: StudyLocus,\n    summary_statistics: SummaryStatistics,\n    collect_locus_distance: int,\n) -&gt; StudyLocus:\n    \"\"\"Annotates study locus with summary statistics in the specified distance around the position.\n\n    Args:\n        summary_statistics (SummaryStatistics): Summary statistics to be used for annotation.\n        collect_locus_distance (int): distance from variant defining window for inclusion of variants in locus.\n\n    Returns:\n        StudyLocus: Study locus annotated with summary statistics in `locus` column. If no statistics are found, the `locus` column will be empty.\n    \"\"\"\n    # The clumps will be used several times (persisting)\n    self.df.persist()\n    # Renaming columns:\n    sumstats_renamed = summary_statistics.df.selectExpr(\n        *[f\"{col} as tag_{col}\" for col in summary_statistics.df.columns]\n    ).alias(\"sumstat\")\n\n    locus_df = (\n        sumstats_renamed\n        # Joining the two datasets together:\n        .join(\n            f.broadcast(\n                self.df.alias(\"clumped\").select(\n                    \"position\", \"chromosome\", \"studyId\", \"studyLocusId\"\n                )\n            ),\n            on=[\n                (f.col(\"sumstat.tag_studyId\") == f.col(\"clumped.studyId\"))\n                &amp; (f.col(\"sumstat.tag_chromosome\") == f.col(\"clumped.chromosome\"))\n                &amp; (\n                    f.col(\"sumstat.tag_position\")\n                    &gt;= (f.col(\"clumped.position\") - collect_locus_distance)\n                )\n                &amp; (\n                    f.col(\"sumstat.tag_position\")\n                    &lt;= (f.col(\"clumped.position\") + collect_locus_distance)\n                )\n            ],\n            how=\"inner\",\n        )\n        .withColumn(\n            \"locus\",\n            f.struct(\n                f.col(\"tag_variantId\").alias(\"variantId\"),\n                f.col(\"tag_beta\").alias(\"beta\"),\n                f.col(\"tag_pValueMantissa\").alias(\"pValueMantissa\"),\n                f.col(\"tag_pValueExponent\").alias(\"pValueExponent\"),\n                f.col(\"tag_standardError\").alias(\"standardError\"),\n            ),\n        )\n        .groupBy(\"studyLocusId\")\n        .agg(\n            f.collect_list(f.col(\"locus\")).alias(\"locus\"),\n        )\n    )\n\n    self.df = self.df.drop(\"locus\").join(\n        locus_df,\n        on=\"studyLocusId\",\n        how=\"left\",\n    )\n\n    return self\n</code></pre>"},{"location":"python_api/datasets/study_locus/#gentropy.dataset.study_locus.StudyLocus.assign_study_locus_id","title":"<code>assign_study_locus_id(study_id_col: Column, variant_id_col: Column) -&gt; Column</code>  <code>staticmethod</code>","text":"<p>Hashes a column with a variant ID and a study ID to extract a consistent studyLocusId.</p> <p>Parameters:</p> Name Type Description Default <code>study_id_col</code> <code>Column</code> <p>column name with a study ID</p> required <code>variant_id_col</code> <code>Column</code> <p>column name with a variant ID</p> required <p>Returns:</p> Name Type Description <code>Column</code> <code>Column</code> <p>column with a study locus ID</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; df = spark.createDataFrame([(\"GCST000001\", \"1_1000_A_C\"), (\"GCST000002\", \"1_1000_A_C\")]).toDF(\"studyId\", \"variantId\")\n&gt;&gt;&gt; df.withColumn(\"study_locus_id\", StudyLocus.assign_study_locus_id(f.col(\"studyId\"), f.col(\"variantId\"))).show()\n+----------+----------+-------------------+\n|   studyId| variantId|     study_locus_id|\n+----------+----------+-------------------+\n|GCST000001|1_1000_A_C|1553357789130151995|\n|GCST000002|1_1000_A_C|-415050894682709184|\n+----------+----------+-------------------+\n</code></pre> Source code in <code>src/gentropy/dataset/study_locus.py</code> <pre><code>@staticmethod\ndef assign_study_locus_id(study_id_col: Column, variant_id_col: Column) -&gt; Column:\n    \"\"\"Hashes a column with a variant ID and a study ID to extract a consistent studyLocusId.\n\n    Args:\n        study_id_col (Column): column name with a study ID\n        variant_id_col (Column): column name with a variant ID\n\n    Returns:\n        Column: column with a study locus ID\n\n    Examples:\n        &gt;&gt;&gt; df = spark.createDataFrame([(\"GCST000001\", \"1_1000_A_C\"), (\"GCST000002\", \"1_1000_A_C\")]).toDF(\"studyId\", \"variantId\")\n        &gt;&gt;&gt; df.withColumn(\"study_locus_id\", StudyLocus.assign_study_locus_id(f.col(\"studyId\"), f.col(\"variantId\"))).show()\n        +----------+----------+-------------------+\n        |   studyId| variantId|     study_locus_id|\n        +----------+----------+-------------------+\n        |GCST000001|1_1000_A_C|1553357789130151995|\n        |GCST000002|1_1000_A_C|-415050894682709184|\n        +----------+----------+-------------------+\n        &lt;BLANKLINE&gt;\n    \"\"\"\n    variant_id_col = f.coalesce(variant_id_col, f.rand().cast(\"string\"))\n    return f.xxhash64(study_id_col, variant_id_col).alias(\"studyLocusId\")\n</code></pre>"},{"location":"python_api/datasets/study_locus/#gentropy.dataset.study_locus.StudyLocus.calculate_credible_set_log10bf","title":"<code>calculate_credible_set_log10bf(logbfs: Column) -&gt; Column</code>  <code>classmethod</code>","text":"<p>Calculate Bayes factor for the entire credible set. The Bayes factor is calculated as the logsumexp of the logBF values of the variants in the locus.</p> <p>Parameters:</p> Name Type Description Default <code>logbfs</code> <code>Column</code> <p>Array column with the logBF values of the variants in the locus.</p> required <p>Returns:</p> Name Type Description <code>Column</code> <code>Column</code> <p>log10 Bayes factor for the entire credible set.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; spark.createDataFrame([([0.2, 0.1, 0.05, 0.0],)]).toDF(\"logBF\").select(f.round(StudyLocus.calculate_credible_set_log10bf(f.col(\"logBF\")), 7).alias(\"credibleSetlog10BF\")).show()\n+------------------+\n|credibleSetlog10BF|\n+------------------+\n|         1.4765565|\n+------------------+\n</code></pre> Source code in <code>src/gentropy/dataset/study_locus.py</code> <pre><code>@classmethod\ndef calculate_credible_set_log10bf(cls: type[StudyLocus], logbfs: Column) -&gt; Column:\n    \"\"\"Calculate Bayes factor for the entire credible set. The Bayes factor is calculated as the logsumexp of the logBF values of the variants in the locus.\n\n    Args:\n        logbfs (Column): Array column with the logBF values of the variants in the locus.\n\n    Returns:\n        Column: log10 Bayes factor for the entire credible set.\n\n    Examples:\n        &gt;&gt;&gt; spark.createDataFrame([([0.2, 0.1, 0.05, 0.0],)]).toDF(\"logBF\").select(f.round(StudyLocus.calculate_credible_set_log10bf(f.col(\"logBF\")), 7).alias(\"credibleSetlog10BF\")).show()\n        +------------------+\n        |credibleSetlog10BF|\n        +------------------+\n        |         1.4765565|\n        +------------------+\n        &lt;BLANKLINE&gt;\n    \"\"\"\n    logsumexp_udf = f.udf(lambda x: get_logsum(x), FloatType())\n    return logsumexp_udf(logbfs).cast(\"double\").alias(\"credibleSetlog10BF\")\n</code></pre>"},{"location":"python_api/datasets/study_locus/#gentropy.dataset.study_locus.StudyLocus.clump","title":"<code>clump() -&gt; StudyLocus</code>","text":"<p>Perform LD clumping of the studyLocus.</p> <p>Evaluates whether a lead variant is linked to a tag (with lowest p-value) in the same studyLocus dataset.</p> <p>Returns:</p> Name Type Description <code>StudyLocus</code> <code>StudyLocus</code> <p>with empty credible sets for linked variants and QC flag.</p> Source code in <code>src/gentropy/dataset/study_locus.py</code> <pre><code>def clump(self: StudyLocus) -&gt; StudyLocus:\n    \"\"\"Perform LD clumping of the studyLocus.\n\n    Evaluates whether a lead variant is linked to a tag (with lowest p-value) in the same studyLocus dataset.\n\n    Returns:\n        StudyLocus: with empty credible sets for linked variants and QC flag.\n    \"\"\"\n    self.df = (\n        self.df.withColumn(\n            \"is_lead_linked\",\n            LDclumping._is_lead_linked(\n                self.df.studyId,\n                self.df.variantId,\n                self.df.pValueExponent,\n                self.df.pValueMantissa,\n                self.df.ldSet,\n            ),\n        )\n        .withColumn(\n            \"ldSet\",\n            f.when(f.col(\"is_lead_linked\"), f.array()).otherwise(f.col(\"ldSet\")),\n        )\n        .withColumn(\n            \"qualityControls\",\n            StudyLocus.update_quality_flag(\n                f.col(\"qualityControls\"),\n                f.col(\"is_lead_linked\"),\n                StudyLocusQualityCheck.LD_CLUMPED,\n            ),\n        )\n        .drop(\"is_lead_linked\")\n    )\n    return self\n</code></pre>"},{"location":"python_api/datasets/study_locus/#gentropy.dataset.study_locus.StudyLocus.filter_by_study_type","title":"<code>filter_by_study_type(study_type: str, study_index: StudyIndex) -&gt; StudyLocus</code>","text":"<p>Creates a new StudyLocus dataset filtered by study type.</p> <p>Parameters:</p> Name Type Description Default <code>study_type</code> <code>str</code> <p>Study type to filter for. Can be one of <code>gwas</code>, <code>eqtl</code>, <code>pqtl</code>, <code>eqtl</code>.</p> required <code>study_index</code> <code>StudyIndex</code> <p>Study index to resolve study types.</p> required <p>Returns:</p> Name Type Description <code>StudyLocus</code> <code>StudyLocus</code> <p>Filtered study-locus dataset.</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>If study type is not supported.</p> Source code in <code>src/gentropy/dataset/study_locus.py</code> <pre><code>def filter_by_study_type(\n    self: StudyLocus, study_type: str, study_index: StudyIndex\n) -&gt; StudyLocus:\n    \"\"\"Creates a new StudyLocus dataset filtered by study type.\n\n    Args:\n        study_type (str): Study type to filter for. Can be one of `gwas`, `eqtl`, `pqtl`, `eqtl`.\n        study_index (StudyIndex): Study index to resolve study types.\n\n    Returns:\n        StudyLocus: Filtered study-locus dataset.\n\n    Raises:\n        ValueError: If study type is not supported.\n    \"\"\"\n    if study_type not in [\"gwas\", \"eqtl\", \"pqtl\", \"sqtl\"]:\n        raise ValueError(\n            f\"Study type {study_type} not supported. Supported types are: gwas, eqtl, pqtl, sqtl.\"\n        )\n    new_df = (\n        self.df.join(study_index.study_type_lut(), on=\"studyId\", how=\"inner\")\n        .filter(f.col(\"studyType\") == study_type)\n        .drop(\"studyType\")\n    )\n    return StudyLocus(\n        _df=new_df,\n        _schema=self._schema,\n    )\n</code></pre>"},{"location":"python_api/datasets/study_locus/#gentropy.dataset.study_locus.StudyLocus.filter_credible_set","title":"<code>filter_credible_set(credible_interval: CredibleInterval) -&gt; StudyLocus</code>","text":"<p>Filter study-locus tag variants based on given credible interval.</p> <p>Parameters:</p> Name Type Description Default <code>credible_interval</code> <code>CredibleInterval</code> <p>Credible interval to filter for.</p> required <p>Returns:</p> Name Type Description <code>StudyLocus</code> <code>StudyLocus</code> <p>Filtered study-locus dataset.</p> Source code in <code>src/gentropy/dataset/study_locus.py</code> <pre><code>def filter_credible_set(\n    self: StudyLocus,\n    credible_interval: CredibleInterval,\n) -&gt; StudyLocus:\n    \"\"\"Filter study-locus tag variants based on given credible interval.\n\n    Args:\n        credible_interval (CredibleInterval): Credible interval to filter for.\n\n    Returns:\n        StudyLocus: Filtered study-locus dataset.\n    \"\"\"\n    self.df = self._df.withColumn(\n        \"locus\",\n        f.filter(\n            f.col(\"locus\"),\n            lambda tag: (tag[credible_interval.value]),\n        ),\n    )\n    return self\n</code></pre>"},{"location":"python_api/datasets/study_locus/#gentropy.dataset.study_locus.StudyLocus.find_overlaps","title":"<code>find_overlaps(study_index: StudyIndex, intra_study_overlap: bool = False) -&gt; StudyLocusOverlap</code>","text":"<p>Calculate overlapping study-locus.</p> <p>Find overlapping study-locus that share at least one tagging variant. All GWAS-GWAS and all GWAS-Molecular traits are computed with the Molecular traits always appearing on the right side.</p> <p>Parameters:</p> Name Type Description Default <code>study_index</code> <code>StudyIndex</code> <p>Study index to resolve study types.</p> required <code>intra_study_overlap</code> <code>bool</code> <p>If True, finds intra-study overlaps for credible set deduplication. Default is False.</p> <code>False</code> <p>Returns:</p> Name Type Description <code>StudyLocusOverlap</code> <code>StudyLocusOverlap</code> <p>Pairs of overlapping study-locus with aligned tags.</p> Source code in <code>src/gentropy/dataset/study_locus.py</code> <pre><code>def find_overlaps(\n    self: StudyLocus, study_index: StudyIndex, intra_study_overlap: bool = False\n) -&gt; StudyLocusOverlap:\n    \"\"\"Calculate overlapping study-locus.\n\n    Find overlapping study-locus that share at least one tagging variant. All GWAS-GWAS and all GWAS-Molecular traits are computed with the Molecular traits always\n    appearing on the right side.\n\n    Args:\n        study_index (StudyIndex): Study index to resolve study types.\n        intra_study_overlap (bool): If True, finds intra-study overlaps for credible set deduplication. Default is False.\n\n    Returns:\n        StudyLocusOverlap: Pairs of overlapping study-locus with aligned tags.\n    \"\"\"\n    loci_to_overlap = (\n        self.df.join(study_index.study_type_lut(), on=\"studyId\", how=\"inner\")\n        .withColumn(\"locus\", f.explode(\"locus\"))\n        .select(\n            \"studyLocusId\",\n            \"studyId\",\n            \"studyType\",\n            \"chromosome\",\n            \"region\",\n            f.col(\"locus.variantId\").alias(\"tagVariantId\"),\n            f.col(\"locus.logBF\").alias(\"logBF\"),\n            f.col(\"locus.posteriorProbability\").alias(\"posteriorProbability\"),\n            f.col(\"locus.pValueMantissa\").alias(\"pValueMantissa\"),\n            f.col(\"locus.pValueExponent\").alias(\"pValueExponent\"),\n            f.col(\"locus.beta\").alias(\"beta\"),\n        )\n        .persist()\n    )\n\n    # overlapping study-locus\n    peak_overlaps = self._overlapping_peaks(loci_to_overlap, intra_study_overlap)\n\n    # study-locus overlap by aligning overlapping variants\n    return self._align_overlapping_tags(loci_to_overlap, peak_overlaps)\n</code></pre>"},{"location":"python_api/datasets/study_locus/#gentropy.dataset.study_locus.StudyLocus.get_schema","title":"<code>get_schema() -&gt; StructType</code>  <code>classmethod</code>","text":"<p>Provides the schema for the StudyLocus dataset.</p> <p>Returns:</p> Name Type Description <code>StructType</code> <code>StructType</code> <p>schema for the StudyLocus dataset.</p> Source code in <code>src/gentropy/dataset/study_locus.py</code> <pre><code>@classmethod\ndef get_schema(cls: type[StudyLocus]) -&gt; StructType:\n    \"\"\"Provides the schema for the StudyLocus dataset.\n\n    Returns:\n        StructType: schema for the StudyLocus dataset.\n    \"\"\"\n    return parse_spark_schema(\"study_locus.json\")\n</code></pre>"},{"location":"python_api/datasets/study_locus/#gentropy.dataset.study_locus.StudyLocus.neglog_pvalue","title":"<code>neglog_pvalue() -&gt; Column</code>","text":"<p>Returns the negative log p-value.</p> <p>Returns:</p> Name Type Description <code>Column</code> <code>Column</code> <p>Negative log p-value</p> Source code in <code>src/gentropy/dataset/study_locus.py</code> <pre><code>def neglog_pvalue(self: StudyLocus) -&gt; Column:\n    \"\"\"Returns the negative log p-value.\n\n    Returns:\n        Column: Negative log p-value\n    \"\"\"\n    return calculate_neglog_pvalue(\n        self.df.pValueMantissa,\n        self.df.pValueExponent,\n    )\n</code></pre>"},{"location":"python_api/datasets/study_locus/#gentropy.dataset.study_locus.StudyLocus.unique_variants_in_locus","title":"<code>unique_variants_in_locus() -&gt; DataFrame</code>","text":"<p>All unique variants collected in a <code>StudyLocus</code> dataframe.</p> <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>A dataframe containing <code>variantId</code> and <code>chromosome</code> columns.</p> Source code in <code>src/gentropy/dataset/study_locus.py</code> <pre><code>def unique_variants_in_locus(self: StudyLocus) -&gt; DataFrame:\n    \"\"\"All unique variants collected in a `StudyLocus` dataframe.\n\n    Returns:\n        DataFrame: A dataframe containing `variantId` and `chromosome` columns.\n    \"\"\"\n    return (\n        self.df.withColumn(\n            \"variantId\",\n            # Joint array of variants in that studylocus. Locus can be null\n            f.explode(\n                f.array_union(\n                    f.array(f.col(\"variantId\")),\n                    f.coalesce(f.col(\"locus.variantId\"), f.array()),\n                )\n            ),\n        )\n        .select(\n            \"variantId\", f.split(f.col(\"variantId\"), \"_\")[0].alias(\"chromosome\")\n        )\n        .distinct()\n    )\n</code></pre>"},{"location":"python_api/datasets/study_locus/#gentropy.dataset.study_locus.StudyLocus.update_quality_flag","title":"<code>update_quality_flag(qc: Column, flag_condition: Column, flag_text: StudyLocusQualityCheck) -&gt; Column</code>  <code>staticmethod</code>","text":"<p>Update the provided quality control list with a new flag if condition is met.</p> <p>Parameters:</p> Name Type Description Default <code>qc</code> <code>Column</code> <p>Array column with the current list of qc flags.</p> required <code>flag_condition</code> <code>Column</code> <p>This is a column of booleans, signing which row should be flagged</p> required <code>flag_text</code> <code>StudyLocusQualityCheck</code> <p>Text for the new quality control flag</p> required <p>Returns:</p> Name Type Description <code>Column</code> <code>Column</code> <p>Array column with the updated list of qc flags.</p> Source code in <code>src/gentropy/dataset/study_locus.py</code> <pre><code>@staticmethod\ndef update_quality_flag(\n    qc: Column, flag_condition: Column, flag_text: StudyLocusQualityCheck\n) -&gt; Column:\n    \"\"\"Update the provided quality control list with a new flag if condition is met.\n\n    Args:\n        qc (Column): Array column with the current list of qc flags.\n        flag_condition (Column): This is a column of booleans, signing which row should be flagged\n        flag_text (StudyLocusQualityCheck): Text for the new quality control flag\n\n    Returns:\n        Column: Array column with the updated list of qc flags.\n    \"\"\"\n    qc = f.when(qc.isNull(), f.array()).otherwise(qc)\n    return f.when(\n        flag_condition,\n        f.array_union(qc, f.array(f.lit(flag_text.value))),\n    ).otherwise(qc)\n</code></pre>"},{"location":"python_api/datasets/study_locus/#gentropy.dataset.study_locus.StudyLocusQualityCheck","title":"<code>gentropy.dataset.study_locus.StudyLocusQualityCheck</code>","text":"<p>             Bases: <code>Enum</code></p> <p>Study-Locus quality control options listing concerns on the quality of the association.</p> <p>Attributes:</p> Name Type Description <code>SUBSIGNIFICANT_FLAG</code> <code>str</code> <p>p-value below significance threshold</p> <code>NO_GENOMIC_LOCATION_FLAG</code> <code>str</code> <p>Incomplete genomic mapping</p> <code>COMPOSITE_FLAG</code> <code>str</code> <p>Composite association due to variant x variant interactions</p> <code>INCONSISTENCY_FLAG</code> <code>str</code> <p>Inconsistencies in the reported variants</p> <code>NON_MAPPED_VARIANT_FLAG</code> <code>str</code> <p>Variant not mapped to GnomAd</p> <code>PALINDROMIC_ALLELE_FLAG</code> <code>str</code> <p>Alleles are palindromic - cannot harmonize</p> <code>AMBIGUOUS_STUDY</code> <code>str</code> <p>Association with ambiguous study</p> <code>UNRESOLVED_LD</code> <code>str</code> <p>Variant not found in LD reference</p> <code>LD_CLUMPED</code> <code>str</code> <p>Explained by a more significant variant in high LD (clumped)</p> <code>NO_POPULATION</code> <code>str</code> <p>Study does not have population annotation to resolve LD</p> <code>NOT_QUALIFYING_LD_BLOCK</code> <code>str</code> <p>LD block does not contain variants at the required R^2 threshold</p> Source code in <code>src/gentropy/dataset/study_locus.py</code> <pre><code>class StudyLocusQualityCheck(Enum):\n    \"\"\"Study-Locus quality control options listing concerns on the quality of the association.\n\n    Attributes:\n        SUBSIGNIFICANT_FLAG (str): p-value below significance threshold\n        NO_GENOMIC_LOCATION_FLAG (str): Incomplete genomic mapping\n        COMPOSITE_FLAG (str): Composite association due to variant x variant interactions\n        INCONSISTENCY_FLAG (str): Inconsistencies in the reported variants\n        NON_MAPPED_VARIANT_FLAG (str): Variant not mapped to GnomAd\n        PALINDROMIC_ALLELE_FLAG (str): Alleles are palindromic - cannot harmonize\n        AMBIGUOUS_STUDY (str): Association with ambiguous study\n        UNRESOLVED_LD (str): Variant not found in LD reference\n        LD_CLUMPED (str): Explained by a more significant variant in high LD (clumped)\n        NO_POPULATION (str): Study does not have population annotation to resolve LD\n        NOT_QUALIFYING_LD_BLOCK (str): LD block does not contain variants at the required R^2 threshold\n    \"\"\"\n\n    SUBSIGNIFICANT_FLAG = \"Subsignificant p-value\"\n    NO_GENOMIC_LOCATION_FLAG = \"Incomplete genomic mapping\"\n    COMPOSITE_FLAG = \"Composite association\"\n    INCONSISTENCY_FLAG = \"Variant inconsistency\"\n    NON_MAPPED_VARIANT_FLAG = \"No mapping in GnomAd\"\n    PALINDROMIC_ALLELE_FLAG = \"Palindrome alleles - cannot harmonize\"\n    AMBIGUOUS_STUDY = \"Association with ambiguous study\"\n    UNRESOLVED_LD = \"Variant not found in LD reference\"\n    LD_CLUMPED = \"Explained by a more significant variant in high LD (clumped)\"\n    NO_POPULATION = \"Study does not have population annotation to resolve LD\"\n    NOT_QUALIFYING_LD_BLOCK = (\n        \"LD block does not contain variants at the required R^2 threshold\"\n    )\n</code></pre>"},{"location":"python_api/datasets/study_locus/#gentropy.dataset.study_locus.CredibleInterval","title":"<code>gentropy.dataset.study_locus.CredibleInterval</code>","text":"<p>             Bases: <code>Enum</code></p> <p>Credible interval enum.</p> <p>Interval within which an unobserved parameter value falls with a particular probability.</p> <p>Attributes:</p> Name Type Description <code>IS95</code> <code>str</code> <p>95% credible interval</p> <code>IS99</code> <code>str</code> <p>99% credible interval</p> Source code in <code>src/gentropy/dataset/study_locus.py</code> <pre><code>class CredibleInterval(Enum):\n    \"\"\"Credible interval enum.\n\n    Interval within which an unobserved parameter value falls with a particular probability.\n\n    Attributes:\n        IS95 (str): 95% credible interval\n        IS99 (str): 99% credible interval\n    \"\"\"\n\n    IS95 = \"is95CredibleSet\"\n    IS99 = \"is99CredibleSet\"\n</code></pre>"},{"location":"python_api/datasets/study_locus/#schema","title":"Schema","text":"<pre><code>root\n |-- studyLocusId: long (nullable = false)\n |-- variantId: string (nullable = false)\n |-- chromosome: string (nullable = true)\n |-- position: integer (nullable = true)\n |-- region: string (nullable = true)\n |-- studyId: string (nullable = false)\n |-- beta: double (nullable = true)\n |-- zScore: double (nullable = true)\n |-- pValueMantissa: float (nullable = true)\n |-- pValueExponent: integer (nullable = true)\n |-- effectAlleleFrequencyFromSource: float (nullable = true)\n |-- standardError: double (nullable = true)\n |-- subStudyDescription: string (nullable = true)\n |-- qualityControls: array (nullable = true)\n |    |-- element: string (containsNull = false)\n |-- finemappingMethod: string (nullable = true)\n |-- credibleSetIndex: integer (nullable = true)\n |-- credibleSetlog10BF: double (nullable = true)\n |-- purityMeanR2: double (nullable = true)\n |-- purityMinR2: double (nullable = true)\n |-- sampleSize: integer (nullable = true)\n |-- ldSet: array (nullable = true)\n |    |-- element: struct (containsNull = true)\n |    |    |-- tagVariantId: string (nullable = true)\n |    |    |-- r2Overall: double (nullable = true)\n |-- locus: array (nullable = true)\n |    |-- element: struct (containsNull = true)\n |    |    |-- is95CredibleSet: boolean (nullable = true)\n |    |    |-- is99CredibleSet: boolean (nullable = true)\n |    |    |-- logBF: double (nullable = true)\n |    |    |-- posteriorProbability: double (nullable = true)\n |    |    |-- variantId: string (nullable = true)\n |    |    |-- pValueMantissa: float (nullable = true)\n |    |    |-- pValueExponent: integer (nullable = true)\n |    |    |-- beta: double (nullable = true)\n |    |    |-- standardError: double (nullable = true)\n |    |    |-- r2Overall: double (nullable = true)\n</code></pre>"},{"location":"python_api/datasets/study_locus_overlap/","title":"Study Locus Overlap","text":""},{"location":"python_api/datasets/study_locus_overlap/#gentropy.dataset.study_locus_overlap.StudyLocusOverlap","title":"<code>gentropy.dataset.study_locus_overlap.StudyLocusOverlap</code>  <code>dataclass</code>","text":"<p>             Bases: <code>Dataset</code></p> <p>Study-Locus overlap.</p> <p>This dataset captures pairs of overlapping <code>StudyLocus</code>: that is associations whose credible sets share at least one tagging variant.</p> <p>Note</p> <p>This is a helpful dataset for other downstream analyses, such as colocalisation. This dataset will contain the overlapping signals between studyLocus associations once they have been clumped and fine-mapped.</p> Source code in <code>src/gentropy/dataset/study_locus_overlap.py</code> <pre><code>@dataclass\nclass StudyLocusOverlap(Dataset):\n    \"\"\"Study-Locus overlap.\n\n    This dataset captures pairs of overlapping `StudyLocus`: that is associations whose credible sets share at least one tagging variant.\n\n    !!! note\n\n        This is a helpful dataset for other downstream analyses, such as colocalisation. This dataset will contain the overlapping signals between studyLocus associations once they have been clumped and fine-mapped.\n    \"\"\"\n\n    @classmethod\n    def get_schema(cls: type[StudyLocusOverlap]) -&gt; StructType:\n        \"\"\"Provides the schema for the StudyLocusOverlap dataset.\n\n        Returns:\n            StructType: Schema for the StudyLocusOverlap dataset\n        \"\"\"\n        return parse_spark_schema(\"study_locus_overlap.json\")\n\n    @classmethod\n    def from_associations(\n        cls: type[StudyLocusOverlap], study_locus: StudyLocus, study_index: StudyIndex\n    ) -&gt; StudyLocusOverlap:\n        \"\"\"Find the overlapping signals in a particular set of associations (StudyLocus dataset).\n\n        Args:\n            study_locus (StudyLocus): Study-locus associations to find the overlapping signals\n            study_index (StudyIndex): Study index to find the overlapping signals\n\n        Returns:\n            StudyLocusOverlap: Study-locus overlap dataset\n        \"\"\"\n        return study_locus.find_overlaps(study_index)\n\n    def _convert_to_square_matrix(self: StudyLocusOverlap) -&gt; StudyLocusOverlap:\n        \"\"\"Convert the dataset to a square matrix.\n\n        Returns:\n            StudyLocusOverlap: Square matrix of the dataset\n        \"\"\"\n        return StudyLocusOverlap(\n            _df=self.df.unionByName(\n                self.df.selectExpr(\n                    \"leftStudyLocusId as rightStudyLocusId\",\n                    \"rightStudyLocusId as leftStudyLocusId\",\n                    \"tagVariantId\",\n                )\n            ).distinct(),\n            _schema=self.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasets/study_locus_overlap/#gentropy.dataset.study_locus_overlap.StudyLocusOverlap.from_associations","title":"<code>from_associations(study_locus: StudyLocus, study_index: StudyIndex) -&gt; StudyLocusOverlap</code>  <code>classmethod</code>","text":"<p>Find the overlapping signals in a particular set of associations (StudyLocus dataset).</p> <p>Parameters:</p> Name Type Description Default <code>study_locus</code> <code>StudyLocus</code> <p>Study-locus associations to find the overlapping signals</p> required <code>study_index</code> <code>StudyIndex</code> <p>Study index to find the overlapping signals</p> required <p>Returns:</p> Name Type Description <code>StudyLocusOverlap</code> <code>StudyLocusOverlap</code> <p>Study-locus overlap dataset</p> Source code in <code>src/gentropy/dataset/study_locus_overlap.py</code> <pre><code>@classmethod\ndef from_associations(\n    cls: type[StudyLocusOverlap], study_locus: StudyLocus, study_index: StudyIndex\n) -&gt; StudyLocusOverlap:\n    \"\"\"Find the overlapping signals in a particular set of associations (StudyLocus dataset).\n\n    Args:\n        study_locus (StudyLocus): Study-locus associations to find the overlapping signals\n        study_index (StudyIndex): Study index to find the overlapping signals\n\n    Returns:\n        StudyLocusOverlap: Study-locus overlap dataset\n    \"\"\"\n    return study_locus.find_overlaps(study_index)\n</code></pre>"},{"location":"python_api/datasets/study_locus_overlap/#gentropy.dataset.study_locus_overlap.StudyLocusOverlap.get_schema","title":"<code>get_schema() -&gt; StructType</code>  <code>classmethod</code>","text":"<p>Provides the schema for the StudyLocusOverlap dataset.</p> <p>Returns:</p> Name Type Description <code>StructType</code> <code>StructType</code> <p>Schema for the StudyLocusOverlap dataset</p> Source code in <code>src/gentropy/dataset/study_locus_overlap.py</code> <pre><code>@classmethod\ndef get_schema(cls: type[StudyLocusOverlap]) -&gt; StructType:\n    \"\"\"Provides the schema for the StudyLocusOverlap dataset.\n\n    Returns:\n        StructType: Schema for the StudyLocusOverlap dataset\n    \"\"\"\n    return parse_spark_schema(\"study_locus_overlap.json\")\n</code></pre>"},{"location":"python_api/datasets/study_locus_overlap/#schema","title":"Schema","text":"<pre><code>root\n |-- leftStudyLocusId: long (nullable = false)\n |-- rightStudyLocusId: long (nullable = false)\n |-- chromosome: string (nullable = true)\n |-- tagVariantId: string (nullable = false)\n |-- statistics: struct (nullable = true)\n |    |-- left_pValueMantissa: float (nullable = true)\n |    |-- left_pValueExponent: integer (nullable = true)\n |    |-- right_pValueMantissa: float (nullable = true)\n |    |-- right_pValueExponent: integer (nullable = true)\n |    |-- left_beta: double (nullable = true)\n |    |-- right_beta: double (nullable = true)\n |    |-- left_logBF: double (nullable = true)\n |    |-- right_logBF: double (nullable = true)\n |    |-- left_posteriorProbability: double (nullable = true)\n |    |-- right_posteriorProbability: double (nullable = true)\n</code></pre>"},{"location":"python_api/datasets/summary_statistics/","title":"Summary Statistics","text":""},{"location":"python_api/datasets/summary_statistics/#gentropy.dataset.summary_statistics.SummaryStatistics","title":"<code>gentropy.dataset.summary_statistics.SummaryStatistics</code>  <code>dataclass</code>","text":"<p>             Bases: <code>Dataset</code></p> <p>Summary Statistics dataset.</p> <p>A summary statistics dataset contains all single point statistics resulting from a GWAS.</p> Source code in <code>src/gentropy/dataset/summary_statistics.py</code> <pre><code>@dataclass\nclass SummaryStatistics(Dataset):\n    \"\"\"Summary Statistics dataset.\n\n    A summary statistics dataset contains all single point statistics resulting from a GWAS.\n    \"\"\"\n\n    @classmethod\n    def get_schema(cls: type[SummaryStatistics]) -&gt; StructType:\n        \"\"\"Provides the schema for the SummaryStatistics dataset.\n\n        Returns:\n            StructType: Schema for the SummaryStatistics dataset\n        \"\"\"\n        return parse_spark_schema(\"summary_statistics.json\")\n\n    def pvalue_filter(self: SummaryStatistics, pvalue: float) -&gt; SummaryStatistics:\n        \"\"\"Filter summary statistics based on the provided p-value threshold.\n\n        Args:\n            pvalue (float): upper limit of the p-value to be filtered upon.\n\n        Returns:\n            SummaryStatistics: summary statistics object containing single point associations with p-values at least as significant as the provided threshold.\n        \"\"\"\n        # Converting p-value to mantissa and exponent:\n        (mantissa, exponent) = split_pvalue(pvalue)\n\n        # Applying filter:\n        df = self._df.filter(\n            (f.col(\"pValueExponent\") &lt; exponent)\n            | (\n                (f.col(\"pValueExponent\") == exponent)\n                &amp; (f.col(\"pValueMantissa\") &lt;= mantissa)\n            )\n        )\n        return SummaryStatistics(_df=df, _schema=self._schema)\n\n    def window_based_clumping(\n        self: SummaryStatistics,\n        distance: int = 500_000,\n        gwas_significance: float = 5e-8,\n    ) -&gt; StudyLocus:\n        \"\"\"Generate study-locus from summary statistics using window-based clumping.\n\n        For more info, see [`WindowBasedClumping`][gentropy.method.window_based_clumping.WindowBasedClumping]\n\n        Args:\n            distance (int): Distance in base pairs to be used for clumping. Defaults to 500_000.\n            gwas_significance (float, optional): GWAS significance threshold. Defaults to 5e-8.\n\n        Returns:\n            StudyLocus: Clumped study-locus optionally containing variants based on window.\n        \"\"\"\n        from gentropy.method.window_based_clumping import WindowBasedClumping\n\n        return WindowBasedClumping.clump(\n            self,\n            distance=distance,\n            gwas_significance=gwas_significance,\n        )\n\n    def exclude_region(self: SummaryStatistics, region: str) -&gt; SummaryStatistics:\n        \"\"\"Exclude a region from the summary stats dataset.\n\n        Args:\n            region (str): region given in \"chr##:#####-####\" format\n\n        Returns:\n            SummaryStatistics: filtered summary statistics.\n        \"\"\"\n        (chromosome, start_position, end_position) = parse_region(region)\n\n        return SummaryStatistics(\n            _df=(\n                self.df.filter(\n                    ~(\n                        (f.col(\"chromosome\") == chromosome)\n                        &amp; (\n                            (f.col(\"position\") &gt;= start_position)\n                            &amp; (f.col(\"position\") &lt;= end_position)\n                        )\n                    )\n                )\n            ),\n            _schema=SummaryStatistics.get_schema(),\n        )\n\n    def sanity_filter(self: SummaryStatistics) -&gt; SummaryStatistics:\n        \"\"\"The function filters the summary statistics by sanity filters.\n\n        The function filters the summary statistics by the following filters:\n            - The p-value should not be eqaul 1.\n            - The beta and se should not be equal 0.\n            - The p-value, beta and se should not be NaN.\n\n        Returns:\n            SummaryStatistics: The filtered summary statistics.\n        \"\"\"\n        gwas_df = self._df\n        gwas_df = gwas_df.dropna(\n            subset=[\"beta\", \"standardError\", \"pValueMantissa\", \"pValueExponent\"]\n        )\n\n        gwas_df = gwas_df.filter((f.col(\"beta\") != 0) &amp; (f.col(\"standardError\") != 0))\n        gwas_df = gwas_df.filter(\n            f.col(\"pValueMantissa\") * 10 ** f.col(\"pValueExponent\") != 1\n        )\n\n        return SummaryStatistics(\n            _df=gwas_df,\n            _schema=SummaryStatistics.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasets/summary_statistics/#gentropy.dataset.summary_statistics.SummaryStatistics.exclude_region","title":"<code>exclude_region(region: str) -&gt; SummaryStatistics</code>","text":"<p>Exclude a region from the summary stats dataset.</p> <p>Parameters:</p> Name Type Description Default <code>region</code> <code>str</code> <p>region given in \"chr##:#####-####\" format</p> required <p>Returns:</p> Name Type Description <code>SummaryStatistics</code> <code>SummaryStatistics</code> <p>filtered summary statistics.</p> Source code in <code>src/gentropy/dataset/summary_statistics.py</code> <pre><code>def exclude_region(self: SummaryStatistics, region: str) -&gt; SummaryStatistics:\n    \"\"\"Exclude a region from the summary stats dataset.\n\n    Args:\n        region (str): region given in \"chr##:#####-####\" format\n\n    Returns:\n        SummaryStatistics: filtered summary statistics.\n    \"\"\"\n    (chromosome, start_position, end_position) = parse_region(region)\n\n    return SummaryStatistics(\n        _df=(\n            self.df.filter(\n                ~(\n                    (f.col(\"chromosome\") == chromosome)\n                    &amp; (\n                        (f.col(\"position\") &gt;= start_position)\n                        &amp; (f.col(\"position\") &lt;= end_position)\n                    )\n                )\n            )\n        ),\n        _schema=SummaryStatistics.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasets/summary_statistics/#gentropy.dataset.summary_statistics.SummaryStatistics.get_schema","title":"<code>get_schema() -&gt; StructType</code>  <code>classmethod</code>","text":"<p>Provides the schema for the SummaryStatistics dataset.</p> <p>Returns:</p> Name Type Description <code>StructType</code> <code>StructType</code> <p>Schema for the SummaryStatistics dataset</p> Source code in <code>src/gentropy/dataset/summary_statistics.py</code> <pre><code>@classmethod\ndef get_schema(cls: type[SummaryStatistics]) -&gt; StructType:\n    \"\"\"Provides the schema for the SummaryStatistics dataset.\n\n    Returns:\n        StructType: Schema for the SummaryStatistics dataset\n    \"\"\"\n    return parse_spark_schema(\"summary_statistics.json\")\n</code></pre>"},{"location":"python_api/datasets/summary_statistics/#gentropy.dataset.summary_statistics.SummaryStatistics.pvalue_filter","title":"<code>pvalue_filter(pvalue: float) -&gt; SummaryStatistics</code>","text":"<p>Filter summary statistics based on the provided p-value threshold.</p> <p>Parameters:</p> Name Type Description Default <code>pvalue</code> <code>float</code> <p>upper limit of the p-value to be filtered upon.</p> required <p>Returns:</p> Name Type Description <code>SummaryStatistics</code> <code>SummaryStatistics</code> <p>summary statistics object containing single point associations with p-values at least as significant as the provided threshold.</p> Source code in <code>src/gentropy/dataset/summary_statistics.py</code> <pre><code>def pvalue_filter(self: SummaryStatistics, pvalue: float) -&gt; SummaryStatistics:\n    \"\"\"Filter summary statistics based on the provided p-value threshold.\n\n    Args:\n        pvalue (float): upper limit of the p-value to be filtered upon.\n\n    Returns:\n        SummaryStatistics: summary statistics object containing single point associations with p-values at least as significant as the provided threshold.\n    \"\"\"\n    # Converting p-value to mantissa and exponent:\n    (mantissa, exponent) = split_pvalue(pvalue)\n\n    # Applying filter:\n    df = self._df.filter(\n        (f.col(\"pValueExponent\") &lt; exponent)\n        | (\n            (f.col(\"pValueExponent\") == exponent)\n            &amp; (f.col(\"pValueMantissa\") &lt;= mantissa)\n        )\n    )\n    return SummaryStatistics(_df=df, _schema=self._schema)\n</code></pre>"},{"location":"python_api/datasets/summary_statistics/#gentropy.dataset.summary_statistics.SummaryStatistics.sanity_filter","title":"<code>sanity_filter() -&gt; SummaryStatistics</code>","text":"<p>The function filters the summary statistics by sanity filters.</p> The function filters the summary statistics by the following filters <ul> <li>The p-value should not be eqaul 1.</li> <li>The beta and se should not be equal 0.</li> <li>The p-value, beta and se should not be NaN.</li> </ul> <p>Returns:</p> Name Type Description <code>SummaryStatistics</code> <code>SummaryStatistics</code> <p>The filtered summary statistics.</p> Source code in <code>src/gentropy/dataset/summary_statistics.py</code> <pre><code>def sanity_filter(self: SummaryStatistics) -&gt; SummaryStatistics:\n    \"\"\"The function filters the summary statistics by sanity filters.\n\n    The function filters the summary statistics by the following filters:\n        - The p-value should not be eqaul 1.\n        - The beta and se should not be equal 0.\n        - The p-value, beta and se should not be NaN.\n\n    Returns:\n        SummaryStatistics: The filtered summary statistics.\n    \"\"\"\n    gwas_df = self._df\n    gwas_df = gwas_df.dropna(\n        subset=[\"beta\", \"standardError\", \"pValueMantissa\", \"pValueExponent\"]\n    )\n\n    gwas_df = gwas_df.filter((f.col(\"beta\") != 0) &amp; (f.col(\"standardError\") != 0))\n    gwas_df = gwas_df.filter(\n        f.col(\"pValueMantissa\") * 10 ** f.col(\"pValueExponent\") != 1\n    )\n\n    return SummaryStatistics(\n        _df=gwas_df,\n        _schema=SummaryStatistics.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasets/summary_statistics/#gentropy.dataset.summary_statistics.SummaryStatistics.window_based_clumping","title":"<code>window_based_clumping(distance: int = 500000, gwas_significance: float = 5e-08) -&gt; StudyLocus</code>","text":"<p>Generate study-locus from summary statistics using window-based clumping.</p> <p>For more info, see <code>WindowBasedClumping</code></p> <p>Parameters:</p> Name Type Description Default <code>distance</code> <code>int</code> <p>Distance in base pairs to be used for clumping. Defaults to 500_000.</p> <code>500000</code> <code>gwas_significance</code> <code>float</code> <p>GWAS significance threshold. Defaults to 5e-8.</p> <code>5e-08</code> <p>Returns:</p> Name Type Description <code>StudyLocus</code> <code>StudyLocus</code> <p>Clumped study-locus optionally containing variants based on window.</p> Source code in <code>src/gentropy/dataset/summary_statistics.py</code> <pre><code>def window_based_clumping(\n    self: SummaryStatistics,\n    distance: int = 500_000,\n    gwas_significance: float = 5e-8,\n) -&gt; StudyLocus:\n    \"\"\"Generate study-locus from summary statistics using window-based clumping.\n\n    For more info, see [`WindowBasedClumping`][gentropy.method.window_based_clumping.WindowBasedClumping]\n\n    Args:\n        distance (int): Distance in base pairs to be used for clumping. Defaults to 500_000.\n        gwas_significance (float, optional): GWAS significance threshold. Defaults to 5e-8.\n\n    Returns:\n        StudyLocus: Clumped study-locus optionally containing variants based on window.\n    \"\"\"\n    from gentropy.method.window_based_clumping import WindowBasedClumping\n\n    return WindowBasedClumping.clump(\n        self,\n        distance=distance,\n        gwas_significance=gwas_significance,\n    )\n</code></pre>"},{"location":"python_api/datasets/summary_statistics/#schema","title":"Schema","text":"<pre><code>root\n |-- studyId: string (nullable = false)\n |-- variantId: string (nullable = false)\n |-- chromosome: string (nullable = false)\n |-- position: integer (nullable = false)\n |-- beta: double (nullable = false)\n |-- sampleSize: integer (nullable = true)\n |-- pValueMantissa: float (nullable = false)\n |-- pValueExponent: integer (nullable = false)\n |-- effectAlleleFrequencyFromSource: float (nullable = true)\n |-- standardError: double (nullable = true)\n</code></pre>"},{"location":"python_api/datasets/variant_annotation/","title":"Variant annotation","text":""},{"location":"python_api/datasets/variant_annotation/#gentropy.dataset.variant_annotation.VariantAnnotation","title":"<code>gentropy.dataset.variant_annotation.VariantAnnotation</code>  <code>dataclass</code>","text":"<p>             Bases: <code>Dataset</code></p> <p>Dataset with variant-level annotations.</p> Source code in <code>src/gentropy/dataset/variant_annotation.py</code> <pre><code>@dataclass\nclass VariantAnnotation(Dataset):\n    \"\"\"Dataset with variant-level annotations.\"\"\"\n\n    @classmethod\n    def get_schema(cls: type[VariantAnnotation]) -&gt; StructType:\n        \"\"\"Provides the schema for the VariantAnnotation dataset.\n\n        Returns:\n            StructType: Schema for the VariantAnnotation dataset\n        \"\"\"\n        return parse_spark_schema(\"variant_annotation.json\")\n\n    def max_maf(self: VariantAnnotation) -&gt; Column:\n        \"\"\"Maximum minor allele frequency accross all populations.\n\n        Returns:\n            Column: Maximum minor allele frequency accross all populations.\n        \"\"\"\n        return f.array_max(\n            f.transform(\n                self.df.alleleFrequencies,\n                lambda af: f.when(\n                    af.alleleFrequency &gt; 0.5, 1 - af.alleleFrequency\n                ).otherwise(af.alleleFrequency),\n            )\n        )\n\n    def filter_by_variant_df(\n        self: VariantAnnotation, df: DataFrame\n    ) -&gt; VariantAnnotation:\n        \"\"\"Filter variant annotation dataset by a variant dataframe.\n\n        Args:\n            df (DataFrame): A dataframe of variants\n\n        Returns:\n            VariantAnnotation: A filtered variant annotation dataset\n        \"\"\"\n        self.df = self._df.join(\n            f.broadcast(df.select(\"variantId\", \"chromosome\")),\n            on=[\"variantId\", \"chromosome\"],\n            how=\"inner\",\n        )\n        return self\n\n    def get_transcript_consequence_df(\n        self: VariantAnnotation, gene_index: GeneIndex | None = None\n    ) -&gt; DataFrame:\n        \"\"\"Dataframe of exploded transcript consequences.\n\n        Optionally the trancript consequences can be reduced to the universe of a gene index.\n\n        Args:\n            gene_index (GeneIndex | None): A gene index. Defaults to None.\n\n        Returns:\n            DataFrame: A dataframe exploded by transcript consequences with the columns variantId, chromosome, transcriptConsequence\n        \"\"\"\n        # exploding the array removes records without VEP annotation\n        transript_consequences = self.df.withColumn(\n            \"transcriptConsequence\", f.explode(\"vep.transcriptConsequences\")\n        ).select(\n            \"variantId\",\n            \"chromosome\",\n            \"position\",\n            \"transcriptConsequence\",\n            f.col(\"transcriptConsequence.geneId\").alias(\"geneId\"),\n        )\n        if gene_index:\n            transript_consequences = transript_consequences.join(\n                f.broadcast(gene_index.df),\n                on=[\"chromosome\", \"geneId\"],\n            )\n        return transript_consequences.persist()\n\n    def get_most_severe_vep_v2g(\n        self: VariantAnnotation,\n        vep_consequences: DataFrame,\n        gene_index: GeneIndex,\n    ) -&gt; V2G:\n        \"\"\"Creates a dataset with variant to gene assignments based on VEP's predicted consequence of the transcript.\n\n        Optionally the trancript consequences can be reduced to the universe of a gene index.\n\n        Args:\n            vep_consequences (DataFrame): A dataframe of VEP consequences\n            gene_index (GeneIndex): A gene index to filter by. Defaults to None.\n\n        Returns:\n            V2G: High and medium severity variant to gene assignments\n        \"\"\"\n        return V2G(\n            _df=self.get_transcript_consequence_df(gene_index)\n            .select(\n                \"variantId\",\n                \"chromosome\",\n                f.col(\"transcriptConsequence.geneId\").alias(\"geneId\"),\n                f.explode(\"transcriptConsequence.consequenceTerms\").alias(\"label\"),\n                f.lit(\"vep\").alias(\"datatypeId\"),\n                f.lit(\"variantConsequence\").alias(\"datasourceId\"),\n            )\n            .join(\n                f.broadcast(vep_consequences),\n                on=\"label\",\n                how=\"inner\",\n            )\n            .drop(\"label\")\n            .filter(f.col(\"score\") != 0)\n            # A variant can have multiple predicted consequences on a transcript, the most severe one is selected\n            .transform(\n                lambda df: get_record_with_maximum_value(\n                    df, [\"variantId\", \"geneId\"], \"score\"\n                )\n            ),\n            _schema=V2G.get_schema(),\n        )\n\n    def get_plof_v2g(self: VariantAnnotation, gene_index: GeneIndex) -&gt; V2G:\n        \"\"\"Creates a dataset with variant to gene assignments with a flag indicating if the variant is predicted to be a loss-of-function variant by the LOFTEE algorithm.\n\n        Optionally the trancript consequences can be reduced to the universe of a gene index.\n\n        Args:\n            gene_index (GeneIndex): A gene index to filter by.\n\n        Returns:\n            V2G: variant to gene assignments from the LOFTEE algorithm\n        \"\"\"\n        return V2G(\n            _df=(\n                self.get_transcript_consequence_df(gene_index)\n                .filter(f.col(\"transcriptConsequence.lof\").isNotNull())\n                .withColumn(\n                    \"isHighQualityPlof\",\n                    f.when(f.col(\"transcriptConsequence.lof\") == \"HC\", True).when(\n                        f.col(\"transcriptConsequence.lof\") == \"LC\", False\n                    ),\n                )\n                .withColumn(\n                    \"score\",\n                    f.when(f.col(\"isHighQualityPlof\"), 1.0).when(\n                        ~f.col(\"isHighQualityPlof\"), 0\n                    ),\n                )\n                .select(\n                    \"variantId\",\n                    \"chromosome\",\n                    \"geneId\",\n                    \"isHighQualityPlof\",\n                    f.col(\"score\"),\n                    f.lit(\"vep\").alias(\"datatypeId\"),\n                    f.lit(\"loftee\").alias(\"datasourceId\"),\n                )\n            ),\n            _schema=V2G.get_schema(),\n        )\n\n    def get_distance_to_tss(\n        self: VariantAnnotation,\n        gene_index: GeneIndex,\n        max_distance: int = 500_000,\n    ) -&gt; V2G:\n        \"\"\"Extracts variant to gene assignments for variants falling within a window of a gene's TSS.\n\n        Args:\n            gene_index (GeneIndex): A gene index to filter by.\n            max_distance (int): The maximum distance from the TSS to consider. Defaults to 500_000.\n\n        Returns:\n            V2G: variant to gene assignments with their distance to the TSS\n        \"\"\"\n        return V2G(\n            _df=(\n                self.df.alias(\"variant\")\n                .join(\n                    f.broadcast(gene_index.locations_lut()).alias(\"gene\"),\n                    on=[\n                        f.col(\"variant.chromosome\") == f.col(\"gene.chromosome\"),\n                        f.abs(f.col(\"variant.position\") - f.col(\"gene.tss\"))\n                        &lt;= max_distance,\n                    ],\n                    how=\"inner\",\n                )\n                .withColumn(\n                    \"distance\", f.abs(f.col(\"variant.position\") - f.col(\"gene.tss\"))\n                )\n                .withColumn(\n                    \"inverse_distance\",\n                    max_distance - f.col(\"distance\"),\n                )\n                .transform(lambda df: normalise_column(df, \"inverse_distance\", \"score\"))\n                .select(\n                    \"variantId\",\n                    f.col(\"variant.chromosome\").alias(\"chromosome\"),\n                    \"distance\",\n                    \"geneId\",\n                    \"score\",\n                    f.lit(\"distance\").alias(\"datatypeId\"),\n                    f.lit(\"canonical_tss\").alias(\"datasourceId\"),\n                )\n            ),\n            _schema=V2G.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasets/variant_annotation/#gentropy.dataset.variant_annotation.VariantAnnotation.filter_by_variant_df","title":"<code>filter_by_variant_df(df: DataFrame) -&gt; VariantAnnotation</code>","text":"<p>Filter variant annotation dataset by a variant dataframe.</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>A dataframe of variants</p> required <p>Returns:</p> Name Type Description <code>VariantAnnotation</code> <code>VariantAnnotation</code> <p>A filtered variant annotation dataset</p> Source code in <code>src/gentropy/dataset/variant_annotation.py</code> <pre><code>def filter_by_variant_df(\n    self: VariantAnnotation, df: DataFrame\n) -&gt; VariantAnnotation:\n    \"\"\"Filter variant annotation dataset by a variant dataframe.\n\n    Args:\n        df (DataFrame): A dataframe of variants\n\n    Returns:\n        VariantAnnotation: A filtered variant annotation dataset\n    \"\"\"\n    self.df = self._df.join(\n        f.broadcast(df.select(\"variantId\", \"chromosome\")),\n        on=[\"variantId\", \"chromosome\"],\n        how=\"inner\",\n    )\n    return self\n</code></pre>"},{"location":"python_api/datasets/variant_annotation/#gentropy.dataset.variant_annotation.VariantAnnotation.get_distance_to_tss","title":"<code>get_distance_to_tss(gene_index: GeneIndex, max_distance: int = 500000) -&gt; V2G</code>","text":"<p>Extracts variant to gene assignments for variants falling within a window of a gene's TSS.</p> <p>Parameters:</p> Name Type Description Default <code>gene_index</code> <code>GeneIndex</code> <p>A gene index to filter by.</p> required <code>max_distance</code> <code>int</code> <p>The maximum distance from the TSS to consider. Defaults to 500_000.</p> <code>500000</code> <p>Returns:</p> Name Type Description <code>V2G</code> <code>V2G</code> <p>variant to gene assignments with their distance to the TSS</p> Source code in <code>src/gentropy/dataset/variant_annotation.py</code> <pre><code>def get_distance_to_tss(\n    self: VariantAnnotation,\n    gene_index: GeneIndex,\n    max_distance: int = 500_000,\n) -&gt; V2G:\n    \"\"\"Extracts variant to gene assignments for variants falling within a window of a gene's TSS.\n\n    Args:\n        gene_index (GeneIndex): A gene index to filter by.\n        max_distance (int): The maximum distance from the TSS to consider. Defaults to 500_000.\n\n    Returns:\n        V2G: variant to gene assignments with their distance to the TSS\n    \"\"\"\n    return V2G(\n        _df=(\n            self.df.alias(\"variant\")\n            .join(\n                f.broadcast(gene_index.locations_lut()).alias(\"gene\"),\n                on=[\n                    f.col(\"variant.chromosome\") == f.col(\"gene.chromosome\"),\n                    f.abs(f.col(\"variant.position\") - f.col(\"gene.tss\"))\n                    &lt;= max_distance,\n                ],\n                how=\"inner\",\n            )\n            .withColumn(\n                \"distance\", f.abs(f.col(\"variant.position\") - f.col(\"gene.tss\"))\n            )\n            .withColumn(\n                \"inverse_distance\",\n                max_distance - f.col(\"distance\"),\n            )\n            .transform(lambda df: normalise_column(df, \"inverse_distance\", \"score\"))\n            .select(\n                \"variantId\",\n                f.col(\"variant.chromosome\").alias(\"chromosome\"),\n                \"distance\",\n                \"geneId\",\n                \"score\",\n                f.lit(\"distance\").alias(\"datatypeId\"),\n                f.lit(\"canonical_tss\").alias(\"datasourceId\"),\n            )\n        ),\n        _schema=V2G.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasets/variant_annotation/#gentropy.dataset.variant_annotation.VariantAnnotation.get_most_severe_vep_v2g","title":"<code>get_most_severe_vep_v2g(vep_consequences: DataFrame, gene_index: GeneIndex) -&gt; V2G</code>","text":"<p>Creates a dataset with variant to gene assignments based on VEP's predicted consequence of the transcript.</p> <p>Optionally the trancript consequences can be reduced to the universe of a gene index.</p> <p>Parameters:</p> Name Type Description Default <code>vep_consequences</code> <code>DataFrame</code> <p>A dataframe of VEP consequences</p> required <code>gene_index</code> <code>GeneIndex</code> <p>A gene index to filter by. Defaults to None.</p> required <p>Returns:</p> Name Type Description <code>V2G</code> <code>V2G</code> <p>High and medium severity variant to gene assignments</p> Source code in <code>src/gentropy/dataset/variant_annotation.py</code> <pre><code>def get_most_severe_vep_v2g(\n    self: VariantAnnotation,\n    vep_consequences: DataFrame,\n    gene_index: GeneIndex,\n) -&gt; V2G:\n    \"\"\"Creates a dataset with variant to gene assignments based on VEP's predicted consequence of the transcript.\n\n    Optionally the trancript consequences can be reduced to the universe of a gene index.\n\n    Args:\n        vep_consequences (DataFrame): A dataframe of VEP consequences\n        gene_index (GeneIndex): A gene index to filter by. Defaults to None.\n\n    Returns:\n        V2G: High and medium severity variant to gene assignments\n    \"\"\"\n    return V2G(\n        _df=self.get_transcript_consequence_df(gene_index)\n        .select(\n            \"variantId\",\n            \"chromosome\",\n            f.col(\"transcriptConsequence.geneId\").alias(\"geneId\"),\n            f.explode(\"transcriptConsequence.consequenceTerms\").alias(\"label\"),\n            f.lit(\"vep\").alias(\"datatypeId\"),\n            f.lit(\"variantConsequence\").alias(\"datasourceId\"),\n        )\n        .join(\n            f.broadcast(vep_consequences),\n            on=\"label\",\n            how=\"inner\",\n        )\n        .drop(\"label\")\n        .filter(f.col(\"score\") != 0)\n        # A variant can have multiple predicted consequences on a transcript, the most severe one is selected\n        .transform(\n            lambda df: get_record_with_maximum_value(\n                df, [\"variantId\", \"geneId\"], \"score\"\n            )\n        ),\n        _schema=V2G.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasets/variant_annotation/#gentropy.dataset.variant_annotation.VariantAnnotation.get_plof_v2g","title":"<code>get_plof_v2g(gene_index: GeneIndex) -&gt; V2G</code>","text":"<p>Creates a dataset with variant to gene assignments with a flag indicating if the variant is predicted to be a loss-of-function variant by the LOFTEE algorithm.</p> <p>Optionally the trancript consequences can be reduced to the universe of a gene index.</p> <p>Parameters:</p> Name Type Description Default <code>gene_index</code> <code>GeneIndex</code> <p>A gene index to filter by.</p> required <p>Returns:</p> Name Type Description <code>V2G</code> <code>V2G</code> <p>variant to gene assignments from the LOFTEE algorithm</p> Source code in <code>src/gentropy/dataset/variant_annotation.py</code> <pre><code>def get_plof_v2g(self: VariantAnnotation, gene_index: GeneIndex) -&gt; V2G:\n    \"\"\"Creates a dataset with variant to gene assignments with a flag indicating if the variant is predicted to be a loss-of-function variant by the LOFTEE algorithm.\n\n    Optionally the trancript consequences can be reduced to the universe of a gene index.\n\n    Args:\n        gene_index (GeneIndex): A gene index to filter by.\n\n    Returns:\n        V2G: variant to gene assignments from the LOFTEE algorithm\n    \"\"\"\n    return V2G(\n        _df=(\n            self.get_transcript_consequence_df(gene_index)\n            .filter(f.col(\"transcriptConsequence.lof\").isNotNull())\n            .withColumn(\n                \"isHighQualityPlof\",\n                f.when(f.col(\"transcriptConsequence.lof\") == \"HC\", True).when(\n                    f.col(\"transcriptConsequence.lof\") == \"LC\", False\n                ),\n            )\n            .withColumn(\n                \"score\",\n                f.when(f.col(\"isHighQualityPlof\"), 1.0).when(\n                    ~f.col(\"isHighQualityPlof\"), 0\n                ),\n            )\n            .select(\n                \"variantId\",\n                \"chromosome\",\n                \"geneId\",\n                \"isHighQualityPlof\",\n                f.col(\"score\"),\n                f.lit(\"vep\").alias(\"datatypeId\"),\n                f.lit(\"loftee\").alias(\"datasourceId\"),\n            )\n        ),\n        _schema=V2G.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasets/variant_annotation/#gentropy.dataset.variant_annotation.VariantAnnotation.get_schema","title":"<code>get_schema() -&gt; StructType</code>  <code>classmethod</code>","text":"<p>Provides the schema for the VariantAnnotation dataset.</p> <p>Returns:</p> Name Type Description <code>StructType</code> <code>StructType</code> <p>Schema for the VariantAnnotation dataset</p> Source code in <code>src/gentropy/dataset/variant_annotation.py</code> <pre><code>@classmethod\ndef get_schema(cls: type[VariantAnnotation]) -&gt; StructType:\n    \"\"\"Provides the schema for the VariantAnnotation dataset.\n\n    Returns:\n        StructType: Schema for the VariantAnnotation dataset\n    \"\"\"\n    return parse_spark_schema(\"variant_annotation.json\")\n</code></pre>"},{"location":"python_api/datasets/variant_annotation/#gentropy.dataset.variant_annotation.VariantAnnotation.get_transcript_consequence_df","title":"<code>get_transcript_consequence_df(gene_index: GeneIndex | None = None) -&gt; DataFrame</code>","text":"<p>Dataframe of exploded transcript consequences.</p> <p>Optionally the trancript consequences can be reduced to the universe of a gene index.</p> <p>Parameters:</p> Name Type Description Default <code>gene_index</code> <code>GeneIndex | None</code> <p>A gene index. Defaults to None.</p> <code>None</code> <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>A dataframe exploded by transcript consequences with the columns variantId, chromosome, transcriptConsequence</p> Source code in <code>src/gentropy/dataset/variant_annotation.py</code> <pre><code>def get_transcript_consequence_df(\n    self: VariantAnnotation, gene_index: GeneIndex | None = None\n) -&gt; DataFrame:\n    \"\"\"Dataframe of exploded transcript consequences.\n\n    Optionally the trancript consequences can be reduced to the universe of a gene index.\n\n    Args:\n        gene_index (GeneIndex | None): A gene index. Defaults to None.\n\n    Returns:\n        DataFrame: A dataframe exploded by transcript consequences with the columns variantId, chromosome, transcriptConsequence\n    \"\"\"\n    # exploding the array removes records without VEP annotation\n    transript_consequences = self.df.withColumn(\n        \"transcriptConsequence\", f.explode(\"vep.transcriptConsequences\")\n    ).select(\n        \"variantId\",\n        \"chromosome\",\n        \"position\",\n        \"transcriptConsequence\",\n        f.col(\"transcriptConsequence.geneId\").alias(\"geneId\"),\n    )\n    if gene_index:\n        transript_consequences = transript_consequences.join(\n            f.broadcast(gene_index.df),\n            on=[\"chromosome\", \"geneId\"],\n        )\n    return transript_consequences.persist()\n</code></pre>"},{"location":"python_api/datasets/variant_annotation/#gentropy.dataset.variant_annotation.VariantAnnotation.max_maf","title":"<code>max_maf() -&gt; Column</code>","text":"<p>Maximum minor allele frequency accross all populations.</p> <p>Returns:</p> Name Type Description <code>Column</code> <code>Column</code> <p>Maximum minor allele frequency accross all populations.</p> Source code in <code>src/gentropy/dataset/variant_annotation.py</code> <pre><code>def max_maf(self: VariantAnnotation) -&gt; Column:\n    \"\"\"Maximum minor allele frequency accross all populations.\n\n    Returns:\n        Column: Maximum minor allele frequency accross all populations.\n    \"\"\"\n    return f.array_max(\n        f.transform(\n            self.df.alleleFrequencies,\n            lambda af: f.when(\n                af.alleleFrequency &gt; 0.5, 1 - af.alleleFrequency\n            ).otherwise(af.alleleFrequency),\n        )\n    )\n</code></pre>"},{"location":"python_api/datasets/variant_annotation/#schema","title":"Schema","text":"<pre><code>root\n |-- variantId: string (nullable = false)\n |-- chromosome: string (nullable = false)\n |-- position: integer (nullable = false)\n |-- referenceAllele: string (nullable = false)\n |-- alternateAllele: string (nullable = false)\n |-- chromosomeB37: string (nullable = true)\n |-- positionB37: integer (nullable = true)\n |-- alleleType: string (nullable = true)\n |-- rsIds: array (nullable = true)\n |    |-- element: string (containsNull = true)\n |-- alleleFrequencies: array (nullable = false)\n |    |-- element: struct (containsNull = true)\n |    |    |-- populationName: string (nullable = true)\n |    |    |-- alleleFrequency: double (nullable = true)\n |-- inSilicoPredictors: struct (nullable = false)\n |    |-- cadd: struct (nullable = true)\n |    |    |-- raw: float (nullable = true)\n |    |    |-- phred: float (nullable = true)\n |    |-- revelMax: double (nullable = true)\n |    |-- spliceaiDsMax: float (nullable = true)\n |    |-- pangolinLargestDs: double (nullable = true)\n |    |-- phylop: double (nullable = true)\n |    |-- siftMax: double (nullable = true)\n |    |-- polyphenMax: double (nullable = true)\n |-- vep: struct (nullable = false)\n |    |-- mostSevereConsequence: string (nullable = true)\n |    |-- transcriptConsequences: array (nullable = true)\n |    |    |-- element: struct (containsNull = true)\n |    |    |    |-- aminoAcids: string (nullable = true)\n |    |    |    |-- consequenceTerms: array (nullable = true)\n |    |    |    |    |-- element: string (containsNull = true)\n |    |    |    |-- geneId: string (nullable = true)\n |    |    |    |-- lof: string (nullable = true)\n</code></pre>"},{"location":"python_api/datasets/variant_index/","title":"Variant index","text":""},{"location":"python_api/datasets/variant_index/#gentropy.dataset.variant_index.VariantIndex","title":"<code>gentropy.dataset.variant_index.VariantIndex</code>  <code>dataclass</code>","text":"<p>             Bases: <code>Dataset</code></p> <p>Variant index dataset.</p> <p>Variant index dataset is the result of intersecting the variant annotation dataset with the variants with V2D available information.</p> Source code in <code>src/gentropy/dataset/variant_index.py</code> <pre><code>@dataclass\nclass VariantIndex(Dataset):\n    \"\"\"Variant index dataset.\n\n    Variant index dataset is the result of intersecting the variant annotation dataset with the variants with V2D available information.\n    \"\"\"\n\n    @classmethod\n    def get_schema(cls: type[VariantIndex]) -&gt; StructType:\n        \"\"\"Provides the schema for the VariantIndex dataset.\n\n        Returns:\n            StructType: Schema for the VariantIndex dataset\n        \"\"\"\n        return parse_spark_schema(\"variant_index.json\")\n\n    @classmethod\n    def from_variant_annotation(\n        cls: type[VariantIndex],\n        variant_annotation: VariantAnnotation,\n        study_locus: StudyLocus,\n    ) -&gt; VariantIndex:\n        \"\"\"Initialise VariantIndex from pre-existing variant annotation dataset.\n\n        Args:\n            variant_annotation (VariantAnnotation): Variant annotation dataset\n            study_locus (StudyLocus): Study locus dataset with the variants to intersect with the variant annotation dataset\n\n        Returns:\n            VariantIndex: Variant index dataset\n        \"\"\"\n        unchanged_cols = [\n            \"variantId\",\n            \"chromosome\",\n            \"position\",\n            \"referenceAllele\",\n            \"alternateAllele\",\n            \"chromosomeB37\",\n            \"positionB37\",\n            \"alleleType\",\n            \"alleleFrequencies\",\n            \"inSilicoPredictors\",\n        ]\n        va_slimmed = variant_annotation.filter_by_variant_df(\n            study_locus.unique_variants_in_locus()\n        )\n        return cls(\n            _df=(\n                va_slimmed.df.select(\n                    *unchanged_cols,\n                    f.col(\"vep.mostSevereConsequence\").alias(\"mostSevereConsequence\"),\n                    # filters/rsid are arrays that can be empty, in this case we convert them to null\n                    nullify_empty_array(f.col(\"rsIds\")).alias(\"rsIds\"),\n                )\n                .repartition(400, \"chromosome\")\n                .sortWithinPartitions(\"chromosome\", \"position\")\n            ),\n            _schema=cls.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasets/variant_index/#gentropy.dataset.variant_index.VariantIndex.from_variant_annotation","title":"<code>from_variant_annotation(variant_annotation: VariantAnnotation, study_locus: StudyLocus) -&gt; VariantIndex</code>  <code>classmethod</code>","text":"<p>Initialise VariantIndex from pre-existing variant annotation dataset.</p> <p>Parameters:</p> Name Type Description Default <code>variant_annotation</code> <code>VariantAnnotation</code> <p>Variant annotation dataset</p> required <code>study_locus</code> <code>StudyLocus</code> <p>Study locus dataset with the variants to intersect with the variant annotation dataset</p> required <p>Returns:</p> Name Type Description <code>VariantIndex</code> <code>VariantIndex</code> <p>Variant index dataset</p> Source code in <code>src/gentropy/dataset/variant_index.py</code> <pre><code>@classmethod\ndef from_variant_annotation(\n    cls: type[VariantIndex],\n    variant_annotation: VariantAnnotation,\n    study_locus: StudyLocus,\n) -&gt; VariantIndex:\n    \"\"\"Initialise VariantIndex from pre-existing variant annotation dataset.\n\n    Args:\n        variant_annotation (VariantAnnotation): Variant annotation dataset\n        study_locus (StudyLocus): Study locus dataset with the variants to intersect with the variant annotation dataset\n\n    Returns:\n        VariantIndex: Variant index dataset\n    \"\"\"\n    unchanged_cols = [\n        \"variantId\",\n        \"chromosome\",\n        \"position\",\n        \"referenceAllele\",\n        \"alternateAllele\",\n        \"chromosomeB37\",\n        \"positionB37\",\n        \"alleleType\",\n        \"alleleFrequencies\",\n        \"inSilicoPredictors\",\n    ]\n    va_slimmed = variant_annotation.filter_by_variant_df(\n        study_locus.unique_variants_in_locus()\n    )\n    return cls(\n        _df=(\n            va_slimmed.df.select(\n                *unchanged_cols,\n                f.col(\"vep.mostSevereConsequence\").alias(\"mostSevereConsequence\"),\n                # filters/rsid are arrays that can be empty, in this case we convert them to null\n                nullify_empty_array(f.col(\"rsIds\")).alias(\"rsIds\"),\n            )\n            .repartition(400, \"chromosome\")\n            .sortWithinPartitions(\"chromosome\", \"position\")\n        ),\n        _schema=cls.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasets/variant_index/#gentropy.dataset.variant_index.VariantIndex.get_schema","title":"<code>get_schema() -&gt; StructType</code>  <code>classmethod</code>","text":"<p>Provides the schema for the VariantIndex dataset.</p> <p>Returns:</p> Name Type Description <code>StructType</code> <code>StructType</code> <p>Schema for the VariantIndex dataset</p> Source code in <code>src/gentropy/dataset/variant_index.py</code> <pre><code>@classmethod\ndef get_schema(cls: type[VariantIndex]) -&gt; StructType:\n    \"\"\"Provides the schema for the VariantIndex dataset.\n\n    Returns:\n        StructType: Schema for the VariantIndex dataset\n    \"\"\"\n    return parse_spark_schema(\"variant_index.json\")\n</code></pre>"},{"location":"python_api/datasets/variant_index/#schema","title":"Schema","text":"<pre><code>root\n |-- variantId: string (nullable = false)\n |-- chromosome: string (nullable = false)\n |-- position: integer (nullable = false)\n |-- referenceAllele: string (nullable = false)\n |-- alternateAllele: string (nullable = false)\n |-- chromosomeB37: string (nullable = true)\n |-- positionB37: integer (nullable = true)\n |-- alleleType: string (nullable = false)\n |-- alleleFrequencies: array (nullable = false)\n |    |-- element: struct (containsNull = true)\n |    |    |-- populationName: string (nullable = true)\n |    |    |-- alleleFrequency: double (nullable = true)\n |-- inSilicoPredictors: struct (nullable = false)\n |    |-- cadd: struct (nullable = true)\n |    |    |-- raw: float (nullable = true)\n |    |    |-- phred: float (nullable = true)\n |    |-- revelMax: double (nullable = true)\n |    |-- spliceaiDsMax: float (nullable = true)\n |    |-- pangolinLargestDs: double (nullable = true)\n |    |-- phylop: double (nullable = true)\n |    |-- siftMax: double (nullable = true)\n |    |-- polyphenMax: double (nullable = true)\n |-- mostSevereConsequence: string (nullable = true)\n |-- rsIds: array (nullable = true)\n |    |-- element: string (containsNull = true)\n</code></pre>"},{"location":"python_api/datasets/variant_to_gene/","title":"Variant-to-gene","text":""},{"location":"python_api/datasets/variant_to_gene/#gentropy.dataset.v2g.V2G","title":"<code>gentropy.dataset.v2g.V2G</code>  <code>dataclass</code>","text":"<p>             Bases: <code>Dataset</code></p> <p>Variant-to-gene (V2G) evidence dataset.</p> <p>A variant-to-gene (V2G) evidence is understood as any piece of evidence that supports the association of a variant with a likely causal gene. The evidence can sometimes be context-specific and refer to specific <code>biofeatures</code> (e.g. cell types)</p> Source code in <code>src/gentropy/dataset/v2g.py</code> <pre><code>@dataclass\nclass V2G(Dataset):\n    \"\"\"Variant-to-gene (V2G) evidence dataset.\n\n    A variant-to-gene (V2G) evidence is understood as any piece of evidence that supports the association of a variant with a likely causal gene. The evidence can sometimes be context-specific and refer to specific `biofeatures` (e.g. cell types)\n    \"\"\"\n\n    @classmethod\n    def get_schema(cls: type[V2G]) -&gt; StructType:\n        \"\"\"Provides the schema for the V2G dataset.\n\n        Returns:\n            StructType: Schema for the V2G dataset\n        \"\"\"\n        return parse_spark_schema(\"v2g.json\")\n\n    def filter_by_genes(self: V2G, genes: GeneIndex) -&gt; V2G:\n        \"\"\"Filter V2G dataset by genes.\n\n        Args:\n            genes (GeneIndex): Gene index dataset to filter by\n\n        Returns:\n            V2G: V2G dataset filtered by genes\n        \"\"\"\n        self.df = self._df.join(genes.df.select(\"geneId\"), on=\"geneId\", how=\"inner\")\n        return self\n\n    def extract_distance_tss_minimum(self: V2G) -&gt; None:\n        \"\"\"Extract minimum distance to TSS.\"\"\"\n        self.df = self._df.filter(f.col(\"distance\")).withColumn(\n            \"distanceTssMinimum\",\n            f.expr(\"min(distTss) OVER (PARTITION BY studyLocusId)\"),\n        )\n</code></pre>"},{"location":"python_api/datasets/variant_to_gene/#gentropy.dataset.v2g.V2G.extract_distance_tss_minimum","title":"<code>extract_distance_tss_minimum() -&gt; None</code>","text":"<p>Extract minimum distance to TSS.</p> Source code in <code>src/gentropy/dataset/v2g.py</code> <pre><code>def extract_distance_tss_minimum(self: V2G) -&gt; None:\n    \"\"\"Extract minimum distance to TSS.\"\"\"\n    self.df = self._df.filter(f.col(\"distance\")).withColumn(\n        \"distanceTssMinimum\",\n        f.expr(\"min(distTss) OVER (PARTITION BY studyLocusId)\"),\n    )\n</code></pre>"},{"location":"python_api/datasets/variant_to_gene/#gentropy.dataset.v2g.V2G.filter_by_genes","title":"<code>filter_by_genes(genes: GeneIndex) -&gt; V2G</code>","text":"<p>Filter V2G dataset by genes.</p> <p>Parameters:</p> Name Type Description Default <code>genes</code> <code>GeneIndex</code> <p>Gene index dataset to filter by</p> required <p>Returns:</p> Name Type Description <code>V2G</code> <code>V2G</code> <p>V2G dataset filtered by genes</p> Source code in <code>src/gentropy/dataset/v2g.py</code> <pre><code>def filter_by_genes(self: V2G, genes: GeneIndex) -&gt; V2G:\n    \"\"\"Filter V2G dataset by genes.\n\n    Args:\n        genes (GeneIndex): Gene index dataset to filter by\n\n    Returns:\n        V2G: V2G dataset filtered by genes\n    \"\"\"\n    self.df = self._df.join(genes.df.select(\"geneId\"), on=\"geneId\", how=\"inner\")\n    return self\n</code></pre>"},{"location":"python_api/datasets/variant_to_gene/#gentropy.dataset.v2g.V2G.get_schema","title":"<code>get_schema() -&gt; StructType</code>  <code>classmethod</code>","text":"<p>Provides the schema for the V2G dataset.</p> <p>Returns:</p> Name Type Description <code>StructType</code> <code>StructType</code> <p>Schema for the V2G dataset</p> Source code in <code>src/gentropy/dataset/v2g.py</code> <pre><code>@classmethod\ndef get_schema(cls: type[V2G]) -&gt; StructType:\n    \"\"\"Provides the schema for the V2G dataset.\n\n    Returns:\n        StructType: Schema for the V2G dataset\n    \"\"\"\n    return parse_spark_schema(\"v2g.json\")\n</code></pre>"},{"location":"python_api/datasets/variant_to_gene/#schema","title":"Schema","text":"<pre><code>root\n |-- geneId: string (nullable = false)\n |-- variantId: string (nullable = false)\n |-- distance: long (nullable = true)\n |-- chromosome: string (nullable = false)\n |-- datatypeId: string (nullable = false)\n |-- datasourceId: string (nullable = false)\n |-- score: double (nullable = true)\n |-- resourceScore: double (nullable = true)\n |-- pmid: string (nullable = true)\n |-- biofeature: string (nullable = true)\n |-- variantFunctionalConsequenceId: string (nullable = true)\n |-- isHighQualityPlof: boolean (nullable = true)\n</code></pre>"},{"location":"python_api/datasources/_datasources/","title":"Data Sources","text":"<p>This section contains information about the data source harmonisation tools available in Open Targets Gentropy.</p>"},{"location":"python_api/datasources/_datasources/#gwas-study-sources","title":"GWAS study sources","text":"<ol> <li>GWAS Catalog (with or without full summary statistics)</li> <li>FinnGen</li> </ol>"},{"location":"python_api/datasources/_datasources/#molecular-qtls","title":"Molecular QTLs","text":"<ol> <li>GTEx (eQTL catalogue)</li> </ol>"},{"location":"python_api/datasources/_datasources/#interaction-interval-based-experiments","title":"Interaction / Interval-based Experiments","text":"<ol> <li>Intervals-based datasets, informing about the relationships between genetic elements and their functional implications.</li> </ol>"},{"location":"python_api/datasources/_datasources/#variant-annotationvalidation","title":"Variant annotation/validation","text":"<ol> <li>GnomAD v4.0</li> <li>GWAS catalog harmonisation pipeline more info</li> </ol>"},{"location":"python_api/datasources/_datasources/#linkage-desiquilibrium","title":"Linkage desiquilibrium","text":"<ol> <li>GnomAD v2.1.1 LD matrixes (7 ancestries)</li> </ol>"},{"location":"python_api/datasources/_datasources/#locus-to-gene-gold-standard","title":"Locus-to-gene gold standard","text":"<ol> <li>Open Targets training set</li> </ol>"},{"location":"python_api/datasources/_datasources/#gene-annotation","title":"Gene annotation","text":"<ol> <li>Open Targets Platform Target Dataset (derived from Ensembl)</li> </ol>"},{"location":"python_api/datasources/eqtl_catalogue/_eqtl_catalogue/","title":"eQTL Catalogue","text":"<p>The eQTL Catalogue aims to provide unified gene, protein expression and splicing Quantitative Trait Loci (QTLs) from all available human public studies.</p> <p>It serves as the ultimate resource of molecular QTLs that we use for colocalization and target prioritization.</p>"},{"location":"python_api/datasources/eqtl_catalogue/finemapping/","title":"Fine mapping results","text":""},{"location":"python_api/datasources/eqtl_catalogue/finemapping/#gentropy.datasource.eqtl_catalogue.finemapping.EqtlCatalogueFinemapping","title":"<code>gentropy.datasource.eqtl_catalogue.finemapping.EqtlCatalogueFinemapping</code>  <code>dataclass</code>","text":"<p>SuSIE finemapping dataset for eQTL Catalogue.</p> <p>Credible sets from SuSIE are extracted and transformed into StudyLocus objects: - A study ID is defined as a triad between: the publication, the tissue, and the measured trait (e.g. Braineac2_substantia_nigra_ENSG00000248275) - Each row in the <code>credible_sets.tsv.gz</code> files is represented by molecular_trait_id/variant/rsid trios relevant for a given tissue. Each have their own finemapping statistics - log Bayes Factors are available for all variants in the <code>lbf_variable.txt</code> files</p> Source code in <code>src/gentropy/datasource/eqtl_catalogue/finemapping.py</code> <pre><code>@dataclass\nclass EqtlCatalogueFinemapping:\n    \"\"\"SuSIE finemapping dataset for eQTL Catalogue.\n\n    Credible sets from SuSIE are extracted and transformed into StudyLocus objects:\n    - A study ID is defined as a triad between: the publication, the tissue, and the measured trait (e.g. Braineac2_substantia_nigra_ENSG00000248275)\n    - Each row in the `credible_sets.tsv.gz` files is represented by molecular_trait_id/variant/rsid trios relevant for a given tissue. Each have their own finemapping statistics\n    - log Bayes Factors are available for all variants in the `lbf_variable.txt` files\n    \"\"\"\n\n    raw_credible_set_schema: StructType = StructType(\n        [\n            StructField(\"molecular_trait_id\", StringType(), True),\n            StructField(\"gene_id\", StringType(), True),\n            StructField(\"cs_id\", StringType(), True),\n            StructField(\"variant\", StringType(), True),\n            StructField(\"rsid\", StringType(), True),\n            StructField(\"cs_size\", IntegerType(), True),\n            StructField(\"pip\", DoubleType(), True),\n            StructField(\"pvalue\", DoubleType(), True),\n            StructField(\"beta\", DoubleType(), True),\n            StructField(\"se\", DoubleType(), True),\n            StructField(\"z\", DoubleType(), True),\n            StructField(\"cs_min_r2\", DoubleType(), True),\n            StructField(\"region\", StringType(), True),\n        ]\n    )\n    raw_lbf_schema: StructType = StructType(\n        [\n            StructField(\"molecular_trait_id\", StringType(), True),\n            StructField(\"region\", StringType(), True),\n            StructField(\"variant\", StringType(), True),\n            StructField(\"chromosome\", StringType(), True),\n            StructField(\"position\", IntegerType(), True),\n            StructField(\"lbf_variable1\", DoubleType(), True),\n            StructField(\"lbf_variable2\", DoubleType(), True),\n            StructField(\"lbf_variable3\", DoubleType(), True),\n            StructField(\"lbf_variable4\", DoubleType(), True),\n            StructField(\"lbf_variable5\", DoubleType(), True),\n            StructField(\"lbf_variable6\", DoubleType(), True),\n            StructField(\"lbf_variable7\", DoubleType(), True),\n            StructField(\"lbf_variable8\", DoubleType(), True),\n            StructField(\"lbf_variable9\", DoubleType(), True),\n            StructField(\"lbf_variable10\", DoubleType(), True),\n        ]\n    )\n\n    @classmethod\n    def _extract_credible_set_index(\n        cls: type[EqtlCatalogueFinemapping], cs_id: Column\n    ) -&gt; Column:\n        \"\"\"Extract the credible set index from the cs_id.\n\n        Args:\n            cs_id (Column): column with the credible set id as defined in the eQTL Catalogue.\n\n        Returns:\n            Column: The credible set index.\n\n        Examples:\n            &gt;&gt;&gt; spark.createDataFrame([(\"QTD000046_L1\",)], [\"cs_id\"]).select(EqtlCatalogueFinemapping._extract_credible_set_index(f.col(\"cs_id\"))).show()\n            +----------------+\n            |credibleSetIndex|\n            +----------------+\n            |               1|\n            +----------------+\n            &lt;BLANKLINE&gt;\n        \"\"\"\n        return f.split(cs_id, \"_L\")[1].cast(IntegerType()).alias(\"credibleSetIndex\")\n\n    @classmethod\n    def _extract_dataset_id_from_file_path(\n        cls: type[EqtlCatalogueFinemapping], file_path: Column\n    ) -&gt; Column:\n        \"\"\"Extract the dataset_id from the file_path. The dataset_id follows the pattern QTD{6}.\n\n        Args:\n            file_path (Column): A column containing the file path.\n\n        Returns:\n            Column: The dataset_id.\n\n        Examples:\n            &gt;&gt;&gt; spark.createDataFrame([(\"QTD000046.credible_sets.tsv\",)], [\"filename\"]).select(EqtlCatalogueFinemapping._extract_dataset_id_from_file_path(f.col(\"filename\"))).show()\n            +----------+\n            |dataset_id|\n            +----------+\n            | QTD000046|\n            +----------+\n            &lt;BLANKLINE&gt;\n        \"\"\"\n        return f.regexp_extract(file_path, r\"QTD\\d{6}\", 0).alias(\"dataset_id\")\n\n    @classmethod\n    def parse_susie_results(\n        cls: type[EqtlCatalogueFinemapping],\n        credible_sets: DataFrame,\n        lbf: DataFrame,\n        studies_metadata: DataFrame,\n    ) -&gt; DataFrame:\n        \"\"\"Parse the SuSIE results into a DataFrame containing the finemapping statistics and metadata about the studies.\n\n        Args:\n            credible_sets (DataFrame): DataFrame containing raw statistics of all variants in the credible sets.\n            lbf (DataFrame): DataFrame containing the raw log Bayes Factors for all variants.\n            studies_metadata (DataFrame): DataFrame containing the study metadata.\n\n        Returns:\n            DataFrame: Processed SuSIE results to contain metadata about the studies and the finemapping statistics.\n        \"\"\"\n        ss_ftp_path_template = \"https://ftp.ebi.ac.uk/pub/databases/spot/eQTL/sumstats\"\n        return (\n            lbf.withColumn(\n                \"dataset_id\",\n                cls._extract_dataset_id_from_file_path(f.input_file_name()),\n            )\n            .join(\n                (\n                    credible_sets.withColumn(\n                        \"dataset_id\",\n                        cls._extract_dataset_id_from_file_path(f.input_file_name()),\n                    )\n                    .withColumn(\n                        \"credibleSetIndex\",\n                        cls._extract_credible_set_index(f.col(\"cs_id\")),\n                    )\n                    .join(f.broadcast(studies_metadata), on=\"dataset_id\")\n                ),\n                on=[\"molecular_trait_id\", \"region\", \"variant\", \"dataset_id\"],\n                how=\"inner\",\n            )\n            .withColumn(\n                \"logBF\",\n                f.when(f.col(\"credibleSetIndex\") == 1, f.col(\"lbf_variable1\"))\n                .when(f.col(\"credibleSetIndex\") == 2, f.col(\"lbf_variable2\"))\n                .when(f.col(\"credibleSetIndex\") == 3, f.col(\"lbf_variable3\"))\n                .when(f.col(\"credibleSetIndex\") == 4, f.col(\"lbf_variable4\"))\n                .when(f.col(\"credibleSetIndex\") == 5, f.col(\"lbf_variable5\"))\n                .when(f.col(\"credibleSetIndex\") == 6, f.col(\"lbf_variable6\"))\n                .when(f.col(\"credibleSetIndex\") == 7, f.col(\"lbf_variable7\"))\n                .when(f.col(\"credibleSetIndex\") == 8, f.col(\"lbf_variable8\"))\n                .when(f.col(\"credibleSetIndex\") == 9, f.col(\"lbf_variable9\"))\n                .when(f.col(\"credibleSetIndex\") == 10, f.col(\"lbf_variable10\")),\n            )\n            .select(\n                f.regexp_replace(f.col(\"variant\"), r\"chr\", \"\").alias(\"variantId\"),\n                f.col(\"region\"),\n                f.col(\"chromosome\"),\n                f.col(\"position\"),\n                f.col(\"pip\").alias(\"posteriorProbability\"),\n                *parse_pvalue(f.col(\"pvalue\")),\n                f.col(\"sample_size\").alias(\"nSamples\"),\n                f.col(\"beta\"),\n                f.col(\"se\").alias(\"standardError\"),\n                f.col(\"credibleSetIndex\"),\n                f.col(\"logBF\"),\n                f.lit(\"SuSie\").alias(\"finemappingMethod\"),\n                # Study metadata\n                f.col(\"molecular_trait_id\").alias(\"traitFromSource\"),\n                f.col(\"gene_id\").alias(\"geneId\"),\n                f.col(\"dataset_id\"),\n                f.concat_ws(\n                    \"_\",\n                    f.col(\"study_label\"),\n                    f.col(\"sample_group\"),\n                    f.col(\"molecular_trait_id\"),\n                ).alias(\"studyId\"),\n                f.col(\"tissue_id\").alias(\"tissueFromSourceId\"),\n                EqtlCatalogueStudyIndex._identify_study_type(\n                    f.col(\"quant_method\")\n                ).alias(\"studyType\"),\n                f.col(\"study_label\").alias(\"projectId\"),\n                f.concat_ws(\n                    \"/\",\n                    f.lit(ss_ftp_path_template),\n                    f.col(\"study_id\"),\n                    f.col(\"dataset_id\"),\n                ).alias(\"summarystatsLocation\"),\n                f.lit(True).alias(\"hasSumstats\"),\n                f.col(\"molecular_trait_id\"),\n            )\n        )\n\n    @classmethod\n    def from_susie_results(\n        cls: type[EqtlCatalogueFinemapping], processed_finemapping_df: DataFrame\n    ) -&gt; StudyLocus:\n        \"\"\"Create a StudyLocus object from the processed SuSIE results.\n\n        Args:\n            processed_finemapping_df (DataFrame): DataFrame containing the processed SuSIE results.\n\n        Returns:\n            StudyLocus: eQTL Catalogue credible sets.\n        \"\"\"\n        lead_w = Window.partitionBy(\n            \"dataset_id\", \"molecular_trait_id\", \"region\", \"credibleSetIndex\"\n        )\n        study_locus_cols = [\n            field.name\n            for field in StudyLocus.get_schema().fields\n            if field.name in processed_finemapping_df.columns\n        ] + [\"locus\"]\n        return StudyLocus(\n            _df=(\n                processed_finemapping_df.withColumn(\n                    \"isLead\",\n                    f.row_number().over(lead_w.orderBy(f.desc(\"posteriorProbability\")))\n                    == f.lit(1),\n                )\n                .withColumn(\n                    # Collecting all variants that constitute the credible set brings as many variants as the credible set size\n                    \"locus\",\n                    f.when(\n                        f.col(\"isLead\"),\n                        f.collect_list(\n                            f.struct(\n                                \"variantId\",\n                                \"posteriorProbability\",\n                                \"pValueMantissa\",\n                                \"pValueExponent\",\n                                \"logBF\",\n                                \"beta\",\n                                \"standardError\",\n                            )\n                        ).over(lead_w),\n                    ),\n                )\n                .filter(f.col(\"isLead\"))\n                .drop(\"isLead\")\n                .select(\n                    *study_locus_cols,\n                    StudyLocus.assign_study_locus_id(\n                        f.col(\"studyId\"), f.col(\"variantId\")\n                    ),\n                    StudyLocus.calculate_credible_set_log10bf(\n                        f.col(\"locus.logBF\")\n                    ).alias(\"credibleSetlog10BF\"),\n                )\n            ),\n            _schema=StudyLocus.get_schema(),\n        ).annotate_credible_sets()\n\n    @classmethod\n    def read_credible_set_from_source(\n        cls: type[EqtlCatalogueFinemapping],\n        session: Session,\n        credible_set_path: str | list[str],\n    ) -&gt; DataFrame:\n        \"\"\"Load raw credible sets from eQTL Catalogue.\n\n        Args:\n            session (Session): Spark session.\n            credible_set_path (str | list[str]): Path to raw table(s) containing finemapping results for any variant belonging to a credible set.\n\n        Returns:\n            DataFrame: Credible sets DataFrame.\n        \"\"\"\n        return session.spark.read.csv(\n            credible_set_path,\n            sep=\"\\t\",\n            header=True,\n            schema=cls.raw_credible_set_schema,\n        )\n\n    @classmethod\n    def read_lbf_from_source(\n        cls: type[EqtlCatalogueFinemapping],\n        session: Session,\n        lbf_path: str | list[str],\n    ) -&gt; DataFrame:\n        \"\"\"Load raw log Bayes Factors from eQTL Catalogue.\n\n        Args:\n            session (Session): Spark session.\n            lbf_path (str | list[str]): Path to raw table(s) containing Log Bayes Factors for each variant.\n\n        Returns:\n            DataFrame: Log Bayes Factors DataFrame.\n        \"\"\"\n        return session.spark.read.csv(\n            lbf_path,\n            sep=\"\\t\",\n            header=True,\n            schema=cls.raw_lbf_schema,\n        )\n</code></pre>"},{"location":"python_api/datasources/eqtl_catalogue/finemapping/#gentropy.datasource.eqtl_catalogue.finemapping.EqtlCatalogueFinemapping.from_susie_results","title":"<code>from_susie_results(processed_finemapping_df: DataFrame) -&gt; StudyLocus</code>  <code>classmethod</code>","text":"<p>Create a StudyLocus object from the processed SuSIE results.</p> <p>Parameters:</p> Name Type Description Default <code>processed_finemapping_df</code> <code>DataFrame</code> <p>DataFrame containing the processed SuSIE results.</p> required <p>Returns:</p> Name Type Description <code>StudyLocus</code> <code>StudyLocus</code> <p>eQTL Catalogue credible sets.</p> Source code in <code>src/gentropy/datasource/eqtl_catalogue/finemapping.py</code> <pre><code>@classmethod\ndef from_susie_results(\n    cls: type[EqtlCatalogueFinemapping], processed_finemapping_df: DataFrame\n) -&gt; StudyLocus:\n    \"\"\"Create a StudyLocus object from the processed SuSIE results.\n\n    Args:\n        processed_finemapping_df (DataFrame): DataFrame containing the processed SuSIE results.\n\n    Returns:\n        StudyLocus: eQTL Catalogue credible sets.\n    \"\"\"\n    lead_w = Window.partitionBy(\n        \"dataset_id\", \"molecular_trait_id\", \"region\", \"credibleSetIndex\"\n    )\n    study_locus_cols = [\n        field.name\n        for field in StudyLocus.get_schema().fields\n        if field.name in processed_finemapping_df.columns\n    ] + [\"locus\"]\n    return StudyLocus(\n        _df=(\n            processed_finemapping_df.withColumn(\n                \"isLead\",\n                f.row_number().over(lead_w.orderBy(f.desc(\"posteriorProbability\")))\n                == f.lit(1),\n            )\n            .withColumn(\n                # Collecting all variants that constitute the credible set brings as many variants as the credible set size\n                \"locus\",\n                f.when(\n                    f.col(\"isLead\"),\n                    f.collect_list(\n                        f.struct(\n                            \"variantId\",\n                            \"posteriorProbability\",\n                            \"pValueMantissa\",\n                            \"pValueExponent\",\n                            \"logBF\",\n                            \"beta\",\n                            \"standardError\",\n                        )\n                    ).over(lead_w),\n                ),\n            )\n            .filter(f.col(\"isLead\"))\n            .drop(\"isLead\")\n            .select(\n                *study_locus_cols,\n                StudyLocus.assign_study_locus_id(\n                    f.col(\"studyId\"), f.col(\"variantId\")\n                ),\n                StudyLocus.calculate_credible_set_log10bf(\n                    f.col(\"locus.logBF\")\n                ).alias(\"credibleSetlog10BF\"),\n            )\n        ),\n        _schema=StudyLocus.get_schema(),\n    ).annotate_credible_sets()\n</code></pre>"},{"location":"python_api/datasources/eqtl_catalogue/finemapping/#gentropy.datasource.eqtl_catalogue.finemapping.EqtlCatalogueFinemapping.parse_susie_results","title":"<code>parse_susie_results(credible_sets: DataFrame, lbf: DataFrame, studies_metadata: DataFrame) -&gt; DataFrame</code>  <code>classmethod</code>","text":"<p>Parse the SuSIE results into a DataFrame containing the finemapping statistics and metadata about the studies.</p> <p>Parameters:</p> Name Type Description Default <code>credible_sets</code> <code>DataFrame</code> <p>DataFrame containing raw statistics of all variants in the credible sets.</p> required <code>lbf</code> <code>DataFrame</code> <p>DataFrame containing the raw log Bayes Factors for all variants.</p> required <code>studies_metadata</code> <code>DataFrame</code> <p>DataFrame containing the study metadata.</p> required <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>Processed SuSIE results to contain metadata about the studies and the finemapping statistics.</p> Source code in <code>src/gentropy/datasource/eqtl_catalogue/finemapping.py</code> <pre><code>@classmethod\ndef parse_susie_results(\n    cls: type[EqtlCatalogueFinemapping],\n    credible_sets: DataFrame,\n    lbf: DataFrame,\n    studies_metadata: DataFrame,\n) -&gt; DataFrame:\n    \"\"\"Parse the SuSIE results into a DataFrame containing the finemapping statistics and metadata about the studies.\n\n    Args:\n        credible_sets (DataFrame): DataFrame containing raw statistics of all variants in the credible sets.\n        lbf (DataFrame): DataFrame containing the raw log Bayes Factors for all variants.\n        studies_metadata (DataFrame): DataFrame containing the study metadata.\n\n    Returns:\n        DataFrame: Processed SuSIE results to contain metadata about the studies and the finemapping statistics.\n    \"\"\"\n    ss_ftp_path_template = \"https://ftp.ebi.ac.uk/pub/databases/spot/eQTL/sumstats\"\n    return (\n        lbf.withColumn(\n            \"dataset_id\",\n            cls._extract_dataset_id_from_file_path(f.input_file_name()),\n        )\n        .join(\n            (\n                credible_sets.withColumn(\n                    \"dataset_id\",\n                    cls._extract_dataset_id_from_file_path(f.input_file_name()),\n                )\n                .withColumn(\n                    \"credibleSetIndex\",\n                    cls._extract_credible_set_index(f.col(\"cs_id\")),\n                )\n                .join(f.broadcast(studies_metadata), on=\"dataset_id\")\n            ),\n            on=[\"molecular_trait_id\", \"region\", \"variant\", \"dataset_id\"],\n            how=\"inner\",\n        )\n        .withColumn(\n            \"logBF\",\n            f.when(f.col(\"credibleSetIndex\") == 1, f.col(\"lbf_variable1\"))\n            .when(f.col(\"credibleSetIndex\") == 2, f.col(\"lbf_variable2\"))\n            .when(f.col(\"credibleSetIndex\") == 3, f.col(\"lbf_variable3\"))\n            .when(f.col(\"credibleSetIndex\") == 4, f.col(\"lbf_variable4\"))\n            .when(f.col(\"credibleSetIndex\") == 5, f.col(\"lbf_variable5\"))\n            .when(f.col(\"credibleSetIndex\") == 6, f.col(\"lbf_variable6\"))\n            .when(f.col(\"credibleSetIndex\") == 7, f.col(\"lbf_variable7\"))\n            .when(f.col(\"credibleSetIndex\") == 8, f.col(\"lbf_variable8\"))\n            .when(f.col(\"credibleSetIndex\") == 9, f.col(\"lbf_variable9\"))\n            .when(f.col(\"credibleSetIndex\") == 10, f.col(\"lbf_variable10\")),\n        )\n        .select(\n            f.regexp_replace(f.col(\"variant\"), r\"chr\", \"\").alias(\"variantId\"),\n            f.col(\"region\"),\n            f.col(\"chromosome\"),\n            f.col(\"position\"),\n            f.col(\"pip\").alias(\"posteriorProbability\"),\n            *parse_pvalue(f.col(\"pvalue\")),\n            f.col(\"sample_size\").alias(\"nSamples\"),\n            f.col(\"beta\"),\n            f.col(\"se\").alias(\"standardError\"),\n            f.col(\"credibleSetIndex\"),\n            f.col(\"logBF\"),\n            f.lit(\"SuSie\").alias(\"finemappingMethod\"),\n            # Study metadata\n            f.col(\"molecular_trait_id\").alias(\"traitFromSource\"),\n            f.col(\"gene_id\").alias(\"geneId\"),\n            f.col(\"dataset_id\"),\n            f.concat_ws(\n                \"_\",\n                f.col(\"study_label\"),\n                f.col(\"sample_group\"),\n                f.col(\"molecular_trait_id\"),\n            ).alias(\"studyId\"),\n            f.col(\"tissue_id\").alias(\"tissueFromSourceId\"),\n            EqtlCatalogueStudyIndex._identify_study_type(\n                f.col(\"quant_method\")\n            ).alias(\"studyType\"),\n            f.col(\"study_label\").alias(\"projectId\"),\n            f.concat_ws(\n                \"/\",\n                f.lit(ss_ftp_path_template),\n                f.col(\"study_id\"),\n                f.col(\"dataset_id\"),\n            ).alias(\"summarystatsLocation\"),\n            f.lit(True).alias(\"hasSumstats\"),\n            f.col(\"molecular_trait_id\"),\n        )\n    )\n</code></pre>"},{"location":"python_api/datasources/eqtl_catalogue/finemapping/#gentropy.datasource.eqtl_catalogue.finemapping.EqtlCatalogueFinemapping.read_credible_set_from_source","title":"<code>read_credible_set_from_source(session: Session, credible_set_path: str | list[str]) -&gt; DataFrame</code>  <code>classmethod</code>","text":"<p>Load raw credible sets from eQTL Catalogue.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Spark session.</p> required <code>credible_set_path</code> <code>str | list[str]</code> <p>Path to raw table(s) containing finemapping results for any variant belonging to a credible set.</p> required <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>Credible sets DataFrame.</p> Source code in <code>src/gentropy/datasource/eqtl_catalogue/finemapping.py</code> <pre><code>@classmethod\ndef read_credible_set_from_source(\n    cls: type[EqtlCatalogueFinemapping],\n    session: Session,\n    credible_set_path: str | list[str],\n) -&gt; DataFrame:\n    \"\"\"Load raw credible sets from eQTL Catalogue.\n\n    Args:\n        session (Session): Spark session.\n        credible_set_path (str | list[str]): Path to raw table(s) containing finemapping results for any variant belonging to a credible set.\n\n    Returns:\n        DataFrame: Credible sets DataFrame.\n    \"\"\"\n    return session.spark.read.csv(\n        credible_set_path,\n        sep=\"\\t\",\n        header=True,\n        schema=cls.raw_credible_set_schema,\n    )\n</code></pre>"},{"location":"python_api/datasources/eqtl_catalogue/finemapping/#gentropy.datasource.eqtl_catalogue.finemapping.EqtlCatalogueFinemapping.read_lbf_from_source","title":"<code>read_lbf_from_source(session: Session, lbf_path: str | list[str]) -&gt; DataFrame</code>  <code>classmethod</code>","text":"<p>Load raw log Bayes Factors from eQTL Catalogue.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Spark session.</p> required <code>lbf_path</code> <code>str | list[str]</code> <p>Path to raw table(s) containing Log Bayes Factors for each variant.</p> required <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>Log Bayes Factors DataFrame.</p> Source code in <code>src/gentropy/datasource/eqtl_catalogue/finemapping.py</code> <pre><code>@classmethod\ndef read_lbf_from_source(\n    cls: type[EqtlCatalogueFinemapping],\n    session: Session,\n    lbf_path: str | list[str],\n) -&gt; DataFrame:\n    \"\"\"Load raw log Bayes Factors from eQTL Catalogue.\n\n    Args:\n        session (Session): Spark session.\n        lbf_path (str | list[str]): Path to raw table(s) containing Log Bayes Factors for each variant.\n\n    Returns:\n        DataFrame: Log Bayes Factors DataFrame.\n    \"\"\"\n    return session.spark.read.csv(\n        lbf_path,\n        sep=\"\\t\",\n        header=True,\n        schema=cls.raw_lbf_schema,\n    )\n</code></pre>"},{"location":"python_api/datasources/eqtl_catalogue/study_index/","title":"Study Index","text":""},{"location":"python_api/datasources/eqtl_catalogue/study_index/#gentropy.datasource.eqtl_catalogue.study_index.EqtlCatalogueStudyIndex","title":"<code>gentropy.datasource.eqtl_catalogue.study_index.EqtlCatalogueStudyIndex</code>","text":"<p>Study index dataset from eQTL Catalogue.</p> <p>We extract study level metadata from eQTL Catalogue's fine mapping results. All available studies can be found here.</p> <p>One study from the eQTL Catalogue clusters together all the molecular QTLs (mQTLs) that were found:</p> <pre><code>- in the same publication (e.g. Alasoo_2018)\n- in the same cell type or tissue (e.g. monocytes)\n- and for the same measured molecular trait (e.g. ENSG00000141510)\n</code></pre> Source code in <code>src/gentropy/datasource/eqtl_catalogue/study_index.py</code> <pre><code>class EqtlCatalogueStudyIndex:\n    \"\"\"Study index dataset from eQTL Catalogue.\n\n    We extract study level metadata from eQTL Catalogue's fine mapping results. All available studies can be found [here](https://www.ebi.ac.uk/eqtl/Studies/).\n\n    One study from the eQTL Catalogue clusters together all the molecular QTLs (mQTLs) that were found:\n\n        - in the same publication (e.g. Alasoo_2018)\n        - in the same cell type or tissue (e.g. monocytes)\n        - and for the same measured molecular trait (e.g. ENSG00000141510)\n\n    \"\"\"\n\n    raw_studies_metadata_schema: StructType = StructType(\n        [\n            StructField(\"study_id\", StringType(), True),\n            StructField(\"dataset_id\", StringType(), True),\n            StructField(\"study_label\", StringType(), True),\n            StructField(\"sample_group\", StringType(), True),\n            StructField(\"tissue_id\", StringType(), True),\n            StructField(\"tissue_label\", StringType(), True),\n            StructField(\"condition_label\", StringType(), True),\n            StructField(\"sample_size\", IntegerType(), True),\n            StructField(\"quant_method\", StringType(), True),\n        ]\n    )\n    raw_studies_metadata_path = \"https://raw.githubusercontent.com/eQTL-Catalogue/eQTL-Catalogue-resources/19929ff6a99bf402194292a14f96f9615b35f65f/data_tables/dataset_metadata.tsv\"\n\n    @classmethod\n    def _identify_study_type(\n        cls: type[EqtlCatalogueStudyIndex], quantification_method_col: Column\n    ) -&gt; Column:\n        \"\"\"Identify the study type based on the method to quantify the trait.\n\n        Args:\n            quantification_method_col (Column): column with the label of the method to quantify the trait. Available methods are [here](https://www.ebi.ac.uk/eqtl/Methods/)\n\n        Returns:\n            Column: The study type.\n\n        Examples:\n            &gt;&gt;&gt; df = spark.createDataFrame([(\"ge\",), (\"exon\",), (\"tx\",)], [\"quant_method\"])\n            &gt;&gt;&gt; df.withColumn(\"study_type\", EqtlCatalogueStudyIndex._identify_study_type(f.col(\"quant_method\"))).show()\n            +------------+----------+\n            |quant_method|study_type|\n            +------------+----------+\n            |          ge|      eqtl|\n            |        exon|      eqtl|\n            |          tx|      eqtl|\n            +------------+----------+\n            &lt;BLANKLINE&gt;\n        \"\"\"\n        method_to_study_type_mapping = {\n            \"ge\": \"eqtl\",\n            \"exon\": \"eqtl\",\n            \"tx\": \"eqtl\",\n            \"microarray\": \"eqtl\",\n            \"leafcutter\": \"sqtl\",\n            \"aptamer\": \"pqtl\",\n            \"txrev\": \"tuqtl\",\n        }\n        map_expr = f.create_map(\n            *[f.lit(x) for x in chain(*method_to_study_type_mapping.items())]\n        )\n        return map_expr.getItem(quantification_method_col)\n\n    @classmethod\n    def get_studies_of_interest(\n        cls: type[EqtlCatalogueStudyIndex], studies_metadata: DataFrame\n    ) -&gt; list[str]:\n        \"\"\"Filter studies of interest from the raw studies metadata.\n\n        Args:\n            studies_metadata (DataFrame): raw studies metadata filtered with studies of interest.\n\n        Returns:\n            list[str]: QTD IDs defining the studies of interest for ingestion.\n        \"\"\"\n        return (\n            studies_metadata.select(\"dataset_id\")\n            .distinct()\n            .toPandas()[\"dataset_id\"]\n            .tolist()\n        )\n\n    @classmethod\n    def from_susie_results(\n        cls: type[EqtlCatalogueStudyIndex],\n        processed_finemapping_df: DataFrame,\n    ) -&gt; StudyIndex:\n        \"\"\"Ingest study level metadata from eQTL Catalogue.\n\n        Args:\n            processed_finemapping_df (DataFrame): processed fine mapping results with study metadata.\n\n        Returns:\n            StudyIndex: eQTL Catalogue study index dataset derived from the selected SuSIE results.\n        \"\"\"\n        study_index_cols = [\n            field.name\n            for field in StudyIndex.get_schema().fields\n            if field.name in processed_finemapping_df.columns\n        ]\n        return StudyIndex(\n            _df=processed_finemapping_df.select(study_index_cols).distinct(),\n            _schema=StudyIndex.get_schema(),\n        )\n\n    @classmethod\n    def read_studies_from_source(\n        cls: type[EqtlCatalogueStudyIndex],\n        session: Session,\n        mqtl_quantification_methods_blacklist: list[str],\n    ) -&gt; DataFrame:\n        \"\"\"Read raw studies metadata from eQTL Catalogue.\n\n        Args:\n            session (Session): Spark session.\n            mqtl_quantification_methods_blacklist (list[str]): Molecular trait quantification methods that we don't want to ingest. Available options in https://github.com/eQTL-Catalogue/eQTL-Catalogue-resources/blob/master/data_tables/dataset_metadata.tsv\n\n        Returns:\n            DataFrame: raw studies metadata.\n        \"\"\"\n        pd.DataFrame.iteritems = pd.DataFrame.items\n        return session.spark.createDataFrame(\n            pd.read_csv(cls.raw_studies_metadata_path, sep=\"\\t\"),\n            schema=cls.raw_studies_metadata_schema,\n        ).filter(~(f.col(\"quant_method\").isin(mqtl_quantification_methods_blacklist)))\n</code></pre>"},{"location":"python_api/datasources/eqtl_catalogue/study_index/#gentropy.datasource.eqtl_catalogue.study_index.EqtlCatalogueStudyIndex.from_susie_results","title":"<code>from_susie_results(processed_finemapping_df: DataFrame) -&gt; StudyIndex</code>  <code>classmethod</code>","text":"<p>Ingest study level metadata from eQTL Catalogue.</p> <p>Parameters:</p> Name Type Description Default <code>processed_finemapping_df</code> <code>DataFrame</code> <p>processed fine mapping results with study metadata.</p> required <p>Returns:</p> Name Type Description <code>StudyIndex</code> <code>StudyIndex</code> <p>eQTL Catalogue study index dataset derived from the selected SuSIE results.</p> Source code in <code>src/gentropy/datasource/eqtl_catalogue/study_index.py</code> <pre><code>@classmethod\ndef from_susie_results(\n    cls: type[EqtlCatalogueStudyIndex],\n    processed_finemapping_df: DataFrame,\n) -&gt; StudyIndex:\n    \"\"\"Ingest study level metadata from eQTL Catalogue.\n\n    Args:\n        processed_finemapping_df (DataFrame): processed fine mapping results with study metadata.\n\n    Returns:\n        StudyIndex: eQTL Catalogue study index dataset derived from the selected SuSIE results.\n    \"\"\"\n    study_index_cols = [\n        field.name\n        for field in StudyIndex.get_schema().fields\n        if field.name in processed_finemapping_df.columns\n    ]\n    return StudyIndex(\n        _df=processed_finemapping_df.select(study_index_cols).distinct(),\n        _schema=StudyIndex.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasources/eqtl_catalogue/study_index/#gentropy.datasource.eqtl_catalogue.study_index.EqtlCatalogueStudyIndex.get_studies_of_interest","title":"<code>get_studies_of_interest(studies_metadata: DataFrame) -&gt; list[str]</code>  <code>classmethod</code>","text":"<p>Filter studies of interest from the raw studies metadata.</p> <p>Parameters:</p> Name Type Description Default <code>studies_metadata</code> <code>DataFrame</code> <p>raw studies metadata filtered with studies of interest.</p> required <p>Returns:</p> Type Description <code>list[str]</code> <p>list[str]: QTD IDs defining the studies of interest for ingestion.</p> Source code in <code>src/gentropy/datasource/eqtl_catalogue/study_index.py</code> <pre><code>@classmethod\ndef get_studies_of_interest(\n    cls: type[EqtlCatalogueStudyIndex], studies_metadata: DataFrame\n) -&gt; list[str]:\n    \"\"\"Filter studies of interest from the raw studies metadata.\n\n    Args:\n        studies_metadata (DataFrame): raw studies metadata filtered with studies of interest.\n\n    Returns:\n        list[str]: QTD IDs defining the studies of interest for ingestion.\n    \"\"\"\n    return (\n        studies_metadata.select(\"dataset_id\")\n        .distinct()\n        .toPandas()[\"dataset_id\"]\n        .tolist()\n    )\n</code></pre>"},{"location":"python_api/datasources/eqtl_catalogue/study_index/#gentropy.datasource.eqtl_catalogue.study_index.EqtlCatalogueStudyIndex.read_studies_from_source","title":"<code>read_studies_from_source(session: Session, mqtl_quantification_methods_blacklist: list[str]) -&gt; DataFrame</code>  <code>classmethod</code>","text":"<p>Read raw studies metadata from eQTL Catalogue.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Spark session.</p> required <code>mqtl_quantification_methods_blacklist</code> <code>list[str]</code> <p>Molecular trait quantification methods that we don't want to ingest. Available options in https://github.com/eQTL-Catalogue/eQTL-Catalogue-resources/blob/master/data_tables/dataset_metadata.tsv</p> required <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>raw studies metadata.</p> Source code in <code>src/gentropy/datasource/eqtl_catalogue/study_index.py</code> <pre><code>@classmethod\ndef read_studies_from_source(\n    cls: type[EqtlCatalogueStudyIndex],\n    session: Session,\n    mqtl_quantification_methods_blacklist: list[str],\n) -&gt; DataFrame:\n    \"\"\"Read raw studies metadata from eQTL Catalogue.\n\n    Args:\n        session (Session): Spark session.\n        mqtl_quantification_methods_blacklist (list[str]): Molecular trait quantification methods that we don't want to ingest. Available options in https://github.com/eQTL-Catalogue/eQTL-Catalogue-resources/blob/master/data_tables/dataset_metadata.tsv\n\n    Returns:\n        DataFrame: raw studies metadata.\n    \"\"\"\n    pd.DataFrame.iteritems = pd.DataFrame.items\n    return session.spark.createDataFrame(\n        pd.read_csv(cls.raw_studies_metadata_path, sep=\"\\t\"),\n        schema=cls.raw_studies_metadata_schema,\n    ).filter(~(f.col(\"quant_method\").isin(mqtl_quantification_methods_blacklist)))\n</code></pre>"},{"location":"python_api/datasources/eqtl_catalogue/summary_stats/","title":"Summary Stats","text":""},{"location":"python_api/datasources/eqtl_catalogue/summary_stats/#gentropy.datasource.eqtl_catalogue.summary_stats.EqtlCatalogueSummaryStats","title":"<code>gentropy.datasource.eqtl_catalogue.summary_stats.EqtlCatalogueSummaryStats</code>  <code>dataclass</code>","text":"<p>Summary statistics dataset for eQTL Catalogue.</p> Source code in <code>src/gentropy/datasource/eqtl_catalogue/summary_stats.py</code> <pre><code>@dataclass\nclass EqtlCatalogueSummaryStats:\n    \"\"\"Summary statistics dataset for eQTL Catalogue.\"\"\"\n\n    @staticmethod\n    def _full_study_id_regexp() -&gt; Column:\n        \"\"\"Constructs a full study ID from the URI.\n\n        Returns:\n            Column: expression to extract a full study ID from the URI.\n        \"\"\"\n        # Example of a URI which is used for parsing:\n        # \"gs://genetics_etl_python_playground/input/preprocess/eqtl_catalogue/imported/GTEx_V8/ge/Adipose_Subcutaneous.tsv.gz\".\n\n        # Regular expession to extract project ID from URI.  Example: \"GTEx_V8\".\n        _project_id = f.regexp_extract(\n            f.input_file_name(),\n            r\"imported/([^/]+)/.*\",\n            1,\n        )\n        # Regular expression to extract QTL group from URI.  Example: \"Adipose_Subcutaneous\".\n        _qtl_group = f.regexp_extract(f.input_file_name(), r\"([^/]+)\\.tsv\\.gz\", 1)\n        # Extracting gene ID from the column.  Example: \"ENSG00000225630\".\n        _gene_id = f.col(\"gene_id\")\n\n        # We can now construct the full study ID based on all fields.\n        # Example: \"GTEx_V8_Adipose_Subcutaneous_ENSG00000225630\".\n        return f.concat(_project_id, f.lit(\"_\"), _qtl_group, f.lit(\"_\"), _gene_id)\n\n    @classmethod\n    def from_source(\n        cls: type[EqtlCatalogueSummaryStats],\n        summary_stats_df: DataFrame,\n    ) -&gt; SummaryStatistics:\n        \"\"\"Ingests all summary stats for all eQTL Catalogue studies.\n\n        Args:\n            summary_stats_df (DataFrame): an ingested but unprocessed summary statistics dataframe from eQTL Catalogue.\n\n        Returns:\n            SummaryStatistics: a processed summary statistics dataframe for eQTL Catalogue.\n        \"\"\"\n        processed_summary_stats_df = (\n            summary_stats_df.select(\n                # Construct study ID from the appropriate columns.\n                cls._full_study_id_regexp().alias(\"studyId\"),\n                # Add variant information.\n                f.concat_ws(\n                    \"_\",\n                    f.col(\"chromosome\"),\n                    f.col(\"position\"),\n                    f.col(\"ref\"),\n                    f.col(\"alt\"),\n                ).alias(\"variantId\"),\n                f.col(\"chromosome\"),\n                f.col(\"position\").cast(t.IntegerType()),\n                # Parse p-value into mantissa and exponent.\n                *parse_pvalue(f.col(\"pvalue\")),\n                # Add beta, standard error, and allele frequency information.\n                f.col(\"beta\").cast(\"double\"),\n                f.col(\"se\").cast(\"double\").alias(\"standardError\"),\n                f.col(\"maf\").cast(\"float\").alias(\"effectAlleleFrequencyFromSource\"),\n            )\n            # Drop rows which don't have proper position or beta value.\n            .filter(\n                f.col(\"position\").cast(t.IntegerType()).isNotNull()\n                &amp; (f.col(\"beta\") != 0)\n            )\n        )\n\n        # Initialise a summary statistics object.\n        return SummaryStatistics(\n            _df=processed_summary_stats_df,\n            _schema=SummaryStatistics.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasources/eqtl_catalogue/summary_stats/#gentropy.datasource.eqtl_catalogue.summary_stats.EqtlCatalogueSummaryStats.from_source","title":"<code>from_source(summary_stats_df: DataFrame) -&gt; SummaryStatistics</code>  <code>classmethod</code>","text":"<p>Ingests all summary stats for all eQTL Catalogue studies.</p> <p>Parameters:</p> Name Type Description Default <code>summary_stats_df</code> <code>DataFrame</code> <p>an ingested but unprocessed summary statistics dataframe from eQTL Catalogue.</p> required <p>Returns:</p> Name Type Description <code>SummaryStatistics</code> <code>SummaryStatistics</code> <p>a processed summary statistics dataframe for eQTL Catalogue.</p> Source code in <code>src/gentropy/datasource/eqtl_catalogue/summary_stats.py</code> <pre><code>@classmethod\ndef from_source(\n    cls: type[EqtlCatalogueSummaryStats],\n    summary_stats_df: DataFrame,\n) -&gt; SummaryStatistics:\n    \"\"\"Ingests all summary stats for all eQTL Catalogue studies.\n\n    Args:\n        summary_stats_df (DataFrame): an ingested but unprocessed summary statistics dataframe from eQTL Catalogue.\n\n    Returns:\n        SummaryStatistics: a processed summary statistics dataframe for eQTL Catalogue.\n    \"\"\"\n    processed_summary_stats_df = (\n        summary_stats_df.select(\n            # Construct study ID from the appropriate columns.\n            cls._full_study_id_regexp().alias(\"studyId\"),\n            # Add variant information.\n            f.concat_ws(\n                \"_\",\n                f.col(\"chromosome\"),\n                f.col(\"position\"),\n                f.col(\"ref\"),\n                f.col(\"alt\"),\n            ).alias(\"variantId\"),\n            f.col(\"chromosome\"),\n            f.col(\"position\").cast(t.IntegerType()),\n            # Parse p-value into mantissa and exponent.\n            *parse_pvalue(f.col(\"pvalue\")),\n            # Add beta, standard error, and allele frequency information.\n            f.col(\"beta\").cast(\"double\"),\n            f.col(\"se\").cast(\"double\").alias(\"standardError\"),\n            f.col(\"maf\").cast(\"float\").alias(\"effectAlleleFrequencyFromSource\"),\n        )\n        # Drop rows which don't have proper position or beta value.\n        .filter(\n            f.col(\"position\").cast(t.IntegerType()).isNotNull()\n            &amp; (f.col(\"beta\") != 0)\n        )\n    )\n\n    # Initialise a summary statistics object.\n    return SummaryStatistics(\n        _df=processed_summary_stats_df,\n        _schema=SummaryStatistics.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasources/finngen/_finngen/","title":"FinnGen","text":"<p>FinnGen is a research project in genomics and personalized medicine, representing a large public-private partnership. The project has collected and analyzed genome and health data from 500,000 Finnish biobank donors to understand the genetic basis of diseases. FinnGen is now expanding its focus to comprehend the progression and biological mechanisms of diseases. This initiative provides a world-class resource for further breakthroughs in disease prevention, diagnosis, and treatment, offering insights into our genetic makeup.</p> <p>For a comprehensive understanding of the dataset and methods, refer to Kurki et al., 2023.</p> <p>We ingested full GWAS summary statistics and SuSiE-based fine-mapping results.</p>"},{"location":"python_api/datasources/finngen/finemapping/","title":"Finemapping","text":""},{"location":"python_api/datasources/finngen/finemapping/#gentropy.datasource.finngen.finemapping.FinnGenFinemapping","title":"<code>gentropy.datasource.finngen.finemapping.FinnGenFinemapping</code>  <code>dataclass</code>","text":"<p>SuSIE finemapping dataset for FinnGen.</p> <p>Credible sets from SuSIE are extracted and transformed into StudyLocus objects:</p> <ul> <li>Study ID in the special format (e.g. FINNGEN_R10_*)</li> <li>Credible set specific finemapping statistics (e.g. LogBayesFactors, Alphas/Posterior)</li> <li>Additional credible set level BayesFactor filtering is applied (LBF &gt; 2)</li> <li>StudyLocusId is annotated for each credible set.</li> </ul> <p>Finemapping method is populated as a constant (\"SuSIE\").</p> Source code in <code>src/gentropy/datasource/finngen/finemapping.py</code> <pre><code>@dataclass\nclass FinnGenFinemapping:\n    \"\"\"SuSIE finemapping dataset for FinnGen.\n\n    Credible sets from SuSIE are extracted and transformed into StudyLocus objects:\n\n    - Study ID in the special format (e.g. FINNGEN_R10_*)\n    - Credible set specific finemapping statistics (e.g. LogBayesFactors, Alphas/Posterior)\n    - Additional credible set level BayesFactor filtering is applied (LBF &gt; 2)\n    - StudyLocusId is annotated for each credible set.\n\n    Finemapping method is populated as a constant (\"SuSIE\").\n    \"\"\"\n\n    raw_schema: t.StructType = StructType(\n        [\n            StructField(\"trait\", StringType(), True),\n            StructField(\"region\", StringType(), True),\n            StructField(\"v\", StringType(), True),\n            StructField(\"rsid\", StringType(), True),\n            StructField(\"chromosome\", StringType(), True),\n            StructField(\"position\", StringType(), True),\n            StructField(\"allele1\", StringType(), True),\n            StructField(\"allele2\", StringType(), True),\n            StructField(\"maf\", StringType(), True),\n            StructField(\"beta\", StringType(), True),\n            StructField(\"se\", StringType(), True),\n            StructField(\"p\", StringType(), True),\n            StructField(\"mean\", StringType(), True),\n            StructField(\"sd\", StringType(), True),\n            StructField(\"prob\", StringType(), True),\n            StructField(\"cs\", StringType(), True),\n            StructField(\"cs_specific_prob\", DoubleType(), True),\n            StructField(\"low_purity\", StringType(), True),\n            StructField(\"lead_r2\", StringType(), True),\n            StructField(\"mean_99\", StringType(), True),\n            StructField(\"sd_99\", StringType(), True),\n            StructField(\"prob_99\", StringType(), True),\n            StructField(\"cs_99\", StringType(), True),\n            StructField(\"cs_specific_prob_99\", StringType(), True),\n            StructField(\"low_purity_99\", StringType(), True),\n            StructField(\"lead_r2_99\", StringType(), True),\n            StructField(\"alpha1\", DoubleType(), True),\n            StructField(\"alpha2\", DoubleType(), True),\n            StructField(\"alpha3\", DoubleType(), True),\n            StructField(\"alpha4\", DoubleType(), True),\n            StructField(\"alpha5\", DoubleType(), True),\n            StructField(\"alpha6\", DoubleType(), True),\n            StructField(\"alpha7\", DoubleType(), True),\n            StructField(\"alpha8\", DoubleType(), True),\n            StructField(\"alpha9\", DoubleType(), True),\n            StructField(\"alpha10\", DoubleType(), True),\n            StructField(\"mean1\", StringType(), True),\n            StructField(\"mean2\", StringType(), True),\n            StructField(\"mean3\", StringType(), True),\n            StructField(\"mean4\", StringType(), True),\n            StructField(\"mean5\", StringType(), True),\n            StructField(\"mean6\", StringType(), True),\n            StructField(\"mean7\", StringType(), True),\n            StructField(\"mean8\", StringType(), True),\n            StructField(\"mean9\", StringType(), True),\n            StructField(\"mean10\", StringType(), True),\n            StructField(\"sd1\", StringType(), True),\n            StructField(\"sd2\", StringType(), True),\n            StructField(\"sd3\", StringType(), True),\n            StructField(\"sd4\", StringType(), True),\n            StructField(\"sd5\", StringType(), True),\n            StructField(\"sd6\", StringType(), True),\n            StructField(\"sd7\", StringType(), True),\n            StructField(\"sd8\", StringType(), True),\n            StructField(\"sd9\", StringType(), True),\n            StructField(\"sd10\", StringType(), True),\n            StructField(\"lbf_variable1\", DoubleType(), True),\n            StructField(\"lbf_variable2\", DoubleType(), True),\n            StructField(\"lbf_variable3\", DoubleType(), True),\n            StructField(\"lbf_variable4\", DoubleType(), True),\n            StructField(\"lbf_variable5\", DoubleType(), True),\n            StructField(\"lbf_variable6\", DoubleType(), True),\n            StructField(\"lbf_variable7\", DoubleType(), True),\n            StructField(\"lbf_variable8\", DoubleType(), True),\n            StructField(\"lbf_variable9\", DoubleType(), True),\n            StructField(\"lbf_variable10\", DoubleType(), True),\n        ]\n    )\n\n    summary_schema: t.StructType = StructType(\n        [\n            StructField(\"trait\", StringType(), True),\n            StructField(\"region\", StringType(), True),\n            StructField(\"cs\", StringType(), True),\n            StructField(\"cs_log10bf\", DoubleType(), True),\n        ]\n    )\n\n    @classmethod\n    def from_finngen_susie_finemapping(\n        cls: type[FinnGenFinemapping],\n        spark: SparkSession,\n        finngen_finemapping_df: (str | list[str]),\n        finngen_finemapping_summaries: (str | list[str]),\n        finngen_release_prefix: str,\n        credset_lbf_threshold: float = 0.8685889638065036,\n    ) -&gt; StudyLocus:\n        \"\"\"Process the SuSIE finemapping output for FinnGen studies.\n\n        Args:\n            spark (SparkSession): Spark session object.\n            finngen_finemapping_df (str | list[str]): SuSIE finemapping output filename(s).\n            finngen_finemapping_summaries (str | list[str]): filename of SuSIE finemapping summaries.\n            finngen_release_prefix (str): FinnGen study prefix.\n            credset_lbf_threshold (float, optional): Filter out credible sets below, Default 0.8685889638065036 == np.log10(np.exp(2)), this is threshold from publication.\n\n        Returns:\n            StudyLocus: Processed SuSIE finemapping output in StudyLocus format.\n        \"\"\"\n        processed_finngen_finemapping_df = (\n            spark.read.schema(cls.raw_schema)\n            .option(\"delimiter\", \"\\t\")\n            .option(\"compression\", \"gzip\")\n            .csv(finngen_finemapping_df, header=True)\n            # Drop rows which don't have proper position.\n            .filter(f.col(\"position\").cast(t.IntegerType()).isNotNull())\n            # Drop non credible set SNPs:\n            .filter(f.col(\"cs\").cast(t.IntegerType()) &gt; 0)\n            .select(\n                # Add study idenfitier.\n                f.concat(f.lit(finngen_release_prefix), f.col(\"trait\"))\n                .cast(t.StringType())\n                .alias(\"studyId\"),\n                f.col(\"region\"),\n                # Add variant information.\n                f.regexp_replace(f.col(\"v\"), \":\", \"_\").alias(\"variantId\"),\n                f.col(\"cs\").cast(\"integer\").alias(\"credibleSetIndex\"),\n                f.regexp_replace(f.col(\"chromosome\"), \"^chr\", \"\")\n                .cast(t.StringType())\n                .alias(\"chromosome\"),\n                f.col(\"position\").cast(t.IntegerType()),\n                f.col(\"allele1\").cast(t.StringType()).alias(\"ref\"),\n                f.col(\"allele2\").cast(t.StringType()).alias(\"alt\"),\n                # Parse p-value into mantissa and exponent.\n                *parse_pvalue(f.col(\"p\")),\n                # Add beta, standard error, and allele frequency information.\n                f.col(\"beta\").cast(\"double\"),\n                f.col(\"se\").cast(\"double\").alias(\"standardError\"),\n                f.col(\"maf\").cast(\"float\").alias(\"effectAlleleFrequencyFromSource\"),\n                f.lit(\"SuSie\").cast(\"string\").alias(\"finemappingMethod\"),\n                *[\n                    f.col(f\"alpha{i}\").cast(t.DoubleType()).alias(f\"alpha_{i}\")\n                    for i in range(1, 11)\n                ],\n                *[\n                    f.col(f\"lbf_variable{i}\").cast(t.DoubleType()).alias(f\"lbf_{i}\")\n                    for i in range(1, 11)\n                ],\n            )\n            .withColumn(\n                \"posteriorProbability\",\n                f.when(f.col(\"credibleSetIndex\") == 1, f.col(\"alpha_1\"))\n                .when(f.col(\"credibleSetIndex\") == 2, f.col(\"alpha_2\"))\n                .when(f.col(\"credibleSetIndex\") == 3, f.col(\"alpha_3\"))\n                .when(f.col(\"credibleSetIndex\") == 4, f.col(\"alpha_4\"))\n                .when(f.col(\"credibleSetIndex\") == 5, f.col(\"alpha_5\"))\n                .when(f.col(\"credibleSetIndex\") == 6, f.col(\"alpha_6\"))\n                .when(f.col(\"credibleSetIndex\") == 7, f.col(\"alpha_7\"))\n                .when(f.col(\"credibleSetIndex\") == 8, f.col(\"alpha_8\"))\n                .when(f.col(\"credibleSetIndex\") == 9, f.col(\"alpha_9\"))\n                .when(f.col(\"credibleSetIndex\") == 10, f.col(\"alpha_10\")),\n            )\n            .drop(\n                \"alpha_1\",\n                \"alpha_2\",\n                \"alpha_3\",\n                \"alpha_4\",\n                \"alpha_5\",\n                \"alpha_6\",\n                \"alpha_7\",\n                \"alpha_8\",\n                \"alpha_9\",\n                \"alpha_10\",\n            )\n            .withColumn(\n                \"logBF\",\n                f.when(f.col(\"credibleSetIndex\") == 1, f.col(\"lbf_1\"))\n                .when(f.col(\"credibleSetIndex\") == 2, f.col(\"lbf_2\"))\n                .when(f.col(\"credibleSetIndex\") == 3, f.col(\"lbf_3\"))\n                .when(f.col(\"credibleSetIndex\") == 4, f.col(\"lbf_4\"))\n                .when(f.col(\"credibleSetIndex\") == 5, f.col(\"lbf_5\"))\n                .when(f.col(\"credibleSetIndex\") == 6, f.col(\"lbf_6\"))\n                .when(f.col(\"credibleSetIndex\") == 7, f.col(\"lbf_7\"))\n                .when(f.col(\"credibleSetIndex\") == 8, f.col(\"lbf_8\"))\n                .when(f.col(\"credibleSetIndex\") == 9, f.col(\"lbf_9\"))\n                .when(f.col(\"credibleSetIndex\") == 10, f.col(\"lbf_10\")),\n            )\n            .drop(\n                \"lbf_1\",\n                \"lbf_2\",\n                \"lbf_3\",\n                \"lbf_4\",\n                \"lbf_5\",\n                \"lbf_6\",\n                \"lbf_7\",\n                \"lbf_8\",\n                \"lbf_9\",\n                \"lbf_10\",\n            )\n        )\n\n        # drop credible sets where logbf &gt; 2. Except when there's only one credible set in region:\n        # 0.8685889638065036 corresponds to np.log10(np.exp(2)), to match the orginal threshold in publication.\n        finngen_finemapping_summaries_df = (\n            # Read credible set level lbf, it is output as a different file which is not ideal.\n            spark.read.schema(cls.summary_schema)\n            .option(\"delimiter\", \"\\t\")\n            .csv(finngen_finemapping_summaries, header=True)\n            .select(\n                f.col(\"region\"),\n                f.col(\"trait\"),\n                f.col(\"cs\").cast(\"integer\").alias(\"credibleSetIndex\"),\n                f.col(\"cs_log10bf\").cast(\"double\").alias(\"credibleSetlog10BF\"),\n            )\n            .filter(\n                (f.col(\"credibleSetlog10BF\") &gt; credset_lbf_threshold)\n                | (f.col(\"credibleSetIndex\") == 1)\n            )\n            .withColumn(\n                \"studyId\", f.concat(f.lit(finngen_release_prefix), f.col(\"trait\"))\n            )\n        )\n\n        processed_finngen_finemapping_df = processed_finngen_finemapping_df.join(\n            finngen_finemapping_summaries_df,\n            on=[\"studyId\", \"region\", \"credibleSetIndex\"],\n            how=\"inner\",\n        )\n\n        toploci_df = get_top_ranked_in_window(\n            processed_finngen_finemapping_df,\n            Window.partitionBy(\"studyId\", \"region\", \"credibleSetIndex\").orderBy(\n                f.desc(\"posteriorProbability\")\n            ),\n        ).select(\n            \"variantId\",\n            \"chromosome\",\n            \"position\",\n            \"studyId\",\n            \"beta\",\n            \"pValueMantissa\",\n            \"pValueExponent\",\n            \"effectAlleleFrequencyFromSource\",\n            \"standardError\",\n            \"region\",\n            \"credibleSetIndex\",\n            \"finemappingMethod\",\n            \"credibleSetlog10BF\",\n        )\n\n        processed_finngen_finemapping_df = (\n            processed_finngen_finemapping_df.groupBy(\n                \"studyId\", \"region\", \"credibleSetIndex\"\n            )\n            .agg(\n                f.collect_list(\n                    f.struct(\n                        f.col(\"variantId\").cast(\"string\").alias(\"variantId\"),\n                        f.col(\"posteriorProbability\")\n                        .cast(\"double\")\n                        .alias(\"posteriorProbability\"),\n                        f.col(\"logBF\").cast(\"double\").alias(\"logBF\"),\n                        f.col(\"pValueMantissa\").cast(\"float\").alias(\"pValueMantissa\"),\n                        f.col(\"pValueExponent\").cast(\"integer\").alias(\"pValueExponent\"),\n                        f.col(\"beta\").cast(\"double\").alias(\"beta\"),\n                        f.col(\"standardError\").cast(\"double\").alias(\"standardError\"),\n                    )\n                ).alias(\"locus\"),\n            )\n            .select(\n                \"studyId\",\n                \"region\",\n                \"credibleSetIndex\",\n                \"locus\",\n            )\n            .join(\n                toploci_df,\n                on=[\"studyId\", \"region\", \"credibleSetIndex\"],\n                how=\"inner\",\n            )\n        ).withColumn(\n            \"studyLocusId\",\n            StudyLocus.assign_study_locus_id(f.col(\"studyId\"), f.col(\"variantId\")),\n        )\n\n        return StudyLocus(\n            _df=processed_finngen_finemapping_df,\n            _schema=StudyLocus.get_schema(),\n        ).annotate_credible_sets()\n</code></pre>"},{"location":"python_api/datasources/finngen/finemapping/#gentropy.datasource.finngen.finemapping.FinnGenFinemapping.from_finngen_susie_finemapping","title":"<code>from_finngen_susie_finemapping(spark: SparkSession, finngen_finemapping_df: str | list[str], finngen_finemapping_summaries: str | list[str], finngen_release_prefix: str, credset_lbf_threshold: float = 0.8685889638065036) -&gt; StudyLocus</code>  <code>classmethod</code>","text":"<p>Process the SuSIE finemapping output for FinnGen studies.</p> <p>Parameters:</p> Name Type Description Default <code>spark</code> <code>SparkSession</code> <p>Spark session object.</p> required <code>finngen_finemapping_df</code> <code>str | list[str]</code> <p>SuSIE finemapping output filename(s).</p> required <code>finngen_finemapping_summaries</code> <code>str | list[str]</code> <p>filename of SuSIE finemapping summaries.</p> required <code>finngen_release_prefix</code> <code>str</code> <p>FinnGen study prefix.</p> required <code>credset_lbf_threshold</code> <code>float</code> <p>Filter out credible sets below, Default 0.8685889638065036 == np.log10(np.exp(2)), this is threshold from publication.</p> <code>0.8685889638065036</code> <p>Returns:</p> Name Type Description <code>StudyLocus</code> <code>StudyLocus</code> <p>Processed SuSIE finemapping output in StudyLocus format.</p> Source code in <code>src/gentropy/datasource/finngen/finemapping.py</code> <pre><code>@classmethod\ndef from_finngen_susie_finemapping(\n    cls: type[FinnGenFinemapping],\n    spark: SparkSession,\n    finngen_finemapping_df: (str | list[str]),\n    finngen_finemapping_summaries: (str | list[str]),\n    finngen_release_prefix: str,\n    credset_lbf_threshold: float = 0.8685889638065036,\n) -&gt; StudyLocus:\n    \"\"\"Process the SuSIE finemapping output for FinnGen studies.\n\n    Args:\n        spark (SparkSession): Spark session object.\n        finngen_finemapping_df (str | list[str]): SuSIE finemapping output filename(s).\n        finngen_finemapping_summaries (str | list[str]): filename of SuSIE finemapping summaries.\n        finngen_release_prefix (str): FinnGen study prefix.\n        credset_lbf_threshold (float, optional): Filter out credible sets below, Default 0.8685889638065036 == np.log10(np.exp(2)), this is threshold from publication.\n\n    Returns:\n        StudyLocus: Processed SuSIE finemapping output in StudyLocus format.\n    \"\"\"\n    processed_finngen_finemapping_df = (\n        spark.read.schema(cls.raw_schema)\n        .option(\"delimiter\", \"\\t\")\n        .option(\"compression\", \"gzip\")\n        .csv(finngen_finemapping_df, header=True)\n        # Drop rows which don't have proper position.\n        .filter(f.col(\"position\").cast(t.IntegerType()).isNotNull())\n        # Drop non credible set SNPs:\n        .filter(f.col(\"cs\").cast(t.IntegerType()) &gt; 0)\n        .select(\n            # Add study idenfitier.\n            f.concat(f.lit(finngen_release_prefix), f.col(\"trait\"))\n            .cast(t.StringType())\n            .alias(\"studyId\"),\n            f.col(\"region\"),\n            # Add variant information.\n            f.regexp_replace(f.col(\"v\"), \":\", \"_\").alias(\"variantId\"),\n            f.col(\"cs\").cast(\"integer\").alias(\"credibleSetIndex\"),\n            f.regexp_replace(f.col(\"chromosome\"), \"^chr\", \"\")\n            .cast(t.StringType())\n            .alias(\"chromosome\"),\n            f.col(\"position\").cast(t.IntegerType()),\n            f.col(\"allele1\").cast(t.StringType()).alias(\"ref\"),\n            f.col(\"allele2\").cast(t.StringType()).alias(\"alt\"),\n            # Parse p-value into mantissa and exponent.\n            *parse_pvalue(f.col(\"p\")),\n            # Add beta, standard error, and allele frequency information.\n            f.col(\"beta\").cast(\"double\"),\n            f.col(\"se\").cast(\"double\").alias(\"standardError\"),\n            f.col(\"maf\").cast(\"float\").alias(\"effectAlleleFrequencyFromSource\"),\n            f.lit(\"SuSie\").cast(\"string\").alias(\"finemappingMethod\"),\n            *[\n                f.col(f\"alpha{i}\").cast(t.DoubleType()).alias(f\"alpha_{i}\")\n                for i in range(1, 11)\n            ],\n            *[\n                f.col(f\"lbf_variable{i}\").cast(t.DoubleType()).alias(f\"lbf_{i}\")\n                for i in range(1, 11)\n            ],\n        )\n        .withColumn(\n            \"posteriorProbability\",\n            f.when(f.col(\"credibleSetIndex\") == 1, f.col(\"alpha_1\"))\n            .when(f.col(\"credibleSetIndex\") == 2, f.col(\"alpha_2\"))\n            .when(f.col(\"credibleSetIndex\") == 3, f.col(\"alpha_3\"))\n            .when(f.col(\"credibleSetIndex\") == 4, f.col(\"alpha_4\"))\n            .when(f.col(\"credibleSetIndex\") == 5, f.col(\"alpha_5\"))\n            .when(f.col(\"credibleSetIndex\") == 6, f.col(\"alpha_6\"))\n            .when(f.col(\"credibleSetIndex\") == 7, f.col(\"alpha_7\"))\n            .when(f.col(\"credibleSetIndex\") == 8, f.col(\"alpha_8\"))\n            .when(f.col(\"credibleSetIndex\") == 9, f.col(\"alpha_9\"))\n            .when(f.col(\"credibleSetIndex\") == 10, f.col(\"alpha_10\")),\n        )\n        .drop(\n            \"alpha_1\",\n            \"alpha_2\",\n            \"alpha_3\",\n            \"alpha_4\",\n            \"alpha_5\",\n            \"alpha_6\",\n            \"alpha_7\",\n            \"alpha_8\",\n            \"alpha_9\",\n            \"alpha_10\",\n        )\n        .withColumn(\n            \"logBF\",\n            f.when(f.col(\"credibleSetIndex\") == 1, f.col(\"lbf_1\"))\n            .when(f.col(\"credibleSetIndex\") == 2, f.col(\"lbf_2\"))\n            .when(f.col(\"credibleSetIndex\") == 3, f.col(\"lbf_3\"))\n            .when(f.col(\"credibleSetIndex\") == 4, f.col(\"lbf_4\"))\n            .when(f.col(\"credibleSetIndex\") == 5, f.col(\"lbf_5\"))\n            .when(f.col(\"credibleSetIndex\") == 6, f.col(\"lbf_6\"))\n            .when(f.col(\"credibleSetIndex\") == 7, f.col(\"lbf_7\"))\n            .when(f.col(\"credibleSetIndex\") == 8, f.col(\"lbf_8\"))\n            .when(f.col(\"credibleSetIndex\") == 9, f.col(\"lbf_9\"))\n            .when(f.col(\"credibleSetIndex\") == 10, f.col(\"lbf_10\")),\n        )\n        .drop(\n            \"lbf_1\",\n            \"lbf_2\",\n            \"lbf_3\",\n            \"lbf_4\",\n            \"lbf_5\",\n            \"lbf_6\",\n            \"lbf_7\",\n            \"lbf_8\",\n            \"lbf_9\",\n            \"lbf_10\",\n        )\n    )\n\n    # drop credible sets where logbf &gt; 2. Except when there's only one credible set in region:\n    # 0.8685889638065036 corresponds to np.log10(np.exp(2)), to match the orginal threshold in publication.\n    finngen_finemapping_summaries_df = (\n        # Read credible set level lbf, it is output as a different file which is not ideal.\n        spark.read.schema(cls.summary_schema)\n        .option(\"delimiter\", \"\\t\")\n        .csv(finngen_finemapping_summaries, header=True)\n        .select(\n            f.col(\"region\"),\n            f.col(\"trait\"),\n            f.col(\"cs\").cast(\"integer\").alias(\"credibleSetIndex\"),\n            f.col(\"cs_log10bf\").cast(\"double\").alias(\"credibleSetlog10BF\"),\n        )\n        .filter(\n            (f.col(\"credibleSetlog10BF\") &gt; credset_lbf_threshold)\n            | (f.col(\"credibleSetIndex\") == 1)\n        )\n        .withColumn(\n            \"studyId\", f.concat(f.lit(finngen_release_prefix), f.col(\"trait\"))\n        )\n    )\n\n    processed_finngen_finemapping_df = processed_finngen_finemapping_df.join(\n        finngen_finemapping_summaries_df,\n        on=[\"studyId\", \"region\", \"credibleSetIndex\"],\n        how=\"inner\",\n    )\n\n    toploci_df = get_top_ranked_in_window(\n        processed_finngen_finemapping_df,\n        Window.partitionBy(\"studyId\", \"region\", \"credibleSetIndex\").orderBy(\n            f.desc(\"posteriorProbability\")\n        ),\n    ).select(\n        \"variantId\",\n        \"chromosome\",\n        \"position\",\n        \"studyId\",\n        \"beta\",\n        \"pValueMantissa\",\n        \"pValueExponent\",\n        \"effectAlleleFrequencyFromSource\",\n        \"standardError\",\n        \"region\",\n        \"credibleSetIndex\",\n        \"finemappingMethod\",\n        \"credibleSetlog10BF\",\n    )\n\n    processed_finngen_finemapping_df = (\n        processed_finngen_finemapping_df.groupBy(\n            \"studyId\", \"region\", \"credibleSetIndex\"\n        )\n        .agg(\n            f.collect_list(\n                f.struct(\n                    f.col(\"variantId\").cast(\"string\").alias(\"variantId\"),\n                    f.col(\"posteriorProbability\")\n                    .cast(\"double\")\n                    .alias(\"posteriorProbability\"),\n                    f.col(\"logBF\").cast(\"double\").alias(\"logBF\"),\n                    f.col(\"pValueMantissa\").cast(\"float\").alias(\"pValueMantissa\"),\n                    f.col(\"pValueExponent\").cast(\"integer\").alias(\"pValueExponent\"),\n                    f.col(\"beta\").cast(\"double\").alias(\"beta\"),\n                    f.col(\"standardError\").cast(\"double\").alias(\"standardError\"),\n                )\n            ).alias(\"locus\"),\n        )\n        .select(\n            \"studyId\",\n            \"region\",\n            \"credibleSetIndex\",\n            \"locus\",\n        )\n        .join(\n            toploci_df,\n            on=[\"studyId\", \"region\", \"credibleSetIndex\"],\n            how=\"inner\",\n        )\n    ).withColumn(\n        \"studyLocusId\",\n        StudyLocus.assign_study_locus_id(f.col(\"studyId\"), f.col(\"variantId\")),\n    )\n\n    return StudyLocus(\n        _df=processed_finngen_finemapping_df,\n        _schema=StudyLocus.get_schema(),\n    ).annotate_credible_sets()\n</code></pre>"},{"location":"python_api/datasources/finngen/study_index/","title":"Study Index","text":""},{"location":"python_api/datasources/finngen/study_index/#gentropy.datasource.finngen.study_index.FinnGenStudyIndex","title":"<code>gentropy.datasource.finngen.study_index.FinnGenStudyIndex</code>","text":"<p>Study index dataset from FinnGen.</p> <p>The following information is aggregated/extracted:</p> <ul> <li>Study ID in the special format (e.g. FINNGEN_R10_*)</li> <li>Trait name (for example, Amoebiasis)</li> <li>Number of cases and controls</li> <li>Link to the summary statistics location</li> </ul> <p>Some fields are also populated as constants, such as study type and the initial sample size.</p> Source code in <code>src/gentropy/datasource/finngen/study_index.py</code> <pre><code>class FinnGenStudyIndex:\n    \"\"\"Study index dataset from FinnGen.\n\n    The following information is aggregated/extracted:\n\n    - Study ID in the special format (e.g. FINNGEN_R10_*)\n    - Trait name (for example, Amoebiasis)\n    - Number of cases and controls\n    - Link to the summary statistics location\n\n    Some fields are also populated as constants, such as study type and the initial sample size.\n    \"\"\"\n\n    finngen_phenotype_table_url: str = \"https://r10.finngen.fi/api/phenos\"\n    finngen_release_prefix: str = \"FINNGEN_R10\"\n    finngen_summary_stats_url_prefix: str = (\n        \"gs://finngen-public-data-r10/summary_stats/finngen_R10_\"\n    )\n    finngen_summary_stats_url_suffix: str = \".gz\"\n\n    @classmethod\n    def from_source(\n        cls: type[FinnGenStudyIndex],\n        spark: SparkSession,\n    ) -&gt; StudyIndex:\n        \"\"\"This function ingests study level metadata from FinnGen.\n\n        Args:\n            spark (SparkSession): Spark session object.\n\n        Returns:\n            StudyIndex: Parsed and annotated FinnGen study table.\n        \"\"\"\n        json_data = urlopen(cls.finngen_phenotype_table_url).read().decode(\"utf-8\")\n        rdd = spark.sparkContext.parallelize([json_data])\n        raw_df = spark.read.json(rdd)\n        return StudyIndex(\n            _df=raw_df.select(\n                f.concat(\n                    f.lit(f\"{cls.finngen_release_prefix}_\"), f.col(\"phenocode\")\n                ).alias(\"studyId\"),\n                f.col(\"phenostring\").alias(\"traitFromSource\"),\n                f.col(\"num_cases\").cast(\"integer\").alias(\"nCases\"),\n                f.col(\"num_controls\").cast(\"integer\").alias(\"nControls\"),\n                (f.col(\"num_cases\") + f.col(\"num_controls\"))\n                .cast(\"integer\")\n                .alias(\"nSamples\"),\n                f.lit(cls.finngen_release_prefix).alias(\"projectId\"),\n                f.lit(\"gwas\").alias(\"studyType\"),\n                f.lit(True).alias(\"hasSumstats\"),\n                f.lit(\"377,277 (210,870 females and 166,407 males)\").alias(\n                    \"initialSampleSize\"\n                ),\n                f.array(\n                    f.struct(\n                        f.lit(377277).cast(\"integer\").alias(\"sampleSize\"),\n                        f.lit(\"Finnish\").alias(\"ancestry\"),\n                    )\n                ).alias(\"discoverySamples\"),\n                # Cohort label is consistent with GWAS Catalog curation.\n                f.array(f.lit(\"FinnGen\")).alias(\"cohorts\"),\n                f.concat(\n                    f.lit(cls.finngen_summary_stats_url_prefix),\n                    f.col(\"phenocode\"),\n                    f.lit(cls.finngen_summary_stats_url_suffix),\n                ).alias(\"summarystatsLocation\"),\n            ).withColumn(\n                \"ldPopulationStructure\",\n                StudyIndex.aggregate_and_map_ancestries(f.col(\"discoverySamples\")),\n            ),\n            _schema=StudyIndex.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasources/finngen/study_index/#gentropy.datasource.finngen.study_index.FinnGenStudyIndex.from_source","title":"<code>from_source(spark: SparkSession) -&gt; StudyIndex</code>  <code>classmethod</code>","text":"<p>This function ingests study level metadata from FinnGen.</p> <p>Parameters:</p> Name Type Description Default <code>spark</code> <code>SparkSession</code> <p>Spark session object.</p> required <p>Returns:</p> Name Type Description <code>StudyIndex</code> <code>StudyIndex</code> <p>Parsed and annotated FinnGen study table.</p> Source code in <code>src/gentropy/datasource/finngen/study_index.py</code> <pre><code>@classmethod\ndef from_source(\n    cls: type[FinnGenStudyIndex],\n    spark: SparkSession,\n) -&gt; StudyIndex:\n    \"\"\"This function ingests study level metadata from FinnGen.\n\n    Args:\n        spark (SparkSession): Spark session object.\n\n    Returns:\n        StudyIndex: Parsed and annotated FinnGen study table.\n    \"\"\"\n    json_data = urlopen(cls.finngen_phenotype_table_url).read().decode(\"utf-8\")\n    rdd = spark.sparkContext.parallelize([json_data])\n    raw_df = spark.read.json(rdd)\n    return StudyIndex(\n        _df=raw_df.select(\n            f.concat(\n                f.lit(f\"{cls.finngen_release_prefix}_\"), f.col(\"phenocode\")\n            ).alias(\"studyId\"),\n            f.col(\"phenostring\").alias(\"traitFromSource\"),\n            f.col(\"num_cases\").cast(\"integer\").alias(\"nCases\"),\n            f.col(\"num_controls\").cast(\"integer\").alias(\"nControls\"),\n            (f.col(\"num_cases\") + f.col(\"num_controls\"))\n            .cast(\"integer\")\n            .alias(\"nSamples\"),\n            f.lit(cls.finngen_release_prefix).alias(\"projectId\"),\n            f.lit(\"gwas\").alias(\"studyType\"),\n            f.lit(True).alias(\"hasSumstats\"),\n            f.lit(\"377,277 (210,870 females and 166,407 males)\").alias(\n                \"initialSampleSize\"\n            ),\n            f.array(\n                f.struct(\n                    f.lit(377277).cast(\"integer\").alias(\"sampleSize\"),\n                    f.lit(\"Finnish\").alias(\"ancestry\"),\n                )\n            ).alias(\"discoverySamples\"),\n            # Cohort label is consistent with GWAS Catalog curation.\n            f.array(f.lit(\"FinnGen\")).alias(\"cohorts\"),\n            f.concat(\n                f.lit(cls.finngen_summary_stats_url_prefix),\n                f.col(\"phenocode\"),\n                f.lit(cls.finngen_summary_stats_url_suffix),\n            ).alias(\"summarystatsLocation\"),\n        ).withColumn(\n            \"ldPopulationStructure\",\n            StudyIndex.aggregate_and_map_ancestries(f.col(\"discoverySamples\")),\n        ),\n        _schema=StudyIndex.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasources/finngen/summary_stats/","title":"Study Index","text":""},{"location":"python_api/datasources/finngen/summary_stats/#gentropy.datasource.finngen.summary_stats.FinnGenSummaryStats","title":"<code>gentropy.datasource.finngen.summary_stats.FinnGenSummaryStats</code>  <code>dataclass</code>","text":"<p>Summary statistics dataset for FinnGen.</p> Source code in <code>src/gentropy/datasource/finngen/summary_stats.py</code> <pre><code>@dataclass\nclass FinnGenSummaryStats:\n    \"\"\"Summary statistics dataset for FinnGen.\"\"\"\n\n    raw_schema: t.StructType = StructType(\n        [\n            StructField(\"#chrom\", StringType(), True),\n            StructField(\"pos\", StringType(), True),\n            StructField(\"ref\", StringType(), True),\n            StructField(\"alt\", StringType(), True),\n            StructField(\"rsids\", StringType(), True),\n            StructField(\"nearest_genes\", StringType(), True),\n            StructField(\"pval\", StringType(), True),\n            StructField(\"mlogp\", StringType(), True),\n            StructField(\"beta\", StringType(), True),\n            StructField(\"sebeta\", StringType(), True),\n            StructField(\"af_alt\", StringType(), True),\n            StructField(\"af_alt_cases\", StringType(), True),\n            StructField(\"af_alt_controls\", StringType(), True),\n        ]\n    )\n\n    @classmethod\n    def from_source(\n        cls: type[FinnGenSummaryStats],\n        spark: SparkSession,\n        raw_file: str,\n    ) -&gt; SummaryStatistics:\n        \"\"\"Ingests all summary statst for all FinnGen studies.\n\n        Args:\n            spark (SparkSession): Spark session object.\n            raw_file (str): Path to raw summary statistics .gz files.\n\n        Returns:\n            SummaryStatistics: Processed summary statistics dataset\n        \"\"\"\n        processed_summary_stats_df = (\n            spark.read.schema(cls.raw_schema)\n            .option(\"delimiter\", \"\\t\")\n            .csv(raw_file, header=True)\n            # Drop rows which don't have proper position.\n            .filter(f.col(\"pos\").cast(t.IntegerType()).isNotNull())\n            .select(\n                # From the full path, extracts just the filename, and converts to upper case to get the study ID.\n                f.upper(\n                    f.regexp_extract(\n                        f.input_file_name(), r\"([^/]+)(\\.tsv\\.gz|\\.gz|\\.tsv)\", 1\n                    )\n                ).alias(\"studyId\"),\n                # Add variant information.\n                f.concat_ws(\n                    \"_\",\n                    f.col(\"#chrom\"),\n                    f.col(\"pos\"),\n                    f.col(\"ref\"),\n                    f.col(\"alt\"),\n                ).alias(\"variantId\"),\n                f.col(\"#chrom\").alias(\"chromosome\"),\n                f.col(\"pos\").cast(t.IntegerType()).alias(\"position\"),\n                # Parse p-value into mantissa and exponent.\n                *parse_pvalue(f.col(\"pval\")),\n                # Add beta, standard error, and allele frequency information.\n                f.col(\"beta\").cast(\"double\"),\n                f.col(\"sebeta\").cast(\"double\").alias(\"standardError\"),\n                f.col(\"af_alt\").cast(\"float\").alias(\"effectAlleleFrequencyFromSource\"),\n            )\n            # Calculating the confidence intervals.\n            .filter(\n                f.col(\"pos\").cast(t.IntegerType()).isNotNull() &amp; (f.col(\"beta\") != 0)\n            )\n            # Average ~20Mb partitions with 30 partitions per study\n            .repartitionByRange(30, \"chromosome\", \"position\")\n            .sortWithinPartitions(\"chromosome\", \"position\")\n        )\n\n        # Initializing summary statistics object:\n        return SummaryStatistics(\n            _df=processed_summary_stats_df,\n            _schema=SummaryStatistics.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasources/finngen/summary_stats/#gentropy.datasource.finngen.summary_stats.FinnGenSummaryStats.from_source","title":"<code>from_source(spark: SparkSession, raw_file: str) -&gt; SummaryStatistics</code>  <code>classmethod</code>","text":"<p>Ingests all summary statst for all FinnGen studies.</p> <p>Parameters:</p> Name Type Description Default <code>spark</code> <code>SparkSession</code> <p>Spark session object.</p> required <code>raw_file</code> <code>str</code> <p>Path to raw summary statistics .gz files.</p> required <p>Returns:</p> Name Type Description <code>SummaryStatistics</code> <code>SummaryStatistics</code> <p>Processed summary statistics dataset</p> Source code in <code>src/gentropy/datasource/finngen/summary_stats.py</code> <pre><code>@classmethod\ndef from_source(\n    cls: type[FinnGenSummaryStats],\n    spark: SparkSession,\n    raw_file: str,\n) -&gt; SummaryStatistics:\n    \"\"\"Ingests all summary statst for all FinnGen studies.\n\n    Args:\n        spark (SparkSession): Spark session object.\n        raw_file (str): Path to raw summary statistics .gz files.\n\n    Returns:\n        SummaryStatistics: Processed summary statistics dataset\n    \"\"\"\n    processed_summary_stats_df = (\n        spark.read.schema(cls.raw_schema)\n        .option(\"delimiter\", \"\\t\")\n        .csv(raw_file, header=True)\n        # Drop rows which don't have proper position.\n        .filter(f.col(\"pos\").cast(t.IntegerType()).isNotNull())\n        .select(\n            # From the full path, extracts just the filename, and converts to upper case to get the study ID.\n            f.upper(\n                f.regexp_extract(\n                    f.input_file_name(), r\"([^/]+)(\\.tsv\\.gz|\\.gz|\\.tsv)\", 1\n                )\n            ).alias(\"studyId\"),\n            # Add variant information.\n            f.concat_ws(\n                \"_\",\n                f.col(\"#chrom\"),\n                f.col(\"pos\"),\n                f.col(\"ref\"),\n                f.col(\"alt\"),\n            ).alias(\"variantId\"),\n            f.col(\"#chrom\").alias(\"chromosome\"),\n            f.col(\"pos\").cast(t.IntegerType()).alias(\"position\"),\n            # Parse p-value into mantissa and exponent.\n            *parse_pvalue(f.col(\"pval\")),\n            # Add beta, standard error, and allele frequency information.\n            f.col(\"beta\").cast(\"double\"),\n            f.col(\"sebeta\").cast(\"double\").alias(\"standardError\"),\n            f.col(\"af_alt\").cast(\"float\").alias(\"effectAlleleFrequencyFromSource\"),\n        )\n        # Calculating the confidence intervals.\n        .filter(\n            f.col(\"pos\").cast(t.IntegerType()).isNotNull() &amp; (f.col(\"beta\") != 0)\n        )\n        # Average ~20Mb partitions with 30 partitions per study\n        .repartitionByRange(30, \"chromosome\", \"position\")\n        .sortWithinPartitions(\"chromosome\", \"position\")\n    )\n\n    # Initializing summary statistics object:\n    return SummaryStatistics(\n        _df=processed_summary_stats_df,\n        _schema=SummaryStatistics.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasources/gnomad/_gnomad/","title":"GnomAD","text":"<p>GnomAD (Genome Aggregation Database) is a comprehensive resource that provides aggregated genomic data from large-scale sequencing projects. It encompasses variants from diverse populations and is widely used for variant annotation and population genetics studies.</p> <p>We use GnomAD v4.0 as a source for variant annotation, offering detailed information about the prevalence and distribution of genetic variants across different populations. This version of GnomAD provides valuable insights into the genomic landscape, aiding in the interpretation of genetic variants and their potential functional implications.</p> <p>Additionally, GnomAD v2.1.1 is utilized as a source for linkage disequilibrium (LD) information.</p>"},{"location":"python_api/datasources/gnomad/gnomad_ld/","title":"LD Matrix","text":""},{"location":"python_api/datasources/gnomad/gnomad_ld/#gentropy.datasource.gnomad.ld.GnomADLDMatrix","title":"<code>gentropy.datasource.gnomad.ld.GnomADLDMatrix</code>  <code>dataclass</code>","text":"<p>Toolset ot interact with GnomAD LD dataset (version: r2.1.1).</p> <p>Datasets are accessed in Hail's native format, as provided by the GnomAD consortium.</p> <p>Attributes:</p> Name Type Description <code>ld_matrix_template</code> <code>str</code> <p>Template for the LD matrix path. Defaults to \"gs://gcp-public-data--gnomad/release/2.1.1/ld/gnomad.genomes.r2.1.1.{POP}.common.adj.ld.bm\".</p> <code>ld_index_raw_template</code> <code>str</code> <p>Template for the LD index path. Defaults to \"gs://gcp-public-data--gnomad/release/2.1.1/ld/gnomad.genomes.r2.1.1.{POP}.common.ld.variant_indices.ht\".</p> <code>grch37_to_grch38_chain_path</code> <code>str</code> <p>Path to the chain file used to lift over the coordinates. Defaults to \"gs://hail-common/references/grch37_to_grch38.over.chain.gz\".</p> <code>ld_populations</code> <code>list[str]</code> <p>List of populations to use to build the LDIndex. Defaults to [\"afr\", \"amr\", \"asj\", \"eas\", \"fin\", \"nfe\", \"nwe\", \"seu\"].</p> Source code in <code>src/gentropy/datasource/gnomad/ld.py</code> <pre><code>@dataclass\nclass GnomADLDMatrix:\n    \"\"\"Toolset ot interact with GnomAD LD dataset (version: r2.1.1).\n\n    Datasets are accessed in Hail's native format, as provided by the [GnomAD consortium](https://gnomad.broadinstitute.org/downloads/#v2-linkage-disequilibrium).\n\n    Attributes:\n        ld_matrix_template (str): Template for the LD matrix path. Defaults to \"gs://gcp-public-data--gnomad/release/2.1.1/ld/gnomad.genomes.r2.1.1.{POP}.common.adj.ld.bm\".\n        ld_index_raw_template (str): Template for the LD index path. Defaults to \"gs://gcp-public-data--gnomad/release/2.1.1/ld/gnomad.genomes.r2.1.1.{POP}.common.ld.variant_indices.ht\".\n        grch37_to_grch38_chain_path (str): Path to the chain file used to lift over the coordinates. Defaults to \"gs://hail-common/references/grch37_to_grch38.over.chain.gz\".\n        ld_populations (list[str]): List of populations to use to build the LDIndex. Defaults to [\"afr\", \"amr\", \"asj\", \"eas\", \"fin\", \"nfe\", \"nwe\", \"seu\"].\n    \"\"\"\n\n    ld_matrix_template: str = \"gs://gcp-public-data--gnomad/release/2.1.1/ld/gnomad.genomes.r2.1.1.{POP}.common.adj.ld.bm\"\n    ld_index_raw_template: str = \"gs://gcp-public-data--gnomad/release/2.1.1/ld/gnomad.genomes.r2.1.1.{POP}.common.ld.variant_indices.ht\"\n    liftover_ht_path: str = \"gs://gcp-public-data--gnomad/release/2.1.1/liftover_grch38/ht/genomes/gnomad.genomes.r2.1.1.sites.liftover_grch38.ht\"\n    grch37_to_grch38_chain_path: str = (\n        \"gs://hail-common/references/grch37_to_grch38.over.chain.gz\"\n    )\n    ld_populations: list[str] = field(\n        default_factory=lambda: [\n            \"afr\",  # African-American\n            \"amr\",  # American Admixed/Latino\n            \"asj\",  # Ashkenazi Jewish\n            \"eas\",  # East Asian\n            \"fin\",  # Finnish\n            \"nfe\",  # Non-Finnish European\n            \"nwe\",  # Northwestern European\n            \"seu\",  # Southeastern European\n        ]\n    )\n\n    @staticmethod\n    def _aggregate_ld_index_across_populations(\n        unaggregated_ld_index: DataFrame,\n    ) -&gt; DataFrame:\n        \"\"\"Aggregate LDIndex across populations.\n\n        Args:\n            unaggregated_ld_index (DataFrame): Unaggregate LDIndex index dataframe  each row is a variant pair in a population\n\n        Returns:\n            DataFrame: Aggregated LDIndex index dataframe  each row is a variant with the LD set across populations\n\n        Examples:\n            &gt;&gt;&gt; data = [(\"1.0\", \"var1\", \"X\", \"var1\", \"pop1\"), (\"1.0\", \"X\", \"var2\", \"var2\", \"pop1\"),\n            ...         (\"0.5\", \"var1\", \"X\", \"var2\", \"pop1\"), (\"0.5\", \"var1\", \"X\", \"var2\", \"pop2\"),\n            ...         (\"0.5\", \"var2\", \"X\", \"var1\", \"pop1\"), (\"0.5\", \"X\", \"var2\", \"var1\", \"pop2\")]\n            &gt;&gt;&gt; df = spark.createDataFrame(data, [\"r\", \"variantId\", \"chromosome\", \"tagvariantId\", \"population\"])\n            &gt;&gt;&gt; GnomADLDMatrix._aggregate_ld_index_across_populations(df).printSchema()\n            root\n             |-- variantId: string (nullable = true)\n             |-- chromosome: string (nullable = true)\n             |-- ldSet: array (nullable = false)\n             |    |-- element: struct (containsNull = false)\n             |    |    |-- tagVariantId: string (nullable = true)\n             |    |    |-- rValues: array (nullable = false)\n             |    |    |    |-- element: struct (containsNull = false)\n             |    |    |    |    |-- population: string (nullable = true)\n             |    |    |    |    |-- r: string (nullable = true)\n            &lt;BLANKLINE&gt;\n        \"\"\"\n        return (\n            unaggregated_ld_index\n            # First level of aggregation: get r/population for each variant/tagVariant pair\n            .withColumn(\"r_pop_struct\", f.struct(\"population\", \"r\"))\n            .groupBy(\"chromosome\", \"variantId\", \"tagVariantId\")\n            .agg(\n                f.collect_set(\"r_pop_struct\").alias(\"rValues\"),\n            )\n            # Second level of aggregation: get r/population for each variant\n            .withColumn(\"r_pop_tag_struct\", f.struct(\"tagVariantId\", \"rValues\"))\n            .groupBy(\"variantId\", \"chromosome\")\n            .agg(\n                f.collect_set(\"r_pop_tag_struct\").alias(\"ldSet\"),\n            )\n        )\n\n    @staticmethod\n    def _convert_ld_matrix_to_table(\n        block_matrix: BlockMatrix, min_r2: float\n    ) -&gt; DataFrame:\n        \"\"\"Convert LD matrix to table.\n\n        Args:\n            block_matrix (BlockMatrix): LD matrix\n            min_r2 (float): Minimum r2 value to keep in the table\n\n        Returns:\n            DataFrame: LD matrix as a Spark DataFrame\n        \"\"\"\n        table = block_matrix.entries(keyed=False)\n        return (\n            table.filter(hl.abs(table.entry) &gt;= min_r2**0.5)\n            .to_spark()\n            .withColumnRenamed(\"entry\", \"r\")\n        )\n\n    @staticmethod\n    def _create_ldindex_for_population(\n        population_id: str,\n        ld_matrix_path: str,\n        ld_index_raw_path: str,\n        grch37_to_grch38_chain_path: str,\n        min_r2: float,\n    ) -&gt; DataFrame:\n        \"\"\"Create LDIndex for a specific population.\n\n        Args:\n            population_id (str): Population ID\n            ld_matrix_path (str): Path to the LD matrix\n            ld_index_raw_path (str): Path to the LD index\n            grch37_to_grch38_chain_path (str): Path to the chain file used to lift over the coordinates\n            min_r2 (float): Minimum r2 value to keep in the table\n\n        Returns:\n            DataFrame: LDIndex for a specific population\n        \"\"\"\n        # Prepare LD Block matrix\n        ld_matrix = GnomADLDMatrix._convert_ld_matrix_to_table(\n            BlockMatrix.read(ld_matrix_path), min_r2\n        )\n\n        # Prepare table with variant indices\n        ld_index = GnomADLDMatrix._process_variant_indices(\n            hl.read_table(ld_index_raw_path),\n            grch37_to_grch38_chain_path,\n        )\n\n        return GnomADLDMatrix._resolve_variant_indices(ld_index, ld_matrix).select(\n            \"*\",\n            f.lit(population_id).alias(\"population\"),\n        )\n\n    @staticmethod\n    def _process_variant_indices(\n        ld_index_raw: hl.Table, grch37_to_grch38_chain_path: str\n    ) -&gt; DataFrame:\n        \"\"\"Creates a look up table between variants and their coordinates in the LD Matrix.\n\n        !!! info \"Gnomad's LD Matrix and Index are based on GRCh37 coordinates. This function will lift over the coordinates to GRCh38 to build the lookup table.\"\n\n        Args:\n            ld_index_raw (hl.Table): LD index table from GnomAD\n            grch37_to_grch38_chain_path (str): Path to the chain file used to lift over the coordinates\n\n        Returns:\n            DataFrame: Look up table between variants in build hg38 and their coordinates in the LD Matrix\n        \"\"\"\n        ld_index_38 = _liftover_loci(\n            ld_index_raw, grch37_to_grch38_chain_path, \"GRCh38\"\n        )\n\n        return (\n            ld_index_38.to_spark()\n            # Filter out variants where the liftover failed\n            .filter(f.col(\"`locus_GRCh38.position`\").isNotNull())\n            .select(\n                f.regexp_replace(\"`locus_GRCh38.contig`\", \"chr\", \"\").alias(\n                    \"chromosome\"\n                ),\n                f.col(\"`locus_GRCh38.position`\").alias(\"position\"),\n                f.concat_ws(\n                    \"_\",\n                    f.regexp_replace(\"`locus_GRCh38.contig`\", \"chr\", \"\"),\n                    f.col(\"`locus_GRCh38.position`\"),\n                    f.col(\"`alleles`\").getItem(0),\n                    f.col(\"`alleles`\").getItem(1),\n                ).alias(\"variantId\"),\n                f.col(\"idx\"),\n            )\n            # Filter out ambiguous liftover results: multiple indices for the same variant\n            .withColumn(\"count\", f.count(\"*\").over(Window.partitionBy([\"variantId\"])))\n            .filter(f.col(\"count\") == 1)\n            .drop(\"count\")\n        )\n\n    @staticmethod\n    def _resolve_variant_indices(\n        ld_index: DataFrame, ld_matrix: DataFrame\n    ) -&gt; DataFrame:\n        \"\"\"Resolve the `i` and `j` indices of the block matrix to variant IDs (build 38).\n\n        Args:\n            ld_index (DataFrame): Dataframe with resolved variant indices\n            ld_matrix (DataFrame): Dataframe with the filtered LD matrix\n\n        Returns:\n            DataFrame: Dataframe with variant IDs instead of `i` and `j` indices\n        \"\"\"\n        ld_index_i = ld_index.selectExpr(\n            \"idx as i\", \"variantId as variantIdI\", \"chromosome\"\n        )\n        ld_index_j = ld_index.selectExpr(\"idx as j\", \"variantId as variantIdJ\")\n        return (\n            ld_matrix.join(ld_index_i, on=\"i\", how=\"inner\")\n            .join(ld_index_j, on=\"j\", how=\"inner\")\n            .drop(\"i\", \"j\")\n        )\n\n    @staticmethod\n    def _transpose_ld_matrix(ld_matrix: DataFrame) -&gt; DataFrame:\n        \"\"\"Transpose LD matrix to a square matrix format.\n\n        Args:\n            ld_matrix (DataFrame): Triangular LD matrix converted to a Spark DataFrame\n\n        Returns:\n            DataFrame: Square LD matrix without diagonal duplicates\n\n        Examples:\n        &gt;&gt;&gt; df = spark.createDataFrame(\n        ...     [\n        ...         (1, 1, 1.0, \"1\", \"AFR\"),\n        ...         (1, 2, 0.5, \"1\", \"AFR\"),\n        ...         (2, 2, 1.0, \"1\", \"AFR\"),\n        ...     ],\n        ...     [\"variantIdI\", \"variantIdJ\", \"r\", \"chromosome\", \"population\"],\n        ... )\n        &gt;&gt;&gt; GnomADLDMatrix._transpose_ld_matrix(df).show()\n        +----------+----------+---+----------+----------+\n        |variantIdI|variantIdJ|  r|chromosome|population|\n        +----------+----------+---+----------+----------+\n        |         1|         2|0.5|         1|       AFR|\n        |         1|         1|1.0|         1|       AFR|\n        |         2|         1|0.5|         1|       AFR|\n        |         2|         2|1.0|         1|       AFR|\n        +----------+----------+---+----------+----------+\n        &lt;BLANKLINE&gt;\n        \"\"\"\n        ld_matrix_transposed = ld_matrix.selectExpr(\n            \"variantIdI as variantIdJ\",\n            \"variantIdJ as variantIdI\",\n            \"r\",\n            \"chromosome\",\n            \"population\",\n        )\n        return ld_matrix.filter(f.col(\"variantIdI\") != f.col(\"variantIdJ\")).unionByName(\n            ld_matrix_transposed\n        )\n\n    def as_ld_index(\n        self: GnomADLDMatrix,\n        min_r2: float,\n    ) -&gt; LDIndex:\n        \"\"\"Create LDIndex dataset aggregating the LD information across a set of populations.\n\n        **The basic steps to generate the LDIndex are:**\n\n        1. Convert LD matrix to a Spark DataFrame.\n        2. Resolve the matrix indices to variant IDs by lifting over the coordinates to GRCh38.\n        3. Aggregate the LD information across populations.\n\n        Args:\n            min_r2 (float): Minimum r2 value to keep in the table\n\n        Returns:\n            LDIndex: LDIndex dataset\n        \"\"\"\n        ld_indices_unaggregated = []\n        for pop in self.ld_populations:\n            try:\n                ld_matrix_path = self.ld_matrix_template.format(POP=pop)\n                ld_index_raw_path = self.ld_index_raw_template.format(POP=pop)\n                pop_ld_index = self._create_ldindex_for_population(\n                    pop,\n                    ld_matrix_path,\n                    ld_index_raw_path.format(pop),\n                    self.grch37_to_grch38_chain_path,\n                    min_r2,\n                )\n                ld_indices_unaggregated.append(pop_ld_index)\n            except Exception as e:\n                print(f\"Failed to create LDIndex for population {pop}: {e}\")  # noqa: T201\n                sys.exit(1)\n\n        ld_index_unaggregated = (\n            GnomADLDMatrix._transpose_ld_matrix(\n                reduce(lambda df1, df2: df1.unionByName(df2), ld_indices_unaggregated)\n            )\n            .withColumnRenamed(\"variantIdI\", \"variantId\")\n            .withColumnRenamed(\"variantIdJ\", \"tagVariantId\")\n        )\n        return LDIndex(\n            _df=self._aggregate_ld_index_across_populations(ld_index_unaggregated),\n            _schema=LDIndex.get_schema(),\n        )\n\n    def get_ld_variants(\n        self: GnomADLDMatrix,\n        gnomad_ancestry: str,\n        chromosome: str,\n        start: int,\n        end: int,\n    ) -&gt; DataFrame | None:\n        \"\"\"Return melted LD table with resolved variant id based on ancestry and genomic location.\n\n        Args:\n            gnomad_ancestry (str): GnomAD major ancestry label eg. `nfe`\n            chromosome (str): chromosome label\n            start (int): window upper bound\n            end (int): window lower bound\n\n        Returns:\n            DataFrame | None: LD table with resolved variant id based on ancestry and genomic location\n        \"\"\"\n        # Extracting locus:\n        ld_index_df = (\n            self._process_variant_indices(\n                hl.read_table(self.ld_index_raw_template.format(POP=gnomad_ancestry)),\n                self.grch37_to_grch38_chain_path,\n            )\n            .filter(\n                (f.col(\"chromosome\") == chromosome)\n                &amp; (f.col(\"position\") &gt;= start)\n                &amp; (f.col(\"position\") &lt;= end)\n            )\n            .select(\"chromosome\", \"position\", \"variantId\", \"idx\")\n        )\n\n        if ld_index_df.limit(1).count() == 0:\n            # If the returned slice from the ld index is empty, return None\n            return None\n\n        # Compute start and end indices\n        start_index = get_value_from_row(\n            get_top_ranked_in_window(\n                ld_index_df, Window.partitionBy().orderBy(f.col(\"position\").asc())\n            ).collect()[0],\n            \"idx\",\n        )\n        end_index = get_value_from_row(\n            get_top_ranked_in_window(\n                ld_index_df, Window.partitionBy().orderBy(f.col(\"position\").desc())\n            ).collect()[0],\n            \"idx\",\n        )\n\n        return self._extract_square_matrix(\n            ld_index_df, gnomad_ancestry, start_index, end_index\n        )\n\n    def _extract_square_matrix(\n        self: GnomADLDMatrix,\n        ld_index_df: DataFrame,\n        gnomad_ancestry: str,\n        start_index: int,\n        end_index: int,\n    ) -&gt; DataFrame:\n        \"\"\"Return LD square matrix for a region where coordinates are normalised.\n\n        Args:\n            ld_index_df (DataFrame): Look up table between a variantId and its index in the LD matrix\n            gnomad_ancestry (str): GnomAD major ancestry label eg. `nfe`\n            start_index (int): start index of the slice\n            end_index (int): end index of the slice\n\n        Returns:\n            DataFrame: square LD matrix resolved to variants.\n        \"\"\"\n        return (\n            self.get_ld_matrix_slice(\n                gnomad_ancestry, start_index=start_index, end_index=end_index\n            )\n            .join(\n                ld_index_df.select(\n                    f.col(\"idx\").alias(\"idx_i\"),\n                    f.col(\"variantId\").alias(\"variantIdI\"),\n                ),\n                on=\"idx_i\",\n                how=\"inner\",\n            )\n            .join(\n                ld_index_df.select(\n                    f.col(\"idx\").alias(\"idx_j\"),\n                    f.col(\"variantId\").alias(\"variantIdJ\"),\n                ),\n                on=\"idx_j\",\n                how=\"inner\",\n            )\n            .select(\"variantIdI\", \"variantIdJ\", \"r\")\n        )\n\n    def get_ld_matrix_slice(\n        self: GnomADLDMatrix,\n        gnomad_ancestry: str,\n        start_index: int,\n        end_index: int,\n    ) -&gt; DataFrame:\n        \"\"\"Extract a slice of the LD matrix based on the provided ancestry and stop and end indices.\n\n        - The half matrix is completed into a full square.\n        - The returned indices are adjusted based on the start index.\n\n        Args:\n            gnomad_ancestry (str): LD population label eg. `nfe`\n            start_index (int): start index of the slice\n            end_index (int): end index of the slice\n\n        Returns:\n            DataFrame: square slice of the LD matrix melted as dataframe with idx_i, idx_j and r columns\n        \"\"\"\n        # Extracting block matrix slice:\n        half_matrix = BlockMatrix.read(\n            self.ld_matrix_template.format(POP=gnomad_ancestry)\n        ).filter(range(start_index, end_index + 1), range(start_index, end_index + 1))\n\n        # Return converted Dataframe:\n        return (\n            (half_matrix + half_matrix.T)\n            .entries()\n            .to_spark()\n            .select(\n                (f.col(\"i\") + start_index).alias(\"idx_i\"),\n                (f.col(\"j\") + start_index).alias(\"idx_j\"),\n                f.when(f.col(\"i\") == f.col(\"j\"), f.col(\"entry\") / 2)\n                .otherwise(f.col(\"entry\"))\n                .alias(\"r\"),\n            )\n        )\n\n    def get_locus_index(\n        self: GnomADLDMatrix,\n        study_locus_row: Row,\n        radius: int = 500_000,\n        major_population: str = \"nfe\",\n    ) -&gt; DataFrame:\n        \"\"\"Extract hail matrix index from StudyLocus rows.\n\n        Args:\n            study_locus_row (Row): Study-locus row\n            radius (int): Locus radius to extract from gnomad matrix\n            major_population (str): Major population to extract from gnomad matrix, default is \"nfe\"\n\n        Returns:\n            DataFrame: Returns the index of the gnomad matrix for the locus\n\n        \"\"\"\n        chromosome = str(\"chr\" + study_locus_row[\"chromosome\"])\n        start = study_locus_row[\"position\"] - radius\n        end = study_locus_row[\"position\"] + radius\n\n        liftover_ht = hl.read_table(self.liftover_ht_path)\n        liftover_ht = (\n            liftover_ht.filter(\n                (liftover_ht.locus.contig == chromosome)\n                &amp; (liftover_ht.locus.position &gt;= start)\n                &amp; (liftover_ht.locus.position &lt;= end)\n            )\n            .key_by()\n            .select(\"locus\", \"alleles\", \"original_locus\")\n            .key_by(\"original_locus\", \"alleles\")\n            .naive_coalesce(20)\n        )\n\n        hail_index = hl.read_table(\n            self.ld_index_raw_template.format(POP=major_population)\n        )\n\n        joined_index = (\n            liftover_ht.join(hail_index, how=\"inner\").order_by(\"idx\").to_spark()\n        )\n\n        return joined_index\n\n    @staticmethod\n    def get_numpy_matrix(\n        locus_index: DataFrame,\n        gnomad_ancestry: str = \"nfe\",\n    ) -&gt; np.ndarray:\n        \"\"\"Extract the LD block matrix for a locus.\n\n        Args:\n            locus_index (DataFrame): hail matrix variant index table\n            gnomad_ancestry (str): GnomAD major ancestry label eg. `nfe`\n\n        Returns:\n            np.ndarray: LD block matrix for the locus\n        \"\"\"\n        idx = [row[\"idx\"] for row in locus_index.select(\"idx\").collect()]\n\n        half_matrix = (\n            BlockMatrix.read(\n                GnomADLDMatrix.ld_matrix_template.format(POP=gnomad_ancestry)\n            )\n            .filter(idx, idx)\n            .to_numpy()\n        )\n\n        return (half_matrix + half_matrix.T) - np.diag(np.diag(half_matrix))\n</code></pre>"},{"location":"python_api/datasources/gnomad/gnomad_ld/#gentropy.datasource.gnomad.ld.GnomADLDMatrix.as_ld_index","title":"<code>as_ld_index(min_r2: float) -&gt; LDIndex</code>","text":"<p>Create LDIndex dataset aggregating the LD information across a set of populations.</p> <p>The basic steps to generate the LDIndex are:</p> <ol> <li>Convert LD matrix to a Spark DataFrame.</li> <li>Resolve the matrix indices to variant IDs by lifting over the coordinates to GRCh38.</li> <li>Aggregate the LD information across populations.</li> </ol> <p>Parameters:</p> Name Type Description Default <code>min_r2</code> <code>float</code> <p>Minimum r2 value to keep in the table</p> required <p>Returns:</p> Name Type Description <code>LDIndex</code> <code>LDIndex</code> <p>LDIndex dataset</p> Source code in <code>src/gentropy/datasource/gnomad/ld.py</code> <pre><code>def as_ld_index(\n    self: GnomADLDMatrix,\n    min_r2: float,\n) -&gt; LDIndex:\n    \"\"\"Create LDIndex dataset aggregating the LD information across a set of populations.\n\n    **The basic steps to generate the LDIndex are:**\n\n    1. Convert LD matrix to a Spark DataFrame.\n    2. Resolve the matrix indices to variant IDs by lifting over the coordinates to GRCh38.\n    3. Aggregate the LD information across populations.\n\n    Args:\n        min_r2 (float): Minimum r2 value to keep in the table\n\n    Returns:\n        LDIndex: LDIndex dataset\n    \"\"\"\n    ld_indices_unaggregated = []\n    for pop in self.ld_populations:\n        try:\n            ld_matrix_path = self.ld_matrix_template.format(POP=pop)\n            ld_index_raw_path = self.ld_index_raw_template.format(POP=pop)\n            pop_ld_index = self._create_ldindex_for_population(\n                pop,\n                ld_matrix_path,\n                ld_index_raw_path.format(pop),\n                self.grch37_to_grch38_chain_path,\n                min_r2,\n            )\n            ld_indices_unaggregated.append(pop_ld_index)\n        except Exception as e:\n            print(f\"Failed to create LDIndex for population {pop}: {e}\")  # noqa: T201\n            sys.exit(1)\n\n    ld_index_unaggregated = (\n        GnomADLDMatrix._transpose_ld_matrix(\n            reduce(lambda df1, df2: df1.unionByName(df2), ld_indices_unaggregated)\n        )\n        .withColumnRenamed(\"variantIdI\", \"variantId\")\n        .withColumnRenamed(\"variantIdJ\", \"tagVariantId\")\n    )\n    return LDIndex(\n        _df=self._aggregate_ld_index_across_populations(ld_index_unaggregated),\n        _schema=LDIndex.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasources/gnomad/gnomad_ld/#gentropy.datasource.gnomad.ld.GnomADLDMatrix.get_ld_matrix_slice","title":"<code>get_ld_matrix_slice(gnomad_ancestry: str, start_index: int, end_index: int) -&gt; DataFrame</code>","text":"<p>Extract a slice of the LD matrix based on the provided ancestry and stop and end indices.</p> <ul> <li>The half matrix is completed into a full square.</li> <li>The returned indices are adjusted based on the start index.</li> </ul> <p>Parameters:</p> Name Type Description Default <code>gnomad_ancestry</code> <code>str</code> <p>LD population label eg. <code>nfe</code></p> required <code>start_index</code> <code>int</code> <p>start index of the slice</p> required <code>end_index</code> <code>int</code> <p>end index of the slice</p> required <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>square slice of the LD matrix melted as dataframe with idx_i, idx_j and r columns</p> Source code in <code>src/gentropy/datasource/gnomad/ld.py</code> <pre><code>def get_ld_matrix_slice(\n    self: GnomADLDMatrix,\n    gnomad_ancestry: str,\n    start_index: int,\n    end_index: int,\n) -&gt; DataFrame:\n    \"\"\"Extract a slice of the LD matrix based on the provided ancestry and stop and end indices.\n\n    - The half matrix is completed into a full square.\n    - The returned indices are adjusted based on the start index.\n\n    Args:\n        gnomad_ancestry (str): LD population label eg. `nfe`\n        start_index (int): start index of the slice\n        end_index (int): end index of the slice\n\n    Returns:\n        DataFrame: square slice of the LD matrix melted as dataframe with idx_i, idx_j and r columns\n    \"\"\"\n    # Extracting block matrix slice:\n    half_matrix = BlockMatrix.read(\n        self.ld_matrix_template.format(POP=gnomad_ancestry)\n    ).filter(range(start_index, end_index + 1), range(start_index, end_index + 1))\n\n    # Return converted Dataframe:\n    return (\n        (half_matrix + half_matrix.T)\n        .entries()\n        .to_spark()\n        .select(\n            (f.col(\"i\") + start_index).alias(\"idx_i\"),\n            (f.col(\"j\") + start_index).alias(\"idx_j\"),\n            f.when(f.col(\"i\") == f.col(\"j\"), f.col(\"entry\") / 2)\n            .otherwise(f.col(\"entry\"))\n            .alias(\"r\"),\n        )\n    )\n</code></pre>"},{"location":"python_api/datasources/gnomad/gnomad_ld/#gentropy.datasource.gnomad.ld.GnomADLDMatrix.get_ld_variants","title":"<code>get_ld_variants(gnomad_ancestry: str, chromosome: str, start: int, end: int) -&gt; DataFrame | None</code>","text":"<p>Return melted LD table with resolved variant id based on ancestry and genomic location.</p> <p>Parameters:</p> Name Type Description Default <code>gnomad_ancestry</code> <code>str</code> <p>GnomAD major ancestry label eg. <code>nfe</code></p> required <code>chromosome</code> <code>str</code> <p>chromosome label</p> required <code>start</code> <code>int</code> <p>window upper bound</p> required <code>end</code> <code>int</code> <p>window lower bound</p> required <p>Returns:</p> Type Description <code>DataFrame | None</code> <p>DataFrame | None: LD table with resolved variant id based on ancestry and genomic location</p> Source code in <code>src/gentropy/datasource/gnomad/ld.py</code> <pre><code>def get_ld_variants(\n    self: GnomADLDMatrix,\n    gnomad_ancestry: str,\n    chromosome: str,\n    start: int,\n    end: int,\n) -&gt; DataFrame | None:\n    \"\"\"Return melted LD table with resolved variant id based on ancestry and genomic location.\n\n    Args:\n        gnomad_ancestry (str): GnomAD major ancestry label eg. `nfe`\n        chromosome (str): chromosome label\n        start (int): window upper bound\n        end (int): window lower bound\n\n    Returns:\n        DataFrame | None: LD table with resolved variant id based on ancestry and genomic location\n    \"\"\"\n    # Extracting locus:\n    ld_index_df = (\n        self._process_variant_indices(\n            hl.read_table(self.ld_index_raw_template.format(POP=gnomad_ancestry)),\n            self.grch37_to_grch38_chain_path,\n        )\n        .filter(\n            (f.col(\"chromosome\") == chromosome)\n            &amp; (f.col(\"position\") &gt;= start)\n            &amp; (f.col(\"position\") &lt;= end)\n        )\n        .select(\"chromosome\", \"position\", \"variantId\", \"idx\")\n    )\n\n    if ld_index_df.limit(1).count() == 0:\n        # If the returned slice from the ld index is empty, return None\n        return None\n\n    # Compute start and end indices\n    start_index = get_value_from_row(\n        get_top_ranked_in_window(\n            ld_index_df, Window.partitionBy().orderBy(f.col(\"position\").asc())\n        ).collect()[0],\n        \"idx\",\n    )\n    end_index = get_value_from_row(\n        get_top_ranked_in_window(\n            ld_index_df, Window.partitionBy().orderBy(f.col(\"position\").desc())\n        ).collect()[0],\n        \"idx\",\n    )\n\n    return self._extract_square_matrix(\n        ld_index_df, gnomad_ancestry, start_index, end_index\n    )\n</code></pre>"},{"location":"python_api/datasources/gnomad/gnomad_ld/#gentropy.datasource.gnomad.ld.GnomADLDMatrix.get_locus_index","title":"<code>get_locus_index(study_locus_row: Row, radius: int = 500000, major_population: str = 'nfe') -&gt; DataFrame</code>","text":"<p>Extract hail matrix index from StudyLocus rows.</p> <p>Parameters:</p> Name Type Description Default <code>study_locus_row</code> <code>Row</code> <p>Study-locus row</p> required <code>radius</code> <code>int</code> <p>Locus radius to extract from gnomad matrix</p> <code>500000</code> <code>major_population</code> <code>str</code> <p>Major population to extract from gnomad matrix, default is \"nfe\"</p> <code>'nfe'</code> <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>Returns the index of the gnomad matrix for the locus</p> Source code in <code>src/gentropy/datasource/gnomad/ld.py</code> <pre><code>def get_locus_index(\n    self: GnomADLDMatrix,\n    study_locus_row: Row,\n    radius: int = 500_000,\n    major_population: str = \"nfe\",\n) -&gt; DataFrame:\n    \"\"\"Extract hail matrix index from StudyLocus rows.\n\n    Args:\n        study_locus_row (Row): Study-locus row\n        radius (int): Locus radius to extract from gnomad matrix\n        major_population (str): Major population to extract from gnomad matrix, default is \"nfe\"\n\n    Returns:\n        DataFrame: Returns the index of the gnomad matrix for the locus\n\n    \"\"\"\n    chromosome = str(\"chr\" + study_locus_row[\"chromosome\"])\n    start = study_locus_row[\"position\"] - radius\n    end = study_locus_row[\"position\"] + radius\n\n    liftover_ht = hl.read_table(self.liftover_ht_path)\n    liftover_ht = (\n        liftover_ht.filter(\n            (liftover_ht.locus.contig == chromosome)\n            &amp; (liftover_ht.locus.position &gt;= start)\n            &amp; (liftover_ht.locus.position &lt;= end)\n        )\n        .key_by()\n        .select(\"locus\", \"alleles\", \"original_locus\")\n        .key_by(\"original_locus\", \"alleles\")\n        .naive_coalesce(20)\n    )\n\n    hail_index = hl.read_table(\n        self.ld_index_raw_template.format(POP=major_population)\n    )\n\n    joined_index = (\n        liftover_ht.join(hail_index, how=\"inner\").order_by(\"idx\").to_spark()\n    )\n\n    return joined_index\n</code></pre>"},{"location":"python_api/datasources/gnomad/gnomad_ld/#gentropy.datasource.gnomad.ld.GnomADLDMatrix.get_numpy_matrix","title":"<code>get_numpy_matrix(locus_index: DataFrame, gnomad_ancestry: str = 'nfe') -&gt; np.ndarray</code>  <code>staticmethod</code>","text":"<p>Extract the LD block matrix for a locus.</p> <p>Parameters:</p> Name Type Description Default <code>locus_index</code> <code>DataFrame</code> <p>hail matrix variant index table</p> required <code>gnomad_ancestry</code> <code>str</code> <p>GnomAD major ancestry label eg. <code>nfe</code></p> <code>'nfe'</code> <p>Returns:</p> Type Description <code>ndarray</code> <p>np.ndarray: LD block matrix for the locus</p> Source code in <code>src/gentropy/datasource/gnomad/ld.py</code> <pre><code>@staticmethod\ndef get_numpy_matrix(\n    locus_index: DataFrame,\n    gnomad_ancestry: str = \"nfe\",\n) -&gt; np.ndarray:\n    \"\"\"Extract the LD block matrix for a locus.\n\n    Args:\n        locus_index (DataFrame): hail matrix variant index table\n        gnomad_ancestry (str): GnomAD major ancestry label eg. `nfe`\n\n    Returns:\n        np.ndarray: LD block matrix for the locus\n    \"\"\"\n    idx = [row[\"idx\"] for row in locus_index.select(\"idx\").collect()]\n\n    half_matrix = (\n        BlockMatrix.read(\n            GnomADLDMatrix.ld_matrix_template.format(POP=gnomad_ancestry)\n        )\n        .filter(idx, idx)\n        .to_numpy()\n    )\n\n    return (half_matrix + half_matrix.T) - np.diag(np.diag(half_matrix))\n</code></pre>"},{"location":"python_api/datasources/gnomad/gnomad_variants/","title":"Variants","text":""},{"location":"python_api/datasources/gnomad/gnomad_variants/#gentropy.datasource.gnomad.variants.GnomADVariants","title":"<code>gentropy.datasource.gnomad.variants.GnomADVariants</code>  <code>dataclass</code>","text":"<p>GnomAD variants included in the GnomAD genomes dataset.</p> <p>Attributes:</p> Name Type Description <code>gnomad_genomes</code> <code>str</code> <p>Path to gnomAD genomes hail table. Defaults to gnomAD's 4.0 release.</p> <code>chain_hail_38_37</code> <code>str</code> <p>Path to GRCh38 to GRCh37 chain file. Defaults to Hail's chain file.</p> <code>populations</code> <code>list[str]</code> <p>List of populations to include. Defaults to all populations.</p> Source code in <code>src/gentropy/datasource/gnomad/variants.py</code> <pre><code>@dataclass\nclass GnomADVariants:\n    \"\"\"GnomAD variants included in the GnomAD genomes dataset.\n\n    Attributes:\n        gnomad_genomes (str): Path to gnomAD genomes hail table. Defaults to gnomAD's 4.0 release.\n        chain_hail_38_37 (str): Path to GRCh38 to GRCh37 chain file. Defaults to Hail's chain file.\n        populations (list[str]): List of populations to include. Defaults to all populations.\n    \"\"\"\n\n    gnomad_genomes: str = \"gs://gcp-public-data--gnomad/release/4.0/ht/genomes/gnomad.genomes.v4.0.sites.ht/\"\n    chain_hail_38_37: str = \"gs://hail-common/references/grch38_to_grch37.over.chain.gz\"\n    populations: list[str] = field(\n        default_factory=lambda: [\n            \"afr\",  # African-American\n            \"amr\",  # American Admixed/Latino\n            \"ami\",  # Amish ancestry\n            \"asj\",  # Ashkenazi Jewish\n            \"eas\",  # East Asian\n            \"fin\",  # Finnish\n            \"nfe\",  # Non-Finnish European\n            \"mid\",  # Middle Eastern\n            \"sas\",  # South Asian\n            \"remaining\",  # Other\n        ]\n    )\n\n    def as_variant_annotation(self: GnomADVariants) -&gt; VariantAnnotation:\n        \"\"\"Generate variant annotation dataset from gnomAD.\n\n        Some relevant modifications to the original dataset are:\n\n        1. The transcript consequences features provided by VEP are filtered to only refer to the Ensembl canonical transcript.\n        2. Genome coordinates are liftovered from GRCh38 to GRCh37 to keep as annotation.\n        3. Field names are converted to camel case to follow the convention.\n\n        Returns:\n            VariantAnnotation: Variant annotation dataset\n        \"\"\"\n        # Load variants dataset\n        ht = hl.read_table(\n            self.gnomad_genomes,\n            _load_refs=False,\n        )\n\n        # Liftover\n        grch37 = hl.get_reference(\"GRCh37\")\n        grch38 = hl.get_reference(\"GRCh38\")\n        grch38.add_liftover(self.chain_hail_38_37, grch37)\n\n        # Drop non biallelic variants\n        ht = ht.filter(ht.alleles.length() == 2)\n        # Liftover\n        ht = ht.annotate(locus_GRCh37=hl.liftover(ht.locus, \"GRCh37\"))\n        # Select relevant fields and nested records to create class\n        return VariantAnnotation(\n            _df=(\n                ht.select(\n                    variantId=hl.str(\"_\").join(\n                        [\n                            ht.locus.contig.replace(\"chr\", \"\"),\n                            hl.str(ht.locus.position),\n                            ht.alleles[0],\n                            ht.alleles[1],\n                        ]\n                    ),\n                    chromosome=ht.locus.contig.replace(\"chr\", \"\"),\n                    position=ht.locus.position,\n                    chromosomeB37=ht.locus_GRCh37.contig.replace(\"chr\", \"\"),\n                    positionB37=ht.locus_GRCh37.position,\n                    referenceAllele=ht.alleles[0],\n                    alternateAllele=ht.alleles[1],\n                    rsIds=ht.rsid,\n                    alleleType=ht.allele_info.allele_type,\n                    alleleFrequencies=hl.set(\n                        [f\"{pop}_adj\" for pop in self.populations]\n                    ).map(\n                        lambda p: hl.struct(\n                            populationName=p,\n                            alleleFrequency=ht.freq[ht.globals.freq_index_dict[p]].AF,\n                        )\n                    ),\n                    vep=hl.struct(\n                        mostSevereConsequence=ht.vep.most_severe_consequence,\n                        transcriptConsequences=hl.map(\n                            lambda x: hl.struct(\n                                aminoAcids=x.amino_acids,\n                                consequenceTerms=x.consequence_terms,\n                                geneId=x.gene_id,\n                                lof=x.lof,\n                            ),\n                            # Only keeping canonical transcripts\n                            ht.vep.transcript_consequences.filter(\n                                lambda x: (x.canonical == 1)\n                                &amp; (x.gene_symbol_source == \"HGNC\")\n                            ),\n                        ),\n                    ),\n                    inSilicoPredictors=hl.struct(\n                        cadd=hl.struct(\n                            phred=ht.in_silico_predictors.cadd.phred,\n                            raw=ht.in_silico_predictors.cadd.raw_score,\n                        ),\n                        revelMax=ht.in_silico_predictors.revel_max,\n                        spliceaiDsMax=ht.in_silico_predictors.spliceai_ds_max,\n                        pangolinLargestDs=ht.in_silico_predictors.pangolin_largest_ds,\n                        phylop=ht.in_silico_predictors.phylop,\n                        siftMax=ht.in_silico_predictors.sift_max,\n                        polyphenMax=ht.in_silico_predictors.polyphen_max,\n                    ),\n                )\n                .key_by(\"chromosome\", \"position\")\n                .drop(\"locus\", \"alleles\")\n                .select_globals()\n                .to_spark(flatten=False)\n            ),\n            _schema=VariantAnnotation.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasources/gnomad/gnomad_variants/#gentropy.datasource.gnomad.variants.GnomADVariants.as_variant_annotation","title":"<code>as_variant_annotation() -&gt; VariantAnnotation</code>","text":"<p>Generate variant annotation dataset from gnomAD.</p> <p>Some relevant modifications to the original dataset are:</p> <ol> <li>The transcript consequences features provided by VEP are filtered to only refer to the Ensembl canonical transcript.</li> <li>Genome coordinates are liftovered from GRCh38 to GRCh37 to keep as annotation.</li> <li>Field names are converted to camel case to follow the convention.</li> </ol> <p>Returns:</p> Name Type Description <code>VariantAnnotation</code> <code>VariantAnnotation</code> <p>Variant annotation dataset</p> Source code in <code>src/gentropy/datasource/gnomad/variants.py</code> <pre><code>def as_variant_annotation(self: GnomADVariants) -&gt; VariantAnnotation:\n    \"\"\"Generate variant annotation dataset from gnomAD.\n\n    Some relevant modifications to the original dataset are:\n\n    1. The transcript consequences features provided by VEP are filtered to only refer to the Ensembl canonical transcript.\n    2. Genome coordinates are liftovered from GRCh38 to GRCh37 to keep as annotation.\n    3. Field names are converted to camel case to follow the convention.\n\n    Returns:\n        VariantAnnotation: Variant annotation dataset\n    \"\"\"\n    # Load variants dataset\n    ht = hl.read_table(\n        self.gnomad_genomes,\n        _load_refs=False,\n    )\n\n    # Liftover\n    grch37 = hl.get_reference(\"GRCh37\")\n    grch38 = hl.get_reference(\"GRCh38\")\n    grch38.add_liftover(self.chain_hail_38_37, grch37)\n\n    # Drop non biallelic variants\n    ht = ht.filter(ht.alleles.length() == 2)\n    # Liftover\n    ht = ht.annotate(locus_GRCh37=hl.liftover(ht.locus, \"GRCh37\"))\n    # Select relevant fields and nested records to create class\n    return VariantAnnotation(\n        _df=(\n            ht.select(\n                variantId=hl.str(\"_\").join(\n                    [\n                        ht.locus.contig.replace(\"chr\", \"\"),\n                        hl.str(ht.locus.position),\n                        ht.alleles[0],\n                        ht.alleles[1],\n                    ]\n                ),\n                chromosome=ht.locus.contig.replace(\"chr\", \"\"),\n                position=ht.locus.position,\n                chromosomeB37=ht.locus_GRCh37.contig.replace(\"chr\", \"\"),\n                positionB37=ht.locus_GRCh37.position,\n                referenceAllele=ht.alleles[0],\n                alternateAllele=ht.alleles[1],\n                rsIds=ht.rsid,\n                alleleType=ht.allele_info.allele_type,\n                alleleFrequencies=hl.set(\n                    [f\"{pop}_adj\" for pop in self.populations]\n                ).map(\n                    lambda p: hl.struct(\n                        populationName=p,\n                        alleleFrequency=ht.freq[ht.globals.freq_index_dict[p]].AF,\n                    )\n                ),\n                vep=hl.struct(\n                    mostSevereConsequence=ht.vep.most_severe_consequence,\n                    transcriptConsequences=hl.map(\n                        lambda x: hl.struct(\n                            aminoAcids=x.amino_acids,\n                            consequenceTerms=x.consequence_terms,\n                            geneId=x.gene_id,\n                            lof=x.lof,\n                        ),\n                        # Only keeping canonical transcripts\n                        ht.vep.transcript_consequences.filter(\n                            lambda x: (x.canonical == 1)\n                            &amp; (x.gene_symbol_source == \"HGNC\")\n                        ),\n                    ),\n                ),\n                inSilicoPredictors=hl.struct(\n                    cadd=hl.struct(\n                        phred=ht.in_silico_predictors.cadd.phred,\n                        raw=ht.in_silico_predictors.cadd.raw_score,\n                    ),\n                    revelMax=ht.in_silico_predictors.revel_max,\n                    spliceaiDsMax=ht.in_silico_predictors.spliceai_ds_max,\n                    pangolinLargestDs=ht.in_silico_predictors.pangolin_largest_ds,\n                    phylop=ht.in_silico_predictors.phylop,\n                    siftMax=ht.in_silico_predictors.sift_max,\n                    polyphenMax=ht.in_silico_predictors.polyphen_max,\n                ),\n            )\n            .key_by(\"chromosome\", \"position\")\n            .drop(\"locus\", \"alleles\")\n            .select_globals()\n            .to_spark(flatten=False)\n        ),\n        _schema=VariantAnnotation.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/_gwas_catalog/","title":"GWAS Catalog","text":"GWAS Catalog <p>The GWAS Catalog is a comprehensive resource that aims to provide a curated collection of Genome-Wide Association Studies (GWAS) (including harmonized full GWAS summary statistics) across various traits and diseases in humans.</p> <p>It serves as a valuable repository of genetic associations identified in diverse populations, offering insights into the genetic basis of complex traits and diseases.</p> <p>We rely on the GWAS Catalog for a rich source of genetic associations, utilizing the data for analysis and interpretation.</p> <p>For detailed information on specific genetic associations, their significance, and associated studies, refer to the GWAS Catalog.</p> <p>Within our analyses, we leverage two different types of studies from the GWAS Catalog:</p> <ol> <li> <p>Studies with (full) GWAS summary stats</p> </li> <li> <p>Studies with top hits only - GWAS curated studies</p> </li> </ol>"},{"location":"python_api/datasources/gwas_catalog/associations/","title":"Associations","text":""},{"location":"python_api/datasources/gwas_catalog/associations/#gentropy.datasource.gwas_catalog.associations.GWASCatalogCuratedAssociationsParser","title":"<code>gentropy.datasource.gwas_catalog.associations.GWASCatalogCuratedAssociationsParser</code>  <code>dataclass</code>","text":"<p>GWAS Catalog curated associations parser.</p> Source code in <code>src/gentropy/datasource/gwas_catalog/associations.py</code> <pre><code>@dataclass\nclass GWASCatalogCuratedAssociationsParser:\n    \"\"\"GWAS Catalog curated associations parser.\"\"\"\n\n    @staticmethod\n    def convert_gnomad_position_to_ensembl(\n        position: Column, reference: Column, alternate: Column\n    ) -&gt; Column:\n        \"\"\"Convert GnomAD variant position to Ensembl variant position.\n\n        For indels (the reference or alternate allele is longer than 1), then adding 1 to the position, for SNPs,\n        the position is unchanged. More info about the problem: https://www.biostars.org/p/84686/\n\n        Args:\n            position (Column): Position of the variant in GnomAD's coordinates system.\n            reference (Column): The reference allele in GnomAD's coordinates system.\n            alternate (Column): The alternate allele in GnomAD's coordinates system.\n\n        Returns:\n            Column: The position of the variant in the Ensembl genome.\n\n        Examples:\n            &gt;&gt;&gt; d = [(1, \"A\", \"C\"), (2, \"AA\", \"C\"), (3, \"A\", \"AA\")]\n            &gt;&gt;&gt; df = spark.createDataFrame(d).toDF(\"position\", \"reference\", \"alternate\")\n            &gt;&gt;&gt; df.withColumn(\"new_position\", GWASCatalogCuratedAssociationsParser.convert_gnomad_position_to_ensembl(f.col(\"position\"), f.col(\"reference\"), f.col(\"alternate\"))).show()\n            +--------+---------+---------+------------+\n            |position|reference|alternate|new_position|\n            +--------+---------+---------+------------+\n            |       1|        A|        C|           1|\n            |       2|       AA|        C|           3|\n            |       3|        A|       AA|           4|\n            +--------+---------+---------+------------+\n            &lt;BLANKLINE&gt;\n        \"\"\"\n        return f.when(\n            (f.length(reference) &gt; 1) | (f.length(alternate) &gt; 1), position + 1\n        ).otherwise(position)\n\n    @staticmethod\n    def _parse_pvalue(pvalue: Column) -&gt; tuple[Column, Column]:\n        \"\"\"Parse p-value column.\n\n        Args:\n            pvalue (Column): p-value [string]\n\n        Returns:\n            tuple[Column, Column]: p-value mantissa and exponent\n\n        Example:\n            &gt;&gt;&gt; import pyspark.sql.types as t\n            &gt;&gt;&gt; d = [(\"1.0\"), (\"0.5\"), (\"1E-20\"), (\"3E-3\"), (\"1E-1000\")]\n            &gt;&gt;&gt; df = spark.createDataFrame(d, t.StringType())\n            &gt;&gt;&gt; df.select('value',*GWASCatalogCuratedAssociationsParser._parse_pvalue(f.col('value'))).show()\n            +-------+--------------+--------------+\n            |  value|pValueMantissa|pValueExponent|\n            +-------+--------------+--------------+\n            |    1.0|           1.0|             1|\n            |    0.5|           0.5|             1|\n            |  1E-20|           1.0|           -20|\n            |   3E-3|           3.0|            -3|\n            |1E-1000|           1.0|         -1000|\n            +-------+--------------+--------------+\n            &lt;BLANKLINE&gt;\n\n        \"\"\"\n        split = f.split(pvalue, \"E\")\n        return split.getItem(0).cast(\"float\").alias(\"pValueMantissa\"), f.coalesce(\n            split.getItem(1).cast(\"integer\"), f.lit(1)\n        ).alias(\"pValueExponent\")\n\n    @staticmethod\n    def _normalise_pvaluetext(p_value_text: Column) -&gt; Column:\n        \"\"\"Normalised p-value text column to a standardised format.\n\n        For cases where there is no mapping, the value is set to null.\n\n        Args:\n            p_value_text (Column): `pValueText` column from GWASCatalog\n\n        Returns:\n            Column: Array column after using GWAS Catalog mappings. There might be multiple mappings for a single p-value text.\n\n        Example:\n            &gt;&gt;&gt; import pyspark.sql.types as t\n            &gt;&gt;&gt; d = [(\"European Ancestry\"), (\"African ancestry\"), (\"Alzheimer\u2019s Disease\"), (\"(progression)\"), (\"\"), (None)]\n            &gt;&gt;&gt; df = spark.createDataFrame(d, t.StringType())\n            &gt;&gt;&gt; df.withColumn('normalised', GWASCatalogCuratedAssociationsParser._normalise_pvaluetext(f.col('value'))).show()\n            +-------------------+----------+\n            |              value|normalised|\n            +-------------------+----------+\n            |  European Ancestry|      [EA]|\n            |   African ancestry|      [AA]|\n            |Alzheimer\u2019s Disease|      [AD]|\n            |      (progression)|      null|\n            |                   |      null|\n            |               null|      null|\n            +-------------------+----------+\n            &lt;BLANKLINE&gt;\n\n        \"\"\"\n        # GWAS Catalog to p-value mapping\n        json_dict = json.loads(\n            pkg_resources.read_text(data, \"gwas_pValueText_map.json\", encoding=\"utf-8\")\n        )\n        map_expr = f.create_map(*[f.lit(x) for x in chain(*json_dict.items())])\n\n        splitted_col = f.split(f.regexp_replace(p_value_text, r\"[\\(\\)]\", \"\"), \",\")\n        mapped_col = f.transform(splitted_col, lambda x: map_expr[x])\n        return f.when(f.forall(mapped_col, lambda x: x.isNull()), None).otherwise(\n            mapped_col\n        )\n\n    @staticmethod\n    def _normalise_risk_allele(risk_allele: Column) -&gt; Column:\n        \"\"\"Normalised risk allele column to a standardised format.\n\n        If multiple risk alleles are present, the first one is returned.\n\n        Args:\n            risk_allele (Column): `riskAllele` column from GWASCatalog\n\n        Returns:\n            Column: mapped using GWAS Catalog mapping\n\n        Example:\n            &gt;&gt;&gt; import pyspark.sql.types as t\n            &gt;&gt;&gt; d = [(\"rs1234-A-G\"), (\"rs1234-A\"), (\"rs1234-A; rs1235-G\")]\n            &gt;&gt;&gt; df = spark.createDataFrame(d, t.StringType())\n            &gt;&gt;&gt; df.withColumn('normalised', GWASCatalogCuratedAssociationsParser._normalise_risk_allele(f.col('value'))).show()\n            +------------------+----------+\n            |             value|normalised|\n            +------------------+----------+\n            |        rs1234-A-G|         A|\n            |          rs1234-A|         A|\n            |rs1234-A; rs1235-G|         A|\n            +------------------+----------+\n            &lt;BLANKLINE&gt;\n\n        \"\"\"\n        # GWAS Catalog to risk allele mapping\n        return f.split(f.split(risk_allele, \"; \").getItem(0), \"-\").getItem(1)\n\n    @staticmethod\n    def _collect_rsids(\n        snp_id: Column, snp_id_current: Column, risk_allele: Column\n    ) -&gt; Column:\n        \"\"\"It takes three columns, and returns an array of distinct values from those columns.\n\n        Args:\n            snp_id (Column): The original snp id from the GWAS catalog.\n            snp_id_current (Column): The current snp id field is just a number at the moment (stored as a string). Adding 'rs' prefix if looks good.\n            risk_allele (Column): The risk allele for the SNP.\n\n        Returns:\n            Column: An array of distinct values.\n        \"\"\"\n        # The current snp id field is just a number at the moment (stored as a string). Adding 'rs' prefix if looks good.\n        snp_id_current = f.when(\n            snp_id_current.rlike(\"^[0-9]*$\"),\n            f.format_string(\"rs%s\", snp_id_current),\n        )\n        # Cleaning risk allele:\n        risk_allele = f.split(risk_allele, \"-\").getItem(0)\n\n        # Collecting all values:\n        return f.array_distinct(f.array(snp_id, snp_id_current, risk_allele))\n\n    @staticmethod\n    def _map_to_variant_annotation_variants(\n        gwas_associations: DataFrame, variant_annotation: VariantAnnotation\n    ) -&gt; DataFrame:\n        \"\"\"Add variant metadata in associations.\n\n        Args:\n            gwas_associations (DataFrame): raw GWAS Catalog associations\n            variant_annotation (VariantAnnotation): variant annotation dataset\n\n        Returns:\n            DataFrame: GWAS Catalog associations data including `variantId`, `referenceAllele`,\n            `alternateAllele`, `chromosome`, `position` with variant metadata\n        \"\"\"\n        # Subset of GWAS Catalog associations required for resolving variant IDs:\n        gwas_associations_subset = gwas_associations.select(\n            \"studyLocusId\",\n            f.col(\"CHR_ID\").alias(\"chromosome\"),\n            # The positions from GWAS Catalog are from ensembl that causes discrepancy for indels:\n            f.col(\"CHR_POS\").cast(IntegerType()).alias(\"ensemblPosition\"),\n            # List of all SNPs associated with the variant\n            GWASCatalogCuratedAssociationsParser._collect_rsids(\n                f.split(f.col(\"SNPS\"), \"; \").getItem(0),\n                f.col(\"SNP_ID_CURRENT\"),\n                f.split(f.col(\"STRONGEST SNP-RISK ALLELE\"), \"; \").getItem(0),\n            ).alias(\"rsIdsGwasCatalog\"),\n            GWASCatalogCuratedAssociationsParser._normalise_risk_allele(\n                f.col(\"STRONGEST SNP-RISK ALLELE\")\n            ).alias(\"riskAllele\"),\n        )\n\n        # Subset of variant annotation required for GWAS Catalog annotations:\n        va_subset = variant_annotation.df.select(\n            \"variantId\",\n            \"chromosome\",\n            # Calculate the position in Ensembl coordinates for indels:\n            GWASCatalogCuratedAssociationsParser.convert_gnomad_position_to_ensembl(\n                f.col(\"position\"), f.col(\"referenceAllele\"), f.col(\"alternateAllele\")\n            ).alias(\"ensemblPosition\"),\n            # Keeping GnomAD position:\n            \"position\",\n            f.col(\"rsIds\").alias(\"rsIdsGnomad\"),\n            \"referenceAllele\",\n            \"alternateAllele\",\n            \"alleleFrequencies\",\n            variant_annotation.max_maf().alias(\"maxMaf\"),\n        ).join(\n            f.broadcast(\n                gwas_associations_subset.select(\n                    \"chromosome\", \"ensemblPosition\"\n                ).distinct()\n            ),\n            on=[\"chromosome\", \"ensemblPosition\"],\n            how=\"inner\",\n        )\n\n        # Semi-resolved ids (still contains duplicates when conclusion was not possible to make\n        # based on rsIds or allele concordance)\n        filtered_associations = (\n            gwas_associations_subset.join(\n                f.broadcast(va_subset),\n                on=[\"chromosome\", \"ensemblPosition\"],\n                how=\"left\",\n            )\n            .withColumn(\n                \"rsIdFilter\",\n                GWASCatalogCuratedAssociationsParser._flag_mappings_to_retain(\n                    f.col(\"studyLocusId\"),\n                    GWASCatalogCuratedAssociationsParser._compare_rsids(\n                        f.col(\"rsIdsGnomad\"), f.col(\"rsIdsGwasCatalog\")\n                    ),\n                ),\n            )\n            .withColumn(\n                \"concordanceFilter\",\n                GWASCatalogCuratedAssociationsParser._flag_mappings_to_retain(\n                    f.col(\"studyLocusId\"),\n                    GWASCatalogCuratedAssociationsParser._check_concordance(\n                        f.col(\"riskAllele\"),\n                        f.col(\"referenceAllele\"),\n                        f.col(\"alternateAllele\"),\n                    ),\n                ),\n            )\n            .filter(\n                # Filter out rows where GWAS Catalog rsId does not match with GnomAD rsId,\n                # but there is corresponding variant for the same association\n                f.col(\"rsIdFilter\")\n                # or filter out rows where GWAS Catalog alleles are not concordant with GnomAD alleles,\n                # but there is corresponding variant for the same association\n                | f.col(\"concordanceFilter\")\n            )\n        )\n\n        # Keep only highest maxMaf variant per studyLocusId\n        fully_mapped_associations = get_record_with_maximum_value(\n            filtered_associations, grouping_col=\"studyLocusId\", sorting_col=\"maxMaf\"\n        ).select(\n            \"studyLocusId\",\n            \"variantId\",\n            \"referenceAllele\",\n            \"alternateAllele\",\n            \"chromosome\",\n            \"position\",\n        )\n\n        return gwas_associations.join(\n            fully_mapped_associations, on=\"studyLocusId\", how=\"left\"\n        )\n\n    @staticmethod\n    def _compare_rsids(gnomad: Column, gwas: Column) -&gt; Column:\n        \"\"\"If the intersection of the two arrays is greater than 0, return True, otherwise return False.\n\n        Args:\n            gnomad (Column): rsids from gnomad\n            gwas (Column): rsids from the GWAS Catalog\n\n        Returns:\n            Column: A boolean column that is true if the GnomAD rsIDs can be found in the GWAS rsIDs.\n\n        Examples:\n            &gt;&gt;&gt; d = [\n            ...    (1, [\"rs123\", \"rs523\"], [\"rs123\"]),\n            ...    (2, [], [\"rs123\"]),\n            ...    (3, [\"rs123\", \"rs523\"], []),\n            ...    (4, [], []),\n            ... ]\n            &gt;&gt;&gt; df = spark.createDataFrame(d, ['associationId', 'gnomad', 'gwas'])\n            &gt;&gt;&gt; df.withColumn(\"rsid_matches\", GWASCatalogCuratedAssociationsParser._compare_rsids(f.col(\"gnomad\"),f.col('gwas'))).show()\n            +-------------+--------------+-------+------------+\n            |associationId|        gnomad|   gwas|rsid_matches|\n            +-------------+--------------+-------+------------+\n            |            1|[rs123, rs523]|[rs123]|        true|\n            |            2|            []|[rs123]|       false|\n            |            3|[rs123, rs523]|     []|       false|\n            |            4|            []|     []|       false|\n            +-------------+--------------+-------+------------+\n            &lt;BLANKLINE&gt;\n\n        \"\"\"\n        return f.when(f.size(f.array_intersect(gnomad, gwas)) &gt; 0, True).otherwise(\n            False\n        )\n\n    @staticmethod\n    def _flag_mappings_to_retain(\n        association_id: Column, filter_column: Column\n    ) -&gt; Column:\n        \"\"\"Flagging mappings to drop for each association.\n\n        Some associations have multiple mappings. Some has matching rsId others don't. We only\n        want to drop the non-matching mappings, when a matching is available for the given association.\n        This logic can be generalised for other measures eg. allele concordance.\n\n        Args:\n            association_id (Column): association identifier column\n            filter_column (Column): boolean col indicating to keep a mapping\n\n        Returns:\n            Column: A column with a boolean value.\n\n        Examples:\n        &gt;&gt;&gt; d = [\n        ...    (1, False),\n        ...    (1, False),\n        ...    (2, False),\n        ...    (2, True),\n        ...    (3, True),\n        ...    (3, True),\n        ... ]\n        &gt;&gt;&gt; df = spark.createDataFrame(d, ['associationId', 'filter'])\n        &gt;&gt;&gt; df.withColumn(\"isConcordant\", GWASCatalogCuratedAssociationsParser._flag_mappings_to_retain(f.col(\"associationId\"),f.col('filter'))).show()\n        +-------------+------+------------+\n        |associationId|filter|isConcordant|\n        +-------------+------+------------+\n        |            1| false|        true|\n        |            1| false|        true|\n        |            2| false|       false|\n        |            2|  true|        true|\n        |            3|  true|        true|\n        |            3|  true|        true|\n        +-------------+------+------------+\n        &lt;BLANKLINE&gt;\n\n        \"\"\"\n        w = Window.partitionBy(association_id)\n\n        # Generating a boolean column informing if the filter column contains true anywhere for the association:\n        aggregated_filter = f.when(\n            f.array_contains(f.collect_set(filter_column).over(w), True), True\n        ).otherwise(False)\n\n        # Generate a filter column:\n        return f.when(aggregated_filter &amp; (~filter_column), False).otherwise(True)\n\n    @staticmethod\n    def _check_concordance(\n        risk_allele: Column, reference_allele: Column, alternate_allele: Column\n    ) -&gt; Column:\n        \"\"\"A function to check if the risk allele is concordant with the alt or ref allele.\n\n        If the risk allele is the same as the reference or alternate allele, or if the reverse complement of\n        the risk allele is the same as the reference or alternate allele, then the allele is concordant.\n        If no mapping is available (ref/alt is null), the function returns True.\n\n        Args:\n            risk_allele (Column): The allele that is associated with the risk of the disease.\n            reference_allele (Column): The reference allele from the GWAS catalog\n            alternate_allele (Column): The alternate allele of the variant.\n\n        Returns:\n            Column: A boolean column that is True if the risk allele is the same as the reference or alternate allele,\n            or if the reverse complement of the risk allele is the same as the reference or alternate allele.\n\n        Examples:\n            &gt;&gt;&gt; d = [\n            ...     ('A', 'A', 'G'),\n            ...     ('A', 'T', 'G'),\n            ...     ('A', 'C', 'G'),\n            ...     ('A', 'A', '?'),\n            ...     (None, None, 'A'),\n            ... ]\n            &gt;&gt;&gt; df = spark.createDataFrame(d, ['riskAllele', 'referenceAllele', 'alternateAllele'])\n            &gt;&gt;&gt; df.withColumn(\"isConcordant\", GWASCatalogCuratedAssociationsParser._check_concordance(f.col(\"riskAllele\"),f.col('referenceAllele'), f.col('alternateAllele'))).show()\n            +----------+---------------+---------------+------------+\n            |riskAllele|referenceAllele|alternateAllele|isConcordant|\n            +----------+---------------+---------------+------------+\n            |         A|              A|              G|        true|\n            |         A|              T|              G|        true|\n            |         A|              C|              G|       false|\n            |         A|              A|              ?|        true|\n            |      null|           null|              A|        true|\n            +----------+---------------+---------------+------------+\n            &lt;BLANKLINE&gt;\n\n        \"\"\"\n        # Calculating the reverse complement of the risk allele:\n        risk_allele_reverse_complement = f.when(\n            risk_allele.rlike(r\"^[ACTG]+$\"),\n            f.reverse(f.translate(risk_allele, \"ACTG\", \"TGAC\")),\n        ).otherwise(risk_allele)\n\n        # OK, is the risk allele or the reverse complent is the same as the mapped alleles:\n        return (\n            f.when(\n                (risk_allele == reference_allele) | (risk_allele == alternate_allele),\n                True,\n            )\n            # If risk allele is found on the negative strand:\n            .when(\n                (risk_allele_reverse_complement == reference_allele)\n                | (risk_allele_reverse_complement == alternate_allele),\n                True,\n            )\n            # If risk allele is ambiguous, still accepted: &lt; This condition could be reconsidered\n            .when(risk_allele == \"?\", True)\n            # If the association could not be mapped we keep it:\n            .when(reference_allele.isNull(), True)\n            # Allele is discordant:\n            .otherwise(False)\n        )\n\n    @staticmethod\n    def _get_reverse_complement(allele_col: Column) -&gt; Column:\n        \"\"\"A function to return the reverse complement of an allele column.\n\n        It takes a string and returns the reverse complement of that string if it's a DNA sequence,\n        otherwise it returns the original string. Assumes alleles in upper case.\n\n        Args:\n            allele_col (Column): The column containing the allele to reverse complement.\n\n        Returns:\n            Column: A column that is the reverse complement of the allele column.\n\n        Examples:\n            &gt;&gt;&gt; d = [{\"allele\": 'A'}, {\"allele\": 'T'},{\"allele\": 'G'}, {\"allele\": 'C'},{\"allele\": 'AC'}, {\"allele\": 'GTaatc'},{\"allele\": '?'}, {\"allele\": None}]\n            &gt;&gt;&gt; df = spark.createDataFrame(d)\n            &gt;&gt;&gt; df.withColumn(\"revcom_allele\", GWASCatalogCuratedAssociationsParser._get_reverse_complement(f.col(\"allele\"))).show()\n            +------+-------------+\n            |allele|revcom_allele|\n            +------+-------------+\n            |     A|            T|\n            |     T|            A|\n            |     G|            C|\n            |     C|            G|\n            |    AC|           GT|\n            |GTaatc|       GATTAC|\n            |     ?|            ?|\n            |  null|         null|\n            +------+-------------+\n            &lt;BLANKLINE&gt;\n\n        \"\"\"\n        allele_col = f.upper(allele_col)\n        return f.when(\n            allele_col.rlike(\"[ACTG]+\"),\n            f.reverse(f.translate(allele_col, \"ACTG\", \"TGAC\")),\n        ).otherwise(allele_col)\n\n    @staticmethod\n    def _effect_needs_harmonisation(\n        risk_allele: Column, reference_allele: Column\n    ) -&gt; Column:\n        \"\"\"A function to check if the effect allele needs to be harmonised.\n\n        Args:\n            risk_allele (Column): Risk allele column\n            reference_allele (Column): Effect allele column\n\n        Returns:\n            Column: A boolean column indicating if the effect allele needs to be harmonised.\n\n        Examples:\n            &gt;&gt;&gt; d = [{\"risk\": 'A', \"reference\": 'A'}, {\"risk\": 'A', \"reference\": 'T'}, {\"risk\": 'AT', \"reference\": 'TA'}, {\"risk\": 'AT', \"reference\": 'AT'}]\n            &gt;&gt;&gt; df = spark.createDataFrame(d)\n            &gt;&gt;&gt; df.withColumn(\"needs_harmonisation\", GWASCatalogCuratedAssociationsParser._effect_needs_harmonisation(f.col(\"risk\"), f.col(\"reference\"))).show()\n            +---------+----+-------------------+\n            |reference|risk|needs_harmonisation|\n            +---------+----+-------------------+\n            |        A|   A|               true|\n            |        T|   A|               true|\n            |       TA|  AT|              false|\n            |       AT|  AT|               true|\n            +---------+----+-------------------+\n            &lt;BLANKLINE&gt;\n\n        \"\"\"\n        return (risk_allele == reference_allele) | (\n            risk_allele\n            == GWASCatalogCuratedAssociationsParser._get_reverse_complement(\n                reference_allele\n            )\n        )\n\n    @staticmethod\n    def _are_alleles_palindromic(\n        reference_allele: Column, alternate_allele: Column\n    ) -&gt; Column:\n        \"\"\"A function to check if the alleles are palindromic.\n\n        Args:\n            reference_allele (Column): Reference allele column\n            alternate_allele (Column): Alternate allele column\n\n        Returns:\n            Column: A boolean column indicating if the alleles are palindromic.\n\n        Examples:\n            &gt;&gt;&gt; d = [{\"reference\": 'A', \"alternate\": 'T'}, {\"reference\": 'AT', \"alternate\": 'AG'}, {\"reference\": 'AT', \"alternate\": 'AT'}, {\"reference\": 'CATATG', \"alternate\": 'CATATG'}, {\"reference\": '-', \"alternate\": None}]\n            &gt;&gt;&gt; df = spark.createDataFrame(d)\n            &gt;&gt;&gt; df.withColumn(\"is_palindromic\", GWASCatalogCuratedAssociationsParser._are_alleles_palindromic(f.col(\"reference\"), f.col(\"alternate\"))).show()\n            +---------+---------+--------------+\n            |alternate|reference|is_palindromic|\n            +---------+---------+--------------+\n            |        T|        A|          true|\n            |       AG|       AT|         false|\n            |       AT|       AT|          true|\n            |   CATATG|   CATATG|          true|\n            |     null|        -|         false|\n            +---------+---------+--------------+\n            &lt;BLANKLINE&gt;\n\n        \"\"\"\n        revcomp = GWASCatalogCuratedAssociationsParser._get_reverse_complement(\n            alternate_allele\n        )\n        return (\n            f.when(reference_allele == revcomp, True)\n            .when(revcomp.isNull(), False)\n            .otherwise(False)\n        )\n\n    @staticmethod\n    def _harmonise_beta(\n        risk_allele: Column,\n        reference_allele: Column,\n        alternate_allele: Column,\n        effect_size: Column,\n        confidence_interval: Column,\n    ) -&gt; Column:\n        \"\"\"A function to extract the beta value from the effect size and confidence interval.\n\n        If the confidence interval contains the word \"increase\" or \"decrease\" it indicates, we are dealing with betas.\n        If it's \"increase\" and the effect size needs to be harmonized, then multiply the effect size by -1\n\n        Args:\n            risk_allele (Column): Risk allele column\n            reference_allele (Column): Reference allele column\n            alternate_allele (Column): Alternate allele column\n            effect_size (Column): GWAS Catalog effect size column\n            confidence_interval (Column): GWAS Catalog confidence interval column\n\n        Returns:\n            Column: A column containing the beta value.\n        \"\"\"\n        return (\n            f.when(\n                GWASCatalogCuratedAssociationsParser._are_alleles_palindromic(\n                    reference_allele, alternate_allele\n                ),\n                None,\n            )\n            .when(\n                (\n                    GWASCatalogCuratedAssociationsParser._effect_needs_harmonisation(\n                        risk_allele, reference_allele\n                    )\n                    &amp; confidence_interval.contains(\"increase\")\n                )\n                | (\n                    ~GWASCatalogCuratedAssociationsParser._effect_needs_harmonisation(\n                        risk_allele, reference_allele\n                    )\n                    &amp; confidence_interval.contains(\"decrease\")\n                ),\n                -effect_size,\n            )\n            .otherwise(effect_size)\n            .cast(DoubleType())\n        )\n\n    @staticmethod\n    def _harmonise_beta_ci(\n        risk_allele: Column,\n        reference_allele: Column,\n        alternate_allele: Column,\n        effect_size: Column,\n        confidence_interval: Column,\n        p_value: Column,\n        direction: str,\n    ) -&gt; Column:\n        \"\"\"Calculating confidence intervals for beta values.\n\n        Args:\n            risk_allele (Column): Risk allele column\n            reference_allele (Column): Reference allele column\n            alternate_allele (Column): Alternate allele column\n            effect_size (Column): GWAS Catalog effect size column\n            confidence_interval (Column): GWAS Catalog confidence interval column\n            p_value (Column): GWAS Catalog p-value column\n            direction (str): This is the direction of the confidence interval. It can be either \"upper\" or \"lower\".\n\n        Returns:\n            Column: The upper and lower bounds of the confidence interval for the beta coefficient.\n        \"\"\"\n        zscore_95 = f.lit(1.96)\n        beta = GWASCatalogCuratedAssociationsParser._harmonise_beta(\n            risk_allele,\n            reference_allele,\n            alternate_allele,\n            effect_size,\n            confidence_interval,\n        )\n        zscore = pvalue_to_zscore(p_value)\n        return (\n            f.when(f.lit(direction) == \"upper\", beta + f.abs(zscore_95 * beta) / zscore)\n            .when(f.lit(direction) == \"lower\", beta - f.abs(zscore_95 * beta) / zscore)\n            .otherwise(None)\n        )\n\n    @staticmethod\n    def _harmonise_odds_ratio(\n        risk_allele: Column,\n        reference_allele: Column,\n        alternate_allele: Column,\n        effect_size: Column,\n        confidence_interval: Column,\n    ) -&gt; Column:\n        \"\"\"Harmonizing odds ratio.\n\n        Args:\n            risk_allele (Column): Risk allele column\n            reference_allele (Column): Reference allele column\n            alternate_allele (Column): Alternate allele column\n            effect_size (Column): GWAS Catalog effect size column\n            confidence_interval (Column): GWAS Catalog confidence interval column\n\n        Returns:\n            Column: A column with the odds ratio, or 1/odds_ratio if harmonization required.\n        \"\"\"\n        return (\n            f.when(\n                GWASCatalogCuratedAssociationsParser._are_alleles_palindromic(\n                    reference_allele, alternate_allele\n                ),\n                None,\n            )\n            .when(\n                (\n                    GWASCatalogCuratedAssociationsParser._effect_needs_harmonisation(\n                        risk_allele, reference_allele\n                    )\n                    &amp; ~confidence_interval.rlike(\"|\".join([\"decrease\", \"increase\"]))\n                ),\n                1 / effect_size,\n            )\n            .otherwise(effect_size)\n            .cast(DoubleType())\n        )\n\n    @staticmethod\n    def _harmonise_odds_ratio_ci(\n        risk_allele: Column,\n        reference_allele: Column,\n        alternate_allele: Column,\n        effect_size: Column,\n        confidence_interval: Column,\n        p_value: Column,\n        direction: str,\n    ) -&gt; Column:\n        \"\"\"Calculating confidence intervals for beta values.\n\n        Args:\n            risk_allele (Column): Risk allele column\n            reference_allele (Column): Reference allele column\n            alternate_allele (Column): Alternate allele column\n            effect_size (Column): GWAS Catalog effect size column\n            confidence_interval (Column): GWAS Catalog confidence interval column\n            p_value (Column): GWAS Catalog p-value column\n            direction (str): This is the direction of the confidence interval. It can be either \"upper\" or \"lower\".\n\n        Returns:\n            Column: The upper and lower bounds of the 95% confidence interval for the odds ratio.\n        \"\"\"\n        zscore_95 = f.lit(1.96)\n        odds_ratio = GWASCatalogCuratedAssociationsParser._harmonise_odds_ratio(\n            risk_allele,\n            reference_allele,\n            alternate_allele,\n            effect_size,\n            confidence_interval,\n        )\n        odds_ratio_estimate = f.log(odds_ratio)\n        zscore = pvalue_to_zscore(p_value)\n        odds_ratio_se = odds_ratio_estimate / zscore\n        return f.when(\n            f.lit(direction) == \"upper\",\n            f.exp(odds_ratio_estimate + f.abs(zscore_95 * odds_ratio_se)),\n        ).when(\n            f.lit(direction) == \"lower\",\n            f.exp(odds_ratio_estimate - f.abs(zscore_95 * odds_ratio_se)),\n        )\n\n    @staticmethod\n    def _concatenate_substudy_description(\n        association_trait: Column, pvalue_text: Column, mapped_trait_uri: Column\n    ) -&gt; Column:\n        \"\"\"Substudy description parsing. Complex string containing metadata about the substudy (e.g. QTL, specific EFO, etc.).\n\n        Args:\n            association_trait (Column): GWAS Catalog association trait column\n            pvalue_text (Column): GWAS Catalog p-value text column\n            mapped_trait_uri (Column): GWAS Catalog mapped trait URI column\n\n        Returns:\n            Column: A column with the substudy description in the shape trait|pvaluetext1_pvaluetext2|EFO1_EFO2.\n\n        Examples:\n        &gt;&gt;&gt; df = spark.createDataFrame([\n        ...    (\"Height\", \"http://www.ebi.ac.uk/efo/EFO_0000001,http://www.ebi.ac.uk/efo/EFO_0000002\", \"European Ancestry\"),\n        ...    (\"Schizophrenia\", \"http://www.ebi.ac.uk/efo/MONDO_0005090\", None)],\n        ...    [\"association_trait\", \"mapped_trait_uri\", \"pvalue_text\"]\n        ... )\n        &gt;&gt;&gt; df.withColumn('substudy_description', GWASCatalogCuratedAssociationsParser._concatenate_substudy_description(df.association_trait, df.pvalue_text, df.mapped_trait_uri)).show(truncate=False)\n        +-----------------+-------------------------------------------------------------------------+-----------------+------------------------------------------+\n        |association_trait|mapped_trait_uri                                                         |pvalue_text      |substudy_description                      |\n        +-----------------+-------------------------------------------------------------------------+-----------------+------------------------------------------+\n        |Height           |http://www.ebi.ac.uk/efo/EFO_0000001,http://www.ebi.ac.uk/efo/EFO_0000002|European Ancestry|Height|EA|EFO_0000001/EFO_0000002         |\n        |Schizophrenia    |http://www.ebi.ac.uk/efo/MONDO_0005090                                   |null             |Schizophrenia|no_pvalue_text|MONDO_0005090|\n        +-----------------+-------------------------------------------------------------------------+-----------------+------------------------------------------+\n        &lt;BLANKLINE&gt;\n        \"\"\"\n        p_value_text = f.coalesce(\n            GWASCatalogCuratedAssociationsParser._normalise_pvaluetext(pvalue_text),\n            f.array(f.lit(\"no_pvalue_text\")),\n        )\n        return f.concat_ws(\n            \"|\",\n            association_trait,\n            f.concat_ws(\n                \"/\",\n                p_value_text,\n            ),\n            f.concat_ws(\n                \"/\",\n                parse_efos(mapped_trait_uri),\n            ),\n        )\n\n    @staticmethod\n    def _qc_all(\n        qc: Column,\n        chromosome: Column,\n        position: Column,\n        reference_allele: Column,\n        alternate_allele: Column,\n        strongest_snp_risk_allele: Column,\n        p_value_mantissa: Column,\n        p_value_exponent: Column,\n        p_value_cutoff: float,\n    ) -&gt; Column:\n        \"\"\"Flag associations that fail any QC.\n\n        Args:\n            qc (Column): QC column\n            chromosome (Column): Chromosome column\n            position (Column): Position column\n            reference_allele (Column): Reference allele column\n            alternate_allele (Column): Alternate allele column\n            strongest_snp_risk_allele (Column): Strongest SNP risk allele column\n            p_value_mantissa (Column): P-value mantissa column\n            p_value_exponent (Column): P-value exponent column\n            p_value_cutoff (float): P-value cutoff\n\n        Returns:\n            Column: Updated QC column with flag.\n        \"\"\"\n        qc = GWASCatalogCuratedAssociationsParser._qc_variant_interactions(\n            qc, strongest_snp_risk_allele\n        )\n        qc = GWASCatalogCuratedAssociationsParser._qc_subsignificant_associations(\n            qc, p_value_mantissa, p_value_exponent, p_value_cutoff\n        )\n        qc = GWASCatalogCuratedAssociationsParser._qc_genomic_location(\n            qc, chromosome, position\n        )\n        qc = GWASCatalogCuratedAssociationsParser._qc_variant_inconsistencies(\n            qc, chromosome, position, strongest_snp_risk_allele\n        )\n        qc = GWASCatalogCuratedAssociationsParser._qc_unmapped_variants(\n            qc, alternate_allele\n        )\n        qc = GWASCatalogCuratedAssociationsParser._qc_palindromic_alleles(\n            qc, reference_allele, alternate_allele\n        )\n        return qc\n\n    @staticmethod\n    def _qc_variant_interactions(\n        qc: Column, strongest_snp_risk_allele: Column\n    ) -&gt; Column:\n        \"\"\"Flag associations based on variant x variant interactions.\n\n        Args:\n            qc (Column): QC column\n            strongest_snp_risk_allele (Column): Column with the strongest SNP risk allele\n\n        Returns:\n            Column: Updated QC column with flag.\n        \"\"\"\n        return StudyLocus.update_quality_flag(\n            qc,\n            strongest_snp_risk_allele.contains(\";\"),\n            StudyLocusQualityCheck.COMPOSITE_FLAG,\n        )\n\n    @staticmethod\n    def _qc_subsignificant_associations(\n        qc: Column,\n        p_value_mantissa: Column,\n        p_value_exponent: Column,\n        pvalue_cutoff: float,\n    ) -&gt; Column:\n        \"\"\"Flag associations below significant threshold.\n\n        Args:\n            qc (Column): QC column\n            p_value_mantissa (Column): P-value mantissa column\n            p_value_exponent (Column): P-value exponent column\n            pvalue_cutoff (float): association p-value cut-off\n\n        Returns:\n            Column: Updated QC column with flag.\n\n        Examples:\n            &gt;&gt;&gt; import pyspark.sql.types as t\n            &gt;&gt;&gt; d = [{'qc': None, 'p_value_mantissa': 1, 'p_value_exponent': -7}, {'qc': None, 'p_value_mantissa': 1, 'p_value_exponent': -8}, {'qc': None, 'p_value_mantissa': 5, 'p_value_exponent': -8}, {'qc': None, 'p_value_mantissa': 1, 'p_value_exponent': -9}]\n            &gt;&gt;&gt; df = spark.createDataFrame(d, t.StructType([t.StructField('qc', t.ArrayType(t.StringType()), True), t.StructField('p_value_mantissa', t.IntegerType()), t.StructField('p_value_exponent', t.IntegerType())]))\n            &gt;&gt;&gt; df.withColumn('qc', GWASCatalogCuratedAssociationsParser._qc_subsignificant_associations(f.col(\"qc\"), f.col(\"p_value_mantissa\"), f.col(\"p_value_exponent\"), 5e-8)).show(truncate = False)\n            +------------------------+----------------+----------------+\n            |qc                      |p_value_mantissa|p_value_exponent|\n            +------------------------+----------------+----------------+\n            |[Subsignificant p-value]|1               |-7              |\n            |[]                      |1               |-8              |\n            |[]                      |5               |-8              |\n            |[]                      |1               |-9              |\n            +------------------------+----------------+----------------+\n            &lt;BLANKLINE&gt;\n\n        \"\"\"\n        return StudyLocus.update_quality_flag(\n            qc,\n            calculate_neglog_pvalue(p_value_mantissa, p_value_exponent)\n            &lt; f.lit(-np.log10(pvalue_cutoff)),\n            StudyLocusQualityCheck.SUBSIGNIFICANT_FLAG,\n        )\n\n    @staticmethod\n    def _qc_genomic_location(\n        qc: Column, chromosome: Column, position: Column\n    ) -&gt; Column:\n        \"\"\"Flag associations without genomic location in GWAS Catalog.\n\n        Args:\n            qc (Column): QC column\n            chromosome (Column): Chromosome column in GWAS Catalog\n            position (Column): Position column in GWAS Catalog\n\n        Returns:\n            Column: Updated QC column with flag.\n\n        Examples:\n            &gt;&gt;&gt; import pyspark.sql.types as t\n            &gt;&gt;&gt; d = [{'qc': None, 'chromosome': None, 'position': None}, {'qc': None, 'chromosome': '1', 'position': None}, {'qc': None, 'chromosome': None, 'position': 1}, {'qc': None, 'chromosome': '1', 'position': 1}]\n            &gt;&gt;&gt; df = spark.createDataFrame(d, schema=t.StructType([t.StructField('qc', t.ArrayType(t.StringType()), True), t.StructField('chromosome', t.StringType()), t.StructField('position', t.IntegerType())]))\n            &gt;&gt;&gt; df.withColumn('qc', GWASCatalogCuratedAssociationsParser._qc_genomic_location(df.qc, df.chromosome, df.position)).show(truncate=False)\n            +----------------------------+----------+--------+\n            |qc                          |chromosome|position|\n            +----------------------------+----------+--------+\n            |[Incomplete genomic mapping]|null      |null    |\n            |[Incomplete genomic mapping]|1         |null    |\n            |[Incomplete genomic mapping]|null      |1       |\n            |[]                          |1         |1       |\n            +----------------------------+----------+--------+\n            &lt;BLANKLINE&gt;\n\n        \"\"\"\n        return StudyLocus.update_quality_flag(\n            qc,\n            position.isNull() | chromosome.isNull(),\n            StudyLocusQualityCheck.NO_GENOMIC_LOCATION_FLAG,\n        )\n\n    @staticmethod\n    def _qc_variant_inconsistencies(\n        qc: Column,\n        chromosome: Column,\n        position: Column,\n        strongest_snp_risk_allele: Column,\n    ) -&gt; Column:\n        \"\"\"Flag associations with inconsistencies in the variant annotation.\n\n        Args:\n            qc (Column): QC column\n            chromosome (Column): Chromosome column in GWAS Catalog\n            position (Column): Position column in GWAS Catalog\n            strongest_snp_risk_allele (Column): Strongest SNP risk allele column in GWAS Catalog\n\n        Returns:\n            Column: Updated QC column with flag.\n        \"\"\"\n        return StudyLocus.update_quality_flag(\n            qc,\n            # Number of chromosomes does not correspond to the number of positions:\n            (f.size(f.split(chromosome, \";\")) != f.size(f.split(position, \";\")))\n            # Number of chromosome values different from riskAllele values:\n            | (\n                f.size(f.split(chromosome, \";\"))\n                != f.size(f.split(strongest_snp_risk_allele, \";\"))\n            ),\n            StudyLocusQualityCheck.INCONSISTENCY_FLAG,\n        )\n\n    @staticmethod\n    def _qc_unmapped_variants(qc: Column, alternate_allele: Column) -&gt; Column:\n        \"\"\"Flag associations with variants not mapped to variantAnnotation.\n\n        Args:\n            qc (Column): QC column\n            alternate_allele (Column): alternate allele\n\n        Returns:\n            Column: Updated QC column with flag.\n\n        Example:\n            &gt;&gt;&gt; import pyspark.sql.types as t\n            &gt;&gt;&gt; d = [{'alternate_allele': 'A', 'qc': None}, {'alternate_allele': None, 'qc': None}]\n            &gt;&gt;&gt; schema = t.StructType([t.StructField('alternate_allele', t.StringType(), True), t.StructField('qc', t.ArrayType(t.StringType()), True)])\n            &gt;&gt;&gt; df = spark.createDataFrame(data=d, schema=schema)\n            &gt;&gt;&gt; df.withColumn(\"new_qc\", GWASCatalogCuratedAssociationsParser._qc_unmapped_variants(f.col(\"qc\"), f.col(\"alternate_allele\"))).show()\n            +----------------+----+--------------------+\n            |alternate_allele|  qc|              new_qc|\n            +----------------+----+--------------------+\n            |               A|null|                  []|\n            |            null|null|[No mapping in Gn...|\n            +----------------+----+--------------------+\n            &lt;BLANKLINE&gt;\n\n        \"\"\"\n        return StudyLocus.update_quality_flag(\n            qc,\n            alternate_allele.isNull(),\n            StudyLocusQualityCheck.NON_MAPPED_VARIANT_FLAG,\n        )\n\n    @staticmethod\n    def _qc_palindromic_alleles(\n        qc: Column, reference_allele: Column, alternate_allele: Column\n    ) -&gt; Column:\n        \"\"\"Flag associations with palindromic variants which effects can not be harmonised.\n\n        Args:\n            qc (Column): QC column\n            reference_allele (Column): reference allele\n            alternate_allele (Column): alternate allele\n\n        Returns:\n            Column: Updated QC column with flag.\n\n        Example:\n            &gt;&gt;&gt; import pyspark.sql.types as t\n            &gt;&gt;&gt; schema = t.StructType([t.StructField('reference_allele', t.StringType(), True), t.StructField('alternate_allele', t.StringType(), True), t.StructField('qc', t.ArrayType(t.StringType()), True)])\n            &gt;&gt;&gt; d = [{'reference_allele': 'A', 'alternate_allele': 'T', 'qc': None}, {'reference_allele': 'AT', 'alternate_allele': 'TA', 'qc': None}, {'reference_allele': 'AT', 'alternate_allele': 'AT', 'qc': None}]\n            &gt;&gt;&gt; df = spark.createDataFrame(data=d, schema=schema)\n            &gt;&gt;&gt; df.withColumn(\"qc\", GWASCatalogCuratedAssociationsParser._qc_palindromic_alleles(f.col(\"qc\"), f.col(\"reference_allele\"), f.col(\"alternate_allele\"))).show(truncate=False)\n            +----------------+----------------+---------------------------------------+\n            |reference_allele|alternate_allele|qc                                     |\n            +----------------+----------------+---------------------------------------+\n            |A               |T               |[Palindrome alleles - cannot harmonize]|\n            |AT              |TA              |[]                                     |\n            |AT              |AT              |[Palindrome alleles - cannot harmonize]|\n            +----------------+----------------+---------------------------------------+\n            &lt;BLANKLINE&gt;\n\n        \"\"\"\n        return StudyLocus.update_quality_flag(\n            qc,\n            GWASCatalogCuratedAssociationsParser._are_alleles_palindromic(\n                reference_allele, alternate_allele\n            ),\n            StudyLocusQualityCheck.PALINDROMIC_ALLELE_FLAG,\n        )\n\n    @classmethod\n    def from_source(\n        cls: type[GWASCatalogCuratedAssociationsParser],\n        gwas_associations: DataFrame,\n        variant_annotation: VariantAnnotation,\n        pvalue_threshold: float = 5e-8,\n    ) -&gt; StudyLocusGWASCatalog:\n        \"\"\"Read GWASCatalog associations.\n\n        It reads the GWAS Catalog association dataset, selects and renames columns, casts columns, and\n        applies some pre-defined filters on the data:\n\n        Args:\n            gwas_associations (DataFrame): GWAS Catalog raw associations dataset\n            variant_annotation (VariantAnnotation): Variant annotation dataset\n            pvalue_threshold (float): P-value threshold for flagging associations\n\n        Returns:\n            StudyLocusGWASCatalog: GWASCatalogAssociations dataset\n        \"\"\"\n        return StudyLocusGWASCatalog(\n            _df=gwas_associations.withColumn(\n                \"studyLocusId\", f.monotonically_increasing_id().cast(LongType())\n            )\n            .transform(\n                # Map/harmonise variants to variant annotation dataset:\n                # This function adds columns: variantId, referenceAllele, alternateAllele, chromosome, position\n                lambda df: GWASCatalogCuratedAssociationsParser._map_to_variant_annotation_variants(\n                    df, variant_annotation\n                )\n            )\n            .withColumn(\n                # Perform all quality control checks:\n                \"qualityControls\",\n                GWASCatalogCuratedAssociationsParser._qc_all(\n                    f.array().alias(\"qualityControls\"),\n                    f.col(\"CHR_ID\"),\n                    f.col(\"CHR_POS\").cast(IntegerType()),\n                    f.col(\"referenceAllele\"),\n                    f.col(\"alternateAllele\"),\n                    f.col(\"STRONGEST SNP-RISK ALLELE\"),\n                    *GWASCatalogCuratedAssociationsParser._parse_pvalue(\n                        f.col(\"P-VALUE\")\n                    ),\n                    pvalue_threshold,\n                ),\n            )\n            .select(\n                # INSIDE STUDY-LOCUS SCHEMA:\n                \"studyLocusId\",\n                \"variantId\",\n                # Mapped genomic location of the variant (; separated list)\n                \"chromosome\",\n                \"position\",\n                f.col(\"STUDY ACCESSION\").alias(\"studyId\"),\n                # beta value of the association\n                GWASCatalogCuratedAssociationsParser._harmonise_beta(\n                    GWASCatalogCuratedAssociationsParser._normalise_risk_allele(\n                        f.col(\"STRONGEST SNP-RISK ALLELE\")\n                    ),\n                    f.col(\"referenceAllele\"),\n                    f.col(\"alternateAllele\"),\n                    f.col(\"OR or BETA\"),\n                    f.col(\"95% CI (TEXT)\"),\n                ).alias(\"beta\"),\n                # p-value of the association, string: split into exponent and mantissa.\n                *GWASCatalogCuratedAssociationsParser._parse_pvalue(f.col(\"P-VALUE\")),\n                # Capturing phenotype granularity at the association level\n                GWASCatalogCuratedAssociationsParser._concatenate_substudy_description(\n                    f.col(\"DISEASE/TRAIT\"),\n                    f.col(\"P-VALUE (TEXT)\"),\n                    f.col(\"MAPPED_TRAIT_URI\"),\n                ).alias(\"subStudyDescription\"),\n                # Quality controls (array of strings)\n                \"qualityControls\",\n            ),\n            _schema=StudyLocusGWASCatalog.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/associations/#gentropy.datasource.gwas_catalog.associations.GWASCatalogCuratedAssociationsParser.convert_gnomad_position_to_ensembl","title":"<code>convert_gnomad_position_to_ensembl(position: Column, reference: Column, alternate: Column) -&gt; Column</code>  <code>staticmethod</code>","text":"<p>Convert GnomAD variant position to Ensembl variant position.</p> <p>For indels (the reference or alternate allele is longer than 1), then adding 1 to the position, for SNPs, the position is unchanged. More info about the problem: https://www.biostars.org/p/84686/</p> <p>Parameters:</p> Name Type Description Default <code>position</code> <code>Column</code> <p>Position of the variant in GnomAD's coordinates system.</p> required <code>reference</code> <code>Column</code> <p>The reference allele in GnomAD's coordinates system.</p> required <code>alternate</code> <code>Column</code> <p>The alternate allele in GnomAD's coordinates system.</p> required <p>Returns:</p> Name Type Description <code>Column</code> <code>Column</code> <p>The position of the variant in the Ensembl genome.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; d = [(1, \"A\", \"C\"), (2, \"AA\", \"C\"), (3, \"A\", \"AA\")]\n&gt;&gt;&gt; df = spark.createDataFrame(d).toDF(\"position\", \"reference\", \"alternate\")\n&gt;&gt;&gt; df.withColumn(\"new_position\", GWASCatalogCuratedAssociationsParser.convert_gnomad_position_to_ensembl(f.col(\"position\"), f.col(\"reference\"), f.col(\"alternate\"))).show()\n+--------+---------+---------+------------+\n|position|reference|alternate|new_position|\n+--------+---------+---------+------------+\n|       1|        A|        C|           1|\n|       2|       AA|        C|           3|\n|       3|        A|       AA|           4|\n+--------+---------+---------+------------+\n</code></pre> Source code in <code>src/gentropy/datasource/gwas_catalog/associations.py</code> <pre><code>@staticmethod\ndef convert_gnomad_position_to_ensembl(\n    position: Column, reference: Column, alternate: Column\n) -&gt; Column:\n    \"\"\"Convert GnomAD variant position to Ensembl variant position.\n\n    For indels (the reference or alternate allele is longer than 1), then adding 1 to the position, for SNPs,\n    the position is unchanged. More info about the problem: https://www.biostars.org/p/84686/\n\n    Args:\n        position (Column): Position of the variant in GnomAD's coordinates system.\n        reference (Column): The reference allele in GnomAD's coordinates system.\n        alternate (Column): The alternate allele in GnomAD's coordinates system.\n\n    Returns:\n        Column: The position of the variant in the Ensembl genome.\n\n    Examples:\n        &gt;&gt;&gt; d = [(1, \"A\", \"C\"), (2, \"AA\", \"C\"), (3, \"A\", \"AA\")]\n        &gt;&gt;&gt; df = spark.createDataFrame(d).toDF(\"position\", \"reference\", \"alternate\")\n        &gt;&gt;&gt; df.withColumn(\"new_position\", GWASCatalogCuratedAssociationsParser.convert_gnomad_position_to_ensembl(f.col(\"position\"), f.col(\"reference\"), f.col(\"alternate\"))).show()\n        +--------+---------+---------+------------+\n        |position|reference|alternate|new_position|\n        +--------+---------+---------+------------+\n        |       1|        A|        C|           1|\n        |       2|       AA|        C|           3|\n        |       3|        A|       AA|           4|\n        +--------+---------+---------+------------+\n        &lt;BLANKLINE&gt;\n    \"\"\"\n    return f.when(\n        (f.length(reference) &gt; 1) | (f.length(alternate) &gt; 1), position + 1\n    ).otherwise(position)\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/associations/#gentropy.datasource.gwas_catalog.associations.GWASCatalogCuratedAssociationsParser.from_source","title":"<code>from_source(gwas_associations: DataFrame, variant_annotation: VariantAnnotation, pvalue_threshold: float = 5e-08) -&gt; StudyLocusGWASCatalog</code>  <code>classmethod</code>","text":"<p>Read GWASCatalog associations.</p> <p>It reads the GWAS Catalog association dataset, selects and renames columns, casts columns, and applies some pre-defined filters on the data:</p> <p>Parameters:</p> Name Type Description Default <code>gwas_associations</code> <code>DataFrame</code> <p>GWAS Catalog raw associations dataset</p> required <code>variant_annotation</code> <code>VariantAnnotation</code> <p>Variant annotation dataset</p> required <code>pvalue_threshold</code> <code>float</code> <p>P-value threshold for flagging associations</p> <code>5e-08</code> <p>Returns:</p> Name Type Description <code>StudyLocusGWASCatalog</code> <code>StudyLocusGWASCatalog</code> <p>GWASCatalogAssociations dataset</p> Source code in <code>src/gentropy/datasource/gwas_catalog/associations.py</code> <pre><code>@classmethod\ndef from_source(\n    cls: type[GWASCatalogCuratedAssociationsParser],\n    gwas_associations: DataFrame,\n    variant_annotation: VariantAnnotation,\n    pvalue_threshold: float = 5e-8,\n) -&gt; StudyLocusGWASCatalog:\n    \"\"\"Read GWASCatalog associations.\n\n    It reads the GWAS Catalog association dataset, selects and renames columns, casts columns, and\n    applies some pre-defined filters on the data:\n\n    Args:\n        gwas_associations (DataFrame): GWAS Catalog raw associations dataset\n        variant_annotation (VariantAnnotation): Variant annotation dataset\n        pvalue_threshold (float): P-value threshold for flagging associations\n\n    Returns:\n        StudyLocusGWASCatalog: GWASCatalogAssociations dataset\n    \"\"\"\n    return StudyLocusGWASCatalog(\n        _df=gwas_associations.withColumn(\n            \"studyLocusId\", f.monotonically_increasing_id().cast(LongType())\n        )\n        .transform(\n            # Map/harmonise variants to variant annotation dataset:\n            # This function adds columns: variantId, referenceAllele, alternateAllele, chromosome, position\n            lambda df: GWASCatalogCuratedAssociationsParser._map_to_variant_annotation_variants(\n                df, variant_annotation\n            )\n        )\n        .withColumn(\n            # Perform all quality control checks:\n            \"qualityControls\",\n            GWASCatalogCuratedAssociationsParser._qc_all(\n                f.array().alias(\"qualityControls\"),\n                f.col(\"CHR_ID\"),\n                f.col(\"CHR_POS\").cast(IntegerType()),\n                f.col(\"referenceAllele\"),\n                f.col(\"alternateAllele\"),\n                f.col(\"STRONGEST SNP-RISK ALLELE\"),\n                *GWASCatalogCuratedAssociationsParser._parse_pvalue(\n                    f.col(\"P-VALUE\")\n                ),\n                pvalue_threshold,\n            ),\n        )\n        .select(\n            # INSIDE STUDY-LOCUS SCHEMA:\n            \"studyLocusId\",\n            \"variantId\",\n            # Mapped genomic location of the variant (; separated list)\n            \"chromosome\",\n            \"position\",\n            f.col(\"STUDY ACCESSION\").alias(\"studyId\"),\n            # beta value of the association\n            GWASCatalogCuratedAssociationsParser._harmonise_beta(\n                GWASCatalogCuratedAssociationsParser._normalise_risk_allele(\n                    f.col(\"STRONGEST SNP-RISK ALLELE\")\n                ),\n                f.col(\"referenceAllele\"),\n                f.col(\"alternateAllele\"),\n                f.col(\"OR or BETA\"),\n                f.col(\"95% CI (TEXT)\"),\n            ).alias(\"beta\"),\n            # p-value of the association, string: split into exponent and mantissa.\n            *GWASCatalogCuratedAssociationsParser._parse_pvalue(f.col(\"P-VALUE\")),\n            # Capturing phenotype granularity at the association level\n            GWASCatalogCuratedAssociationsParser._concatenate_substudy_description(\n                f.col(\"DISEASE/TRAIT\"),\n                f.col(\"P-VALUE (TEXT)\"),\n                f.col(\"MAPPED_TRAIT_URI\"),\n            ).alias(\"subStudyDescription\"),\n            # Quality controls (array of strings)\n            \"qualityControls\",\n        ),\n        _schema=StudyLocusGWASCatalog.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/associations/#gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog","title":"<code>gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog</code>  <code>dataclass</code>","text":"<p>             Bases: <code>StudyLocus</code></p> <p>Study locus Dataset for GWAS Catalog curated associations.</p> <p>A study index dataset captures all the metadata for all studies including GWAS and Molecular QTL.</p> Source code in <code>src/gentropy/datasource/gwas_catalog/associations.py</code> <pre><code>@dataclass\nclass StudyLocusGWASCatalog(StudyLocus):\n    \"\"\"Study locus Dataset for GWAS Catalog curated associations.\n\n    A study index dataset captures all the metadata for all studies including GWAS and Molecular QTL.\n    \"\"\"\n\n    def update_study_id(\n        self: StudyLocusGWASCatalog, study_annotation: DataFrame\n    ) -&gt; StudyLocusGWASCatalog:\n        \"\"\"Update final studyId and studyLocusId with a dataframe containing study annotation.\n\n        Args:\n            study_annotation (DataFrame): Dataframe containing `updatedStudyId` and key columns `studyId` and `subStudyDescription`.\n\n        Returns:\n            StudyLocusGWASCatalog: Updated study locus with new `studyId` and `studyLocusId`.\n        \"\"\"\n        self.df = (\n            self._df.join(\n                study_annotation, on=[\"studyId\", \"subStudyDescription\"], how=\"left\"\n            )\n            .withColumn(\"studyId\", f.coalesce(\"updatedStudyId\", \"studyId\"))\n            .drop(\"subStudyDescription\", \"updatedStudyId\")\n        ).withColumn(\n            \"studyLocusId\",\n            StudyLocus.assign_study_locus_id(f.col(\"studyId\"), f.col(\"variantId\")),\n        )\n        return self\n\n    def qc_ambiguous_study(self: StudyLocusGWASCatalog) -&gt; StudyLocusGWASCatalog:\n        \"\"\"Flag associations with variants that can not be unambiguously associated with one study.\n\n        Returns:\n            StudyLocusGWASCatalog: Updated study locus.\n        \"\"\"\n        assoc_ambiguity_window = Window.partitionBy(\n            f.col(\"studyId\"), f.col(\"variantId\")\n        )\n\n        self._df.withColumn(\n            \"qualityControls\",\n            StudyLocus.update_quality_flag(\n                f.col(\"qualityControls\"),\n                f.count(f.col(\"variantId\")).over(assoc_ambiguity_window) &gt; 1,\n                StudyLocusQualityCheck.AMBIGUOUS_STUDY,\n            ),\n        )\n        return self\n\n    def apply_inclusion_list(\n        self: StudyLocusGWASCatalog, inclusion_list: DataFrame\n    ) -&gt; StudyLocusGWASCatalog:\n        \"\"\"Restricting GWAS Catalog studies based on a list of accpected study ids.\n\n        Args:\n            inclusion_list (DataFrame): List of accepted GWAS Catalog study identifiers\n\n        Returns:\n            StudyLocusGWASCatalog: Filtered dataset.\n        \"\"\"\n        return StudyLocusGWASCatalog(\n            _df=self.df.join(inclusion_list, on=\"studyId\", how=\"inner\"),\n            _schema=StudyLocusGWASCatalog.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/associations/#gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog.apply_inclusion_list","title":"<code>apply_inclusion_list(inclusion_list: DataFrame) -&gt; StudyLocusGWASCatalog</code>","text":"<p>Restricting GWAS Catalog studies based on a list of accpected study ids.</p> <p>Parameters:</p> Name Type Description Default <code>inclusion_list</code> <code>DataFrame</code> <p>List of accepted GWAS Catalog study identifiers</p> required <p>Returns:</p> Name Type Description <code>StudyLocusGWASCatalog</code> <code>StudyLocusGWASCatalog</code> <p>Filtered dataset.</p> Source code in <code>src/gentropy/datasource/gwas_catalog/associations.py</code> <pre><code>def apply_inclusion_list(\n    self: StudyLocusGWASCatalog, inclusion_list: DataFrame\n) -&gt; StudyLocusGWASCatalog:\n    \"\"\"Restricting GWAS Catalog studies based on a list of accpected study ids.\n\n    Args:\n        inclusion_list (DataFrame): List of accepted GWAS Catalog study identifiers\n\n    Returns:\n        StudyLocusGWASCatalog: Filtered dataset.\n    \"\"\"\n    return StudyLocusGWASCatalog(\n        _df=self.df.join(inclusion_list, on=\"studyId\", how=\"inner\"),\n        _schema=StudyLocusGWASCatalog.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/associations/#gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog.qc_ambiguous_study","title":"<code>qc_ambiguous_study() -&gt; StudyLocusGWASCatalog</code>","text":"<p>Flag associations with variants that can not be unambiguously associated with one study.</p> <p>Returns:</p> Name Type Description <code>StudyLocusGWASCatalog</code> <code>StudyLocusGWASCatalog</code> <p>Updated study locus.</p> Source code in <code>src/gentropy/datasource/gwas_catalog/associations.py</code> <pre><code>def qc_ambiguous_study(self: StudyLocusGWASCatalog) -&gt; StudyLocusGWASCatalog:\n    \"\"\"Flag associations with variants that can not be unambiguously associated with one study.\n\n    Returns:\n        StudyLocusGWASCatalog: Updated study locus.\n    \"\"\"\n    assoc_ambiguity_window = Window.partitionBy(\n        f.col(\"studyId\"), f.col(\"variantId\")\n    )\n\n    self._df.withColumn(\n        \"qualityControls\",\n        StudyLocus.update_quality_flag(\n            f.col(\"qualityControls\"),\n            f.count(f.col(\"variantId\")).over(assoc_ambiguity_window) &gt; 1,\n            StudyLocusQualityCheck.AMBIGUOUS_STUDY,\n        ),\n    )\n    return self\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/associations/#gentropy.datasource.gwas_catalog.associations.StudyLocusGWASCatalog.update_study_id","title":"<code>update_study_id(study_annotation: DataFrame) -&gt; StudyLocusGWASCatalog</code>","text":"<p>Update final studyId and studyLocusId with a dataframe containing study annotation.</p> <p>Parameters:</p> Name Type Description Default <code>study_annotation</code> <code>DataFrame</code> <p>Dataframe containing <code>updatedStudyId</code> and key columns <code>studyId</code> and <code>subStudyDescription</code>.</p> required <p>Returns:</p> Name Type Description <code>StudyLocusGWASCatalog</code> <code>StudyLocusGWASCatalog</code> <p>Updated study locus with new <code>studyId</code> and <code>studyLocusId</code>.</p> Source code in <code>src/gentropy/datasource/gwas_catalog/associations.py</code> <pre><code>def update_study_id(\n    self: StudyLocusGWASCatalog, study_annotation: DataFrame\n) -&gt; StudyLocusGWASCatalog:\n    \"\"\"Update final studyId and studyLocusId with a dataframe containing study annotation.\n\n    Args:\n        study_annotation (DataFrame): Dataframe containing `updatedStudyId` and key columns `studyId` and `subStudyDescription`.\n\n    Returns:\n        StudyLocusGWASCatalog: Updated study locus with new `studyId` and `studyLocusId`.\n    \"\"\"\n    self.df = (\n        self._df.join(\n            study_annotation, on=[\"studyId\", \"subStudyDescription\"], how=\"left\"\n        )\n        .withColumn(\"studyId\", f.coalesce(\"updatedStudyId\", \"studyId\"))\n        .drop(\"subStudyDescription\", \"updatedStudyId\")\n    ).withColumn(\n        \"studyLocusId\",\n        StudyLocus.assign_study_locus_id(f.col(\"studyId\"), f.col(\"variantId\")),\n    )\n    return self\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/study_index/","title":"Study Index","text":""},{"location":"python_api/datasources/gwas_catalog/study_index/#gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalogParser","title":"<code>gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalogParser</code>  <code>dataclass</code>","text":"<p>GWAS Catalog study index parser.</p> <p>The following information is harmonised from the GWAS Catalog:</p> <ul> <li>All publication related information retained.</li> <li>Mapped measured and background traits parsed.</li> <li>Flagged if harmonized summary statistics datasets available.</li> <li>If available, the ftp path to these files presented.</li> <li>Ancestries from the discovery and replication stages are structured with sample counts.</li> <li>Case/control counts extracted.</li> <li>The number of samples with European ancestry extracted.</li> </ul> Source code in <code>src/gentropy/datasource/gwas_catalog/study_index.py</code> <pre><code>@dataclass\nclass StudyIndexGWASCatalogParser:\n    \"\"\"GWAS Catalog study index parser.\n\n    The following information is harmonised from the GWAS Catalog:\n\n    - All publication related information retained.\n    - Mapped measured and background traits parsed.\n    - Flagged if harmonized summary statistics datasets available.\n    - If available, the ftp path to these files presented.\n    - Ancestries from the discovery and replication stages are structured with sample counts.\n    - Case/control counts extracted.\n    - The number of samples with European ancestry extracted.\n\n    \"\"\"\n\n    @staticmethod\n    def _parse_discovery_samples(discovery_samples: Column) -&gt; Column:\n        \"\"\"Parse discovery sample sizes from GWAS Catalog.\n\n        This is a curated field. From publication sometimes it is not clear how the samples were split\n        across the reported ancestries. In such cases we are assuming the ancestries were evenly presented\n        and the total sample size is split:\n\n        [\"European, African\", 100] -&gt; [\"European, 50], [\"African\", 50]\n\n        Args:\n            discovery_samples (Column): Raw discovery sample sizes\n\n        Returns:\n            Column: Parsed and de-duplicated list of discovery ancestries with sample size.\n\n        Examples:\n            &gt;&gt;&gt; data = [('s1', \"European\", 10), ('s1', \"African\", 10), ('s2', \"European, African, Asian\", 100), ('s2', \"European\", 50)]\n            &gt;&gt;&gt; df = (\n            ...    spark.createDataFrame(data, ['studyId', 'ancestry', 'sampleSize'])\n            ...    .groupBy('studyId')\n            ...    .agg(\n            ...        f.collect_set(\n            ...            f.struct('ancestry', 'sampleSize')\n            ...        ).alias('discoverySampleSize')\n            ...    )\n            ...    .orderBy('studyId')\n            ...    .withColumn('discoverySampleSize', StudyIndexGWASCatalogParser._parse_discovery_samples(f.col('discoverySampleSize')))\n            ...    .select('discoverySampleSize')\n            ...    .show(truncate=False)\n            ... )\n            +--------------------------------------------+\n            |discoverySampleSize                         |\n            +--------------------------------------------+\n            |[{African, 10}, {European, 10}]             |\n            |[{European, 83}, {African, 33}, {Asian, 33}]|\n            +--------------------------------------------+\n            &lt;BLANKLINE&gt;\n        \"\"\"\n        # To initialize return objects for aggregate functions, schema has to be defined:\n        schema = t.ArrayType(\n            t.StructType(\n                [\n                    t.StructField(\"ancestry\", t.StringType(), True),\n                    t.StructField(\"sampleSize\", t.IntegerType(), True),\n                ]\n            )\n        )\n\n        # Splitting comma separated ancestries:\n        exploded_ancestries = f.transform(\n            discovery_samples,\n            lambda sample: f.split(sample.ancestry, r\",\\s(?![^()]*\\))\"),\n        )\n\n        # Initialize discoverySample object from unique list of ancestries:\n        unique_ancestries = f.transform(\n            f.aggregate(\n                exploded_ancestries,\n                f.array().cast(t.ArrayType(t.StringType())),\n                lambda x, y: f.array_union(x, y),\n                f.array_distinct,\n            ),\n            lambda ancestry: f.struct(\n                ancestry.alias(\"ancestry\"),\n                f.lit(0).alias(\"sampleSize\"),\n            ),\n        )\n\n        # Computing sample sizes for ancestries when splitting is needed:\n        resolved_sample_count = f.transform(\n            f.arrays_zip(\n                f.transform(exploded_ancestries, lambda pop: f.size(pop)).alias(\n                    \"pop_size\"\n                ),\n                f.transform(discovery_samples, lambda pop: pop.sampleSize).alias(\n                    \"pop_count\"\n                ),\n            ),\n            lambda pop: (pop.pop_count / pop.pop_size).cast(t.IntegerType()),\n        )\n\n        # Flattening out ancestries with sample sizes:\n        parsed_sample_size = f.aggregate(\n            f.transform(\n                f.arrays_zip(\n                    exploded_ancestries.alias(\"ancestries\"),\n                    resolved_sample_count.alias(\"sample_count\"),\n                ),\n                StudyIndexGWASCatalogParser._merge_ancestries_and_counts,\n            ),\n            f.array().cast(schema),\n            lambda x, y: f.array_union(x, y),\n        )\n\n        # Normalize ancestries:\n        return f.aggregate(\n            parsed_sample_size,\n            unique_ancestries,\n            StudyIndexGWASCatalogParser._normalize_ancestries,\n        )\n\n    @staticmethod\n    def _normalize_ancestries(merged: Column, ancestry: Column) -&gt; Column:\n        \"\"\"Normalize ancestries from a list of structs.\n\n        As some ancestry label might be repeated with different sample counts,\n        these counts need to be collected.\n\n        Args:\n            merged (Column): Resulting list of struct with unique ancestries.\n            ancestry (Column): One ancestry object coming from raw.\n\n        Returns:\n            Column: Unique list of ancestries with the sample counts.\n        \"\"\"\n        # Iterating over the list of unique ancestries and adding the sample size if label matches:\n        return f.transform(\n            merged,\n            lambda a: f.when(\n                a.ancestry == ancestry.ancestry,\n                f.struct(\n                    a.ancestry.alias(\"ancestry\"),\n                    (a.sampleSize + ancestry.sampleSize)\n                    .cast(t.IntegerType())\n                    .alias(\"sampleSize\"),\n                ),\n            ).otherwise(a),\n        )\n\n    @staticmethod\n    def _merge_ancestries_and_counts(ancestry_group: Column) -&gt; Column:\n        \"\"\"Merge ancestries with sample sizes.\n\n        After splitting ancestry annotations, all resulting ancestries needs to be assigned\n        with the proper sample size.\n\n        Args:\n            ancestry_group (Column): Each element is a struct with `sample_count` (int) and `ancestries` (list)\n\n        Returns:\n            Column: a list of structs with `ancestry` and `sampleSize` fields.\n\n        Examples:\n            &gt;&gt;&gt; data = [(12, ['African', 'European']),(12, ['African'])]\n            &gt;&gt;&gt; (\n            ...     spark.createDataFrame(data, ['sample_count', 'ancestries'])\n            ...     .select(StudyIndexGWASCatalogParser._merge_ancestries_and_counts(f.struct('sample_count', 'ancestries')).alias('test'))\n            ...     .show(truncate=False)\n            ... )\n            +-------------------------------+\n            |test                           |\n            +-------------------------------+\n            |[{African, 12}, {European, 12}]|\n            |[{African, 12}]                |\n            +-------------------------------+\n            &lt;BLANKLINE&gt;\n        \"\"\"\n        # Extract sample size for the ancestry group:\n        count = ancestry_group.sample_count\n\n        # We need to loop through the ancestries:\n        return f.transform(\n            ancestry_group.ancestries,\n            lambda ancestry: f.struct(\n                ancestry.alias(\"ancestry\"),\n                count.alias(\"sampleSize\"),\n            ),\n        )\n\n    @staticmethod\n    def parse_cohorts(raw_cohort: Column) -&gt; Column:\n        \"\"\"Return a list of unique cohort labels from pipe separated list if provided.\n\n        Args:\n            raw_cohort (Column): Cohort list column, where labels are separated by `|` sign.\n\n        Returns:\n            Column: an array colun with string elements.\n\n        Examples:\n        &gt;&gt;&gt; data = [('BioME|CaPS|Estonia|FHS|UKB|GERA|GERA|GERA',),(None,),]\n        &gt;&gt;&gt; spark.createDataFrame(data, ['cohorts']).select(StudyIndexGWASCatalogParser.parse_cohorts(f.col('cohorts')).alias('parsedCohorts')).show(truncate=False)\n        +--------------------------------------+\n        |parsedCohorts                         |\n        +--------------------------------------+\n        |[BioME, CaPS, Estonia, FHS, UKB, GERA]|\n        |[null]                                |\n        +--------------------------------------+\n        &lt;BLANKLINE&gt;\n        \"\"\"\n        return f.when(\n            (raw_cohort.isNull()) | (raw_cohort == \"\"),\n            f.array(f.lit(None).cast(t.StringType())),\n        ).otherwise(f.array_distinct(f.split(raw_cohort, r\"\\|\")))\n\n    @classmethod\n    def _parse_study_table(\n        cls: type[StudyIndexGWASCatalogParser], catalog_studies: DataFrame\n    ) -&gt; StudyIndexGWASCatalog:\n        \"\"\"Harmonise GWASCatalog study table with `StudyIndex` schema.\n\n        Args:\n            catalog_studies (DataFrame): GWAS Catalog study table\n\n        Returns:\n            StudyIndexGWASCatalog: Parsed and annotated GWAS Catalog study table.\n        \"\"\"\n        return StudyIndexGWASCatalog(\n            _df=catalog_studies.select(\n                f.coalesce(\n                    f.col(\"STUDY ACCESSION\"), f.monotonically_increasing_id()\n                ).alias(\"studyId\"),\n                f.lit(\"GCST\").alias(\"projectId\"),\n                f.lit(\"gwas\").alias(\"studyType\"),\n                f.col(\"PUBMED ID\").alias(\"pubmedId\"),\n                f.col(\"FIRST AUTHOR\").alias(\"publicationFirstAuthor\"),\n                f.col(\"DATE\").alias(\"publicationDate\"),\n                f.col(\"JOURNAL\").alias(\"publicationJournal\"),\n                f.col(\"STUDY\").alias(\"publicationTitle\"),\n                f.coalesce(f.col(\"DISEASE/TRAIT\"), f.lit(\"Unreported\")).alias(\n                    \"traitFromSource\"\n                ),\n                f.col(\"INITIAL SAMPLE SIZE\").alias(\"initialSampleSize\"),\n                parse_efos(f.col(\"MAPPED_TRAIT_URI\")).alias(\"traitFromSourceMappedIds\"),\n                parse_efos(f.col(\"MAPPED BACKGROUND TRAIT URI\")).alias(\n                    \"backgroundTraitFromSourceMappedIds\"\n                ),\n                cls.parse_cohorts(f.col(\"COHORT\")).alias(\"cohorts\"),\n            ),\n            _schema=StudyIndexGWASCatalog.get_schema(),\n        )\n\n    @classmethod\n    def from_source(\n        cls: type[StudyIndexGWASCatalogParser],\n        catalog_studies: DataFrame,\n        ancestry_file: DataFrame,\n        sumstats_lut: DataFrame,\n    ) -&gt; StudyIndexGWASCatalog:\n        \"\"\"Ingests study level metadata from the GWAS Catalog.\n\n        Args:\n            catalog_studies (DataFrame): GWAS Catalog raw study table\n            ancestry_file (DataFrame): GWAS Catalog ancestry table.\n            sumstats_lut (DataFrame): GWAS Catalog summary statistics list.\n\n        Returns:\n            StudyIndexGWASCatalog: Parsed and annotated GWAS Catalog study table.\n        \"\"\"\n        # Read GWAS Catalogue raw data\n        return (\n            cls._parse_study_table(catalog_studies)\n            .annotate_ancestries(ancestry_file)\n            .annotate_sumstats_info(sumstats_lut)\n            .annotate_discovery_sample_sizes()\n        )\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/study_index/#gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalogParser.from_source","title":"<code>from_source(catalog_studies: DataFrame, ancestry_file: DataFrame, sumstats_lut: DataFrame) -&gt; StudyIndexGWASCatalog</code>  <code>classmethod</code>","text":"<p>Ingests study level metadata from the GWAS Catalog.</p> <p>Parameters:</p> Name Type Description Default <code>catalog_studies</code> <code>DataFrame</code> <p>GWAS Catalog raw study table</p> required <code>ancestry_file</code> <code>DataFrame</code> <p>GWAS Catalog ancestry table.</p> required <code>sumstats_lut</code> <code>DataFrame</code> <p>GWAS Catalog summary statistics list.</p> required <p>Returns:</p> Name Type Description <code>StudyIndexGWASCatalog</code> <code>StudyIndexGWASCatalog</code> <p>Parsed and annotated GWAS Catalog study table.</p> Source code in <code>src/gentropy/datasource/gwas_catalog/study_index.py</code> <pre><code>@classmethod\ndef from_source(\n    cls: type[StudyIndexGWASCatalogParser],\n    catalog_studies: DataFrame,\n    ancestry_file: DataFrame,\n    sumstats_lut: DataFrame,\n) -&gt; StudyIndexGWASCatalog:\n    \"\"\"Ingests study level metadata from the GWAS Catalog.\n\n    Args:\n        catalog_studies (DataFrame): GWAS Catalog raw study table\n        ancestry_file (DataFrame): GWAS Catalog ancestry table.\n        sumstats_lut (DataFrame): GWAS Catalog summary statistics list.\n\n    Returns:\n        StudyIndexGWASCatalog: Parsed and annotated GWAS Catalog study table.\n    \"\"\"\n    # Read GWAS Catalogue raw data\n    return (\n        cls._parse_study_table(catalog_studies)\n        .annotate_ancestries(ancestry_file)\n        .annotate_sumstats_info(sumstats_lut)\n        .annotate_discovery_sample_sizes()\n    )\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/study_index/#gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalogParser.parse_cohorts","title":"<code>parse_cohorts(raw_cohort: Column) -&gt; Column</code>  <code>staticmethod</code>","text":"<p>Return a list of unique cohort labels from pipe separated list if provided.</p> <p>Parameters:</p> Name Type Description Default <code>raw_cohort</code> <code>Column</code> <p>Cohort list column, where labels are separated by <code>|</code> sign.</p> required <p>Returns:</p> Name Type Description <code>Column</code> <code>Column</code> <p>an array colun with string elements.</p> <p>Examples:</p> <p>data = [('BioME|CaPS|Estonia|FHS|UKB|GERA|GERA|GERA',),(None,),] spark.createDataFrame(data, ['cohorts']).select(StudyIndexGWASCatalogParser.parse_cohorts(f.col('cohorts')).alias('parsedCohorts')).show(truncate=False) +--------------------------------------+ |parsedCohorts                         | +--------------------------------------+ |[BioME, CaPS, Estonia, FHS, UKB, GERA]| |[null]                                | +--------------------------------------+  Source code in <code>src/gentropy/datasource/gwas_catalog/study_index.py</code> <pre><code>@staticmethod\ndef parse_cohorts(raw_cohort: Column) -&gt; Column:\n    \"\"\"Return a list of unique cohort labels from pipe separated list if provided.\n\n    Args:\n        raw_cohort (Column): Cohort list column, where labels are separated by `|` sign.\n\n    Returns:\n        Column: an array colun with string elements.\n\n    Examples:\n    &gt;&gt;&gt; data = [('BioME|CaPS|Estonia|FHS|UKB|GERA|GERA|GERA',),(None,),]\n    &gt;&gt;&gt; spark.createDataFrame(data, ['cohorts']).select(StudyIndexGWASCatalogParser.parse_cohorts(f.col('cohorts')).alias('parsedCohorts')).show(truncate=False)\n    +--------------------------------------+\n    |parsedCohorts                         |\n    +--------------------------------------+\n    |[BioME, CaPS, Estonia, FHS, UKB, GERA]|\n    |[null]                                |\n    +--------------------------------------+\n    &lt;BLANKLINE&gt;\n    \"\"\"\n    return f.when(\n        (raw_cohort.isNull()) | (raw_cohort == \"\"),\n        f.array(f.lit(None).cast(t.StringType())),\n    ).otherwise(f.array_distinct(f.split(raw_cohort, r\"\\|\")))\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/study_index/#gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog","title":"<code>gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog</code>  <code>dataclass</code>","text":"<p>             Bases: <code>StudyIndex</code></p> <p>Study index dataset from GWAS Catalog.</p> <p>A study index dataset captures all the metadata for all studies including GWAS and Molecular QTL.</p> Source code in <code>src/gentropy/datasource/gwas_catalog/study_index.py</code> <pre><code>@dataclass\nclass StudyIndexGWASCatalog(StudyIndex):\n    \"\"\"Study index dataset from GWAS Catalog.\n\n    A study index dataset captures all the metadata for all studies including GWAS and Molecular QTL.\n    \"\"\"\n\n    def update_study_id(\n        self: StudyIndexGWASCatalog, study_annotation: DataFrame\n    ) -&gt; StudyIndexGWASCatalog:\n        \"\"\"Update studyId with a dataframe containing study.\n\n        Args:\n            study_annotation (DataFrame): Dataframe containing `updatedStudyId`, `traitFromSource`, `traitFromSourceMappedIds` and key column `studyId`.\n\n        Returns:\n            StudyIndexGWASCatalog: Updated study table.\n        \"\"\"\n        self.df = (\n            self._df.join(\n                study_annotation.select(\n                    *[\n                        f.col(c).alias(f\"updated{c}\")\n                        if c not in [\"studyId\", \"updatedStudyId\"]\n                        else f.col(c)\n                        for c in study_annotation.columns\n                    ]\n                ),\n                on=\"studyId\",\n                how=\"left\",\n            )\n            .withColumn(\n                \"studyId\",\n                f.coalesce(f.col(\"updatedStudyId\"), f.col(\"studyId\")),\n            )\n            .withColumn(\n                \"traitFromSource\",\n                f.coalesce(f.col(\"updatedtraitFromSource\"), f.col(\"traitFromSource\")),\n            )\n            .withColumn(\n                \"traitFromSourceMappedIds\",\n                f.coalesce(\n                    f.col(\"updatedtraitFromSourceMappedIds\"),\n                    f.col(\"traitFromSourceMappedIds\"),\n                ),\n            )\n            .select(self._df.columns)\n        )\n\n        return self\n\n    def annotate_from_study_curation(\n        self: StudyIndexGWASCatalog, curation_table: DataFrame | None\n    ) -&gt; StudyIndexGWASCatalog:\n        \"\"\"Annotating study index with curation.\n\n        Args:\n            curation_table (DataFrame | None): Curated GWAS Catalog studies with summary statistics\n\n        Returns:\n            StudyIndexGWASCatalog: Updated study index\n        \"\"\"\n        # Providing curation table is optional. However once this method is called, the quality and studyFlag columns are added.\n        if curation_table is None:\n            return self\n\n        columns = self.df.columns\n\n        # Adding prefix to columns in the curation table:\n        curation_table = curation_table.select(\n            *[\n                f.col(column).alias(f\"curation_{column}\")\n                if column != \"studyId\"\n                else f.col(column)\n                for column in curation_table.columns\n            ]\n        )\n\n        # Create expression how to update/create quality controls dataset:\n        qualityControls_expression = (\n            f.col(\"curation_qualityControls\")\n            if \"qualityControls\" not in columns\n            else f.when(\n                f.col(\"curation_qualityControls\").isNotNull(),\n                f.array_union(\n                    f.col(\"qualityControls\"), f.array(f.col(\"curation_qualityControls\"))\n                ),\n            ).otherwise(f.col(\"qualityControls\"))\n        )\n\n        # Create expression how to update/create analysis flag:\n        analysis_expression = (\n            f.col(\"curation_analysisFlags\")\n            if \"analysisFlags\" not in columns\n            else f.when(\n                f.col(\"curation_analysisFlags\").isNotNull(),\n                f.array_union(\n                    f.col(\"analysisFlags\"), f.array(f.col(\"curation_analysisFlags\"))\n                ),\n            ).otherwise(f.col(\"analysisFlags\"))\n        )\n\n        # Updating columns list. We might or might not list columns twice, but that doesn't matter, unique set will generated:\n        columns = list(set(columns + [\"qualityControls\", \"analysisFlags\"]))\n\n        # Based on the curation table, columns needs to be updated:\n        curated_df = (\n            self.df.join(curation_table, on=\"studyId\", how=\"left\")\n            # Updating study type:\n            .withColumn(\n                \"studyType\", f.coalesce(f.col(\"curation_studyType\"), f.col(\"studyType\"))\n            )\n            # Updating quality controls:\n            .withColumn(\"qualityControls\", qualityControls_expression)\n            # Updating study annotation flags:\n            .withColumn(\"analysisFlags\", analysis_expression)\n            # Dropping columns coming from the curation table:\n            .select(*columns)\n        )\n        return StudyIndexGWASCatalog(\n            _df=curated_df, _schema=StudyIndexGWASCatalog.get_schema()\n        )\n\n    def extract_studies_for_curation(\n        self: StudyIndexGWASCatalog, curation: DataFrame | None\n    ) -&gt; DataFrame:\n        \"\"\"Extract studies for curation.\n\n        Args:\n            curation (DataFrame | None): Dataframe with curation.\n\n        Returns:\n            DataFrame: Updated curation table. New studies are have the `isCurated` False.\n        \"\"\"\n        # If no curation table provided, assume all studies needs curation:\n        if curation is None:\n            return (\n                self.df\n                # Curation only applyed on studies with summary statistics:\n                .filter(f.col(\"hasSumstats\"))\n                # Adding columns expected in the curation table - array columns aready flattened:\n                .withColumn(\"studyType\", f.lit(None).cast(t.StringType()))\n                .withColumn(\"analysisFlag\", f.lit(None).cast(t.StringType()))\n                .withColumn(\"qualityControl\", f.lit(None).cast(t.StringType()))\n                .withColumn(\"isCurated\", f.lit(False).cast(t.StringType()))\n            )\n\n        # Adding prefix to columns in the curation table:\n        curation = curation.select(\n            *[\n                f.col(column).alias(f\"curation_{column}\")\n                if column != \"studyId\"\n                else f.col(column)\n                for column in curation.columns\n            ]\n        )\n\n        return (\n            self.df\n            # Curation only applyed on studies with summary statistics:\n            .filter(f.col(\"hasSumstats\"))\n            .join(curation, on=\"studyId\", how=\"left\")\n            .select(\n                \"studyId\",\n                # Propagate existing curation - array columns are being flattened:\n                f.col(\"curation_studyType\").alias(\"studyType\"),\n                f.array_join(f.col(\"curation_analysisFlags\"), \"|\").alias(\n                    \"analysisFlag\"\n                ),\n                f.array_join(f.col(\"curation_qualityControls\"), \"|\").alias(\n                    \"qualityControl\"\n                ),\n                # This boolean flag needs to be casted to string, because saving to tsv would fail otherwise:\n                f.coalesce(f.col(\"curation_isCurated\"), f.lit(False))\n                .cast(t.StringType())\n                .alias(\"isCurated\"),\n                # The following columns are propagated to make curation easier:\n                \"pubmedId\",\n                \"publicationTitle\",\n                \"traitFromSource\",\n            )\n        )\n\n    def annotate_ancestries(\n        self: StudyIndexGWASCatalog, ancestry_lut: DataFrame\n    ) -&gt; StudyIndexGWASCatalog:\n        \"\"\"Extracting sample sizes and ancestry information.\n\n        This function parses the ancestry data. Also get counts for the europeans in the same\n        discovery stage.\n\n        Args:\n            ancestry_lut (DataFrame): Ancestry table as downloaded from the GWAS Catalog\n\n        Returns:\n            StudyIndexGWASCatalog: Slimmed and cleaned version of the ancestry annotation.\n        \"\"\"\n        from gentropy.datasource.gwas_catalog.study_index import (\n            StudyIndexGWASCatalogParser as GWASCatalogStudyIndexParser,\n        )\n\n        ancestry = (\n            ancestry_lut\n            # Convert column headers to camelcase:\n            .transform(\n                lambda df: df.select(\n                    *[f.expr(column2camel_case(x)) for x in df.columns]\n                )\n            ).withColumnRenamed(\n                \"studyAccession\", \"studyId\"\n            )  # studyId has not been split yet\n        )\n\n        # Get a high resolution dataset on experimental stage:\n        ancestry_stages = (\n            ancestry.groupBy(\"studyId\")\n            .pivot(\"stage\")\n            .agg(\n                f.collect_set(\n                    f.struct(\n                        f.col(\"broadAncestralCategory\").alias(\"ancestry\"),\n                        f.col(\"numberOfIndividuals\")\n                        .cast(t.IntegerType())\n                        .alias(\"sampleSize\"),\n                    )\n                )\n            )\n            .withColumn(\n                \"discoverySamples\",\n                GWASCatalogStudyIndexParser._parse_discovery_samples(f.col(\"initial\")),\n            )\n            .withColumnRenamed(\"replication\", \"replicationSamples\")\n            # Mapping discovery stage ancestries to LD reference:\n            .withColumn(\n                \"ldPopulationStructure\",\n                self.aggregate_and_map_ancestries(f.col(\"discoverySamples\")),\n            )\n            .drop(\"initial\")\n            .persist()\n        )\n\n        # Generate information on the ancestry composition of the discovery stage, and calculate\n        # the proportion of the Europeans:\n        europeans_deconvoluted = (\n            ancestry\n            # Focus on discovery stage:\n            .filter(f.col(\"stage\") == \"initial\")\n            # Sorting ancestries if European:\n            .withColumn(\n                \"ancestryFlag\",\n                # Excluding finnish:\n                f.when(\n                    f.col(\"initialSampleDescription\").contains(\"Finnish\"),\n                    f.lit(\"other\"),\n                )\n                # Excluding Icelandic population:\n                .when(\n                    f.col(\"initialSampleDescription\").contains(\"Icelandic\"),\n                    f.lit(\"other\"),\n                )\n                # Including European ancestry:\n                .when(f.col(\"broadAncestralCategory\") == \"European\", f.lit(\"european\"))\n                # Exclude all other population:\n                .otherwise(\"other\"),\n            )\n            # Grouping by study accession and initial sample description:\n            .groupBy(\"studyId\")\n            .pivot(\"ancestryFlag\")\n            .agg(\n                # Summarizing sample sizes for all ancestries:\n                f.sum(f.col(\"numberOfIndividuals\"))\n            )\n            # Do arithmetics to make sure we have the right proportion of european in the set:\n            .withColumn(\n                \"initialSampleCountEuropean\",\n                f.when(f.col(\"european\").isNull(), f.lit(0)).otherwise(\n                    f.col(\"european\")\n                ),\n            )\n            .withColumn(\n                \"initialSampleCountOther\",\n                f.when(f.col(\"other\").isNull(), f.lit(0)).otherwise(f.col(\"other\")),\n            )\n            .withColumn(\n                \"initialSampleCount\",\n                f.col(\"initialSampleCountEuropean\") + f.col(\"other\"),\n            )\n            .drop(\n                \"european\",\n                \"other\",\n                \"initialSampleCount\",\n                \"initialSampleCountEuropean\",\n                \"initialSampleCountOther\",\n            )\n        )\n\n        parsed_ancestry_lut = ancestry_stages.join(\n            europeans_deconvoluted, on=\"studyId\", how=\"outer\"\n        ).select(\n            \"studyId\", \"discoverySamples\", \"ldPopulationStructure\", \"replicationSamples\"\n        )\n        self.df = self.df.join(parsed_ancestry_lut, on=\"studyId\", how=\"left\")\n        return self\n\n    def annotate_sumstats_info(\n        self: StudyIndexGWASCatalog, sumstats_lut: DataFrame\n    ) -&gt; StudyIndexGWASCatalog:\n        \"\"\"Annotate summary stat locations.\n\n        Args:\n            sumstats_lut (DataFrame): listing GWAS Catalog summary stats paths\n\n        Returns:\n            StudyIndexGWASCatalog: including `summarystatsLocation` and `hasSumstats` columns\n\n        Raises:\n            ValueError: if the sumstats_lut table doesn't have the right columns\n        \"\"\"\n        gwas_sumstats_base_uri = (\n            \"ftp://ftp.ebi.ac.uk/pub/databases/gwas/summary_statistics/\"\n        )\n\n        if \"_c0\" not in sumstats_lut.columns:\n            raise ValueError(\n                f'Sumstats look-up table needs to have `_c0` column. However it has: {\",\".join(sumstats_lut.columns)}'\n            )\n\n        parsed_sumstats_lut = sumstats_lut.withColumn(\n            \"summarystatsLocation\",\n            f.concat(\n                f.lit(gwas_sumstats_base_uri),\n                f.regexp_replace(f.col(\"_c0\"), r\"^\\.\\/\", \"\"),\n            ),\n        ).select(\n            self._parse_gwas_catalog_study_id(\"summarystatsLocation\").alias(\"studyId\"),\n            \"summarystatsLocation\",\n            f.lit(True).alias(\"hasSumstats\"),\n        )\n        self.df = (\n            self.df.drop(\"hasSumstats\")\n            .join(parsed_sumstats_lut, on=\"studyId\", how=\"left\")\n            .withColumn(\"hasSumstats\", f.coalesce(f.col(\"hasSumstats\"), f.lit(False)))\n        )\n        return self\n\n    def annotate_discovery_sample_sizes(\n        self: StudyIndexGWASCatalog,\n    ) -&gt; StudyIndexGWASCatalog:\n        \"\"\"Extract the sample size of the discovery stage of the study as annotated in the GWAS Catalog.\n\n        For some studies that measure quantitative traits, nCases and nControls can't be extracted. Therefore, we assume these are 0.\n\n        Returns:\n            StudyIndexGWASCatalog: object with columns `nCases`, `nControls`, and `nSamples` per `studyId` correctly extracted.\n        \"\"\"\n        sample_size_lut = (\n            self.df.select(\n                \"studyId\",\n                f.explode_outer(f.split(f.col(\"initialSampleSize\"), r\",\\s+\")).alias(\n                    \"samples\"\n                ),\n            )\n            # Extracting the sample size from the string:\n            .withColumn(\n                \"sampleSize\",\n                f.regexp_extract(\n                    f.regexp_replace(f.col(\"samples\"), \",\", \"\"), r\"[0-9,]+\", 0\n                ).cast(t.IntegerType()),\n            )\n            .select(\n                \"studyId\",\n                \"sampleSize\",\n                f.when(f.col(\"samples\").contains(\"cases\"), f.col(\"sampleSize\"))\n                .otherwise(f.lit(0))\n                .alias(\"nCases\"),\n                f.when(f.col(\"samples\").contains(\"controls\"), f.col(\"sampleSize\"))\n                .otherwise(f.lit(0))\n                .alias(\"nControls\"),\n            )\n            # Aggregating sample sizes for all ancestries:\n            .groupBy(\"studyId\")  # studyId has not been split yet\n            .agg(\n                f.sum(\"nCases\").cast(\"integer\").alias(\"nCases\"),\n                f.sum(\"nControls\").cast(\"integer\").alias(\"nControls\"),\n                f.sum(\"sampleSize\").cast(\"integer\").alias(\"nSamples\"),\n            )\n        )\n        self.df = self.df.join(sample_size_lut, on=\"studyId\", how=\"left\")\n        return self\n\n    def apply_inclusion_list(\n        self: StudyIndexGWASCatalog, inclusion_list: DataFrame\n    ) -&gt; StudyIndexGWASCatalog:\n        \"\"\"Restricting GWAS Catalog studies based on a list of accepted study identifiers.\n\n        Args:\n            inclusion_list (DataFrame): List of accepted GWAS Catalog study identifiers\n\n        Returns:\n            StudyIndexGWASCatalog: Filtered dataset.\n        \"\"\"\n        return StudyIndexGWASCatalog(\n            _df=self.df.join(inclusion_list, on=\"studyId\", how=\"inner\"),\n            _schema=StudyIndexGWASCatalog.get_schema(),\n        )\n\n    @staticmethod\n    def _parse_gwas_catalog_study_id(sumstats_path_column: str) -&gt; Column:\n        \"\"\"Extract GWAS Catalog study accession from the summary statistics path.\n\n        Args:\n            sumstats_path_column (str): column *name* for the summary statistics path\n\n        Returns:\n            Column: GWAS Catalog study accession.\n\n        Examples:\n            &gt;&gt;&gt; data = [\n            ... ('./GCST90086001-GCST90087000/GCST90086758/harmonised/35078996-GCST90086758-EFO_0007937.h.tsv.gz',),\n            ...    ('gs://open-targets-gwas-summary-stats/harmonised/GCST000568.parquet/',),\n            ...    (None,)\n            ... ]\n            &gt;&gt;&gt; spark.createDataFrame(data, ['testColumn']).select(StudyIndexGWASCatalog._parse_gwas_catalog_study_id('testColumn').alias('accessions')).collect()\n            [Row(accessions='GCST90086758'), Row(accessions='GCST000568'), Row(accessions=None)]\n        \"\"\"\n        accesions = f.expr(rf\"regexp_extract_all({sumstats_path_column}, '(GCST\\\\d+)')\")\n        return accesions[f.size(accesions) - 1]\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/study_index/#gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.annotate_ancestries","title":"<code>annotate_ancestries(ancestry_lut: DataFrame) -&gt; StudyIndexGWASCatalog</code>","text":"<p>Extracting sample sizes and ancestry information.</p> <p>This function parses the ancestry data. Also get counts for the europeans in the same discovery stage.</p> <p>Parameters:</p> Name Type Description Default <code>ancestry_lut</code> <code>DataFrame</code> <p>Ancestry table as downloaded from the GWAS Catalog</p> required <p>Returns:</p> Name Type Description <code>StudyIndexGWASCatalog</code> <code>StudyIndexGWASCatalog</code> <p>Slimmed and cleaned version of the ancestry annotation.</p> Source code in <code>src/gentropy/datasource/gwas_catalog/study_index.py</code> <pre><code>def annotate_ancestries(\n    self: StudyIndexGWASCatalog, ancestry_lut: DataFrame\n) -&gt; StudyIndexGWASCatalog:\n    \"\"\"Extracting sample sizes and ancestry information.\n\n    This function parses the ancestry data. Also get counts for the europeans in the same\n    discovery stage.\n\n    Args:\n        ancestry_lut (DataFrame): Ancestry table as downloaded from the GWAS Catalog\n\n    Returns:\n        StudyIndexGWASCatalog: Slimmed and cleaned version of the ancestry annotation.\n    \"\"\"\n    from gentropy.datasource.gwas_catalog.study_index import (\n        StudyIndexGWASCatalogParser as GWASCatalogStudyIndexParser,\n    )\n\n    ancestry = (\n        ancestry_lut\n        # Convert column headers to camelcase:\n        .transform(\n            lambda df: df.select(\n                *[f.expr(column2camel_case(x)) for x in df.columns]\n            )\n        ).withColumnRenamed(\n            \"studyAccession\", \"studyId\"\n        )  # studyId has not been split yet\n    )\n\n    # Get a high resolution dataset on experimental stage:\n    ancestry_stages = (\n        ancestry.groupBy(\"studyId\")\n        .pivot(\"stage\")\n        .agg(\n            f.collect_set(\n                f.struct(\n                    f.col(\"broadAncestralCategory\").alias(\"ancestry\"),\n                    f.col(\"numberOfIndividuals\")\n                    .cast(t.IntegerType())\n                    .alias(\"sampleSize\"),\n                )\n            )\n        )\n        .withColumn(\n            \"discoverySamples\",\n            GWASCatalogStudyIndexParser._parse_discovery_samples(f.col(\"initial\")),\n        )\n        .withColumnRenamed(\"replication\", \"replicationSamples\")\n        # Mapping discovery stage ancestries to LD reference:\n        .withColumn(\n            \"ldPopulationStructure\",\n            self.aggregate_and_map_ancestries(f.col(\"discoverySamples\")),\n        )\n        .drop(\"initial\")\n        .persist()\n    )\n\n    # Generate information on the ancestry composition of the discovery stage, and calculate\n    # the proportion of the Europeans:\n    europeans_deconvoluted = (\n        ancestry\n        # Focus on discovery stage:\n        .filter(f.col(\"stage\") == \"initial\")\n        # Sorting ancestries if European:\n        .withColumn(\n            \"ancestryFlag\",\n            # Excluding finnish:\n            f.when(\n                f.col(\"initialSampleDescription\").contains(\"Finnish\"),\n                f.lit(\"other\"),\n            )\n            # Excluding Icelandic population:\n            .when(\n                f.col(\"initialSampleDescription\").contains(\"Icelandic\"),\n                f.lit(\"other\"),\n            )\n            # Including European ancestry:\n            .when(f.col(\"broadAncestralCategory\") == \"European\", f.lit(\"european\"))\n            # Exclude all other population:\n            .otherwise(\"other\"),\n        )\n        # Grouping by study accession and initial sample description:\n        .groupBy(\"studyId\")\n        .pivot(\"ancestryFlag\")\n        .agg(\n            # Summarizing sample sizes for all ancestries:\n            f.sum(f.col(\"numberOfIndividuals\"))\n        )\n        # Do arithmetics to make sure we have the right proportion of european in the set:\n        .withColumn(\n            \"initialSampleCountEuropean\",\n            f.when(f.col(\"european\").isNull(), f.lit(0)).otherwise(\n                f.col(\"european\")\n            ),\n        )\n        .withColumn(\n            \"initialSampleCountOther\",\n            f.when(f.col(\"other\").isNull(), f.lit(0)).otherwise(f.col(\"other\")),\n        )\n        .withColumn(\n            \"initialSampleCount\",\n            f.col(\"initialSampleCountEuropean\") + f.col(\"other\"),\n        )\n        .drop(\n            \"european\",\n            \"other\",\n            \"initialSampleCount\",\n            \"initialSampleCountEuropean\",\n            \"initialSampleCountOther\",\n        )\n    )\n\n    parsed_ancestry_lut = ancestry_stages.join(\n        europeans_deconvoluted, on=\"studyId\", how=\"outer\"\n    ).select(\n        \"studyId\", \"discoverySamples\", \"ldPopulationStructure\", \"replicationSamples\"\n    )\n    self.df = self.df.join(parsed_ancestry_lut, on=\"studyId\", how=\"left\")\n    return self\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/study_index/#gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.annotate_discovery_sample_sizes","title":"<code>annotate_discovery_sample_sizes() -&gt; StudyIndexGWASCatalog</code>","text":"<p>Extract the sample size of the discovery stage of the study as annotated in the GWAS Catalog.</p> <p>For some studies that measure quantitative traits, nCases and nControls can't be extracted. Therefore, we assume these are 0.</p> <p>Returns:</p> Name Type Description <code>StudyIndexGWASCatalog</code> <code>StudyIndexGWASCatalog</code> <p>object with columns <code>nCases</code>, <code>nControls</code>, and <code>nSamples</code> per <code>studyId</code> correctly extracted.</p> Source code in <code>src/gentropy/datasource/gwas_catalog/study_index.py</code> <pre><code>def annotate_discovery_sample_sizes(\n    self: StudyIndexGWASCatalog,\n) -&gt; StudyIndexGWASCatalog:\n    \"\"\"Extract the sample size of the discovery stage of the study as annotated in the GWAS Catalog.\n\n    For some studies that measure quantitative traits, nCases and nControls can't be extracted. Therefore, we assume these are 0.\n\n    Returns:\n        StudyIndexGWASCatalog: object with columns `nCases`, `nControls`, and `nSamples` per `studyId` correctly extracted.\n    \"\"\"\n    sample_size_lut = (\n        self.df.select(\n            \"studyId\",\n            f.explode_outer(f.split(f.col(\"initialSampleSize\"), r\",\\s+\")).alias(\n                \"samples\"\n            ),\n        )\n        # Extracting the sample size from the string:\n        .withColumn(\n            \"sampleSize\",\n            f.regexp_extract(\n                f.regexp_replace(f.col(\"samples\"), \",\", \"\"), r\"[0-9,]+\", 0\n            ).cast(t.IntegerType()),\n        )\n        .select(\n            \"studyId\",\n            \"sampleSize\",\n            f.when(f.col(\"samples\").contains(\"cases\"), f.col(\"sampleSize\"))\n            .otherwise(f.lit(0))\n            .alias(\"nCases\"),\n            f.when(f.col(\"samples\").contains(\"controls\"), f.col(\"sampleSize\"))\n            .otherwise(f.lit(0))\n            .alias(\"nControls\"),\n        )\n        # Aggregating sample sizes for all ancestries:\n        .groupBy(\"studyId\")  # studyId has not been split yet\n        .agg(\n            f.sum(\"nCases\").cast(\"integer\").alias(\"nCases\"),\n            f.sum(\"nControls\").cast(\"integer\").alias(\"nControls\"),\n            f.sum(\"sampleSize\").cast(\"integer\").alias(\"nSamples\"),\n        )\n    )\n    self.df = self.df.join(sample_size_lut, on=\"studyId\", how=\"left\")\n    return self\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/study_index/#gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.annotate_from_study_curation","title":"<code>annotate_from_study_curation(curation_table: DataFrame | None) -&gt; StudyIndexGWASCatalog</code>","text":"<p>Annotating study index with curation.</p> <p>Parameters:</p> Name Type Description Default <code>curation_table</code> <code>DataFrame | None</code> <p>Curated GWAS Catalog studies with summary statistics</p> required <p>Returns:</p> Name Type Description <code>StudyIndexGWASCatalog</code> <code>StudyIndexGWASCatalog</code> <p>Updated study index</p> Source code in <code>src/gentropy/datasource/gwas_catalog/study_index.py</code> <pre><code>def annotate_from_study_curation(\n    self: StudyIndexGWASCatalog, curation_table: DataFrame | None\n) -&gt; StudyIndexGWASCatalog:\n    \"\"\"Annotating study index with curation.\n\n    Args:\n        curation_table (DataFrame | None): Curated GWAS Catalog studies with summary statistics\n\n    Returns:\n        StudyIndexGWASCatalog: Updated study index\n    \"\"\"\n    # Providing curation table is optional. However once this method is called, the quality and studyFlag columns are added.\n    if curation_table is None:\n        return self\n\n    columns = self.df.columns\n\n    # Adding prefix to columns in the curation table:\n    curation_table = curation_table.select(\n        *[\n            f.col(column).alias(f\"curation_{column}\")\n            if column != \"studyId\"\n            else f.col(column)\n            for column in curation_table.columns\n        ]\n    )\n\n    # Create expression how to update/create quality controls dataset:\n    qualityControls_expression = (\n        f.col(\"curation_qualityControls\")\n        if \"qualityControls\" not in columns\n        else f.when(\n            f.col(\"curation_qualityControls\").isNotNull(),\n            f.array_union(\n                f.col(\"qualityControls\"), f.array(f.col(\"curation_qualityControls\"))\n            ),\n        ).otherwise(f.col(\"qualityControls\"))\n    )\n\n    # Create expression how to update/create analysis flag:\n    analysis_expression = (\n        f.col(\"curation_analysisFlags\")\n        if \"analysisFlags\" not in columns\n        else f.when(\n            f.col(\"curation_analysisFlags\").isNotNull(),\n            f.array_union(\n                f.col(\"analysisFlags\"), f.array(f.col(\"curation_analysisFlags\"))\n            ),\n        ).otherwise(f.col(\"analysisFlags\"))\n    )\n\n    # Updating columns list. We might or might not list columns twice, but that doesn't matter, unique set will generated:\n    columns = list(set(columns + [\"qualityControls\", \"analysisFlags\"]))\n\n    # Based on the curation table, columns needs to be updated:\n    curated_df = (\n        self.df.join(curation_table, on=\"studyId\", how=\"left\")\n        # Updating study type:\n        .withColumn(\n            \"studyType\", f.coalesce(f.col(\"curation_studyType\"), f.col(\"studyType\"))\n        )\n        # Updating quality controls:\n        .withColumn(\"qualityControls\", qualityControls_expression)\n        # Updating study annotation flags:\n        .withColumn(\"analysisFlags\", analysis_expression)\n        # Dropping columns coming from the curation table:\n        .select(*columns)\n    )\n    return StudyIndexGWASCatalog(\n        _df=curated_df, _schema=StudyIndexGWASCatalog.get_schema()\n    )\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/study_index/#gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.annotate_sumstats_info","title":"<code>annotate_sumstats_info(sumstats_lut: DataFrame) -&gt; StudyIndexGWASCatalog</code>","text":"<p>Annotate summary stat locations.</p> <p>Parameters:</p> Name Type Description Default <code>sumstats_lut</code> <code>DataFrame</code> <p>listing GWAS Catalog summary stats paths</p> required <p>Returns:</p> Name Type Description <code>StudyIndexGWASCatalog</code> <code>StudyIndexGWASCatalog</code> <p>including <code>summarystatsLocation</code> and <code>hasSumstats</code> columns</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>if the sumstats_lut table doesn't have the right columns</p> Source code in <code>src/gentropy/datasource/gwas_catalog/study_index.py</code> <pre><code>def annotate_sumstats_info(\n    self: StudyIndexGWASCatalog, sumstats_lut: DataFrame\n) -&gt; StudyIndexGWASCatalog:\n    \"\"\"Annotate summary stat locations.\n\n    Args:\n        sumstats_lut (DataFrame): listing GWAS Catalog summary stats paths\n\n    Returns:\n        StudyIndexGWASCatalog: including `summarystatsLocation` and `hasSumstats` columns\n\n    Raises:\n        ValueError: if the sumstats_lut table doesn't have the right columns\n    \"\"\"\n    gwas_sumstats_base_uri = (\n        \"ftp://ftp.ebi.ac.uk/pub/databases/gwas/summary_statistics/\"\n    )\n\n    if \"_c0\" not in sumstats_lut.columns:\n        raise ValueError(\n            f'Sumstats look-up table needs to have `_c0` column. However it has: {\",\".join(sumstats_lut.columns)}'\n        )\n\n    parsed_sumstats_lut = sumstats_lut.withColumn(\n        \"summarystatsLocation\",\n        f.concat(\n            f.lit(gwas_sumstats_base_uri),\n            f.regexp_replace(f.col(\"_c0\"), r\"^\\.\\/\", \"\"),\n        ),\n    ).select(\n        self._parse_gwas_catalog_study_id(\"summarystatsLocation\").alias(\"studyId\"),\n        \"summarystatsLocation\",\n        f.lit(True).alias(\"hasSumstats\"),\n    )\n    self.df = (\n        self.df.drop(\"hasSumstats\")\n        .join(parsed_sumstats_lut, on=\"studyId\", how=\"left\")\n        .withColumn(\"hasSumstats\", f.coalesce(f.col(\"hasSumstats\"), f.lit(False)))\n    )\n    return self\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/study_index/#gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.apply_inclusion_list","title":"<code>apply_inclusion_list(inclusion_list: DataFrame) -&gt; StudyIndexGWASCatalog</code>","text":"<p>Restricting GWAS Catalog studies based on a list of accepted study identifiers.</p> <p>Parameters:</p> Name Type Description Default <code>inclusion_list</code> <code>DataFrame</code> <p>List of accepted GWAS Catalog study identifiers</p> required <p>Returns:</p> Name Type Description <code>StudyIndexGWASCatalog</code> <code>StudyIndexGWASCatalog</code> <p>Filtered dataset.</p> Source code in <code>src/gentropy/datasource/gwas_catalog/study_index.py</code> <pre><code>def apply_inclusion_list(\n    self: StudyIndexGWASCatalog, inclusion_list: DataFrame\n) -&gt; StudyIndexGWASCatalog:\n    \"\"\"Restricting GWAS Catalog studies based on a list of accepted study identifiers.\n\n    Args:\n        inclusion_list (DataFrame): List of accepted GWAS Catalog study identifiers\n\n    Returns:\n        StudyIndexGWASCatalog: Filtered dataset.\n    \"\"\"\n    return StudyIndexGWASCatalog(\n        _df=self.df.join(inclusion_list, on=\"studyId\", how=\"inner\"),\n        _schema=StudyIndexGWASCatalog.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/study_index/#gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.extract_studies_for_curation","title":"<code>extract_studies_for_curation(curation: DataFrame | None) -&gt; DataFrame</code>","text":"<p>Extract studies for curation.</p> <p>Parameters:</p> Name Type Description Default <code>curation</code> <code>DataFrame | None</code> <p>Dataframe with curation.</p> required <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>Updated curation table. New studies are have the <code>isCurated</code> False.</p> Source code in <code>src/gentropy/datasource/gwas_catalog/study_index.py</code> <pre><code>def extract_studies_for_curation(\n    self: StudyIndexGWASCatalog, curation: DataFrame | None\n) -&gt; DataFrame:\n    \"\"\"Extract studies for curation.\n\n    Args:\n        curation (DataFrame | None): Dataframe with curation.\n\n    Returns:\n        DataFrame: Updated curation table. New studies are have the `isCurated` False.\n    \"\"\"\n    # If no curation table provided, assume all studies needs curation:\n    if curation is None:\n        return (\n            self.df\n            # Curation only applyed on studies with summary statistics:\n            .filter(f.col(\"hasSumstats\"))\n            # Adding columns expected in the curation table - array columns aready flattened:\n            .withColumn(\"studyType\", f.lit(None).cast(t.StringType()))\n            .withColumn(\"analysisFlag\", f.lit(None).cast(t.StringType()))\n            .withColumn(\"qualityControl\", f.lit(None).cast(t.StringType()))\n            .withColumn(\"isCurated\", f.lit(False).cast(t.StringType()))\n        )\n\n    # Adding prefix to columns in the curation table:\n    curation = curation.select(\n        *[\n            f.col(column).alias(f\"curation_{column}\")\n            if column != \"studyId\"\n            else f.col(column)\n            for column in curation.columns\n        ]\n    )\n\n    return (\n        self.df\n        # Curation only applyed on studies with summary statistics:\n        .filter(f.col(\"hasSumstats\"))\n        .join(curation, on=\"studyId\", how=\"left\")\n        .select(\n            \"studyId\",\n            # Propagate existing curation - array columns are being flattened:\n            f.col(\"curation_studyType\").alias(\"studyType\"),\n            f.array_join(f.col(\"curation_analysisFlags\"), \"|\").alias(\n                \"analysisFlag\"\n            ),\n            f.array_join(f.col(\"curation_qualityControls\"), \"|\").alias(\n                \"qualityControl\"\n            ),\n            # This boolean flag needs to be casted to string, because saving to tsv would fail otherwise:\n            f.coalesce(f.col(\"curation_isCurated\"), f.lit(False))\n            .cast(t.StringType())\n            .alias(\"isCurated\"),\n            # The following columns are propagated to make curation easier:\n            \"pubmedId\",\n            \"publicationTitle\",\n            \"traitFromSource\",\n        )\n    )\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/study_index/#gentropy.datasource.gwas_catalog.study_index.StudyIndexGWASCatalog.update_study_id","title":"<code>update_study_id(study_annotation: DataFrame) -&gt; StudyIndexGWASCatalog</code>","text":"<p>Update studyId with a dataframe containing study.</p> <p>Parameters:</p> Name Type Description Default <code>study_annotation</code> <code>DataFrame</code> <p>Dataframe containing <code>updatedStudyId</code>, <code>traitFromSource</code>, <code>traitFromSourceMappedIds</code> and key column <code>studyId</code>.</p> required <p>Returns:</p> Name Type Description <code>StudyIndexGWASCatalog</code> <code>StudyIndexGWASCatalog</code> <p>Updated study table.</p> Source code in <code>src/gentropy/datasource/gwas_catalog/study_index.py</code> <pre><code>def update_study_id(\n    self: StudyIndexGWASCatalog, study_annotation: DataFrame\n) -&gt; StudyIndexGWASCatalog:\n    \"\"\"Update studyId with a dataframe containing study.\n\n    Args:\n        study_annotation (DataFrame): Dataframe containing `updatedStudyId`, `traitFromSource`, `traitFromSourceMappedIds` and key column `studyId`.\n\n    Returns:\n        StudyIndexGWASCatalog: Updated study table.\n    \"\"\"\n    self.df = (\n        self._df.join(\n            study_annotation.select(\n                *[\n                    f.col(c).alias(f\"updated{c}\")\n                    if c not in [\"studyId\", \"updatedStudyId\"]\n                    else f.col(c)\n                    for c in study_annotation.columns\n                ]\n            ),\n            on=\"studyId\",\n            how=\"left\",\n        )\n        .withColumn(\n            \"studyId\",\n            f.coalesce(f.col(\"updatedStudyId\"), f.col(\"studyId\")),\n        )\n        .withColumn(\n            \"traitFromSource\",\n            f.coalesce(f.col(\"updatedtraitFromSource\"), f.col(\"traitFromSource\")),\n        )\n        .withColumn(\n            \"traitFromSourceMappedIds\",\n            f.coalesce(\n                f.col(\"updatedtraitFromSourceMappedIds\"),\n                f.col(\"traitFromSourceMappedIds\"),\n            ),\n        )\n        .select(self._df.columns)\n    )\n\n    return self\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/study_splitter/","title":"Study Splitter","text":""},{"location":"python_api/datasources/gwas_catalog/study_splitter/#gentropy.datasource.gwas_catalog.study_splitter.GWASCatalogStudySplitter","title":"<code>gentropy.datasource.gwas_catalog.study_splitter.GWASCatalogStudySplitter</code>","text":"<p>Splitting multi-trait GWAS Catalog studies.</p> Source code in <code>src/gentropy/datasource/gwas_catalog/study_splitter.py</code> <pre><code>class GWASCatalogStudySplitter:\n    \"\"\"Splitting multi-trait GWAS Catalog studies.\"\"\"\n\n    @staticmethod\n    def _resolve_trait(\n        study_trait: Column, association_trait: Column, p_value_text: Column\n    ) -&gt; Column:\n        \"\"\"Resolve trait names by consolidating association-level and study-level trait names.\n\n        Args:\n            study_trait (Column): Study-level trait name.\n            association_trait (Column): Association-level trait name.\n            p_value_text (Column): P-value text.\n\n        Returns:\n            Column: Resolved trait name.\n        \"\"\"\n        return (\n            f.when(\n                (p_value_text.isNotNull()) &amp; (p_value_text != (\"no_pvalue_text\")),\n                f.concat(\n                    association_trait,\n                    f.lit(\" [\"),\n                    p_value_text,\n                    f.lit(\"]\"),\n                ),\n            )\n            .when(\n                association_trait.isNotNull(),\n                association_trait,\n            )\n            .otherwise(study_trait)\n        )\n\n    @staticmethod\n    def _resolve_efo(association_efo: Column, study_efo: Column) -&gt; Column:\n        \"\"\"Resolve EFOs by consolidating association-level and study-level EFOs.\n\n        Args:\n            association_efo (Column): EFO column from the association table.\n            study_efo (Column): EFO column from the study table.\n\n        Returns:\n            Column: Consolidated EFO column.\n        \"\"\"\n        return f.coalesce(f.split(association_efo, r\"\\/\"), study_efo)\n\n    @staticmethod\n    def _resolve_study_id(study_id: Column, sub_study_description: Column) -&gt; Column:\n        \"\"\"Resolve study IDs by exploding association-level information (e.g. pvalue_text, EFO).\n\n        Args:\n            study_id (Column): Study ID column.\n            sub_study_description (Column): Sub-study description column from the association table.\n\n        Returns:\n            Column: Resolved study ID column.\n        \"\"\"\n        split_w = Window.partitionBy(study_id).orderBy(sub_study_description)\n        row_number = f.dense_rank().over(split_w)\n        substudy_count = f.approx_count_distinct(row_number).over(split_w)\n        return f.when(substudy_count == 1, study_id).otherwise(\n            f.concat_ws(\"_\", study_id, row_number)\n        )\n\n    @classmethod\n    def split(\n        cls: type[GWASCatalogStudySplitter],\n        studies: StudyIndexGWASCatalog,\n        associations: StudyLocusGWASCatalog,\n    ) -&gt; Tuple[StudyIndexGWASCatalog, StudyLocusGWASCatalog]:\n        \"\"\"Splitting multi-trait GWAS Catalog studies.\n\n        If assigned disease of the study and the association don't agree, we assume the study needs to be split.\n        Then disease EFOs, trait names and study ID are consolidated\n\n        Args:\n            studies (StudyIndexGWASCatalog): GWAS Catalog studies.\n            associations (StudyLocusGWASCatalog): GWAS Catalog associations.\n\n        Returns:\n            Tuple[StudyIndexGWASCatalog, StudyLocusGWASCatalog]: Split studies and associations.\n        \"\"\"\n        # Composite of studies and associations to resolve scattered information\n        st_ass = (\n            associations.df.join(f.broadcast(studies.df), on=\"studyId\", how=\"inner\")\n            .select(\n                \"studyId\",\n                \"subStudyDescription\",\n                cls._resolve_study_id(\n                    f.col(\"studyId\"), f.col(\"subStudyDescription\")\n                ).alias(\"updatedStudyId\"),\n                cls._resolve_trait(\n                    f.col(\"traitFromSource\"),\n                    f.split(\"subStudyDescription\", r\"\\|\").getItem(0),\n                    f.split(\"subStudyDescription\", r\"\\|\").getItem(1),\n                ).alias(\"traitFromSource\"),\n                cls._resolve_efo(\n                    f.split(\"subStudyDescription\", r\"\\|\").getItem(2),\n                    f.col(\"traitFromSourceMappedIds\"),\n                ).alias(\"traitFromSourceMappedIds\"),\n            )\n            .persist()\n        )\n\n        return (\n            studies.update_study_id(\n                st_ass.select(\n                    \"studyId\",\n                    \"updatedStudyId\",\n                    \"traitFromSource\",\n                    \"traitFromSourceMappedIds\",\n                ).distinct()\n            ),\n            associations.update_study_id(\n                st_ass.select(\n                    \"updatedStudyId\", \"studyId\", \"subStudyDescription\"\n                ).distinct()\n            ).qc_ambiguous_study(),\n        )\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/study_splitter/#gentropy.datasource.gwas_catalog.study_splitter.GWASCatalogStudySplitter.split","title":"<code>split(studies: StudyIndexGWASCatalog, associations: StudyLocusGWASCatalog) -&gt; Tuple[StudyIndexGWASCatalog, StudyLocusGWASCatalog]</code>  <code>classmethod</code>","text":"<p>Splitting multi-trait GWAS Catalog studies.</p> <p>If assigned disease of the study and the association don't agree, we assume the study needs to be split. Then disease EFOs, trait names and study ID are consolidated</p> <p>Parameters:</p> Name Type Description Default <code>studies</code> <code>StudyIndexGWASCatalog</code> <p>GWAS Catalog studies.</p> required <code>associations</code> <code>StudyLocusGWASCatalog</code> <p>GWAS Catalog associations.</p> required <p>Returns:</p> Type Description <code>Tuple[StudyIndexGWASCatalog, StudyLocusGWASCatalog]</code> <p>Tuple[StudyIndexGWASCatalog, StudyLocusGWASCatalog]: Split studies and associations.</p> Source code in <code>src/gentropy/datasource/gwas_catalog/study_splitter.py</code> <pre><code>@classmethod\ndef split(\n    cls: type[GWASCatalogStudySplitter],\n    studies: StudyIndexGWASCatalog,\n    associations: StudyLocusGWASCatalog,\n) -&gt; Tuple[StudyIndexGWASCatalog, StudyLocusGWASCatalog]:\n    \"\"\"Splitting multi-trait GWAS Catalog studies.\n\n    If assigned disease of the study and the association don't agree, we assume the study needs to be split.\n    Then disease EFOs, trait names and study ID are consolidated\n\n    Args:\n        studies (StudyIndexGWASCatalog): GWAS Catalog studies.\n        associations (StudyLocusGWASCatalog): GWAS Catalog associations.\n\n    Returns:\n        Tuple[StudyIndexGWASCatalog, StudyLocusGWASCatalog]: Split studies and associations.\n    \"\"\"\n    # Composite of studies and associations to resolve scattered information\n    st_ass = (\n        associations.df.join(f.broadcast(studies.df), on=\"studyId\", how=\"inner\")\n        .select(\n            \"studyId\",\n            \"subStudyDescription\",\n            cls._resolve_study_id(\n                f.col(\"studyId\"), f.col(\"subStudyDescription\")\n            ).alias(\"updatedStudyId\"),\n            cls._resolve_trait(\n                f.col(\"traitFromSource\"),\n                f.split(\"subStudyDescription\", r\"\\|\").getItem(0),\n                f.split(\"subStudyDescription\", r\"\\|\").getItem(1),\n            ).alias(\"traitFromSource\"),\n            cls._resolve_efo(\n                f.split(\"subStudyDescription\", r\"\\|\").getItem(2),\n                f.col(\"traitFromSourceMappedIds\"),\n            ).alias(\"traitFromSourceMappedIds\"),\n        )\n        .persist()\n    )\n\n    return (\n        studies.update_study_id(\n            st_ass.select(\n                \"studyId\",\n                \"updatedStudyId\",\n                \"traitFromSource\",\n                \"traitFromSourceMappedIds\",\n            ).distinct()\n        ),\n        associations.update_study_id(\n            st_ass.select(\n                \"updatedStudyId\", \"studyId\", \"subStudyDescription\"\n            ).distinct()\n        ).qc_ambiguous_study(),\n    )\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/summary_statistics/","title":"Summary statistics","text":""},{"location":"python_api/datasources/gwas_catalog/summary_statistics/#gentropy.datasource.gwas_catalog.summary_statistics.GWASCatalogSummaryStatistics","title":"<code>gentropy.datasource.gwas_catalog.summary_statistics.GWASCatalogSummaryStatistics</code>  <code>dataclass</code>","text":"<p>             Bases: <code>SummaryStatistics</code></p> <p>GWAS Catalog Summary Statistics reader.</p> Source code in <code>src/gentropy/datasource/gwas_catalog/summary_statistics.py</code> <pre><code>@dataclass\nclass GWASCatalogSummaryStatistics(SummaryStatistics):\n    \"\"\"GWAS Catalog Summary Statistics reader.\"\"\"\n\n    @classmethod\n    def from_gwas_harmonized_summary_stats(\n        cls: type[GWASCatalogSummaryStatistics],\n        spark: SparkSession,\n        sumstats_file: str,\n    ) -&gt; GWASCatalogSummaryStatistics:\n        \"\"\"Create summary statistics object from summary statistics flatfile, harmonized by the GWAS Catalog.\n\n        Things got slightly complicated given the GWAS Catalog harmonization pipelines changed recently so we had to accomodate to\n        both formats.\n\n        Args:\n            spark (SparkSession): spark session\n            sumstats_file (str): list of GWAS Catalog summary stat files, with study ids in them.\n\n        Returns:\n            GWASCatalogSummaryStatistics: Summary statistics object.\n        \"\"\"\n        sumstats_df = spark.read.csv(sumstats_file, sep=\"\\t\", header=True).withColumn(\n            # Parsing GWAS Catalog study identifier from filename:\n            \"studyId\",\n            f.lit(filename_to_study_identifier(sumstats_file)),\n        )\n\n        # Parsing variant id fields:\n        chromosome = (\n            f.col(\"hm_chrom\")\n            if \"hm_chrom\" in sumstats_df.columns\n            else f.col(\"chromosome\")\n        ).cast(t.StringType())\n        position = (\n            f.col(\"hm_pos\")\n            if \"hm_pos\" in sumstats_df.columns\n            else f.col(\"base_pair_location\")\n        ).cast(t.IntegerType())\n        ref_allele = (\n            f.col(\"hm_other_allele\")\n            if \"hm_other_allele\" in sumstats_df.columns\n            else f.col(\"other_allele\")\n        )\n        alt_allele = (\n            f.col(\"hm_effect_allele\")\n            if \"hm_effect_allele\" in sumstats_df.columns\n            else f.col(\"effect_allele\")\n        )\n\n        # Parsing p-value (get a tuple with mantissa and exponent):\n        p_value_expression = (\n            parse_pvalue(f.col(\"p_value\"))\n            if \"p_value\" in sumstats_df.columns\n            else neglog_pvalue_to_mantissa_and_exponent(f.col(\"neg_log_10_p_value\"))\n        )\n\n        # The effect allele frequency is an optional column, we have to test if it is there:\n        allele_frequency = (\n            f.col(\"effect_allele_frequency\")\n            if \"effect_allele_frequency\" in sumstats_df.columns\n            else f.lit(None)\n        ).cast(t.FloatType())\n\n        # Do we have sample size? This expression captures 99.7% of sample size columns.\n        sample_size = (f.col(\"n\") if \"n\" in sumstats_df.columns else f.lit(None)).cast(\n            t.IntegerType()\n        )\n\n        # Depending on the input, we might have beta, but the column might not be there at all also old format calls differently:\n        beta_expression = (\n            f.col(\"hm_beta\")\n            if \"hm_beta\" in sumstats_df.columns\n            else f.col(\"beta\")\n            if \"beta\" in sumstats_df.columns\n            # If no column, create one:\n            else f.lit(None)\n        ).cast(t.DoubleType())\n\n        # We might have odds ratio or hazard ratio, wich are basically the same:\n        odds_ratio_expression = (\n            f.col(\"hm_odds_ratio\")\n            if \"hm_odds_ratio\" in sumstats_df.columns\n            else f.col(\"odds_ratio\")\n            if \"odds_ratio\" in sumstats_df.columns\n            else f.col(\"hazard_ratio\")\n            if \"hazard_ratio\" in sumstats_df.columns\n            # If no column, create one:\n            else f.lit(None)\n        ).cast(t.DoubleType())\n\n        # Does the file have standard error column?\n        standard_error = (\n            f.col(\"standard_error\")\n            if \"standard_error\" in sumstats_df.columns\n            else f.lit(None)\n        ).cast(t.DoubleType())\n\n        # Processing columns of interest:\n        processed_sumstats_df = (\n            sumstats_df\n            # Dropping rows which doesn't have proper position:\n            .select(\n                \"studyId\",\n                # Adding variant identifier:\n                f.concat_ws(\n                    \"_\",\n                    chromosome,\n                    position,\n                    ref_allele,\n                    alt_allele,\n                ).alias(\"variantId\"),\n                chromosome.alias(\"chromosome\"),\n                position.alias(\"position\"),\n                # Parsing p-value mantissa and exponent:\n                *p_value_expression,\n                # Converting/calculating effect and confidence interval:\n                *convert_odds_ratio_to_beta(\n                    beta_expression,\n                    odds_ratio_expression,\n                    standard_error,\n                ),\n                allele_frequency.alias(\"effectAlleleFrequencyFromSource\"),\n                sample_size.alias(\"sampleSize\"),\n            )\n            .filter(\n                # Dropping associations where no harmonized position is available:\n                f.col(\"position\").isNotNull()\n                &amp;\n                # We are not interested in associations with zero effect:\n                (f.col(\"beta\") != 0)\n            )\n            .orderBy(f.col(\"chromosome\"), f.col(\"position\"))\n            # median study size is 200Mb, max is 2.6Gb\n            .repartition(20)\n        )\n\n        # Initializing summary statistics object:\n        return cls(\n            _df=processed_sumstats_df,\n            _schema=cls.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasources/gwas_catalog/summary_statistics/#gentropy.datasource.gwas_catalog.summary_statistics.GWASCatalogSummaryStatistics.from_gwas_harmonized_summary_stats","title":"<code>from_gwas_harmonized_summary_stats(spark: SparkSession, sumstats_file: str) -&gt; GWASCatalogSummaryStatistics</code>  <code>classmethod</code>","text":"<p>Create summary statistics object from summary statistics flatfile, harmonized by the GWAS Catalog.</p> <p>Things got slightly complicated given the GWAS Catalog harmonization pipelines changed recently so we had to accomodate to both formats.</p> <p>Parameters:</p> Name Type Description Default <code>spark</code> <code>SparkSession</code> <p>spark session</p> required <code>sumstats_file</code> <code>str</code> <p>list of GWAS Catalog summary stat files, with study ids in them.</p> required <p>Returns:</p> Name Type Description <code>GWASCatalogSummaryStatistics</code> <code>GWASCatalogSummaryStatistics</code> <p>Summary statistics object.</p> Source code in <code>src/gentropy/datasource/gwas_catalog/summary_statistics.py</code> <pre><code>@classmethod\ndef from_gwas_harmonized_summary_stats(\n    cls: type[GWASCatalogSummaryStatistics],\n    spark: SparkSession,\n    sumstats_file: str,\n) -&gt; GWASCatalogSummaryStatistics:\n    \"\"\"Create summary statistics object from summary statistics flatfile, harmonized by the GWAS Catalog.\n\n    Things got slightly complicated given the GWAS Catalog harmonization pipelines changed recently so we had to accomodate to\n    both formats.\n\n    Args:\n        spark (SparkSession): spark session\n        sumstats_file (str): list of GWAS Catalog summary stat files, with study ids in them.\n\n    Returns:\n        GWASCatalogSummaryStatistics: Summary statistics object.\n    \"\"\"\n    sumstats_df = spark.read.csv(sumstats_file, sep=\"\\t\", header=True).withColumn(\n        # Parsing GWAS Catalog study identifier from filename:\n        \"studyId\",\n        f.lit(filename_to_study_identifier(sumstats_file)),\n    )\n\n    # Parsing variant id fields:\n    chromosome = (\n        f.col(\"hm_chrom\")\n        if \"hm_chrom\" in sumstats_df.columns\n        else f.col(\"chromosome\")\n    ).cast(t.StringType())\n    position = (\n        f.col(\"hm_pos\")\n        if \"hm_pos\" in sumstats_df.columns\n        else f.col(\"base_pair_location\")\n    ).cast(t.IntegerType())\n    ref_allele = (\n        f.col(\"hm_other_allele\")\n        if \"hm_other_allele\" in sumstats_df.columns\n        else f.col(\"other_allele\")\n    )\n    alt_allele = (\n        f.col(\"hm_effect_allele\")\n        if \"hm_effect_allele\" in sumstats_df.columns\n        else f.col(\"effect_allele\")\n    )\n\n    # Parsing p-value (get a tuple with mantissa and exponent):\n    p_value_expression = (\n        parse_pvalue(f.col(\"p_value\"))\n        if \"p_value\" in sumstats_df.columns\n        else neglog_pvalue_to_mantissa_and_exponent(f.col(\"neg_log_10_p_value\"))\n    )\n\n    # The effect allele frequency is an optional column, we have to test if it is there:\n    allele_frequency = (\n        f.col(\"effect_allele_frequency\")\n        if \"effect_allele_frequency\" in sumstats_df.columns\n        else f.lit(None)\n    ).cast(t.FloatType())\n\n    # Do we have sample size? This expression captures 99.7% of sample size columns.\n    sample_size = (f.col(\"n\") if \"n\" in sumstats_df.columns else f.lit(None)).cast(\n        t.IntegerType()\n    )\n\n    # Depending on the input, we might have beta, but the column might not be there at all also old format calls differently:\n    beta_expression = (\n        f.col(\"hm_beta\")\n        if \"hm_beta\" in sumstats_df.columns\n        else f.col(\"beta\")\n        if \"beta\" in sumstats_df.columns\n        # If no column, create one:\n        else f.lit(None)\n    ).cast(t.DoubleType())\n\n    # We might have odds ratio or hazard ratio, wich are basically the same:\n    odds_ratio_expression = (\n        f.col(\"hm_odds_ratio\")\n        if \"hm_odds_ratio\" in sumstats_df.columns\n        else f.col(\"odds_ratio\")\n        if \"odds_ratio\" in sumstats_df.columns\n        else f.col(\"hazard_ratio\")\n        if \"hazard_ratio\" in sumstats_df.columns\n        # If no column, create one:\n        else f.lit(None)\n    ).cast(t.DoubleType())\n\n    # Does the file have standard error column?\n    standard_error = (\n        f.col(\"standard_error\")\n        if \"standard_error\" in sumstats_df.columns\n        else f.lit(None)\n    ).cast(t.DoubleType())\n\n    # Processing columns of interest:\n    processed_sumstats_df = (\n        sumstats_df\n        # Dropping rows which doesn't have proper position:\n        .select(\n            \"studyId\",\n            # Adding variant identifier:\n            f.concat_ws(\n                \"_\",\n                chromosome,\n                position,\n                ref_allele,\n                alt_allele,\n            ).alias(\"variantId\"),\n            chromosome.alias(\"chromosome\"),\n            position.alias(\"position\"),\n            # Parsing p-value mantissa and exponent:\n            *p_value_expression,\n            # Converting/calculating effect and confidence interval:\n            *convert_odds_ratio_to_beta(\n                beta_expression,\n                odds_ratio_expression,\n                standard_error,\n            ),\n            allele_frequency.alias(\"effectAlleleFrequencyFromSource\"),\n            sample_size.alias(\"sampleSize\"),\n        )\n        .filter(\n            # Dropping associations where no harmonized position is available:\n            f.col(\"position\").isNotNull()\n            &amp;\n            # We are not interested in associations with zero effect:\n            (f.col(\"beta\") != 0)\n        )\n        .orderBy(f.col(\"chromosome\"), f.col(\"position\"))\n        # median study size is 200Mb, max is 2.6Gb\n        .repartition(20)\n    )\n\n    # Initializing summary statistics object:\n    return cls(\n        _df=processed_sumstats_df,\n        _schema=cls.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasources/intervals/_intervals/","title":"List of Interaction and Interval-based Studies","text":"<p>In this section, we provide a list of studies that focus on interaction and interval-based investigations, shedding light on the intricate relationships between genetic elements and their functional implications.</p> <ol> <li> <p>Promoter Capture Hi-C (Javierre et al., 2016): Title: \"Lineage-Specific Genome Architecture Links Enhancers and Non-coding Disease Variants to Target Gene Promoters\".    This study presents evidence linking genetic variation to genes through the application of Promoter Capture Hi-C across each of the 17 human primary hematopoietic cell types. The method captures interactions between promoters and distal regulatory elements, providing valuable insights into the three-dimensional chromatin architecture. DOI: 10.1016/j.cell.2016.09.037</p> </li> <li> <p>Enhancer-TSS Correlation (Andersson et al., 2014): Title: \"An Atlas of Active Enhancers across Human Cell Types and Tissues\".    This study explores genetic variation's impact on genes by examining the correlation between the transcriptional activity of enhancers and transcription start sites. The findings are documented in the FANTOM5 CAGE expression atlas, offering a comprehensive view of the regulatory landscape. DOI: 10.1038/nature12787</p> </li> <li> <p>DHS-Promoter Correlation (Thurman et al., 2012): Title: \"The accessible chromatin landscape of the human genome\".    Investigating genetic variation's connection to genes, this study employs the correlation of DNase I hypersensitive sites (DHS) and gene promoters. The analysis spans 125 cell and tissue types from the ENCODE project, providing a broad understanding of the regulatory interactions across diverse biological contexts. DOI: 10.1038/nature11232</p> </li> <li> <p>Promoter Capture Hi-C (Jung et al., 2019): Title: \"A compendium of promoter-centered long-range chromatin interactions in the human genome\".    This study compiles a compendium of promoter-centered long-range chromatin interactions in the human genome. By focusing on the three-dimensional organization of chromatin, the research contributes to our understanding of the spatial arrangement of genetic elements and their implications in gene regulation. DOI: 10.1038/s41588-019-0494-8</p> </li> </ol> <p>For in-depth details on each study, you may refer to the respective publications.</p>"},{"location":"python_api/datasources/intervals/andersson/","title":"Andersson et al.","text":""},{"location":"python_api/datasources/intervals/andersson/#gentropy.datasource.intervals.andersson.IntervalsAndersson","title":"<code>gentropy.datasource.intervals.andersson.IntervalsAndersson</code>","text":"<p>Interval dataset from Andersson et al. 2014.</p> Source code in <code>src/gentropy/datasource/intervals/andersson.py</code> <pre><code>class IntervalsAndersson:\n    \"\"\"Interval dataset from Andersson et al. 2014.\"\"\"\n\n    @staticmethod\n    def read(spark: SparkSession, path: str) -&gt; DataFrame:\n        \"\"\"Read andersson2014 dataset.\n\n        Args:\n            spark (SparkSession): Spark session\n            path (str): Path to the dataset\n\n        Returns:\n            DataFrame: Raw Andersson et al. dataframe\n        \"\"\"\n        input_schema = t.StructType.fromJson(\n            json.loads(\n                pkg_resources.read_text(schemas, \"andersson2014.json\", encoding=\"utf-8\")\n            )\n        )\n        return (\n            spark.read.option(\"delimiter\", \"\\t\")\n            .option(\"mode\", \"DROPMALFORMED\")\n            .option(\"header\", \"true\")\n            .schema(input_schema)\n            .csv(path)\n        )\n\n    @classmethod\n    def parse(\n        cls: type[IntervalsAndersson],\n        raw_anderson_df: DataFrame,\n        gene_index: GeneIndex,\n        lift: LiftOverSpark,\n    ) -&gt; Intervals:\n        \"\"\"Parse Andersson et al. 2014 dataset.\n\n        Args:\n            raw_anderson_df (DataFrame): Raw Andersson et al. dataset\n            gene_index (GeneIndex): Gene index\n            lift (LiftOverSpark): LiftOverSpark instance\n\n        Returns:\n            Intervals: Intervals dataset\n        \"\"\"\n        # Constant values:\n        dataset_name = \"andersson2014\"\n        experiment_type = \"fantom5\"\n        pmid = \"24670763\"\n        bio_feature = \"aggregate\"\n        twosided_threshold = 2.45e6  # &lt;-  this needs to phased out. Filter by percentile instead of absolute value.\n\n        # Read the anderson file:\n        parsed_anderson_df = (\n            raw_anderson_df\n            # Parsing score column and casting as float:\n            .withColumn(\"score\", f.col(\"score\").cast(\"float\") / f.lit(1000))\n            # Parsing the 'name' column:\n            .withColumn(\"parsedName\", f.split(f.col(\"name\"), \";\"))\n            .withColumn(\"gene_symbol\", f.col(\"parsedName\")[2])\n            .withColumn(\"location\", f.col(\"parsedName\")[0])\n            .withColumn(\n                \"chrom\",\n                f.regexp_replace(f.split(f.col(\"location\"), \":|-\")[0], \"chr\", \"\"),\n            )\n            .withColumn(\n                \"start\", f.split(f.col(\"location\"), \":|-\")[1].cast(t.IntegerType())\n            )\n            .withColumn(\n                \"end\", f.split(f.col(\"location\"), \":|-\")[2].cast(t.IntegerType())\n            )\n            # Select relevant columns:\n            .select(\"chrom\", \"start\", \"end\", \"gene_symbol\", \"score\")\n            # Drop rows with non-canonical chromosomes:\n            .filter(\n                f.col(\"chrom\").isin([str(x) for x in range(1, 23)] + [\"X\", \"Y\", \"MT\"])\n            )\n            # For each region/gene, keep only one row with the highest score:\n            .groupBy(\"chrom\", \"start\", \"end\", \"gene_symbol\")\n            .agg(f.max(\"score\").alias(\"resourceScore\"))\n            .orderBy(\"chrom\", \"start\")\n        )\n\n        return Intervals(\n            _df=(\n                # Lift over the intervals:\n                lift.convert_intervals(parsed_anderson_df, \"chrom\", \"start\", \"end\")\n                .drop(\"start\", \"end\")\n                .withColumnRenamed(\"mapped_start\", \"start\")\n                .withColumnRenamed(\"mapped_end\", \"end\")\n                .distinct()\n                # Joining with the gene index\n                .alias(\"intervals\")\n                .join(\n                    gene_index.symbols_lut().alias(\"genes\"),\n                    on=[\n                        f.col(\"intervals.gene_symbol\") == f.col(\"genes.geneSymbol\"),\n                        # Drop rows where the TSS is far from the start of the region\n                        f.abs(\n                            (f.col(\"intervals.start\") + f.col(\"intervals.end\")) / 2\n                            - f.col(\"tss\")\n                        )\n                        &lt;= twosided_threshold,\n                    ],\n                    how=\"left\",\n                )\n                # Select relevant columns:\n                .select(\n                    f.col(\"chrom\").alias(\"chromosome\"),\n                    f.col(\"intervals.start\").alias(\"start\"),\n                    f.col(\"intervals.end\").alias(\"end\"),\n                    \"geneId\",\n                    \"resourceScore\",\n                    f.lit(dataset_name).alias(\"datasourceId\"),\n                    f.lit(experiment_type).alias(\"datatypeId\"),\n                    f.lit(pmid).alias(\"pmid\"),\n                    f.lit(bio_feature).alias(\"biofeature\"),\n                )\n            ),\n            _schema=Intervals.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasources/intervals/andersson/#gentropy.datasource.intervals.andersson.IntervalsAndersson.parse","title":"<code>parse(raw_anderson_df: DataFrame, gene_index: GeneIndex, lift: LiftOverSpark) -&gt; Intervals</code>  <code>classmethod</code>","text":"<p>Parse Andersson et al. 2014 dataset.</p> <p>Parameters:</p> Name Type Description Default <code>raw_anderson_df</code> <code>DataFrame</code> <p>Raw Andersson et al. dataset</p> required <code>gene_index</code> <code>GeneIndex</code> <p>Gene index</p> required <code>lift</code> <code>LiftOverSpark</code> <p>LiftOverSpark instance</p> required <p>Returns:</p> Name Type Description <code>Intervals</code> <code>Intervals</code> <p>Intervals dataset</p> Source code in <code>src/gentropy/datasource/intervals/andersson.py</code> <pre><code>@classmethod\ndef parse(\n    cls: type[IntervalsAndersson],\n    raw_anderson_df: DataFrame,\n    gene_index: GeneIndex,\n    lift: LiftOverSpark,\n) -&gt; Intervals:\n    \"\"\"Parse Andersson et al. 2014 dataset.\n\n    Args:\n        raw_anderson_df (DataFrame): Raw Andersson et al. dataset\n        gene_index (GeneIndex): Gene index\n        lift (LiftOverSpark): LiftOverSpark instance\n\n    Returns:\n        Intervals: Intervals dataset\n    \"\"\"\n    # Constant values:\n    dataset_name = \"andersson2014\"\n    experiment_type = \"fantom5\"\n    pmid = \"24670763\"\n    bio_feature = \"aggregate\"\n    twosided_threshold = 2.45e6  # &lt;-  this needs to phased out. Filter by percentile instead of absolute value.\n\n    # Read the anderson file:\n    parsed_anderson_df = (\n        raw_anderson_df\n        # Parsing score column and casting as float:\n        .withColumn(\"score\", f.col(\"score\").cast(\"float\") / f.lit(1000))\n        # Parsing the 'name' column:\n        .withColumn(\"parsedName\", f.split(f.col(\"name\"), \";\"))\n        .withColumn(\"gene_symbol\", f.col(\"parsedName\")[2])\n        .withColumn(\"location\", f.col(\"parsedName\")[0])\n        .withColumn(\n            \"chrom\",\n            f.regexp_replace(f.split(f.col(\"location\"), \":|-\")[0], \"chr\", \"\"),\n        )\n        .withColumn(\n            \"start\", f.split(f.col(\"location\"), \":|-\")[1].cast(t.IntegerType())\n        )\n        .withColumn(\n            \"end\", f.split(f.col(\"location\"), \":|-\")[2].cast(t.IntegerType())\n        )\n        # Select relevant columns:\n        .select(\"chrom\", \"start\", \"end\", \"gene_symbol\", \"score\")\n        # Drop rows with non-canonical chromosomes:\n        .filter(\n            f.col(\"chrom\").isin([str(x) for x in range(1, 23)] + [\"X\", \"Y\", \"MT\"])\n        )\n        # For each region/gene, keep only one row with the highest score:\n        .groupBy(\"chrom\", \"start\", \"end\", \"gene_symbol\")\n        .agg(f.max(\"score\").alias(\"resourceScore\"))\n        .orderBy(\"chrom\", \"start\")\n    )\n\n    return Intervals(\n        _df=(\n            # Lift over the intervals:\n            lift.convert_intervals(parsed_anderson_df, \"chrom\", \"start\", \"end\")\n            .drop(\"start\", \"end\")\n            .withColumnRenamed(\"mapped_start\", \"start\")\n            .withColumnRenamed(\"mapped_end\", \"end\")\n            .distinct()\n            # Joining with the gene index\n            .alias(\"intervals\")\n            .join(\n                gene_index.symbols_lut().alias(\"genes\"),\n                on=[\n                    f.col(\"intervals.gene_symbol\") == f.col(\"genes.geneSymbol\"),\n                    # Drop rows where the TSS is far from the start of the region\n                    f.abs(\n                        (f.col(\"intervals.start\") + f.col(\"intervals.end\")) / 2\n                        - f.col(\"tss\")\n                    )\n                    &lt;= twosided_threshold,\n                ],\n                how=\"left\",\n            )\n            # Select relevant columns:\n            .select(\n                f.col(\"chrom\").alias(\"chromosome\"),\n                f.col(\"intervals.start\").alias(\"start\"),\n                f.col(\"intervals.end\").alias(\"end\"),\n                \"geneId\",\n                \"resourceScore\",\n                f.lit(dataset_name).alias(\"datasourceId\"),\n                f.lit(experiment_type).alias(\"datatypeId\"),\n                f.lit(pmid).alias(\"pmid\"),\n                f.lit(bio_feature).alias(\"biofeature\"),\n            )\n        ),\n        _schema=Intervals.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasources/intervals/andersson/#gentropy.datasource.intervals.andersson.IntervalsAndersson.read","title":"<code>read(spark: SparkSession, path: str) -&gt; DataFrame</code>  <code>staticmethod</code>","text":"<p>Read andersson2014 dataset.</p> <p>Parameters:</p> Name Type Description Default <code>spark</code> <code>SparkSession</code> <p>Spark session</p> required <code>path</code> <code>str</code> <p>Path to the dataset</p> required <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>Raw Andersson et al. dataframe</p> Source code in <code>src/gentropy/datasource/intervals/andersson.py</code> <pre><code>@staticmethod\ndef read(spark: SparkSession, path: str) -&gt; DataFrame:\n    \"\"\"Read andersson2014 dataset.\n\n    Args:\n        spark (SparkSession): Spark session\n        path (str): Path to the dataset\n\n    Returns:\n        DataFrame: Raw Andersson et al. dataframe\n    \"\"\"\n    input_schema = t.StructType.fromJson(\n        json.loads(\n            pkg_resources.read_text(schemas, \"andersson2014.json\", encoding=\"utf-8\")\n        )\n    )\n    return (\n        spark.read.option(\"delimiter\", \"\\t\")\n        .option(\"mode\", \"DROPMALFORMED\")\n        .option(\"header\", \"true\")\n        .schema(input_schema)\n        .csv(path)\n    )\n</code></pre>"},{"location":"python_api/datasources/intervals/javierre/","title":"Javierre et al.","text":""},{"location":"python_api/datasources/intervals/javierre/#gentropy.datasource.intervals.javierre.IntervalsJavierre","title":"<code>gentropy.datasource.intervals.javierre.IntervalsJavierre</code>","text":"<p>Interval dataset from Javierre et al. 2016.</p> Source code in <code>src/gentropy/datasource/intervals/javierre.py</code> <pre><code>class IntervalsJavierre:\n    \"\"\"Interval dataset from Javierre et al. 2016.\"\"\"\n\n    @staticmethod\n    def read(spark: SparkSession, path: str) -&gt; DataFrame:\n        \"\"\"Read Javierre dataset.\n\n        Args:\n            spark (SparkSession): Spark session\n            path (str): Path to dataset\n\n        Returns:\n            DataFrame: Raw Javierre dataset\n        \"\"\"\n        return spark.read.parquet(path)\n\n    @classmethod\n    def parse(\n        cls: type[IntervalsJavierre],\n        javierre_raw: DataFrame,\n        gene_index: GeneIndex,\n        lift: LiftOverSpark,\n    ) -&gt; Intervals:\n        \"\"\"Parse Javierre et al. 2016 dataset.\n\n        Args:\n            javierre_raw (DataFrame): Raw Javierre data\n            gene_index (GeneIndex): Gene index\n            lift (LiftOverSpark): LiftOverSpark instance\n\n        Returns:\n            Intervals: Javierre et al. 2016 interval data\n        \"\"\"\n        # Constant values:\n        dataset_name = \"javierre2016\"\n        experiment_type = \"pchic\"\n        pmid = \"27863249\"\n        twosided_threshold = 2.45e6\n\n        # Read Javierre data:\n        javierre_parsed = (\n            javierre_raw\n            # Splitting name column into chromosome, start, end, and score:\n            .withColumn(\"name_split\", f.split(f.col(\"name\"), r\":|-|,\"))\n            .withColumn(\n                \"name_chr\",\n                f.regexp_replace(f.col(\"name_split\")[0], \"chr\", \"\").cast(\n                    t.StringType()\n                ),\n            )\n            .withColumn(\"name_start\", f.col(\"name_split\")[1].cast(t.IntegerType()))\n            .withColumn(\"name_end\", f.col(\"name_split\")[2].cast(t.IntegerType()))\n            .withColumn(\"name_score\", f.col(\"name_split\")[3].cast(t.FloatType()))\n            # Cleaning up chromosome:\n            .withColumn(\n                \"chrom\",\n                f.regexp_replace(f.col(\"chrom\"), \"chr\", \"\").cast(t.StringType()),\n            )\n            .drop(\"name_split\", \"name\", \"annotation\")\n            # Keep canonical chromosomes and consistent chromosomes with scores:\n            .filter(\n                (f.col(\"name_score\").isNotNull())\n                &amp; (f.col(\"chrom\") == f.col(\"name_chr\"))\n                &amp; f.col(\"name_chr\").isin(\n                    [f\"{x}\" for x in range(1, 23)] + [\"X\", \"Y\", \"MT\"]\n                )\n            )\n        )\n\n        # Lifting over intervals:\n        javierre_remapped = (\n            javierre_parsed\n            # Lifting over to GRCh38 interval 1:\n            .transform(lambda df: lift.convert_intervals(df, \"chrom\", \"start\", \"end\"))\n            .drop(\"start\", \"end\")\n            .withColumnRenamed(\"mapped_chrom\", \"chrom\")\n            .withColumnRenamed(\"mapped_start\", \"start\")\n            .withColumnRenamed(\"mapped_end\", \"end\")\n            # Lifting over interval 2 to GRCh38:\n            .transform(\n                lambda df: lift.convert_intervals(\n                    df, \"name_chr\", \"name_start\", \"name_end\"\n                )\n            )\n            .drop(\"name_start\", \"name_end\")\n            .withColumnRenamed(\"mapped_name_chr\", \"name_chr\")\n            .withColumnRenamed(\"mapped_name_start\", \"name_start\")\n            .withColumnRenamed(\"mapped_name_end\", \"name_end\")\n        )\n\n        # Once the intervals are lifted, extracting the unique intervals:\n        unique_intervals_with_genes = (\n            javierre_remapped.select(\n                f.col(\"chrom\"),\n                f.col(\"start\").cast(t.IntegerType()),\n                f.col(\"end\").cast(t.IntegerType()),\n            )\n            .distinct()\n            .alias(\"intervals\")\n            .join(\n                gene_index.locations_lut().alias(\"genes\"),\n                on=[\n                    f.col(\"intervals.chrom\") == f.col(\"genes.chromosome\"),\n                    (\n                        (f.col(\"intervals.start\") &gt;= f.col(\"genes.start\"))\n                        &amp; (f.col(\"intervals.start\") &lt;= f.col(\"genes.end\"))\n                    )\n                    | (\n                        (f.col(\"intervals.end\") &gt;= f.col(\"genes.start\"))\n                        &amp; (f.col(\"intervals.end\") &lt;= f.col(\"genes.end\"))\n                    ),\n                ],\n                how=\"left\",\n            )\n            .select(\n                f.col(\"intervals.chrom\").alias(\"chrom\"),\n                f.col(\"intervals.start\").alias(\"start\"),\n                f.col(\"intervals.end\").alias(\"end\"),\n                f.col(\"genes.geneId\").alias(\"geneId\"),\n                f.col(\"genes.tss\").alias(\"tss\"),\n            )\n        )\n\n        # Joining back the data:\n        return Intervals(\n            _df=(\n                javierre_remapped.join(\n                    unique_intervals_with_genes,\n                    on=[\"chrom\", \"start\", \"end\"],\n                    how=\"left\",\n                )\n                .filter(\n                    # Drop rows where the TSS is far from the start of the region\n                    f.abs((f.col(\"start\") + f.col(\"end\")) / 2 - f.col(\"tss\"))\n                    &lt;= twosided_threshold\n                )\n                # For each gene, keep only the highest scoring interval:\n                .groupBy(\"name_chr\", \"name_start\", \"name_end\", \"geneId\", \"bio_feature\")\n                .agg(f.max(f.col(\"name_score\")).alias(\"resourceScore\"))\n                # Create the output:\n                .select(\n                    f.col(\"name_chr\").alias(\"chromosome\"),\n                    f.col(\"name_start\").alias(\"start\"),\n                    f.col(\"name_end\").alias(\"end\"),\n                    f.col(\"resourceScore\").cast(t.DoubleType()),\n                    f.col(\"geneId\"),\n                    f.col(\"bio_feature\").alias(\"biofeature\"),\n                    f.lit(dataset_name).alias(\"datasourceId\"),\n                    f.lit(experiment_type).alias(\"datatypeId\"),\n                    f.lit(pmid).alias(\"pmid\"),\n                )\n            ),\n            _schema=Intervals.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasources/intervals/javierre/#gentropy.datasource.intervals.javierre.IntervalsJavierre.parse","title":"<code>parse(javierre_raw: DataFrame, gene_index: GeneIndex, lift: LiftOverSpark) -&gt; Intervals</code>  <code>classmethod</code>","text":"<p>Parse Javierre et al. 2016 dataset.</p> <p>Parameters:</p> Name Type Description Default <code>javierre_raw</code> <code>DataFrame</code> <p>Raw Javierre data</p> required <code>gene_index</code> <code>GeneIndex</code> <p>Gene index</p> required <code>lift</code> <code>LiftOverSpark</code> <p>LiftOverSpark instance</p> required <p>Returns:</p> Name Type Description <code>Intervals</code> <code>Intervals</code> <p>Javierre et al. 2016 interval data</p> Source code in <code>src/gentropy/datasource/intervals/javierre.py</code> <pre><code>@classmethod\ndef parse(\n    cls: type[IntervalsJavierre],\n    javierre_raw: DataFrame,\n    gene_index: GeneIndex,\n    lift: LiftOverSpark,\n) -&gt; Intervals:\n    \"\"\"Parse Javierre et al. 2016 dataset.\n\n    Args:\n        javierre_raw (DataFrame): Raw Javierre data\n        gene_index (GeneIndex): Gene index\n        lift (LiftOverSpark): LiftOverSpark instance\n\n    Returns:\n        Intervals: Javierre et al. 2016 interval data\n    \"\"\"\n    # Constant values:\n    dataset_name = \"javierre2016\"\n    experiment_type = \"pchic\"\n    pmid = \"27863249\"\n    twosided_threshold = 2.45e6\n\n    # Read Javierre data:\n    javierre_parsed = (\n        javierre_raw\n        # Splitting name column into chromosome, start, end, and score:\n        .withColumn(\"name_split\", f.split(f.col(\"name\"), r\":|-|,\"))\n        .withColumn(\n            \"name_chr\",\n            f.regexp_replace(f.col(\"name_split\")[0], \"chr\", \"\").cast(\n                t.StringType()\n            ),\n        )\n        .withColumn(\"name_start\", f.col(\"name_split\")[1].cast(t.IntegerType()))\n        .withColumn(\"name_end\", f.col(\"name_split\")[2].cast(t.IntegerType()))\n        .withColumn(\"name_score\", f.col(\"name_split\")[3].cast(t.FloatType()))\n        # Cleaning up chromosome:\n        .withColumn(\n            \"chrom\",\n            f.regexp_replace(f.col(\"chrom\"), \"chr\", \"\").cast(t.StringType()),\n        )\n        .drop(\"name_split\", \"name\", \"annotation\")\n        # Keep canonical chromosomes and consistent chromosomes with scores:\n        .filter(\n            (f.col(\"name_score\").isNotNull())\n            &amp; (f.col(\"chrom\") == f.col(\"name_chr\"))\n            &amp; f.col(\"name_chr\").isin(\n                [f\"{x}\" for x in range(1, 23)] + [\"X\", \"Y\", \"MT\"]\n            )\n        )\n    )\n\n    # Lifting over intervals:\n    javierre_remapped = (\n        javierre_parsed\n        # Lifting over to GRCh38 interval 1:\n        .transform(lambda df: lift.convert_intervals(df, \"chrom\", \"start\", \"end\"))\n        .drop(\"start\", \"end\")\n        .withColumnRenamed(\"mapped_chrom\", \"chrom\")\n        .withColumnRenamed(\"mapped_start\", \"start\")\n        .withColumnRenamed(\"mapped_end\", \"end\")\n        # Lifting over interval 2 to GRCh38:\n        .transform(\n            lambda df: lift.convert_intervals(\n                df, \"name_chr\", \"name_start\", \"name_end\"\n            )\n        )\n        .drop(\"name_start\", \"name_end\")\n        .withColumnRenamed(\"mapped_name_chr\", \"name_chr\")\n        .withColumnRenamed(\"mapped_name_start\", \"name_start\")\n        .withColumnRenamed(\"mapped_name_end\", \"name_end\")\n    )\n\n    # Once the intervals are lifted, extracting the unique intervals:\n    unique_intervals_with_genes = (\n        javierre_remapped.select(\n            f.col(\"chrom\"),\n            f.col(\"start\").cast(t.IntegerType()),\n            f.col(\"end\").cast(t.IntegerType()),\n        )\n        .distinct()\n        .alias(\"intervals\")\n        .join(\n            gene_index.locations_lut().alias(\"genes\"),\n            on=[\n                f.col(\"intervals.chrom\") == f.col(\"genes.chromosome\"),\n                (\n                    (f.col(\"intervals.start\") &gt;= f.col(\"genes.start\"))\n                    &amp; (f.col(\"intervals.start\") &lt;= f.col(\"genes.end\"))\n                )\n                | (\n                    (f.col(\"intervals.end\") &gt;= f.col(\"genes.start\"))\n                    &amp; (f.col(\"intervals.end\") &lt;= f.col(\"genes.end\"))\n                ),\n            ],\n            how=\"left\",\n        )\n        .select(\n            f.col(\"intervals.chrom\").alias(\"chrom\"),\n            f.col(\"intervals.start\").alias(\"start\"),\n            f.col(\"intervals.end\").alias(\"end\"),\n            f.col(\"genes.geneId\").alias(\"geneId\"),\n            f.col(\"genes.tss\").alias(\"tss\"),\n        )\n    )\n\n    # Joining back the data:\n    return Intervals(\n        _df=(\n            javierre_remapped.join(\n                unique_intervals_with_genes,\n                on=[\"chrom\", \"start\", \"end\"],\n                how=\"left\",\n            )\n            .filter(\n                # Drop rows where the TSS is far from the start of the region\n                f.abs((f.col(\"start\") + f.col(\"end\")) / 2 - f.col(\"tss\"))\n                &lt;= twosided_threshold\n            )\n            # For each gene, keep only the highest scoring interval:\n            .groupBy(\"name_chr\", \"name_start\", \"name_end\", \"geneId\", \"bio_feature\")\n            .agg(f.max(f.col(\"name_score\")).alias(\"resourceScore\"))\n            # Create the output:\n            .select(\n                f.col(\"name_chr\").alias(\"chromosome\"),\n                f.col(\"name_start\").alias(\"start\"),\n                f.col(\"name_end\").alias(\"end\"),\n                f.col(\"resourceScore\").cast(t.DoubleType()),\n                f.col(\"geneId\"),\n                f.col(\"bio_feature\").alias(\"biofeature\"),\n                f.lit(dataset_name).alias(\"datasourceId\"),\n                f.lit(experiment_type).alias(\"datatypeId\"),\n                f.lit(pmid).alias(\"pmid\"),\n            )\n        ),\n        _schema=Intervals.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasources/intervals/javierre/#gentropy.datasource.intervals.javierre.IntervalsJavierre.read","title":"<code>read(spark: SparkSession, path: str) -&gt; DataFrame</code>  <code>staticmethod</code>","text":"<p>Read Javierre dataset.</p> <p>Parameters:</p> Name Type Description Default <code>spark</code> <code>SparkSession</code> <p>Spark session</p> required <code>path</code> <code>str</code> <p>Path to dataset</p> required <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>Raw Javierre dataset</p> Source code in <code>src/gentropy/datasource/intervals/javierre.py</code> <pre><code>@staticmethod\ndef read(spark: SparkSession, path: str) -&gt; DataFrame:\n    \"\"\"Read Javierre dataset.\n\n    Args:\n        spark (SparkSession): Spark session\n        path (str): Path to dataset\n\n    Returns:\n        DataFrame: Raw Javierre dataset\n    \"\"\"\n    return spark.read.parquet(path)\n</code></pre>"},{"location":"python_api/datasources/intervals/jung/","title":"Jung et al.","text":""},{"location":"python_api/datasources/intervals/jung/#gentropy.datasource.intervals.jung.IntervalsJung","title":"<code>gentropy.datasource.intervals.jung.IntervalsJung</code>","text":"<p>Interval dataset from Jung et al. 2019.</p> Source code in <code>src/gentropy/datasource/intervals/jung.py</code> <pre><code>class IntervalsJung:\n    \"\"\"Interval dataset from Jung et al. 2019.\"\"\"\n\n    @staticmethod\n    def read(spark: SparkSession, path: str) -&gt; DataFrame:\n        \"\"\"Read jung dataset.\n\n        Args:\n            spark (SparkSession): Spark session\n            path (str): Path to dataset\n\n        Returns:\n            DataFrame: DataFrame with raw jung data\n        \"\"\"\n        return spark.read.csv(path, sep=\",\", header=True)\n\n    @classmethod\n    def parse(\n        cls: type[IntervalsJung],\n        jung_raw: DataFrame,\n        gene_index: GeneIndex,\n        lift: LiftOverSpark,\n    ) -&gt; Intervals:\n        \"\"\"Parse the Jung et al. 2019 dataset.\n\n        Args:\n            jung_raw (DataFrame): raw Jung et al. 2019 dataset\n            gene_index (GeneIndex): gene index\n            lift (LiftOverSpark): LiftOverSpark instance\n\n        Returns:\n            Intervals: Interval dataset containing Jung et al. 2019 data\n        \"\"\"\n        dataset_name = \"jung2019\"\n        experiment_type = \"pchic\"\n        pmid = \"31501517\"\n\n        # Lifting over the coordinates:\n        return Intervals(\n            _df=(\n                jung_raw.withColumn(\n                    \"interval\", f.split(f.col(\"Interacting_fragment\"), r\"\\.\")\n                )\n                .select(\n                    # Parsing intervals:\n                    f.regexp_replace(f.col(\"interval\")[0], \"chr\", \"\").alias(\"chrom\"),\n                    f.col(\"interval\")[1].cast(t.IntegerType()).alias(\"start\"),\n                    f.col(\"interval\")[2].cast(t.IntegerType()).alias(\"end\"),\n                    # Extract other columns:\n                    f.col(\"Promoter\").alias(\"gene_name\"),\n                    f.col(\"Tissue_type\").alias(\"tissue\"),\n                )\n                # Lifting over to GRCh38 interval 1:\n                .transform(\n                    lambda df: lift.convert_intervals(df, \"chrom\", \"start\", \"end\")\n                )\n                .select(\n                    \"chrom\",\n                    f.col(\"mapped_start\").alias(\"start\"),\n                    f.col(\"mapped_end\").alias(\"end\"),\n                    f.explode(f.split(f.col(\"gene_name\"), \";\")).alias(\"gene_name\"),\n                    \"tissue\",\n                )\n                .alias(\"intervals\")\n                # Joining with genes:\n                .join(\n                    gene_index.symbols_lut().alias(\"genes\"),\n                    on=[f.col(\"intervals.gene_name\") == f.col(\"genes.geneSymbol\")],\n                    how=\"inner\",\n                )\n                # Finalize dataset:\n                .select(\n                    \"chromosome\",\n                    f.col(\"intervals.start\").alias(\"start\"),\n                    f.col(\"intervals.end\").alias(\"end\"),\n                    \"geneId\",\n                    f.col(\"tissue\").alias(\"biofeature\"),\n                    f.lit(1.0).alias(\"score\"),\n                    f.lit(dataset_name).alias(\"datasourceId\"),\n                    f.lit(experiment_type).alias(\"datatypeId\"),\n                    f.lit(pmid).alias(\"pmid\"),\n                )\n                .drop_duplicates()\n            ),\n            _schema=Intervals.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasources/intervals/jung/#gentropy.datasource.intervals.jung.IntervalsJung.parse","title":"<code>parse(jung_raw: DataFrame, gene_index: GeneIndex, lift: LiftOverSpark) -&gt; Intervals</code>  <code>classmethod</code>","text":"<p>Parse the Jung et al. 2019 dataset.</p> <p>Parameters:</p> Name Type Description Default <code>jung_raw</code> <code>DataFrame</code> <p>raw Jung et al. 2019 dataset</p> required <code>gene_index</code> <code>GeneIndex</code> <p>gene index</p> required <code>lift</code> <code>LiftOverSpark</code> <p>LiftOverSpark instance</p> required <p>Returns:</p> Name Type Description <code>Intervals</code> <code>Intervals</code> <p>Interval dataset containing Jung et al. 2019 data</p> Source code in <code>src/gentropy/datasource/intervals/jung.py</code> <pre><code>@classmethod\ndef parse(\n    cls: type[IntervalsJung],\n    jung_raw: DataFrame,\n    gene_index: GeneIndex,\n    lift: LiftOverSpark,\n) -&gt; Intervals:\n    \"\"\"Parse the Jung et al. 2019 dataset.\n\n    Args:\n        jung_raw (DataFrame): raw Jung et al. 2019 dataset\n        gene_index (GeneIndex): gene index\n        lift (LiftOverSpark): LiftOverSpark instance\n\n    Returns:\n        Intervals: Interval dataset containing Jung et al. 2019 data\n    \"\"\"\n    dataset_name = \"jung2019\"\n    experiment_type = \"pchic\"\n    pmid = \"31501517\"\n\n    # Lifting over the coordinates:\n    return Intervals(\n        _df=(\n            jung_raw.withColumn(\n                \"interval\", f.split(f.col(\"Interacting_fragment\"), r\"\\.\")\n            )\n            .select(\n                # Parsing intervals:\n                f.regexp_replace(f.col(\"interval\")[0], \"chr\", \"\").alias(\"chrom\"),\n                f.col(\"interval\")[1].cast(t.IntegerType()).alias(\"start\"),\n                f.col(\"interval\")[2].cast(t.IntegerType()).alias(\"end\"),\n                # Extract other columns:\n                f.col(\"Promoter\").alias(\"gene_name\"),\n                f.col(\"Tissue_type\").alias(\"tissue\"),\n            )\n            # Lifting over to GRCh38 interval 1:\n            .transform(\n                lambda df: lift.convert_intervals(df, \"chrom\", \"start\", \"end\")\n            )\n            .select(\n                \"chrom\",\n                f.col(\"mapped_start\").alias(\"start\"),\n                f.col(\"mapped_end\").alias(\"end\"),\n                f.explode(f.split(f.col(\"gene_name\"), \";\")).alias(\"gene_name\"),\n                \"tissue\",\n            )\n            .alias(\"intervals\")\n            # Joining with genes:\n            .join(\n                gene_index.symbols_lut().alias(\"genes\"),\n                on=[f.col(\"intervals.gene_name\") == f.col(\"genes.geneSymbol\")],\n                how=\"inner\",\n            )\n            # Finalize dataset:\n            .select(\n                \"chromosome\",\n                f.col(\"intervals.start\").alias(\"start\"),\n                f.col(\"intervals.end\").alias(\"end\"),\n                \"geneId\",\n                f.col(\"tissue\").alias(\"biofeature\"),\n                f.lit(1.0).alias(\"score\"),\n                f.lit(dataset_name).alias(\"datasourceId\"),\n                f.lit(experiment_type).alias(\"datatypeId\"),\n                f.lit(pmid).alias(\"pmid\"),\n            )\n            .drop_duplicates()\n        ),\n        _schema=Intervals.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasources/intervals/jung/#gentropy.datasource.intervals.jung.IntervalsJung.read","title":"<code>read(spark: SparkSession, path: str) -&gt; DataFrame</code>  <code>staticmethod</code>","text":"<p>Read jung dataset.</p> <p>Parameters:</p> Name Type Description Default <code>spark</code> <code>SparkSession</code> <p>Spark session</p> required <code>path</code> <code>str</code> <p>Path to dataset</p> required <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>DataFrame with raw jung data</p> Source code in <code>src/gentropy/datasource/intervals/jung.py</code> <pre><code>@staticmethod\ndef read(spark: SparkSession, path: str) -&gt; DataFrame:\n    \"\"\"Read jung dataset.\n\n    Args:\n        spark (SparkSession): Spark session\n        path (str): Path to dataset\n\n    Returns:\n        DataFrame: DataFrame with raw jung data\n    \"\"\"\n    return spark.read.csv(path, sep=\",\", header=True)\n</code></pre>"},{"location":"python_api/datasources/intervals/thurman/","title":"Thurman et al.","text":""},{"location":"python_api/datasources/intervals/thurman/#gentropy.datasource.intervals.thurman.IntervalsThurman","title":"<code>gentropy.datasource.intervals.thurman.IntervalsThurman</code>","text":"<p>Interval dataset from Thurman et al. 2012.</p> Source code in <code>src/gentropy/datasource/intervals/thurman.py</code> <pre><code>class IntervalsThurman:\n    \"\"\"Interval dataset from Thurman et al. 2012.\"\"\"\n\n    @staticmethod\n    def read(spark: SparkSession, path: str) -&gt; DataFrame:\n        \"\"\"Read thurman dataset.\n\n        Args:\n            spark (SparkSession): Spark session\n            path (str): Path to dataset\n\n        Returns:\n            DataFrame: DataFrame with raw thurman data\n        \"\"\"\n        thurman_schema = t.StructType(\n            [\n                t.StructField(\"gene_chr\", t.StringType(), False),\n                t.StructField(\"gene_start\", t.IntegerType(), False),\n                t.StructField(\"gene_end\", t.IntegerType(), False),\n                t.StructField(\"gene_name\", t.StringType(), False),\n                t.StructField(\"chrom\", t.StringType(), False),\n                t.StructField(\"start\", t.IntegerType(), False),\n                t.StructField(\"end\", t.IntegerType(), False),\n                t.StructField(\"score\", t.FloatType(), False),\n            ]\n        )\n        return spark.read.csv(path, sep=\"\\t\", header=False, schema=thurman_schema)\n\n    @classmethod\n    def parse(\n        cls: type[IntervalsThurman],\n        thurman_raw: DataFrame,\n        gene_index: GeneIndex,\n        lift: LiftOverSpark,\n    ) -&gt; Intervals:\n        \"\"\"Parse the Thurman et al. 2012 dataset.\n\n        Args:\n            thurman_raw (DataFrame): raw Thurman et al. 2019 dataset\n            gene_index (GeneIndex): gene index\n            lift (LiftOverSpark): LiftOverSpark instance\n\n        Returns:\n            Intervals: Interval dataset containing Thurman et al. 2012 data\n        \"\"\"\n        dataset_name = \"thurman2012\"\n        experiment_type = \"dhscor\"\n        pmid = \"22955617\"\n\n        return Intervals(\n            _df=(\n                thurman_raw.select(\n                    f.regexp_replace(f.col(\"chrom\"), \"chr\", \"\").alias(\"chrom\"),\n                    \"start\",\n                    \"end\",\n                    \"gene_name\",\n                    \"score\",\n                )\n                # Lift over to the GRCh38 build:\n                .transform(\n                    lambda df: lift.convert_intervals(df, \"chrom\", \"start\", \"end\")\n                )\n                .alias(\"intervals\")\n                # Map gene names to gene IDs:\n                .join(\n                    gene_index.symbols_lut().alias(\"genes\"),\n                    on=[\n                        f.col(\"intervals.gene_name\") == f.col(\"genes.geneSymbol\"),\n                        f.col(\"intervals.chrom\") == f.col(\"genes.chromosome\"),\n                    ],\n                    how=\"inner\",\n                )\n                # Select relevant columns and add constant columns:\n                .select(\n                    f.col(\"chrom\").alias(\"chromosome\"),\n                    f.col(\"mapped_start\").alias(\"start\"),\n                    f.col(\"mapped_end\").alias(\"end\"),\n                    \"geneId\",\n                    f.col(\"score\").cast(t.DoubleType()).alias(\"resourceScore\"),\n                    f.lit(dataset_name).alias(\"datasourceId\"),\n                    f.lit(experiment_type).alias(\"datatypeId\"),\n                    f.lit(pmid).alias(\"pmid\"),\n                )\n                .distinct()\n            ),\n            _schema=Intervals.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasources/intervals/thurman/#gentropy.datasource.intervals.thurman.IntervalsThurman.parse","title":"<code>parse(thurman_raw: DataFrame, gene_index: GeneIndex, lift: LiftOverSpark) -&gt; Intervals</code>  <code>classmethod</code>","text":"<p>Parse the Thurman et al. 2012 dataset.</p> <p>Parameters:</p> Name Type Description Default <code>thurman_raw</code> <code>DataFrame</code> <p>raw Thurman et al. 2019 dataset</p> required <code>gene_index</code> <code>GeneIndex</code> <p>gene index</p> required <code>lift</code> <code>LiftOverSpark</code> <p>LiftOverSpark instance</p> required <p>Returns:</p> Name Type Description <code>Intervals</code> <code>Intervals</code> <p>Interval dataset containing Thurman et al. 2012 data</p> Source code in <code>src/gentropy/datasource/intervals/thurman.py</code> <pre><code>@classmethod\ndef parse(\n    cls: type[IntervalsThurman],\n    thurman_raw: DataFrame,\n    gene_index: GeneIndex,\n    lift: LiftOverSpark,\n) -&gt; Intervals:\n    \"\"\"Parse the Thurman et al. 2012 dataset.\n\n    Args:\n        thurman_raw (DataFrame): raw Thurman et al. 2019 dataset\n        gene_index (GeneIndex): gene index\n        lift (LiftOverSpark): LiftOverSpark instance\n\n    Returns:\n        Intervals: Interval dataset containing Thurman et al. 2012 data\n    \"\"\"\n    dataset_name = \"thurman2012\"\n    experiment_type = \"dhscor\"\n    pmid = \"22955617\"\n\n    return Intervals(\n        _df=(\n            thurman_raw.select(\n                f.regexp_replace(f.col(\"chrom\"), \"chr\", \"\").alias(\"chrom\"),\n                \"start\",\n                \"end\",\n                \"gene_name\",\n                \"score\",\n            )\n            # Lift over to the GRCh38 build:\n            .transform(\n                lambda df: lift.convert_intervals(df, \"chrom\", \"start\", \"end\")\n            )\n            .alias(\"intervals\")\n            # Map gene names to gene IDs:\n            .join(\n                gene_index.symbols_lut().alias(\"genes\"),\n                on=[\n                    f.col(\"intervals.gene_name\") == f.col(\"genes.geneSymbol\"),\n                    f.col(\"intervals.chrom\") == f.col(\"genes.chromosome\"),\n                ],\n                how=\"inner\",\n            )\n            # Select relevant columns and add constant columns:\n            .select(\n                f.col(\"chrom\").alias(\"chromosome\"),\n                f.col(\"mapped_start\").alias(\"start\"),\n                f.col(\"mapped_end\").alias(\"end\"),\n                \"geneId\",\n                f.col(\"score\").cast(t.DoubleType()).alias(\"resourceScore\"),\n                f.lit(dataset_name).alias(\"datasourceId\"),\n                f.lit(experiment_type).alias(\"datatypeId\"),\n                f.lit(pmid).alias(\"pmid\"),\n            )\n            .distinct()\n        ),\n        _schema=Intervals.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasources/intervals/thurman/#gentropy.datasource.intervals.thurman.IntervalsThurman.read","title":"<code>read(spark: SparkSession, path: str) -&gt; DataFrame</code>  <code>staticmethod</code>","text":"<p>Read thurman dataset.</p> <p>Parameters:</p> Name Type Description Default <code>spark</code> <code>SparkSession</code> <p>Spark session</p> required <code>path</code> <code>str</code> <p>Path to dataset</p> required <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>DataFrame with raw thurman data</p> Source code in <code>src/gentropy/datasource/intervals/thurman.py</code> <pre><code>@staticmethod\ndef read(spark: SparkSession, path: str) -&gt; DataFrame:\n    \"\"\"Read thurman dataset.\n\n    Args:\n        spark (SparkSession): Spark session\n        path (str): Path to dataset\n\n    Returns:\n        DataFrame: DataFrame with raw thurman data\n    \"\"\"\n    thurman_schema = t.StructType(\n        [\n            t.StructField(\"gene_chr\", t.StringType(), False),\n            t.StructField(\"gene_start\", t.IntegerType(), False),\n            t.StructField(\"gene_end\", t.IntegerType(), False),\n            t.StructField(\"gene_name\", t.StringType(), False),\n            t.StructField(\"chrom\", t.StringType(), False),\n            t.StructField(\"start\", t.IntegerType(), False),\n            t.StructField(\"end\", t.IntegerType(), False),\n            t.StructField(\"score\", t.FloatType(), False),\n        ]\n    )\n    return spark.read.csv(path, sep=\"\\t\", header=False, schema=thurman_schema)\n</code></pre>"},{"location":"python_api/datasources/open_targets/_open_targets/","title":"Open Targets","text":"<p>The Open Targets Platform is a comprehensive resource that aims to aggregate and harmonize various types of data to facilitate the identification, prioritization, and validation of drug targets. By integrating publicly available datasets, including data generated by the Open Targets consortium, the Platform builds and scores target-disease associations to assist in drug target identification and prioritization. It also integrates relevant annotation information about targets, diseases, phenotypes, and drugs, as well as their most relevant relationships.</p> <p>Within our analyses, we utilize Open Targets to infer two datasets:</p> <ol> <li> <p>The list of targets:    This dataset provides a compilation of targets. In the Open Targets Platform, a target is understood as any naturally-occurring molecule that can be targeted by a medicinal product. The EMBL-EBI Ensembl database serves as the source for human targets in the Platform, with the Ensembl gene ID as the primary identifier. For more details, refer to this link.</p> </li> <li> <p>The list of Gold Standard Positives:    We use this dataset for training the Locus-to-Gene model. The current list contains 496 Gold Standard Positives.</p> </li> </ol>"},{"location":"python_api/datasources/open_targets/l2g_gold_standard/","title":"L2G Gold Standard","text":""},{"location":"python_api/datasources/open_targets/l2g_gold_standard/#gentropy.datasource.open_targets.l2g_gold_standard.OpenTargetsL2GGoldStandard","title":"<code>gentropy.datasource.open_targets.l2g_gold_standard.OpenTargetsL2GGoldStandard</code>","text":"<p>Parser for OTGenetics locus to gene gold standards curation.</p> The curation is processed to generate a dataset with 2 labels <ul> <li>Gold Standard Positive (GSP): When the lead variant is part of a curated list of GWAS loci with known gene-trait associations.</li> <li>Gold Standard Negative (GSN): When the lead variant is not part of a curated list of GWAS loci with known gene-trait associations but is in the vicinity of a gene's TSS.</li> </ul> Source code in <code>src/gentropy/datasource/open_targets/l2g_gold_standard.py</code> <pre><code>class OpenTargetsL2GGoldStandard:\n    \"\"\"Parser for OTGenetics locus to gene gold standards curation.\n\n    The curation is processed to generate a dataset with 2 labels:\n        - Gold Standard Positive (GSP): When the lead variant is part of a curated list of GWAS loci with known gene-trait associations.\n        - Gold Standard Negative (GSN): When the lead variant is not part of a curated list of GWAS loci with known gene-trait associations but is in the vicinity of a gene's TSS.\n    \"\"\"\n\n    LOCUS_TO_GENE_WINDOW = 500_000\n\n    @classmethod\n    def parse_positive_curation(\n        cls: Type[OpenTargetsL2GGoldStandard], gold_standard_curation: DataFrame\n    ) -&gt; DataFrame:\n        \"\"\"Parse positive set from gold standard curation.\n\n        Args:\n            gold_standard_curation (DataFrame): Gold standard curation dataframe\n\n        Returns:\n            DataFrame: Positive set\n        \"\"\"\n        return (\n            gold_standard_curation.filter(\n                f.col(\"gold_standard_info.highest_confidence\").isin([\"High\", \"Medium\"])\n            )\n            .select(\n                f.col(\"association_info.otg_id\").alias(\"studyId\"),\n                f.col(\"gold_standard_info.gene_id\").alias(\"geneId\"),\n                f.concat_ws(\n                    \"_\",\n                    f.col(\"sentinel_variant.locus_GRCh38.chromosome\"),\n                    f.col(\"sentinel_variant.locus_GRCh38.position\"),\n                    f.col(\"sentinel_variant.alleles.reference\"),\n                    f.col(\"sentinel_variant.alleles.alternative\"),\n                ).alias(\"variantId\"),\n                f.col(\"metadata.set_label\").alias(\"source\"),\n            )\n            .withColumn(\n                \"studyLocusId\",\n                StudyLocus.assign_study_locus_id(f.col(\"studyId\"), f.col(\"variantId\")),\n            )\n            .groupBy(\"studyLocusId\", \"studyId\", \"variantId\", \"geneId\")\n            .agg(f.collect_set(\"source\").alias(\"sources\"))\n        )\n\n    @classmethod\n    def expand_gold_standard_with_negatives(\n        cls: Type[OpenTargetsL2GGoldStandard], positive_set: DataFrame, v2g: V2G\n    ) -&gt; DataFrame:\n        \"\"\"Create full set of positive and negative evidence of locus to gene associations.\n\n        Negative evidence consists of all genes within a window of 500kb of the lead variant that are not in the positive set.\n\n        Args:\n            positive_set (DataFrame): Positive set from curation\n            v2g (V2G): Variant to gene dataset to bring distance between a variant and a gene's TSS\n\n        Returns:\n            DataFrame: Full set of positive and negative evidence of locus to gene associations\n        \"\"\"\n        return (\n            positive_set.withColumnRenamed(\"geneId\", \"curated_geneId\")\n            .join(\n                v2g.df.selectExpr(\n                    \"variantId\", \"geneId as non_curated_geneId\", \"distance\"\n                ).filter(f.col(\"distance\") &lt;= cls.LOCUS_TO_GENE_WINDOW),\n                on=\"variantId\",\n                how=\"left\",\n            )\n            .withColumn(\n                \"goldStandardSet\",\n                f.when(\n                    (f.col(\"curated_geneId\") == f.col(\"non_curated_geneId\"))\n                    # to keep the positives that are outside the v2g dataset\n                    | (f.col(\"non_curated_geneId\").isNull()),\n                    f.lit(L2GGoldStandard.GS_POSITIVE_LABEL),\n                ).otherwise(L2GGoldStandard.GS_NEGATIVE_LABEL),\n            )\n            .withColumn(\n                \"geneId\",\n                f.when(\n                    f.col(\"goldStandardSet\") == L2GGoldStandard.GS_POSITIVE_LABEL,\n                    f.col(\"curated_geneId\"),\n                ).otherwise(f.col(\"non_curated_geneId\")),\n            )\n            .drop(\"distance\", \"curated_geneId\", \"non_curated_geneId\")\n        )\n\n    @classmethod\n    def as_l2g_gold_standard(\n        cls: type[OpenTargetsL2GGoldStandard],\n        gold_standard_curation: DataFrame,\n        v2g: V2G,\n    ) -&gt; L2GGoldStandard:\n        \"\"\"Initialise L2GGoldStandard from source dataset.\n\n        Args:\n            gold_standard_curation (DataFrame): Gold standard curation dataframe, extracted from https://github.com/opentargets/genetics-gold-standards\n            v2g (V2G): Variant to gene dataset to bring distance between a variant and a gene's TSS\n\n        Returns:\n            L2GGoldStandard: L2G Gold Standard dataset. False negatives have not yet been removed.\n        \"\"\"\n        return L2GGoldStandard(\n            _df=cls.parse_positive_curation(gold_standard_curation).transform(\n                cls.expand_gold_standard_with_negatives, v2g\n            ),\n            _schema=L2GGoldStandard.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasources/open_targets/l2g_gold_standard/#gentropy.datasource.open_targets.l2g_gold_standard.OpenTargetsL2GGoldStandard.as_l2g_gold_standard","title":"<code>as_l2g_gold_standard(gold_standard_curation: DataFrame, v2g: V2G) -&gt; L2GGoldStandard</code>  <code>classmethod</code>","text":"<p>Initialise L2GGoldStandard from source dataset.</p> <p>Parameters:</p> Name Type Description Default <code>gold_standard_curation</code> <code>DataFrame</code> <p>Gold standard curation dataframe, extracted from https://github.com/opentargets/genetics-gold-standards</p> required <code>v2g</code> <code>V2G</code> <p>Variant to gene dataset to bring distance between a variant and a gene's TSS</p> required <p>Returns:</p> Name Type Description <code>L2GGoldStandard</code> <code>L2GGoldStandard</code> <p>L2G Gold Standard dataset. False negatives have not yet been removed.</p> Source code in <code>src/gentropy/datasource/open_targets/l2g_gold_standard.py</code> <pre><code>@classmethod\ndef as_l2g_gold_standard(\n    cls: type[OpenTargetsL2GGoldStandard],\n    gold_standard_curation: DataFrame,\n    v2g: V2G,\n) -&gt; L2GGoldStandard:\n    \"\"\"Initialise L2GGoldStandard from source dataset.\n\n    Args:\n        gold_standard_curation (DataFrame): Gold standard curation dataframe, extracted from https://github.com/opentargets/genetics-gold-standards\n        v2g (V2G): Variant to gene dataset to bring distance between a variant and a gene's TSS\n\n    Returns:\n        L2GGoldStandard: L2G Gold Standard dataset. False negatives have not yet been removed.\n    \"\"\"\n    return L2GGoldStandard(\n        _df=cls.parse_positive_curation(gold_standard_curation).transform(\n            cls.expand_gold_standard_with_negatives, v2g\n        ),\n        _schema=L2GGoldStandard.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/datasources/open_targets/l2g_gold_standard/#gentropy.datasource.open_targets.l2g_gold_standard.OpenTargetsL2GGoldStandard.expand_gold_standard_with_negatives","title":"<code>expand_gold_standard_with_negatives(positive_set: DataFrame, v2g: V2G) -&gt; DataFrame</code>  <code>classmethod</code>","text":"<p>Create full set of positive and negative evidence of locus to gene associations.</p> <p>Negative evidence consists of all genes within a window of 500kb of the lead variant that are not in the positive set.</p> <p>Parameters:</p> Name Type Description Default <code>positive_set</code> <code>DataFrame</code> <p>Positive set from curation</p> required <code>v2g</code> <code>V2G</code> <p>Variant to gene dataset to bring distance between a variant and a gene's TSS</p> required <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>Full set of positive and negative evidence of locus to gene associations</p> Source code in <code>src/gentropy/datasource/open_targets/l2g_gold_standard.py</code> <pre><code>@classmethod\ndef expand_gold_standard_with_negatives(\n    cls: Type[OpenTargetsL2GGoldStandard], positive_set: DataFrame, v2g: V2G\n) -&gt; DataFrame:\n    \"\"\"Create full set of positive and negative evidence of locus to gene associations.\n\n    Negative evidence consists of all genes within a window of 500kb of the lead variant that are not in the positive set.\n\n    Args:\n        positive_set (DataFrame): Positive set from curation\n        v2g (V2G): Variant to gene dataset to bring distance between a variant and a gene's TSS\n\n    Returns:\n        DataFrame: Full set of positive and negative evidence of locus to gene associations\n    \"\"\"\n    return (\n        positive_set.withColumnRenamed(\"geneId\", \"curated_geneId\")\n        .join(\n            v2g.df.selectExpr(\n                \"variantId\", \"geneId as non_curated_geneId\", \"distance\"\n            ).filter(f.col(\"distance\") &lt;= cls.LOCUS_TO_GENE_WINDOW),\n            on=\"variantId\",\n            how=\"left\",\n        )\n        .withColumn(\n            \"goldStandardSet\",\n            f.when(\n                (f.col(\"curated_geneId\") == f.col(\"non_curated_geneId\"))\n                # to keep the positives that are outside the v2g dataset\n                | (f.col(\"non_curated_geneId\").isNull()),\n                f.lit(L2GGoldStandard.GS_POSITIVE_LABEL),\n            ).otherwise(L2GGoldStandard.GS_NEGATIVE_LABEL),\n        )\n        .withColumn(\n            \"geneId\",\n            f.when(\n                f.col(\"goldStandardSet\") == L2GGoldStandard.GS_POSITIVE_LABEL,\n                f.col(\"curated_geneId\"),\n            ).otherwise(f.col(\"non_curated_geneId\")),\n        )\n        .drop(\"distance\", \"curated_geneId\", \"non_curated_geneId\")\n    )\n</code></pre>"},{"location":"python_api/datasources/open_targets/l2g_gold_standard/#gentropy.datasource.open_targets.l2g_gold_standard.OpenTargetsL2GGoldStandard.parse_positive_curation","title":"<code>parse_positive_curation(gold_standard_curation: DataFrame) -&gt; DataFrame</code>  <code>classmethod</code>","text":"<p>Parse positive set from gold standard curation.</p> <p>Parameters:</p> Name Type Description Default <code>gold_standard_curation</code> <code>DataFrame</code> <p>Gold standard curation dataframe</p> required <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>Positive set</p> Source code in <code>src/gentropy/datasource/open_targets/l2g_gold_standard.py</code> <pre><code>@classmethod\ndef parse_positive_curation(\n    cls: Type[OpenTargetsL2GGoldStandard], gold_standard_curation: DataFrame\n) -&gt; DataFrame:\n    \"\"\"Parse positive set from gold standard curation.\n\n    Args:\n        gold_standard_curation (DataFrame): Gold standard curation dataframe\n\n    Returns:\n        DataFrame: Positive set\n    \"\"\"\n    return (\n        gold_standard_curation.filter(\n            f.col(\"gold_standard_info.highest_confidence\").isin([\"High\", \"Medium\"])\n        )\n        .select(\n            f.col(\"association_info.otg_id\").alias(\"studyId\"),\n            f.col(\"gold_standard_info.gene_id\").alias(\"geneId\"),\n            f.concat_ws(\n                \"_\",\n                f.col(\"sentinel_variant.locus_GRCh38.chromosome\"),\n                f.col(\"sentinel_variant.locus_GRCh38.position\"),\n                f.col(\"sentinel_variant.alleles.reference\"),\n                f.col(\"sentinel_variant.alleles.alternative\"),\n            ).alias(\"variantId\"),\n            f.col(\"metadata.set_label\").alias(\"source\"),\n        )\n        .withColumn(\n            \"studyLocusId\",\n            StudyLocus.assign_study_locus_id(f.col(\"studyId\"), f.col(\"variantId\")),\n        )\n        .groupBy(\"studyLocusId\", \"studyId\", \"variantId\", \"geneId\")\n        .agg(f.collect_set(\"source\").alias(\"sources\"))\n    )\n</code></pre>"},{"location":"python_api/datasources/open_targets/target/","title":"Target","text":""},{"location":"python_api/datasources/open_targets/target/#gentropy.datasource.open_targets.target.OpenTargetsTarget","title":"<code>gentropy.datasource.open_targets.target.OpenTargetsTarget</code>","text":"<p>Parser for OTPlatform target dataset.</p> <p>Genomic data from Open Targets provides gene identification and genomic coordinates that are integrated into the gene index of our ETL pipeline.</p> <p>The EMBL-EBI Ensembl database is used as a source for human targets in the Platform, with the Ensembl gene ID as the primary identifier. The criteria for target inclusion is: - Genes from all biotypes encoded in canonical chromosomes - Genes in alternative assemblies encoding for a reviewed protein product.</p> Source code in <code>src/gentropy/datasource/open_targets/target.py</code> <pre><code>class OpenTargetsTarget:\n    \"\"\"Parser for OTPlatform target dataset.\n\n    Genomic data from Open Targets provides gene identification and genomic coordinates that are integrated into the gene index of our ETL pipeline.\n\n    The EMBL-EBI Ensembl database is used as a source for human targets in the Platform, with the Ensembl gene ID as the primary identifier. The criteria for target inclusion is:\n    - Genes from all biotypes encoded in canonical chromosomes\n    - Genes in alternative assemblies encoding for a reviewed protein product.\n    \"\"\"\n\n    @staticmethod\n    def _get_gene_tss(strand_col: Column, start_col: Column, end_col: Column) -&gt; Column:\n        \"\"\"Returns the TSS of a gene based on its orientation.\n\n        Args:\n            strand_col (Column): Column containing 1 if the coding strand of the gene is forward, and -1 if it is reverse.\n            start_col (Column): Column containing the start position of the gene.\n            end_col (Column): Column containing the end position of the gene.\n\n        Returns:\n            Column: Column containing the TSS of the gene.\n\n        Examples:\n            &gt;&gt;&gt; df = spark.createDataFrame([{\"strand\": 1, \"start\": 100, \"end\": 200}, {\"strand\": -1, \"start\": 100, \"end\": 200}])\n            &gt;&gt;&gt; df.withColumn(\"tss\", OpenTargetsTarget._get_gene_tss(f.col(\"strand\"), f.col(\"start\"), f.col(\"end\"))).show()\n            +---+-----+------+---+\n            |end|start|strand|tss|\n            +---+-----+------+---+\n            |200|  100|     1|100|\n            |200|  100|    -1|200|\n            +---+-----+------+---+\n            &lt;BLANKLINE&gt;\n\n        \"\"\"\n        return f.when(strand_col == 1, start_col).when(strand_col == -1, end_col)\n\n    @classmethod\n    def as_gene_index(\n        cls: type[OpenTargetsTarget], target_index: DataFrame\n    ) -&gt; GeneIndex:\n        \"\"\"Initialise GeneIndex from source dataset.\n\n        Args:\n            target_index (DataFrame): Target index dataframe\n\n        Returns:\n            GeneIndex: Gene index dataset\n        \"\"\"\n        return GeneIndex(\n            _df=target_index.select(\n                f.coalesce(f.col(\"id\"), f.lit(\"unknown\")).alias(\"geneId\"),\n                \"approvedSymbol\",\n                \"approvedName\",\n                \"biotype\",\n                f.col(\"obsoleteSymbols.label\").alias(\"obsoleteSymbols\"),\n                f.coalesce(f.col(\"genomicLocation.chromosome\"), f.lit(\"unknown\")).alias(\n                    \"chromosome\"\n                ),\n                OpenTargetsTarget._get_gene_tss(\n                    f.col(\"genomicLocation.strand\"),\n                    f.col(\"genomicLocation.start\"),\n                    f.col(\"genomicLocation.end\"),\n                ).alias(\"tss\"),\n                f.col(\"genomicLocation.start\").alias(\"start\"),\n                f.col(\"genomicLocation.end\").alias(\"end\"),\n                f.col(\"genomicLocation.strand\").alias(\"strand\"),\n            ),\n            _schema=GeneIndex.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/datasources/open_targets/target/#gentropy.datasource.open_targets.target.OpenTargetsTarget.as_gene_index","title":"<code>as_gene_index(target_index: DataFrame) -&gt; GeneIndex</code>  <code>classmethod</code>","text":"<p>Initialise GeneIndex from source dataset.</p> <p>Parameters:</p> Name Type Description Default <code>target_index</code> <code>DataFrame</code> <p>Target index dataframe</p> required <p>Returns:</p> Name Type Description <code>GeneIndex</code> <code>GeneIndex</code> <p>Gene index dataset</p> Source code in <code>src/gentropy/datasource/open_targets/target.py</code> <pre><code>@classmethod\ndef as_gene_index(\n    cls: type[OpenTargetsTarget], target_index: DataFrame\n) -&gt; GeneIndex:\n    \"\"\"Initialise GeneIndex from source dataset.\n\n    Args:\n        target_index (DataFrame): Target index dataframe\n\n    Returns:\n        GeneIndex: Gene index dataset\n    \"\"\"\n    return GeneIndex(\n        _df=target_index.select(\n            f.coalesce(f.col(\"id\"), f.lit(\"unknown\")).alias(\"geneId\"),\n            \"approvedSymbol\",\n            \"approvedName\",\n            \"biotype\",\n            f.col(\"obsoleteSymbols.label\").alias(\"obsoleteSymbols\"),\n            f.coalesce(f.col(\"genomicLocation.chromosome\"), f.lit(\"unknown\")).alias(\n                \"chromosome\"\n            ),\n            OpenTargetsTarget._get_gene_tss(\n                f.col(\"genomicLocation.strand\"),\n                f.col(\"genomicLocation.start\"),\n                f.col(\"genomicLocation.end\"),\n            ).alias(\"tss\"),\n            f.col(\"genomicLocation.start\").alias(\"start\"),\n            f.col(\"genomicLocation.end\").alias(\"end\"),\n            f.col(\"genomicLocation.strand\").alias(\"strand\"),\n        ),\n        _schema=GeneIndex.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/methods/_methods/","title":"Methods","text":"<p>This section consists of all the methods available in the package. It provides detailed explanations and usage examples for each method. Developers can refer to this section to understand how to use the methods effectively in their code. The list of methods is constantly updated.</p>"},{"location":"python_api/methods/carma/","title":"CARMA","text":"<p>CARMA is the method of the fine-mapping and outlier detection, originally implemented in R (CARMA on GitHub).</p> <p>The full repository for the reimplementation of CARMA in Python can be found here.</p> <p>This is a simplified version of CARMA with the following features:</p> <ol> <li>It uses only Spike-slab effect size priors and Poisson model priors.</li> <li>C++ is re-implemented in Python.</li> <li>The way of storing the configuration list is changed. It uses a string with the list of indexes for causal SNPs instead of a sparse matrix.</li> <li>Fixed bugs in PIP calculation.</li> <li>No credible models.</li> <li>No credible sets, only PIPs.</li> <li>No functional annotations.</li> <li>Removed unnecessary parameters.</li> </ol>"},{"location":"python_api/methods/carma/#gentropy.method.carma.CARMA","title":"<code>gentropy.method.carma.CARMA</code>","text":"<p>Implementation of CARMA outlier detection method.</p> Source code in <code>src/gentropy/method/carma.py</code> <pre><code>class CARMA:\n    \"\"\"Implementation of CARMA outlier detection method.\"\"\"\n\n    @staticmethod\n    def time_limited_CARMA_spike_slab_noEM(\n        z: np.ndarray, ld: np.ndarray, sec_threshold: float = 600\n    ) -&gt; dict[str, Any]:\n        \"\"\"The wrapper for the CARMA_spike_slab_noEM function that runs the function in a separate thread and terminates it if it takes too long.\n\n        Args:\n            z (np.ndarray): Numeric vector representing z-scores.\n            ld (np.ndarray): Numeric matrix representing the linkage disequilibrium (LD) matrix.\n            sec_threshold (float): The time threshold in seconds.\n\n        Returns:\n            dict[str, Any]: A dictionary containing the following results:\n                - PIPs: A numeric vector of posterior inclusion probabilities (PIPs) for all SNPs or None.\n                - B_list: A dataframe containing the marginal likelihoods and the corresponding model space or None.\n                - Outliers: A list of outlier SNPs or None.\n        \"\"\"\n        # Ignore pandas future warnings\n        warnings.simplefilter(action=\"ignore\", category=FutureWarning)\n        try:\n            # Execute CARMA.CARMA_spike_slab_noEM with a timeout\n            with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:\n                future = executor.submit(CARMA.CARMA_spike_slab_noEM, z=z, ld=ld)\n                result = future.result(timeout=sec_threshold)\n        except concurrent.futures.TimeoutError:\n            # If execution exceeds the timeout, return None\n            result = {\"PIPs\": None, \"B_list\": None, \"Outliers\": None}\n\n        return result\n\n    @staticmethod\n    def CARMA_spike_slab_noEM(\n        z: np.ndarray,\n        ld: np.ndarray,\n        lambda_val: float = 1,\n        Max_Model_Dim: int = 200_000,\n        all_iter: int = 1,\n        all_inner_iter: int = 10,\n        epsilon_threshold: float = 1e-5,\n        num_causal: int = 10,\n        tau: float = 0.04,\n        outlier_switch: bool = True,\n        outlier_BF_index: float = 1 / 3.2,\n    ) -&gt; dict[str, Any]:\n        \"\"\"Perform CARMA analysis using a Spike-and-Slab prior without Expectation-Maximization (EM).\n\n        Args:\n            z (np.ndarray): Numeric vector representing z-scores.\n            ld (np.ndarray): Numeric matrix representing the linkage disequilibrium (LD) matrix.\n            lambda_val (float): Regularization parameter controlling the strength of the L1 penalty.\n            Max_Model_Dim (int): Maximum allowed dimension for the causal models.\n            all_iter (int): The total number of iterations to run the CARMA analysis.\n            all_inner_iter (int): The number of inner iterations in each CARMA iteration.\n            epsilon_threshold (float): Threshold for convergence in CARMA iterations.\n            num_causal (int): Maximal number of causal variants to be selected in the final model.\n            tau (float): Tuning parameter controlling the degree of sparsity in the Spike-and-Slab prior.\n            outlier_switch (bool): Whether to consider outlier detection in the analysis.\n            outlier_BF_index (float): Bayes Factor threshold for identifying outliers.\n\n        Returns:\n            dict[str, Any]: A dictionary containing the following results:\n                - PIPs: A numeric vector of posterior inclusion probabilities (PIPs) for all SNPs.\n                - B_list: A dataframe containing the marginal likelihoods and the corresponding model space.\n                - Outliers: A list of outlier SNPs.\n        \"\"\"\n        p_snp = len(z)\n        epsilon_list = epsilon_threshold * p_snp\n        all_epsilon_threshold = epsilon_threshold * p_snp\n\n        # Zero step\n        all_C_list = CARMA._MCS_modified(\n            z=z,\n            ld_matrix=ld,\n            epsilon=epsilon_list,\n            Max_Model_Dim=Max_Model_Dim,\n            lambda_val=lambda_val,\n            outlier_switch=outlier_switch,\n            tau=tau,\n            num_causal=num_causal,\n            inner_all_iter=all_inner_iter,\n            outlier_BF_index=outlier_BF_index,\n        )\n\n        # Main steps\n        for _ in range(0, all_iter):\n            ac1 = all_C_list[\"B_list\"][\"set_gamma_margin\"]\n            previous_result = np.mean(ac1[0 : round(len(ac1) / 4)])\n\n            all_C_list = CARMA._MCS_modified(\n                z=z,\n                ld_matrix=ld,\n                input_conditional_S_list=all_C_list[\"conditional_S_list\"],\n                Max_Model_Dim=Max_Model_Dim,\n                num_causal=num_causal,\n                epsilon=epsilon_list,\n                outlier_switch=outlier_switch,\n                tau=tau,\n                lambda_val=lambda_val,\n                inner_all_iter=all_inner_iter,\n                outlier_BF_index=outlier_BF_index,\n            )\n\n            ac1 = all_C_list[\"B_list\"][\"set_gamma_margin\"]\n            difference = np.abs(previous_result - np.mean(ac1[0 : round(len(ac1) / 4)]))\n            if difference &lt; all_epsilon_threshold:\n                break\n\n        # Calculate PIPs and Credible Set\n        pip = CARMA._PIP_func(\n            likeli=all_C_list[\"B_list\"][\"set_gamma_margin\"],\n            model_space=all_C_list[\"B_list\"][\"matrix_gamma\"],\n            p=p_snp,\n            num_causal=num_causal,\n        )\n\n        results_list = {\n            \"PIPs\": pip,\n            \"B_list\": all_C_list[\"B_list\"],\n            \"Outliers\": all_C_list[\"conditional_S_list\"],\n        }\n\n        return results_list\n\n    @staticmethod\n    def _ind_normal_sigma_fixed_marginal_fun_indi(\n        zSigmaz_S: np.ndarray, tau: float, p_S: int, det_S: float\n    ) -&gt; float:\n        \"\"\"Internal function for calculating the marginal likelihood of configuration model.\n\n        Args:\n            zSigmaz_S (np.ndarray): The zSigmaz_S value.\n            tau (float): The tau value.\n            p_S (int): The number of SNPs.\n            det_S (float): The det_S value.\n\n        Returns:\n            float: The marginal likelihood of a model.\n\n        Examples:\n            &gt;&gt;&gt; zSigmaz_S = 0.1\n            &gt;&gt;&gt; tau = 1 / 0.05**2\n            &gt;&gt;&gt; p_S = 3\n            &gt;&gt;&gt; det_S = 0.1\n            &gt;&gt;&gt; np.round(CARMA._ind_normal_sigma_fixed_marginal_fun_indi(zSigmaz_S, tau, p_S, det_S),decimals=5)\n            10.18849\n        \"\"\"\n        return p_S / 2.0 * np.log(tau) - 0.5 * np.log(det_S) + zSigmaz_S / 2.0\n\n    @staticmethod\n    def _ind_Normal_fixed_sigma_marginal_external(\n        index_vec_input: np.ndarray,\n        Sigma: np.ndarray,\n        z: np.ndarray,\n        tau: float,\n        p_S: int,\n    ) -&gt; float:\n        \"\"\"Marginal likelihood of configuration model.\n\n        Args:\n            index_vec_input (np.ndarray): The index vector.\n            Sigma (np.ndarray): The Sigma matrix.\n            z (np.ndarray): The z vector.\n            tau (float): The tau value.\n            p_S (int): The number of SNPs.\n\n        Returns:\n            float: The marginal likelihood of a model.\n\n        Examples:\n            &gt;&gt;&gt; index_vec_input = np.array([1, 2])\n            &gt;&gt;&gt; Sigma = np.array([[1, 0.5, 0.2], [0.5, 1, 0.3], [0.2, 0.3, 1]])\n            &gt;&gt;&gt; z = np.array([10, 11, 10])\n            &gt;&gt;&gt; tau = 1\n            &gt;&gt;&gt; p_S = 2\n            &gt;&gt;&gt; np.round(CARMA._ind_Normal_fixed_sigma_marginal_external(index_vec_input, Sigma, z, tau, p_S),decimals=5)\n            43.60579\n        \"\"\"\n        index_vec = index_vec_input - 1\n        Sigma_S = Sigma[np.ix_(index_vec, index_vec)]\n        A = tau * np.eye(p_S)\n\n        det_S = det(Sigma_S + A)\n        Sigma_S_inv = inv(Sigma_S + A)\n\n        sub_z = z[index_vec]\n        zSigmaz_S = np.dot(sub_z.T, np.dot(Sigma_S_inv, sub_z))\n\n        b = CARMA._ind_normal_sigma_fixed_marginal_fun_indi(zSigmaz_S, tau, p_S, det_S)\n\n        results = b\n\n        return results\n\n    @staticmethod\n    def _outlier_ind_Normal_marginal_external(\n        index_vec_input: np.ndarray,\n        Sigma: np.ndarray,\n        z: np.ndarray,\n        tau: float,\n        p_S: int,\n    ) -&gt; float:\n        \"\"\"Likehood of outlier model.\n\n        Args:\n            index_vec_input (np.ndarray): The index vector.\n            Sigma (np.ndarray): The Sigma matrix.\n            z (np.ndarray): The z vector.\n            tau (float): The tau value.\n            p_S (int): The number of SNPs.\n\n        Returns:\n            float: The likelihood of a model.\n\n        Examples:\n            &gt;&gt;&gt; index_vec_input = np.array([1, 2, 3])\n            &gt;&gt;&gt; Sigma = np.array([[1, 0.5, 0.2], [0.5, 1, 0.3], [0.2, 0.3, 1]])\n            &gt;&gt;&gt; z = np.array([0.1, 0.2, 0.3])\n            &gt;&gt;&gt; tau = 1 / 0.05**2\n            &gt;&gt;&gt; p_S = 3\n            &gt;&gt;&gt; np.round(CARMA._outlier_ind_Normal_marginal_external(index_vec_input, Sigma, z, tau, p_S),decimals=5)\n            -8.8497\n        \"\"\"\n        index_vec = index_vec_input - 1\n\n        Sigma_S = Sigma[np.ix_(index_vec, index_vec)]\n        A = tau * np.eye(p_S)\n\n        Sigma_S_I_inv = pinv(Sigma_S + A, rtol=0.00001)\n        Sigma_S_inv = pinv(Sigma_S, rtol=0.00001)\n\n        det_S = np.abs(det(Sigma_S_inv))\n        det_I_S = np.abs(det(Sigma_S_I_inv))\n\n        sub_z = z[index_vec]\n        zSigmaz_S = np.dot(sub_z, np.dot(Sigma_S_inv, sub_z))\n        zSigmaz_I_S = np.dot(sub_z, np.dot(Sigma_S_I_inv, sub_z))\n\n        b = 0.5 * (np.log(det_S) + np.log(det_I_S)) - 0.5 * (zSigmaz_S - zSigmaz_I_S)\n        results = b\n\n        return results\n\n    @staticmethod\n    def _add_function(S_sub: np.ndarray, y: Any) -&gt; np.ndarray:\n        \"\"\"Concatenate two arrays and sort the result.\n\n        Args:\n            S_sub (np.ndarray): The first array.\n            y (Any): The second array.\n\n        Returns:\n            np.ndarray: The concatenated and sorted array.\n\n        Examples:\n            &gt;&gt;&gt; S_sub = np.array([3, 4])\n            &gt;&gt;&gt; y = np.array([1, 2])\n            &gt;&gt;&gt; CARMA._add_function(S_sub, y)\n            array([[1, 2, 3],\n                   [1, 2, 4]])\n        \"\"\"\n        return np.array([np.sort(np.concatenate(([x], y))) for x in S_sub])\n\n    @staticmethod\n    def _set_gamma_func_base(S: Any, p: int) -&gt; dict[int, np.ndarray]:\n        \"\"\"Creates a dictionary of sets of configurations assuming no conditional set.\n\n        Args:\n            S (Any): The input set.\n            p (int): The number of SNPs.\n\n        Returns:\n            dict[int, np.ndarray]: A dictionary of sets of configurations.\n\n        Examples:\n        &gt;&gt;&gt; S = [0,1]\n        &gt;&gt;&gt; p = 4\n        &gt;&gt;&gt; CARMA._set_gamma_func_base(S, p)\n        {0: array([[0],\n               [1]]), 1: array([[0, 1, 2],\n               [0, 1, 3]]), 2: array([[0, 2],\n               [0, 3],\n               [1, 2],\n               [1, 3]])}\n\n        &gt;&gt;&gt; S = [0]\n        &gt;&gt;&gt; p = 2\n        &gt;&gt;&gt; CARMA._set_gamma_func_base(S, p)\n        {0: None, 1: array([[0, 1]]), 2: array([[1]])}\n\n        &gt;&gt;&gt; S = []\n        &gt;&gt;&gt; p = 2\n        &gt;&gt;&gt; CARMA._set_gamma_func_base(S, p)\n        {0: None, 1: array([[0],\n               [1]]), 2: None}\n        \"\"\"\n        set_gamma: dict[int, Any] = {}\n\n        if len(S) == 0:\n            set_gamma[0] = None\n            set_gamma[1] = np.arange(0, p).reshape(-1, 1)\n            set_gamma[2] = None\n\n        if len(S) == 1:\n            S_sub = np.setdiff1d(np.arange(0, p), S)\n            set_gamma[0] = None\n            set_gamma[1] = CARMA._add_function(S_sub, S)\n            set_gamma[2] = S_sub.reshape(-1, 1)\n\n        if len(S) &gt; 1:\n            S_sub = np.setdiff1d(np.arange(0, p), S)\n            S = np.sort(S)\n            set_gamma[0] = np.array(list(combinations(S, len(S) - 1)))\n            set_gamma[1] = CARMA._add_function(S_sub, S)\n            xs = np.vstack([CARMA._add_function(S_sub, row) for row in set_gamma[0]])\n            set_gamma[2] = xs\n\n        return set_gamma\n\n    @staticmethod\n    def _set_gamma_func_conditional(\n        input_S: Any, condition_index: list[int], p: int\n    ) -&gt; dict[int, np.ndarray]:\n        \"\"\"Creates a dictionary of sets of configurations assuming conditional set.\n\n        Args:\n            input_S (Any): The input set.\n            condition_index (list[int]): The conditional set.\n            p (int): The number of SNPs.\n\n        Returns:\n            dict[int, np.ndarray]: A dictionary of sets of configurations.\n\n        Examples:\n        &gt;&gt;&gt; input_S = [0,1,2]\n        &gt;&gt;&gt; condition_index = [2]\n        &gt;&gt;&gt; p = 4\n        &gt;&gt;&gt; CARMA._set_gamma_func_conditional(input_S, condition_index, p)\n        {0: array([[0],\n               [1]]), 1: array([[0, 1, 3]]), 2: array([[0, 3],\n               [1, 3]])}\n        \"\"\"\n        set_gamma: dict[int, Any] = {}\n        S = np.setdiff1d(input_S, condition_index)\n\n        # set of gamma-\n        if len(S) == 0:\n            S_sub = np.setdiff1d(np.arange(0, p), condition_index)\n            set_gamma[0] = None\n            set_gamma[1] = S_sub.reshape(-1, 1)\n            set_gamma[2] = None\n\n        if len(S) == 1:\n            S_sub = np.setdiff1d(np.arange(0, p), input_S)\n            set_gamma[0] = None\n            set_gamma[1] = CARMA._add_function(S_sub, S)\n            set_gamma[2] = S_sub.reshape(-1, 1)\n\n        if len(S) &gt; 1:\n            S_sub = np.setdiff1d(np.arange(0, p), input_S)\n            S = np.sort(S)\n            set_gamma[0] = np.array(list(combinations(S, len(S) - 1)))\n            set_gamma[1] = CARMA._add_function(S_sub, S)\n            xs = np.vstack([CARMA._add_function(S_sub, row) for row in set_gamma[0]])\n            set_gamma[2] = xs\n\n        return set_gamma\n\n    @staticmethod\n    def _set_gamma_func(\n        input_S: Any, p: int, condition_index: list[int] | None = None\n    ) -&gt; dict[int, np.ndarray]:\n        \"\"\"Creates a dictionary of sets of configurations.\n\n        Args:\n            input_S (Any): The input set.\n            p (int): The number of SNPs.\n            condition_index (list[int] | None): The conditional set. Defaults to None.\n\n        Returns:\n            dict[int, np.ndarray]: A dictionary of sets of configurations.\n\n        Examples:\n        &gt;&gt;&gt; input_S = [0,1,2]\n        &gt;&gt;&gt; condition_index=[2]\n        &gt;&gt;&gt; p = 4\n        &gt;&gt;&gt; CARMA._set_gamma_func(input_S, p, condition_index)\n        {0: array([[0],\n               [1]]), 1: array([[0, 1, 3]]), 2: array([[0, 3],\n               [1, 3]])}\n        \"\"\"\n        if condition_index is None:\n            results = CARMA._set_gamma_func_base(input_S, p)\n        else:\n            results = CARMA._set_gamma_func_conditional(input_S, condition_index, p)\n        return results\n\n    @staticmethod\n    def _index_fun_internal(x: np.ndarray) -&gt; str:\n        \"\"\"Convert an array of causal SNP indexes to comma-separated string.\n\n        Args:\n            x (np.ndarray): The input array.\n\n        Returns:\n            str: The comma-separated string.\n\n        Examples:\n        &gt;&gt;&gt; x = np.array([1,2,3])\n        &gt;&gt;&gt; CARMA._index_fun_internal(x)\n        '1,2,3'\n        \"\"\"\n        y = np.sort(x)\n        y = y.astype(str)\n        return \",\".join(y)\n\n    @staticmethod\n    def _index_fun(y: np.ndarray) -&gt; np.ndarray:\n        \"\"\"Convert an array of causal SNP indexes to comma-separated string.\n\n        Args:\n            y (np.ndarray): The input array.\n\n        Returns:\n            np.ndarray: The comma-separated string.\n\n        Examples:\n        &gt;&gt;&gt; y = np.array([[1,2,3],[4,5,6]])\n        &gt;&gt;&gt; CARMA._index_fun(y)\n        array(['1,2,3', '4,5,6'], dtype='&lt;U5')\n        \"\"\"\n        return np.array([CARMA._index_fun_internal(x) for x in y])\n\n    @staticmethod\n    def _ridge_fun(\n        x: float,\n        Sigma: np.ndarray,\n        modi_ld_S: np.ndarray,\n        test_S: np.ndarray,\n        z: np.ndarray,\n        outlier_tau: float,\n        outlier_likelihood: Any,\n    ) -&gt; float:\n        \"\"\"Estimate the matrix shrinkage parameter for outlier detection.\n\n        Args:\n            x (float): The input parameter.\n            Sigma (np.ndarray): The Sigma matrix.\n            modi_ld_S (np.ndarray): The modi_ld_S matrix.\n            test_S (np.ndarray): The test_S matrix.\n            z (np.ndarray): The z vector.\n            outlier_tau (float): The outlier_tau value.\n            outlier_likelihood (Any): The outlier_likelihood function.\n\n        Returns:\n            float: The estimated matrix shrinkage parameter.\n\n        Examples:\n        &gt;&gt;&gt; x = 0.5\n        &gt;&gt;&gt; Sigma = np.array([[1, 0.5, 0.2], [0.5, 1, 0.3], [0.2, 0.3, 1]])\n        &gt;&gt;&gt; modi_ld_S = np.array([[1, 0.5], [0.5, 1]])\n        &gt;&gt;&gt; test_S = np.array([1, 2])\n        &gt;&gt;&gt; z = np.array([0.1, 0.2, 0.3])\n        &gt;&gt;&gt; outlier_tau = 1 / 0.05**2\n        &gt;&gt;&gt; outlier_likelihood = CARMA._outlier_ind_Normal_marginal_external\n        &gt;&gt;&gt; np.round(CARMA._ridge_fun(x, Sigma, modi_ld_S, test_S, z, outlier_tau, outlier_likelihood),decimals=5)\n        6.01486\n        \"\"\"\n        temp_Sigma = Sigma.copy()\n        temp_ld_S = x * modi_ld_S + (1 - x) * np.eye(len(modi_ld_S))\n        temp_Sigma[np.ix_(test_S, test_S)] = temp_ld_S\n        return -outlier_likelihood(\n            index_vec_input=test_S + 1,\n            Sigma=temp_Sigma,\n            z=z,\n            tau=outlier_tau,\n            p_S=len(test_S),\n        )\n\n    @staticmethod\n    def _prior_dist(t: str, lambda_val: float, p: int) -&gt; float:\n        \"\"\"Estimate the priors for the given configurations.\n\n        Args:\n            t (str): The input string for the given configuration.\n            lambda_val (float): The lambda value.\n            p (int): The number of SNPs.\n\n        Returns:\n            float: The estimated prior.\n\n        Examples:\n        &gt;&gt;&gt; t = \"1,2,3\"\n        &gt;&gt;&gt; lambda_val = 1\n        &gt;&gt;&gt; p = 4\n        &gt;&gt;&gt; np.round(CARMA._prior_dist(t, lambda_val, p),decimals=5)\n        -3.17805\n        \"\"\"\n        index_array = t.split(\",\")\n        dim_model = len(index_array)\n        if t == \"\":\n            dim_model = 0\n        return (\n            dim_model * np.log(lambda_val) + lgamma(p - dim_model + 1) - lgamma(p + 1)\n        )\n\n    @staticmethod\n    def _PIP_func(\n        likeli: pd.DataFrame, model_space: pd.DataFrame, p: int, num_causal: int\n    ) -&gt; np.ndarray:\n        \"\"\"Estimates the posterior inclusion probabilities (PIPs) for all SNPs.\n\n        Args:\n            likeli (pd.DataFrame): The marginal likelihoods.\n            model_space (pd.DataFrame): The corresponding model space.\n            p (int): The number of SNPs.\n            num_causal (int): The maximal number of causal SNPs.\n\n        Returns:\n            np.ndarray: The posterior inclusion probabilities (PIPs) for all SNPs.\n\n        Examples:\n        &gt;&gt;&gt; likeli = pd.DataFrame([10, 10, 5,11,0], columns=['likeli']).squeeze()\n        &gt;&gt;&gt; model_space = pd.DataFrame(['0', '1', '2','0,1',''], columns=['config']).squeeze()\n        &gt;&gt;&gt; p = 3\n        &gt;&gt;&gt; num_causal = 2\n        &gt;&gt;&gt; CARMA._PIP_func(likeli, model_space, p, num_causal)\n        array([0.7869271, 0.7869271, 0.001426 ])\n        \"\"\"\n        likeli = likeli.reset_index(drop=True)\n        model_space = model_space.reset_index(drop=True)\n\n        model_space_matrix = np.zeros((len(model_space), p), dtype=int)\n\n        for i in range(len(model_space)):\n            if model_space.iloc[i] != \"\":\n                ind = list(map(int, model_space.iloc[i].split(\",\")))\n                if len(ind) &gt; 0:\n                    model_space_matrix[i, ind] = 1\n\n        infi_index = np.where(np.isinf(likeli))[0]\n        if len(infi_index) != 0:\n            likeli = likeli.drop(infi_index).reset_index(drop=True)\n            model_space_matrix = np.delete(model_space_matrix, infi_index, axis=0)\n\n        na_index = np.where(np.isnan(likeli))[0]\n        if len(na_index) != 0:\n            likeli = likeli.drop(na_index).reset_index(drop=True)\n            model_space_matrix = np.delete(model_space_matrix, na_index, axis=0)\n\n        row_sums = np.sum(model_space_matrix, axis=1)\n        model_space_matrix = model_space_matrix[row_sums &lt;= num_causal]\n        likeli = likeli[row_sums &lt;= num_causal]\n\n        aa = likeli - max(likeli)\n        prob_sum = np.sum(np.exp(aa))\n\n        result_prob = np.zeros(p)\n        for i in range(p):\n            result_prob[i] = (\n                np.sum(np.exp(aa[model_space_matrix[:, i] == 1])) / prob_sum\n            )\n\n        return result_prob\n\n    @staticmethod\n    def _MCS_modified(  # noqa: C901\n        z: np.ndarray,\n        ld_matrix: np.ndarray,\n        Max_Model_Dim: int = 10_000,\n        lambda_val: float = 1,\n        num_causal: int = 10,\n        outlier_switch: bool = True,\n        input_conditional_S_list: list[int] | None = None,\n        tau: float = 1 / 0.05**2,\n        epsilon: float = 1e-3,\n        inner_all_iter: int = 10,\n        outlier_BF_index: float | None = None,\n    ) -&gt; dict[str, Any]:\n        \"\"\"Modified Monte Carlo shotgun sampling (MCS) algorithm.\n\n        Args:\n            z (np.ndarray): Numeric vector representing z-scores.\n            ld_matrix (np.ndarray): Numeric matrix representing the linkage disequilibrium (LD) matrix.\n            Max_Model_Dim (int): Maximum allowed dimension for the causal models.\n            lambda_val (float): Regularization parameter controlling the strength of the L1 penalty.\n            num_causal (int): Maximal number of causal variants to be selected in the final model.\n            outlier_switch (bool): Whether to consider outlier detection in the analysis.\n            input_conditional_S_list (list[int] | None): The conditional set. Defaults to None.\n            tau (float): Tuning parameter controlling the degree of sparsity in the Spike-and-Slab prior.\n            epsilon (float): Threshold for convergence in CARMA iterations.\n            inner_all_iter (int): The number of inner iterations in each CARMA iteration.\n            outlier_BF_index (float | None): Bayes Factor threshold for identifying outliers. Defaults to None.\n\n        Returns:\n            dict[str, Any]: A dictionary containing the following results:\n                - B_list: A dataframe containing the marginal likelihoods and the corresponding model space.\n                - conditional_S_list: A list of outliers.\n\n        Examples:\n        &gt;&gt;&gt; z = np.array([0.1, 0.2, 0.3])\n        &gt;&gt;&gt; ld_matrix = np.array([[1, 0.5, 0.2], [0.5, 1, 0.3], [0.2, 0.3, 1]])\n        &gt;&gt;&gt; Max_Model_Dim = 10_000\n        &gt;&gt;&gt; lambda_val = 1\n\n        &gt;&gt;&gt; num_causal = 10\n        &gt;&gt;&gt; outlier_switch = True\n        \"\"\"\n        p = len(z)\n        marginal_likelihood = CARMA._ind_Normal_fixed_sigma_marginal_external\n        tau_sample = tau\n        if outlier_switch:\n            outlier_likelihood = CARMA._outlier_ind_Normal_marginal_external\n            outlier_tau = tau\n\n        B = Max_Model_Dim\n        stored_bf = 0\n        Sigma = ld_matrix\n\n        S = []\n\n        null_model = \"\"\n        null_margin = CARMA._prior_dist(null_model, lambda_val=lambda_val, p=p)\n\n        B_list = pd.DataFrame({\"set_gamma_margin\": [null_margin], \"matrix_gamma\": [\"\"]})\n\n        if input_conditional_S_list is None:\n            conditional_S = []\n        else:\n            conditional_S = input_conditional_S_list\n            S = conditional_S\n\n        for _i in range(0, inner_all_iter):\n            for _j in range(0, 10):\n                set_gamma = CARMA._set_gamma_func(\n                    input_S=S, p=p, condition_index=conditional_S\n                )\n\n                if conditional_S is None:\n                    working_S = S\n                else:\n                    working_S = np.sort(np.setdiff1d(S, conditional_S)).astype(int)\n\n                set_gamma_margin: list[Any] = [None, None, None]\n                set_gamma_prior: list[Any] = [None, None, None]\n                matrix_gamma: list[Any] = [None, None, None]\n\n                for i in range(0, len(set_gamma)):\n                    if set_gamma[i] is not None:\n                        matrix_gamma[i] = CARMA._index_fun(set_gamma[i])\n                        p_S = set_gamma[i].shape[1]\n                        set_gamma_margin[i] = np.apply_along_axis(\n                            marginal_likelihood,\n                            1,\n                            set_gamma[i] + 1,\n                            Sigma=Sigma,\n                            z=z,\n                            tau=tau_sample,\n                            p_S=p_S,\n                        )\n                        set_gamma_prior[i] = np.array(\n                            [\n                                CARMA._prior_dist(model, lambda_val=lambda_val, p=p)\n                                for model in matrix_gamma[i]\n                            ]\n                        )\n                        set_gamma_margin[i] = set_gamma_prior[i] + set_gamma_margin[i]\n                    else:\n                        set_gamma_margin[i] = np.array(null_margin)\n                        set_gamma_prior[i] = 0\n                        matrix_gamma[i] = np.array(null_model)\n\n                columns = [\"set_gamma_margin\", \"matrix_gamma\"]\n                add_B = pd.DataFrame(columns=columns)\n\n                for i in range(len(set_gamma)):\n                    if isinstance(set_gamma_margin[i].tolist(), list):\n                        new_row = pd.DataFrame(\n                            {\n                                \"set_gamma_margin\": set_gamma_margin[i].tolist(),\n                                \"matrix_gamma\": matrix_gamma[i].tolist(),\n                            }\n                        )\n                        add_B = pd.concat([add_B, new_row], ignore_index=True)\n                    else:\n                        new_row = pd.DataFrame(\n                            {\n                                \"set_gamma_margin\": [set_gamma_margin[i].tolist()],\n                                \"matrix_gamma\": [matrix_gamma[i].tolist()],\n                            }\n                        )\n                        add_B = pd.concat([add_B, new_row], ignore_index=True)\n\n                # Add visited models into the storage space of models\n                B_list = pd.concat([B_list, add_B], ignore_index=True)\n                B_list = B_list.drop_duplicates(\n                    subset=\"matrix_gamma\", ignore_index=True\n                )\n                B_list = B_list.sort_values(\n                    by=\"set_gamma_margin\", ignore_index=True, ascending=False\n                )\n\n                if len(working_S) == 0:\n                    # Create a DataFrame set.star\n                    set_star = pd.DataFrame(\n                        {\n                            \"set_index\": [0, 1, 2],\n                            \"gamma_set_index\": [np.nan, np.nan, np.nan],\n                            \"margin\": [np.nan, np.nan, np.nan],\n                        }\n                    )\n\n                    # Assuming set.gamma.margin and current.log.margin are defined\n                    aa = set_gamma_margin[1]\n                    aa = aa - aa[np.argmax(aa)]\n\n                    min_half_len = min(len(aa), floor(p / 2))\n                    decr_ind = np.argsort(np.exp(aa))[::-1]\n                    decr_half_ind = decr_ind[:min_half_len]\n\n                    probs = np.exp(aa)[decr_half_ind]\n\n                    chosen_index = np.random.choice(\n                        decr_half_ind, 1, p=probs / np.sum(probs)\n                    )\n                    set_star.at[1, \"gamma_set_index\"] = chosen_index[0]\n                    set_star.at[1, \"margin\"] = set_gamma_margin[1][chosen_index[0]]\n\n                    S = set_gamma[1][chosen_index[0]].tolist()\n\n                else:\n                    set_star = pd.DataFrame(\n                        {\n                            \"set_index\": [0, 1, 2],\n                            \"gamma_set_index\": [np.nan, np.nan, np.nan],\n                            \"margin\": [np.nan, np.nan, np.nan],\n                        }\n                    )\n                    for i in range(0, 3):\n                        aa = set_gamma_margin[i]\n                        if np.size(aa) &gt; 1:\n                            aa = aa - aa[np.argmax(aa)]\n                            chosen_index = np.random.choice(\n                                range(0, np.size(set_gamma_margin[i])),\n                                1,\n                                p=np.exp(aa) / np.sum(np.exp(aa)),\n                            )\n                            set_star.at[i, \"gamma_set_index\"] = chosen_index\n                            set_star.at[i, \"margin\"] = set_gamma_margin[i][chosen_index]\n                        else:\n                            set_star.at[i, \"gamma_set_index\"] = 0\n                            set_star.at[i, \"margin\"] = set_gamma_margin[i]\n\n                    if outlier_switch:\n                        for i in range(1, len(set_gamma)):\n                            test_log_BF: float = 100\n                            while True:\n                                aa = set_gamma_margin[i]\n                                aa = aa - aa[np.argmax(aa)]\n                                chosen_index = np.random.choice(\n                                    range(0, np.size(set_gamma_margin[i])),\n                                    1,\n                                    p=np.exp(aa) / np.sum(np.exp(aa)),\n                                )\n                                set_star.at[i, \"gamma_set_index\"] = chosen_index\n                                set_star.at[i, \"margin\"] = set_gamma_margin[i][\n                                    chosen_index\n                                ]\n\n                                test_S = set_gamma[i][int(chosen_index), :]\n\n                                modi_Sigma = Sigma.copy()\n                                if np.size(test_S) &gt; 1:\n                                    modi_ld_S = modi_Sigma[test_S][:, test_S]\n\n                                    result = minimize_scalar(\n                                        CARMA._ridge_fun,\n                                        bounds=(0, 1),\n                                        args=(\n                                            Sigma,\n                                            modi_ld_S,\n                                            test_S,\n                                            z,\n                                            outlier_tau,\n                                            outlier_likelihood,\n                                        ),\n                                        method=\"bounded\",\n                                    )\n                                    modi_ld_S = result.x * modi_ld_S + (\n                                        1 - result.x\n                                    ) * np.eye(len(modi_ld_S))\n\n                                    modi_Sigma[np.ix_(test_S, test_S)] = modi_ld_S\n\n                                    test_log_BF = outlier_likelihood(\n                                        test_S + 1, Sigma, z, outlier_tau, len(test_S)\n                                    ) - outlier_likelihood(\n                                        test_S + 1,\n                                        modi_Sigma,\n                                        z,\n                                        outlier_tau,\n                                        len(test_S),\n                                    )\n                                    test_log_BF = -np.abs(test_log_BF)\n\n                                if np.exp(test_log_BF) &lt; outlier_BF_index:\n                                    set_gamma[i] = np.delete(\n                                        set_gamma[i],\n                                        int(set_star[\"gamma_set_index\"][i]),\n                                        axis=0,\n                                    )\n                                    set_gamma_margin[i] = np.delete(\n                                        set_gamma_margin[i],\n                                        int(set_star[\"gamma_set_index\"][i]),\n                                        axis=0,\n                                    )\n                                    conditional_S = np.concatenate(\n                                        [conditional_S, np.setdiff1d(test_S, working_S)]\n                                    )\n                                    conditional_S = (\n                                        np.unique(conditional_S).astype(int).tolist()\n                                    )\n                                else:\n                                    break\n\n                    if len(working_S) == num_causal:\n                        set_star = set_star.drop(1)\n                        aa = set_star[\"margin\"] - max(set_star[\"margin\"])\n                        sec_sample = np.random.choice(\n                            [0, 2], 1, p=np.exp(aa) / np.sum(np.exp(aa))\n                        )\n                        ind_sec = int(\n                            set_star[\"gamma_set_index\"][\n                                set_star[\"set_index\"] == int(sec_sample)\n                            ]\n                        )\n                        S = set_gamma[sec_sample[0]][ind_sec].tolist()\n                    else:\n                        aa = set_star[\"margin\"] - max(set_star[\"margin\"])\n                        sec_sample = np.random.choice(\n                            range(0, 3), 1, p=np.exp(aa) / np.sum(np.exp(aa))\n                        )\n                        if set_gamma[sec_sample[0]] is not None:\n                            S = set_gamma[sec_sample[0]][\n                                int(set_star[\"gamma_set_index\"][sec_sample[0]])\n                            ].tolist()\n                        else:\n                            sec_sample = np.random.choice(\n                                range(1, 3),\n                                1,\n                                p=np.exp(aa)[[1, 2]] / np.sum(np.exp(aa)[[1, 2]]),\n                            )\n                            S = set_gamma[sec_sample[0]][\n                                int(set_star[\"gamma_set_index\"][sec_sample[0]])\n                            ].tolist()\n\n                for item in conditional_S:\n                    if item not in S:\n                        S.append(item)\n            # END h_ind loop\n            #\n            if conditional_S is not None:\n                all_c_index = []\n                index_array = [s.split(\",\") for s in B_list[\"matrix_gamma\"]]\n                for tt in conditional_S:\n                    tt_str = str(tt)\n                    ind = [\n                        i for i, sublist in enumerate(index_array) if tt_str in sublist\n                    ]\n                    all_c_index.extend(ind)\n\n                all_c_index = list(set(all_c_index))\n\n                if len(all_c_index) &gt; 0:\n                    temp_B_list = B_list.copy()\n                    temp_B_list = B_list.drop(all_c_index)\n                else:\n                    temp_B_list = B_list.copy()\n            else:\n                temp_B_list = B_list.copy()\n\n            result_B_list = temp_B_list[: min(int(B), len(temp_B_list))]\n\n            rb1 = result_B_list[\"set_gamma_margin\"]\n\n            difference = abs(rb1[: (len(rb1) // 4)].mean() - stored_bf)\n\n            if difference &lt; epsilon:\n                break\n            else:\n                stored_bf = rb1[: (len(rb1) // 4)].mean()\n\n        out = {\"B_list\": result_B_list, \"conditional_S_list\": conditional_S}\n\n        return out\n</code></pre>"},{"location":"python_api/methods/carma/#gentropy.method.carma.CARMA.CARMA_spike_slab_noEM","title":"<code>CARMA_spike_slab_noEM(z: np.ndarray, ld: np.ndarray, lambda_val: float = 1, Max_Model_Dim: int = 200000, all_iter: int = 1, all_inner_iter: int = 10, epsilon_threshold: float = 1e-05, num_causal: int = 10, tau: float = 0.04, outlier_switch: bool = True, outlier_BF_index: float = 1 / 3.2) -&gt; dict[str, Any]</code>  <code>staticmethod</code>","text":"<p>Perform CARMA analysis using a Spike-and-Slab prior without Expectation-Maximization (EM).</p> <p>Parameters:</p> Name Type Description Default <code>z</code> <code>ndarray</code> <p>Numeric vector representing z-scores.</p> required <code>ld</code> <code>ndarray</code> <p>Numeric matrix representing the linkage disequilibrium (LD) matrix.</p> required <code>lambda_val</code> <code>float</code> <p>Regularization parameter controlling the strength of the L1 penalty.</p> <code>1</code> <code>Max_Model_Dim</code> <code>int</code> <p>Maximum allowed dimension for the causal models.</p> <code>200000</code> <code>all_iter</code> <code>int</code> <p>The total number of iterations to run the CARMA analysis.</p> <code>1</code> <code>all_inner_iter</code> <code>int</code> <p>The number of inner iterations in each CARMA iteration.</p> <code>10</code> <code>epsilon_threshold</code> <code>float</code> <p>Threshold for convergence in CARMA iterations.</p> <code>1e-05</code> <code>num_causal</code> <code>int</code> <p>Maximal number of causal variants to be selected in the final model.</p> <code>10</code> <code>tau</code> <code>float</code> <p>Tuning parameter controlling the degree of sparsity in the Spike-and-Slab prior.</p> <code>0.04</code> <code>outlier_switch</code> <code>bool</code> <p>Whether to consider outlier detection in the analysis.</p> <code>True</code> <code>outlier_BF_index</code> <code>float</code> <p>Bayes Factor threshold for identifying outliers.</p> <code>1 / 3.2</code> <p>Returns:</p> Type Description <code>dict[str, Any]</code> <p>dict[str, Any]: A dictionary containing the following results: - PIPs: A numeric vector of posterior inclusion probabilities (PIPs) for all SNPs. - B_list: A dataframe containing the marginal likelihoods and the corresponding model space. - Outliers: A list of outlier SNPs.</p> Source code in <code>src/gentropy/method/carma.py</code> <pre><code>@staticmethod\ndef CARMA_spike_slab_noEM(\n    z: np.ndarray,\n    ld: np.ndarray,\n    lambda_val: float = 1,\n    Max_Model_Dim: int = 200_000,\n    all_iter: int = 1,\n    all_inner_iter: int = 10,\n    epsilon_threshold: float = 1e-5,\n    num_causal: int = 10,\n    tau: float = 0.04,\n    outlier_switch: bool = True,\n    outlier_BF_index: float = 1 / 3.2,\n) -&gt; dict[str, Any]:\n    \"\"\"Perform CARMA analysis using a Spike-and-Slab prior without Expectation-Maximization (EM).\n\n    Args:\n        z (np.ndarray): Numeric vector representing z-scores.\n        ld (np.ndarray): Numeric matrix representing the linkage disequilibrium (LD) matrix.\n        lambda_val (float): Regularization parameter controlling the strength of the L1 penalty.\n        Max_Model_Dim (int): Maximum allowed dimension for the causal models.\n        all_iter (int): The total number of iterations to run the CARMA analysis.\n        all_inner_iter (int): The number of inner iterations in each CARMA iteration.\n        epsilon_threshold (float): Threshold for convergence in CARMA iterations.\n        num_causal (int): Maximal number of causal variants to be selected in the final model.\n        tau (float): Tuning parameter controlling the degree of sparsity in the Spike-and-Slab prior.\n        outlier_switch (bool): Whether to consider outlier detection in the analysis.\n        outlier_BF_index (float): Bayes Factor threshold for identifying outliers.\n\n    Returns:\n        dict[str, Any]: A dictionary containing the following results:\n            - PIPs: A numeric vector of posterior inclusion probabilities (PIPs) for all SNPs.\n            - B_list: A dataframe containing the marginal likelihoods and the corresponding model space.\n            - Outliers: A list of outlier SNPs.\n    \"\"\"\n    p_snp = len(z)\n    epsilon_list = epsilon_threshold * p_snp\n    all_epsilon_threshold = epsilon_threshold * p_snp\n\n    # Zero step\n    all_C_list = CARMA._MCS_modified(\n        z=z,\n        ld_matrix=ld,\n        epsilon=epsilon_list,\n        Max_Model_Dim=Max_Model_Dim,\n        lambda_val=lambda_val,\n        outlier_switch=outlier_switch,\n        tau=tau,\n        num_causal=num_causal,\n        inner_all_iter=all_inner_iter,\n        outlier_BF_index=outlier_BF_index,\n    )\n\n    # Main steps\n    for _ in range(0, all_iter):\n        ac1 = all_C_list[\"B_list\"][\"set_gamma_margin\"]\n        previous_result = np.mean(ac1[0 : round(len(ac1) / 4)])\n\n        all_C_list = CARMA._MCS_modified(\n            z=z,\n            ld_matrix=ld,\n            input_conditional_S_list=all_C_list[\"conditional_S_list\"],\n            Max_Model_Dim=Max_Model_Dim,\n            num_causal=num_causal,\n            epsilon=epsilon_list,\n            outlier_switch=outlier_switch,\n            tau=tau,\n            lambda_val=lambda_val,\n            inner_all_iter=all_inner_iter,\n            outlier_BF_index=outlier_BF_index,\n        )\n\n        ac1 = all_C_list[\"B_list\"][\"set_gamma_margin\"]\n        difference = np.abs(previous_result - np.mean(ac1[0 : round(len(ac1) / 4)]))\n        if difference &lt; all_epsilon_threshold:\n            break\n\n    # Calculate PIPs and Credible Set\n    pip = CARMA._PIP_func(\n        likeli=all_C_list[\"B_list\"][\"set_gamma_margin\"],\n        model_space=all_C_list[\"B_list\"][\"matrix_gamma\"],\n        p=p_snp,\n        num_causal=num_causal,\n    )\n\n    results_list = {\n        \"PIPs\": pip,\n        \"B_list\": all_C_list[\"B_list\"],\n        \"Outliers\": all_C_list[\"conditional_S_list\"],\n    }\n\n    return results_list\n</code></pre>"},{"location":"python_api/methods/carma/#gentropy.method.carma.CARMA.time_limited_CARMA_spike_slab_noEM","title":"<code>time_limited_CARMA_spike_slab_noEM(z: np.ndarray, ld: np.ndarray, sec_threshold: float = 600) -&gt; dict[str, Any]</code>  <code>staticmethod</code>","text":"<p>The wrapper for the CARMA_spike_slab_noEM function that runs the function in a separate thread and terminates it if it takes too long.</p> <p>Parameters:</p> Name Type Description Default <code>z</code> <code>ndarray</code> <p>Numeric vector representing z-scores.</p> required <code>ld</code> <code>ndarray</code> <p>Numeric matrix representing the linkage disequilibrium (LD) matrix.</p> required <code>sec_threshold</code> <code>float</code> <p>The time threshold in seconds.</p> <code>600</code> <p>Returns:</p> Type Description <code>dict[str, Any]</code> <p>dict[str, Any]: A dictionary containing the following results: - PIPs: A numeric vector of posterior inclusion probabilities (PIPs) for all SNPs or None. - B_list: A dataframe containing the marginal likelihoods and the corresponding model space or None. - Outliers: A list of outlier SNPs or None.</p> Source code in <code>src/gentropy/method/carma.py</code> <pre><code>@staticmethod\ndef time_limited_CARMA_spike_slab_noEM(\n    z: np.ndarray, ld: np.ndarray, sec_threshold: float = 600\n) -&gt; dict[str, Any]:\n    \"\"\"The wrapper for the CARMA_spike_slab_noEM function that runs the function in a separate thread and terminates it if it takes too long.\n\n    Args:\n        z (np.ndarray): Numeric vector representing z-scores.\n        ld (np.ndarray): Numeric matrix representing the linkage disequilibrium (LD) matrix.\n        sec_threshold (float): The time threshold in seconds.\n\n    Returns:\n        dict[str, Any]: A dictionary containing the following results:\n            - PIPs: A numeric vector of posterior inclusion probabilities (PIPs) for all SNPs or None.\n            - B_list: A dataframe containing the marginal likelihoods and the corresponding model space or None.\n            - Outliers: A list of outlier SNPs or None.\n    \"\"\"\n    # Ignore pandas future warnings\n    warnings.simplefilter(action=\"ignore\", category=FutureWarning)\n    try:\n        # Execute CARMA.CARMA_spike_slab_noEM with a timeout\n        with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:\n            future = executor.submit(CARMA.CARMA_spike_slab_noEM, z=z, ld=ld)\n            result = future.result(timeout=sec_threshold)\n    except concurrent.futures.TimeoutError:\n        # If execution exceeds the timeout, return None\n        result = {\"PIPs\": None, \"B_list\": None, \"Outliers\": None}\n\n    return result\n</code></pre>"},{"location":"python_api/methods/clumping/","title":"Clumping","text":"<p>Clumping is a commonly used post-processing method that allows for the identification of independent association signals from GWAS summary statistics and curated associations. This process is critical because of the complex linkage disequilibrium (LD) structure in human populations, which can result in multiple statistically significant associations within the same genomic region. Clumping methods help reduce redundancy in GWAS results and ensure that each reported association represents an independent signal.</p> <p>We have implemented two clumping methods:</p> <ol> <li>Distance-based clumping: Uses genomic window to clump the significant SNPs into one hit.</li> <li>LD-based clumping: Uses genomic window and LD to clump the significant SNPs into one hit.</li> </ol> <p>The algorithmic logic is similar to classic clumping approaches from PLINK (Reference: PLINK Clump Documentation). See details below:</p>"},{"location":"python_api/methods/clumping/#distance-based-clumping","title":"Distance-based clumping","text":""},{"location":"python_api/methods/clumping/#gentropy.method.window_based_clumping.WindowBasedClumping","title":"<code>gentropy.method.window_based_clumping.WindowBasedClumping</code>","text":"<p>Get semi-lead snps from summary statistics using a window based function.</p> Source code in <code>src/gentropy/method/window_based_clumping.py</code> <pre><code>class WindowBasedClumping:\n    \"\"\"Get semi-lead snps from summary statistics using a window based function.\"\"\"\n\n    @staticmethod\n    def _cluster_peaks(\n        study: Column, chromosome: Column, position: Column, window_length: int\n    ) -&gt; Column:\n        \"\"\"Cluster GWAS significant variants, were clusters are separated by a defined distance.\n\n        !! Important to note that the length of the clusters can be arbitrarily big.\n\n        Args:\n            study (Column): study identifier\n            chromosome (Column): chromosome identifier\n            position (Column): position of the variant\n            window_length (int): window length in basepair\n\n        Returns:\n            Column: containing cluster identifier\n\n        Examples:\n            &gt;&gt;&gt; data = [\n            ...     # Cluster 1:\n            ...     ('s1', 'chr1', 2),\n            ...     ('s1', 'chr1', 4),\n            ...     ('s1', 'chr1', 12),\n            ...     # Cluster 2 - Same chromosome:\n            ...     ('s1', 'chr1', 31),\n            ...     ('s1', 'chr1', 38),\n            ...     ('s1', 'chr1', 42),\n            ...     # Cluster 3 - New chromosome:\n            ...     ('s1', 'chr2', 41),\n            ...     ('s1', 'chr2', 44),\n            ...     ('s1', 'chr2', 50),\n            ...     # Cluster 4 - other study:\n            ...     ('s2', 'chr2', 55),\n            ...     ('s2', 'chr2', 62),\n            ...     ('s2', 'chr2', 70),\n            ... ]\n            &gt;&gt;&gt; window_length = 10\n            &gt;&gt;&gt; (\n            ...     spark.createDataFrame(data, ['studyId', 'chromosome', 'position'])\n            ...     .withColumn(\"cluster_id\",\n            ...         WindowBasedClumping._cluster_peaks(\n            ...             f.col('studyId'),\n            ...             f.col('chromosome'),\n            ...             f.col('position'),\n            ...             window_length\n            ...         )\n            ...     ).show()\n            ... )\n            +-------+----------+--------+----------+\n            |studyId|chromosome|position|cluster_id|\n            +-------+----------+--------+----------+\n            |     s1|      chr1|       2| s1_chr1_2|\n            |     s1|      chr1|       4| s1_chr1_2|\n            |     s1|      chr1|      12| s1_chr1_2|\n            |     s1|      chr1|      31|s1_chr1_31|\n            |     s1|      chr1|      38|s1_chr1_31|\n            |     s1|      chr1|      42|s1_chr1_31|\n            |     s1|      chr2|      41|s1_chr2_41|\n            |     s1|      chr2|      44|s1_chr2_41|\n            |     s1|      chr2|      50|s1_chr2_41|\n            |     s2|      chr2|      55|s2_chr2_55|\n            |     s2|      chr2|      62|s2_chr2_55|\n            |     s2|      chr2|      70|s2_chr2_55|\n            +-------+----------+--------+----------+\n            &lt;BLANKLINE&gt;\n\n        \"\"\"\n        # By adding previous position, the cluster boundary can be identified:\n        previous_position = f.lag(position).over(\n            Window.partitionBy(study, chromosome).orderBy(position)\n        )\n        # We consider a cluster boudary if subsequent snps are further than the defined window:\n        cluster_id = f.when(\n            (previous_position.isNull())\n            | (position - previous_position &gt; window_length),\n            f.concat_ws(\"_\", study, chromosome, position),\n        )\n        # The cluster identifier is propagated across every variant of the cluster:\n        return f.when(\n            cluster_id.isNull(),\n            f.last(cluster_id, ignorenulls=True).over(\n                Window.partitionBy(study, chromosome)\n                .orderBy(position)\n                .rowsBetween(Window.unboundedPreceding, Window.currentRow)\n            ),\n        ).otherwise(cluster_id)\n\n    @staticmethod\n    def _prune_peak(position: NDArray[np.float64], window_size: int) -&gt; DenseVector:\n        \"\"\"Establish lead snps based on their positions listed by p-value.\n\n        The function `find_peak` assigns lead SNPs based on their positions listed by p-value within a specified window size.\n\n        Args:\n            position (NDArray[np.float64]): positions of the SNPs sorted by p-value.\n            window_size (int): the distance in bp within which associations are clumped together around the lead snp.\n\n        Returns:\n            DenseVector: binary vector where 1 indicates a lead SNP and 0 indicates a non-lead SNP.\n\n        Examples:\n            &gt;&gt;&gt; from pyspark.ml import functions as fml\n            &gt;&gt;&gt; from pyspark.ml.linalg import DenseVector\n            &gt;&gt;&gt; WindowBasedClumping._prune_peak(np.array((3, 9, 8, 4, 6)), 2)\n            DenseVector([1.0, 1.0, 0.0, 0.0, 1.0])\n\n        \"\"\"\n        # Initializing the lead list with zeroes:\n        is_lead = np.zeros(len(position))\n\n        # List containing indices of leads:\n        lead_indices: list[int] = []\n\n        # Looping through all positions:\n        for index in range(len(position)):\n            # Looping through leads to find out if they are within a window:\n            for lead_index in lead_indices:\n                # If any of the leads within the window:\n                if abs(position[lead_index] - position[index]) &lt; window_size:\n                    # Skipping further checks:\n                    break\n            else:\n                # None of the leads were within the window:\n                lead_indices.append(index)\n                is_lead[index] = 1\n\n        return DenseVector(is_lead)\n\n    @staticmethod\n    def clump(\n        summary_statistics: SummaryStatistics,\n        distance: int = 500_000,\n        gwas_significance: float = 5e-8,\n    ) -&gt; StudyLocus:\n        \"\"\"Clump significant signals from summary statistics based on window.\n\n        Args:\n            summary_statistics (SummaryStatistics): Summary statistics to be used for clumping.\n            distance (int): Distance in base pairs to be used for clumping. Defaults to 500_000.\n            gwas_significance (float): GWAS significance threshold. Defaults to 5e-8.\n\n        Returns:\n            StudyLocus: clumped summary statistics (without locus collection)\n        \"\"\"\n        # Create window for locus clusters\n        # - variants where the distance between subsequent variants is below the defined threshold.\n        # - Variants are sorted by descending significance\n        cluster_window = Window.partitionBy(\n            \"studyId\", \"chromosome\", \"cluster_id\"\n        ).orderBy(f.col(\"pValueExponent\").asc(), f.col(\"pValueMantissa\").asc())\n\n        return StudyLocus(\n            _df=(\n                summary_statistics\n                # Dropping snps below significance - all subsequent steps are done on significant variants:\n                .pvalue_filter(gwas_significance)\n                .df\n                # Clustering summary variants for efficient windowing (complexity reduction):\n                .withColumn(\n                    \"cluster_id\",\n                    WindowBasedClumping._cluster_peaks(\n                        f.col(\"studyId\"),\n                        f.col(\"chromosome\"),\n                        f.col(\"position\"),\n                        distance,\n                    ),\n                )\n                # Within each cluster variants are ranked by significance:\n                .withColumn(\"pvRank\", f.row_number().over(cluster_window))\n                # Collect positions in cluster for the most significant variant (complexity reduction):\n                .withColumn(\n                    \"collectedPositions\",\n                    f.when(\n                        f.col(\"pvRank\") == 1,\n                        f.collect_list(f.col(\"position\")).over(\n                            cluster_window.rowsBetween(\n                                Window.currentRow, Window.unboundedFollowing\n                            )\n                        ),\n                    ).otherwise(f.array()),\n                )\n                # Get semi indices only ONCE per cluster:\n                .withColumn(\n                    \"semiIndices\",\n                    f.when(\n                        f.size(f.col(\"collectedPositions\")) &gt; 0,\n                        fml.vector_to_array(\n                            f.udf(WindowBasedClumping._prune_peak, VectorUDT())(\n                                fml.array_to_vector(f.col(\"collectedPositions\")),\n                                f.lit(distance),\n                            )\n                        ),\n                    ),\n                )\n                # Propagating the result of the above calculation for all rows:\n                .withColumn(\n                    \"semiIndices\",\n                    f.when(\n                        f.col(\"semiIndices\").isNull(),\n                        f.first(f.col(\"semiIndices\"), ignorenulls=True).over(\n                            cluster_window\n                        ),\n                    ).otherwise(f.col(\"semiIndices\")),\n                )\n                # Keeping semi indices only:\n                .filter(f.col(\"semiIndices\")[f.col(\"pvRank\") - 1] &gt; 0)\n                .drop(\"pvRank\", \"collectedPositions\", \"semiIndices\", \"cluster_id\")\n                # Adding study-locus id:\n                .withColumn(\n                    \"studyLocusId\",\n                    StudyLocus.assign_study_locus_id(\n                        f.col(\"studyId\"), f.col(\"variantId\")\n                    ),\n                )\n                # Initialize QC column as array of strings:\n                .withColumn(\n                    \"qualityControls\", f.array().cast(t.ArrayType(t.StringType()))\n                )\n            ),\n            _schema=StudyLocus.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/methods/clumping/#gentropy.method.window_based_clumping.WindowBasedClumping.clump","title":"<code>clump(summary_statistics: SummaryStatistics, distance: int = 500000, gwas_significance: float = 5e-08) -&gt; StudyLocus</code>  <code>staticmethod</code>","text":"<p>Clump significant signals from summary statistics based on window.</p> <p>Parameters:</p> Name Type Description Default <code>summary_statistics</code> <code>SummaryStatistics</code> <p>Summary statistics to be used for clumping.</p> required <code>distance</code> <code>int</code> <p>Distance in base pairs to be used for clumping. Defaults to 500_000.</p> <code>500000</code> <code>gwas_significance</code> <code>float</code> <p>GWAS significance threshold. Defaults to 5e-8.</p> <code>5e-08</code> <p>Returns:</p> Name Type Description <code>StudyLocus</code> <code>StudyLocus</code> <p>clumped summary statistics (without locus collection)</p> Source code in <code>src/gentropy/method/window_based_clumping.py</code> <pre><code>@staticmethod\ndef clump(\n    summary_statistics: SummaryStatistics,\n    distance: int = 500_000,\n    gwas_significance: float = 5e-8,\n) -&gt; StudyLocus:\n    \"\"\"Clump significant signals from summary statistics based on window.\n\n    Args:\n        summary_statistics (SummaryStatistics): Summary statistics to be used for clumping.\n        distance (int): Distance in base pairs to be used for clumping. Defaults to 500_000.\n        gwas_significance (float): GWAS significance threshold. Defaults to 5e-8.\n\n    Returns:\n        StudyLocus: clumped summary statistics (without locus collection)\n    \"\"\"\n    # Create window for locus clusters\n    # - variants where the distance between subsequent variants is below the defined threshold.\n    # - Variants are sorted by descending significance\n    cluster_window = Window.partitionBy(\n        \"studyId\", \"chromosome\", \"cluster_id\"\n    ).orderBy(f.col(\"pValueExponent\").asc(), f.col(\"pValueMantissa\").asc())\n\n    return StudyLocus(\n        _df=(\n            summary_statistics\n            # Dropping snps below significance - all subsequent steps are done on significant variants:\n            .pvalue_filter(gwas_significance)\n            .df\n            # Clustering summary variants for efficient windowing (complexity reduction):\n            .withColumn(\n                \"cluster_id\",\n                WindowBasedClumping._cluster_peaks(\n                    f.col(\"studyId\"),\n                    f.col(\"chromosome\"),\n                    f.col(\"position\"),\n                    distance,\n                ),\n            )\n            # Within each cluster variants are ranked by significance:\n            .withColumn(\"pvRank\", f.row_number().over(cluster_window))\n            # Collect positions in cluster for the most significant variant (complexity reduction):\n            .withColumn(\n                \"collectedPositions\",\n                f.when(\n                    f.col(\"pvRank\") == 1,\n                    f.collect_list(f.col(\"position\")).over(\n                        cluster_window.rowsBetween(\n                            Window.currentRow, Window.unboundedFollowing\n                        )\n                    ),\n                ).otherwise(f.array()),\n            )\n            # Get semi indices only ONCE per cluster:\n            .withColumn(\n                \"semiIndices\",\n                f.when(\n                    f.size(f.col(\"collectedPositions\")) &gt; 0,\n                    fml.vector_to_array(\n                        f.udf(WindowBasedClumping._prune_peak, VectorUDT())(\n                            fml.array_to_vector(f.col(\"collectedPositions\")),\n                            f.lit(distance),\n                        )\n                    ),\n                ),\n            )\n            # Propagating the result of the above calculation for all rows:\n            .withColumn(\n                \"semiIndices\",\n                f.when(\n                    f.col(\"semiIndices\").isNull(),\n                    f.first(f.col(\"semiIndices\"), ignorenulls=True).over(\n                        cluster_window\n                    ),\n                ).otherwise(f.col(\"semiIndices\")),\n            )\n            # Keeping semi indices only:\n            .filter(f.col(\"semiIndices\")[f.col(\"pvRank\") - 1] &gt; 0)\n            .drop(\"pvRank\", \"collectedPositions\", \"semiIndices\", \"cluster_id\")\n            # Adding study-locus id:\n            .withColumn(\n                \"studyLocusId\",\n                StudyLocus.assign_study_locus_id(\n                    f.col(\"studyId\"), f.col(\"variantId\")\n                ),\n            )\n            # Initialize QC column as array of strings:\n            .withColumn(\n                \"qualityControls\", f.array().cast(t.ArrayType(t.StringType()))\n            )\n        ),\n        _schema=StudyLocus.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/methods/clumping/#ld-based-clumping","title":"LD-based clumping:","text":""},{"location":"python_api/methods/clumping/#gentropy.method.clump.LDclumping","title":"<code>gentropy.method.clump.LDclumping</code>","text":"<p>LD clumping reports the most significant genetic associations in a region in terms of a smaller number of \u201cclumps\u201d of genetically linked SNPs.</p> Source code in <code>src/gentropy/method/clump.py</code> <pre><code>class LDclumping:\n    \"\"\"LD clumping reports the most significant genetic associations in a region in terms of a smaller number of \u201cclumps\u201d of genetically linked SNPs.\"\"\"\n\n    @staticmethod\n    def _is_lead_linked(\n        study_id: Column,\n        variant_id: Column,\n        p_value_exponent: Column,\n        p_value_mantissa: Column,\n        ld_set: Column,\n    ) -&gt; Column:\n        \"\"\"Evaluates whether a lead variant is linked to a tag (with lowest p-value) in the same studyLocus dataset.\n\n        Args:\n            study_id (Column): studyId\n            variant_id (Column): Lead variant id\n            p_value_exponent (Column): p-value exponent\n            p_value_mantissa (Column): p-value mantissa\n            ld_set (Column): Array of variants in LD with the lead variant\n\n        Returns:\n            Column: Boolean in which True indicates that the lead is linked to another tag in the same dataset.\n        \"\"\"\n        leads_in_study = f.collect_set(variant_id).over(Window.partitionBy(study_id))\n        tags_in_studylocus = f.array_union(\n            # Get all tag variants from the credible set per studyLocusId\n            f.transform(ld_set, lambda x: x.tagVariantId),\n            # And append the lead variant so that the intersection is the same for all studyLocusIds in a study\n            f.array(variant_id),\n        )\n        intersect_lead_tags = f.array_sort(\n            f.array_intersect(leads_in_study, tags_in_studylocus)\n        )\n        return (\n            # If the lead is in the credible set, we rank the peaks by p-value\n            f.when(\n                f.size(intersect_lead_tags) &gt; 0,\n                f.row_number().over(\n                    Window.partitionBy(study_id, intersect_lead_tags).orderBy(\n                        p_value_exponent, p_value_mantissa\n                    )\n                )\n                &gt; 1,\n            )\n            # If the intersection is empty (lead is not in the credible set or cred set is empty), the association is not linked\n            .otherwise(f.lit(False))\n        )\n\n    @classmethod\n    def clump(cls: type[LDclumping], associations: StudyLocus) -&gt; StudyLocus:\n        \"\"\"Perform clumping on studyLocus dataset.\n\n        Args:\n            associations (StudyLocus): StudyLocus dataset\n\n        Returns:\n            StudyLocus: including flag and removing locus information for LD clumped loci.\n        \"\"\"\n        return associations.clump()\n</code></pre>"},{"location":"python_api/methods/clumping/#gentropy.method.clump.LDclumping.clump","title":"<code>clump(associations: StudyLocus) -&gt; StudyLocus</code>  <code>classmethod</code>","text":"<p>Perform clumping on studyLocus dataset.</p> <p>Parameters:</p> Name Type Description Default <code>associations</code> <code>StudyLocus</code> <p>StudyLocus dataset</p> required <p>Returns:</p> Name Type Description <code>StudyLocus</code> <code>StudyLocus</code> <p>including flag and removing locus information for LD clumped loci.</p> Source code in <code>src/gentropy/method/clump.py</code> <pre><code>@classmethod\ndef clump(cls: type[LDclumping], associations: StudyLocus) -&gt; StudyLocus:\n    \"\"\"Perform clumping on studyLocus dataset.\n\n    Args:\n        associations (StudyLocus): StudyLocus dataset\n\n    Returns:\n        StudyLocus: including flag and removing locus information for LD clumped loci.\n    \"\"\"\n    return associations.clump()\n</code></pre>"},{"location":"python_api/methods/coloc/","title":"Coloc","text":""},{"location":"python_api/methods/coloc/#gentropy.method.colocalisation.Coloc","title":"<code>gentropy.method.colocalisation.Coloc</code>","text":"<p>Calculate bayesian colocalisation based on overlapping signals from credible sets.</p> <p>Based on the R COLOC package, which uses the Bayes factors from the credible set to estimate the posterior probability of colocalisation. This method makes the simplifying assumption that only one single causal variant exists for any given trait in any genomic region.</p> Hypothesis Description H<sub>0</sub> no association with either trait in the region H<sub>1</sub> association with trait 1 only H<sub>2</sub> association with trait 2 only H<sub>3</sub> both traits are associated, but have different single causal variants H<sub>4</sub> both traits are associated and share the same single causal variant <p>Bayes factors required</p> <p>Coloc requires the availability of Bayes factors (BF) for each variant in the credible set (<code>logBF</code> column).</p> <p>Attributes:</p> Name Type Description <code>PSEUDOCOUNT</code> <code>float</code> <p>Pseudocount to avoid log(0). Defaults to 1e-10.</p> Source code in <code>src/gentropy/method/colocalisation.py</code> <pre><code>class Coloc:\n    \"\"\"Calculate bayesian colocalisation based on overlapping signals from credible sets.\n\n    Based on the [R COLOC package](https://github.com/chr1swallace/coloc/blob/main/R/claudia.R), which uses the Bayes factors from the credible set to estimate the posterior probability of colocalisation. This method makes the simplifying assumption that **only one single causal variant** exists for any given trait in any genomic region.\n\n    | Hypothesis    | Description                                                           |\n    | ------------- | --------------------------------------------------------------------- |\n    | H&lt;sub&gt;0&lt;/sub&gt; | no association with either trait in the region                        |\n    | H&lt;sub&gt;1&lt;/sub&gt; | association with trait 1 only                                         |\n    | H&lt;sub&gt;2&lt;/sub&gt; | association with trait 2 only                                         |\n    | H&lt;sub&gt;3&lt;/sub&gt; | both traits are associated, but have different single causal variants |\n    | H&lt;sub&gt;4&lt;/sub&gt; | both traits are associated and share the same single causal variant   |\n\n    !!! warning \"Bayes factors required\"\n\n        Coloc requires the availability of Bayes factors (BF) for each variant in the credible set (`logBF` column).\n\n    Attributes:\n        PSEUDOCOUNT (float): Pseudocount to avoid log(0). Defaults to 1e-10.\n    \"\"\"\n\n    METHOD_NAME: str = \"COLOC\"\n    METHOD_METRIC: str = \"llr\"\n    PSEUDOCOUNT: float = 1e-10\n\n    @staticmethod\n    def _get_posteriors(all_bfs: NDArray[np.float64]) -&gt; DenseVector:\n        \"\"\"Calculate posterior probabilities for each hypothesis.\n\n        Args:\n            all_bfs (NDArray[np.float64]): h0-h4 bayes factors\n\n        Returns:\n            DenseVector: Posterior\n\n        Example:\n            &gt;&gt;&gt; l = np.array([0.2, 0.1, 0.05, 0])\n            &gt;&gt;&gt; Coloc._get_posteriors(l)\n            DenseVector([0.279, 0.2524, 0.2401, 0.2284])\n        \"\"\"\n        diff = all_bfs - get_logsum(all_bfs)\n        bfs_posteriors = np.exp(diff)\n        return Vectors.dense(bfs_posteriors)\n\n    @classmethod\n    def colocalise(\n        cls: type[Coloc],\n        overlapping_signals: StudyLocusOverlap,\n        priorc1: float = 1e-4,\n        priorc2: float = 1e-4,\n        priorc12: float = 1e-5,\n    ) -&gt; Colocalisation:\n        \"\"\"Calculate bayesian colocalisation based on overlapping signals.\n\n        Args:\n            overlapping_signals (StudyLocusOverlap): overlapping peaks\n\n            priorc1 (float): Prior on variant being causal for trait 1. Defaults to 1e-4.\n            priorc2 (float): Prior on variant being causal for trait 2. Defaults to 1e-4.\n            priorc12 (float): Prior on variant being causal for traits 1 and 2. Defaults to 1e-5.\n\n        Returns:\n            Colocalisation: Colocalisation results\n        \"\"\"\n        # register udfs\n        logsum = f.udf(get_logsum, DoubleType())\n        posteriors = f.udf(Coloc._get_posteriors, VectorUDT())\n        return Colocalisation(\n            _df=(\n                overlapping_signals.df.select(\"*\", \"statistics.*\")\n                # Before summing log_BF columns nulls need to be filled with 0:\n                .fillna(0, subset=[\"left_logBF\", \"right_logBF\"])\n                # Sum of log_BFs for each pair of signals\n                .withColumn(\n                    \"sum_log_bf\",\n                    f.col(\"left_logBF\") + f.col(\"right_logBF\"),\n                )\n                # Group by overlapping peak and generating dense vectors of log_BF:\n                .groupBy(\"chromosome\", \"leftStudyLocusId\", \"rightStudyLocusId\")\n                .agg(\n                    f.count(\"*\").alias(\"numberColocalisingVariants\"),\n                    fml.array_to_vector(f.collect_list(f.col(\"left_logBF\"))).alias(\n                        \"left_logBF\"\n                    ),\n                    fml.array_to_vector(f.collect_list(f.col(\"right_logBF\"))).alias(\n                        \"right_logBF\"\n                    ),\n                    fml.array_to_vector(f.collect_list(f.col(\"sum_log_bf\"))).alias(\n                        \"sum_log_bf\"\n                    ),\n                )\n                .withColumn(\"logsum1\", logsum(f.col(\"left_logBF\")))\n                .withColumn(\"logsum2\", logsum(f.col(\"right_logBF\")))\n                .withColumn(\"logsum12\", logsum(f.col(\"sum_log_bf\")))\n                .drop(\"left_logBF\", \"right_logBF\", \"sum_log_bf\")\n                # Add priors\n                # priorc1 Prior on variant being causal for trait 1\n                .withColumn(\"priorc1\", f.lit(priorc1))\n                # priorc2 Prior on variant being causal for trait 2\n                .withColumn(\"priorc2\", f.lit(priorc2))\n                # priorc12 Prior on variant being causal for traits 1 and 2\n                .withColumn(\"priorc12\", f.lit(priorc12))\n                # h0-h2\n                .withColumn(\"lH0bf\", f.lit(0))\n                .withColumn(\"lH1bf\", f.log(f.col(\"priorc1\")) + f.col(\"logsum1\"))\n                .withColumn(\"lH2bf\", f.log(f.col(\"priorc2\")) + f.col(\"logsum2\"))\n                # h3\n                .withColumn(\"sumlogsum\", f.col(\"logsum1\") + f.col(\"logsum2\"))\n                .withColumn(\"max\", f.greatest(\"sumlogsum\", \"logsum12\"))\n                .withColumn(\n                    \"logdiff\",\n                    f.when(\n                        f.col(\"sumlogsum\") == f.col(\"logsum12\"), Coloc.PSEUDOCOUNT\n                    ).otherwise(\n                        f.col(\"max\")\n                        + f.log(\n                            f.exp(f.col(\"sumlogsum\") - f.col(\"max\"))\n                            - f.exp(f.col(\"logsum12\") - f.col(\"max\"))\n                        )\n                    ),\n                )\n                .withColumn(\n                    \"lH3bf\",\n                    f.log(f.col(\"priorc1\"))\n                    + f.log(f.col(\"priorc2\"))\n                    + f.col(\"logdiff\"),\n                )\n                .drop(\"right_logsum\", \"left_logsum\", \"sumlogsum\", \"max\", \"logdiff\")\n                # h4\n                .withColumn(\"lH4bf\", f.log(f.col(\"priorc12\")) + f.col(\"logsum12\"))\n                # cleaning\n                .drop(\n                    \"priorc1\", \"priorc2\", \"priorc12\", \"logsum1\", \"logsum2\", \"logsum12\"\n                )\n                # posteriors\n                .withColumn(\n                    \"allBF\",\n                    fml.array_to_vector(\n                        f.array(\n                            f.col(\"lH0bf\"),\n                            f.col(\"lH1bf\"),\n                            f.col(\"lH2bf\"),\n                            f.col(\"lH3bf\"),\n                            f.col(\"lH4bf\"),\n                        )\n                    ),\n                )\n                .withColumn(\n                    \"posteriors\", fml.vector_to_array(posteriors(f.col(\"allBF\")))\n                )\n                .withColumn(\"h0\", f.col(\"posteriors\").getItem(0))\n                .withColumn(\"h1\", f.col(\"posteriors\").getItem(1))\n                .withColumn(\"h2\", f.col(\"posteriors\").getItem(2))\n                .withColumn(\"h3\", f.col(\"posteriors\").getItem(3))\n                .withColumn(\"h4\", f.col(\"posteriors\").getItem(4))\n                .withColumn(\"h4h3\", f.col(\"h4\") / f.col(\"h3\"))\n                .withColumn(\"log2h4h3\", f.log2(f.col(\"h4h3\")))\n                # clean up\n                .drop(\n                    \"posteriors\",\n                    \"allBF\",\n                    \"h4h3\",\n                    \"lH0bf\",\n                    \"lH1bf\",\n                    \"lH2bf\",\n                    \"lH3bf\",\n                    \"lH4bf\",\n                )\n                .withColumn(\"colocalisationMethod\", f.lit(cls.METHOD_NAME))\n            ),\n            _schema=Colocalisation.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/methods/coloc/#gentropy.method.colocalisation.Coloc.colocalise","title":"<code>colocalise(overlapping_signals: StudyLocusOverlap, priorc1: float = 0.0001, priorc2: float = 0.0001, priorc12: float = 1e-05) -&gt; Colocalisation</code>  <code>classmethod</code>","text":"<p>Calculate bayesian colocalisation based on overlapping signals.</p> <p>Parameters:</p> Name Type Description Default <code>overlapping_signals</code> <code>StudyLocusOverlap</code> <p>overlapping peaks</p> required <code>priorc1</code> <code>float</code> <p>Prior on variant being causal for trait 1. Defaults to 1e-4.</p> <code>0.0001</code> <code>priorc2</code> <code>float</code> <p>Prior on variant being causal for trait 2. Defaults to 1e-4.</p> <code>0.0001</code> <code>priorc12</code> <code>float</code> <p>Prior on variant being causal for traits 1 and 2. Defaults to 1e-5.</p> <code>1e-05</code> <p>Returns:</p> Name Type Description <code>Colocalisation</code> <code>Colocalisation</code> <p>Colocalisation results</p> Source code in <code>src/gentropy/method/colocalisation.py</code> <pre><code>@classmethod\ndef colocalise(\n    cls: type[Coloc],\n    overlapping_signals: StudyLocusOverlap,\n    priorc1: float = 1e-4,\n    priorc2: float = 1e-4,\n    priorc12: float = 1e-5,\n) -&gt; Colocalisation:\n    \"\"\"Calculate bayesian colocalisation based on overlapping signals.\n\n    Args:\n        overlapping_signals (StudyLocusOverlap): overlapping peaks\n\n        priorc1 (float): Prior on variant being causal for trait 1. Defaults to 1e-4.\n        priorc2 (float): Prior on variant being causal for trait 2. Defaults to 1e-4.\n        priorc12 (float): Prior on variant being causal for traits 1 and 2. Defaults to 1e-5.\n\n    Returns:\n        Colocalisation: Colocalisation results\n    \"\"\"\n    # register udfs\n    logsum = f.udf(get_logsum, DoubleType())\n    posteriors = f.udf(Coloc._get_posteriors, VectorUDT())\n    return Colocalisation(\n        _df=(\n            overlapping_signals.df.select(\"*\", \"statistics.*\")\n            # Before summing log_BF columns nulls need to be filled with 0:\n            .fillna(0, subset=[\"left_logBF\", \"right_logBF\"])\n            # Sum of log_BFs for each pair of signals\n            .withColumn(\n                \"sum_log_bf\",\n                f.col(\"left_logBF\") + f.col(\"right_logBF\"),\n            )\n            # Group by overlapping peak and generating dense vectors of log_BF:\n            .groupBy(\"chromosome\", \"leftStudyLocusId\", \"rightStudyLocusId\")\n            .agg(\n                f.count(\"*\").alias(\"numberColocalisingVariants\"),\n                fml.array_to_vector(f.collect_list(f.col(\"left_logBF\"))).alias(\n                    \"left_logBF\"\n                ),\n                fml.array_to_vector(f.collect_list(f.col(\"right_logBF\"))).alias(\n                    \"right_logBF\"\n                ),\n                fml.array_to_vector(f.collect_list(f.col(\"sum_log_bf\"))).alias(\n                    \"sum_log_bf\"\n                ),\n            )\n            .withColumn(\"logsum1\", logsum(f.col(\"left_logBF\")))\n            .withColumn(\"logsum2\", logsum(f.col(\"right_logBF\")))\n            .withColumn(\"logsum12\", logsum(f.col(\"sum_log_bf\")))\n            .drop(\"left_logBF\", \"right_logBF\", \"sum_log_bf\")\n            # Add priors\n            # priorc1 Prior on variant being causal for trait 1\n            .withColumn(\"priorc1\", f.lit(priorc1))\n            # priorc2 Prior on variant being causal for trait 2\n            .withColumn(\"priorc2\", f.lit(priorc2))\n            # priorc12 Prior on variant being causal for traits 1 and 2\n            .withColumn(\"priorc12\", f.lit(priorc12))\n            # h0-h2\n            .withColumn(\"lH0bf\", f.lit(0))\n            .withColumn(\"lH1bf\", f.log(f.col(\"priorc1\")) + f.col(\"logsum1\"))\n            .withColumn(\"lH2bf\", f.log(f.col(\"priorc2\")) + f.col(\"logsum2\"))\n            # h3\n            .withColumn(\"sumlogsum\", f.col(\"logsum1\") + f.col(\"logsum2\"))\n            .withColumn(\"max\", f.greatest(\"sumlogsum\", \"logsum12\"))\n            .withColumn(\n                \"logdiff\",\n                f.when(\n                    f.col(\"sumlogsum\") == f.col(\"logsum12\"), Coloc.PSEUDOCOUNT\n                ).otherwise(\n                    f.col(\"max\")\n                    + f.log(\n                        f.exp(f.col(\"sumlogsum\") - f.col(\"max\"))\n                        - f.exp(f.col(\"logsum12\") - f.col(\"max\"))\n                    )\n                ),\n            )\n            .withColumn(\n                \"lH3bf\",\n                f.log(f.col(\"priorc1\"))\n                + f.log(f.col(\"priorc2\"))\n                + f.col(\"logdiff\"),\n            )\n            .drop(\"right_logsum\", \"left_logsum\", \"sumlogsum\", \"max\", \"logdiff\")\n            # h4\n            .withColumn(\"lH4bf\", f.log(f.col(\"priorc12\")) + f.col(\"logsum12\"))\n            # cleaning\n            .drop(\n                \"priorc1\", \"priorc2\", \"priorc12\", \"logsum1\", \"logsum2\", \"logsum12\"\n            )\n            # posteriors\n            .withColumn(\n                \"allBF\",\n                fml.array_to_vector(\n                    f.array(\n                        f.col(\"lH0bf\"),\n                        f.col(\"lH1bf\"),\n                        f.col(\"lH2bf\"),\n                        f.col(\"lH3bf\"),\n                        f.col(\"lH4bf\"),\n                    )\n                ),\n            )\n            .withColumn(\n                \"posteriors\", fml.vector_to_array(posteriors(f.col(\"allBF\")))\n            )\n            .withColumn(\"h0\", f.col(\"posteriors\").getItem(0))\n            .withColumn(\"h1\", f.col(\"posteriors\").getItem(1))\n            .withColumn(\"h2\", f.col(\"posteriors\").getItem(2))\n            .withColumn(\"h3\", f.col(\"posteriors\").getItem(3))\n            .withColumn(\"h4\", f.col(\"posteriors\").getItem(4))\n            .withColumn(\"h4h3\", f.col(\"h4\") / f.col(\"h3\"))\n            .withColumn(\"log2h4h3\", f.log2(f.col(\"h4h3\")))\n            # clean up\n            .drop(\n                \"posteriors\",\n                \"allBF\",\n                \"h4h3\",\n                \"lH0bf\",\n                \"lH1bf\",\n                \"lH2bf\",\n                \"lH3bf\",\n                \"lH4bf\",\n            )\n            .withColumn(\"colocalisationMethod\", f.lit(cls.METHOD_NAME))\n        ),\n        _schema=Colocalisation.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/methods/ecaviar/","title":"eCAVIAR","text":""},{"location":"python_api/methods/ecaviar/#gentropy.method.colocalisation.ECaviar","title":"<code>gentropy.method.colocalisation.ECaviar</code>","text":"<p>ECaviar-based colocalisation analysis.</p> <p>It extends CAVIAR\u00a0framework to explicitly estimate the posterior probability that the same variant is causal in 2 studies while accounting for the uncertainty of LD. eCAVIAR computes the colocalization posterior probability (CLPP) by utilizing the marginal posterior probabilities. This framework allows for multiple variants to be causal in a single locus.</p> Source code in <code>src/gentropy/method/colocalisation.py</code> <pre><code>class ECaviar:\n    \"\"\"ECaviar-based colocalisation analysis.\n\n    It extends [CAVIAR](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5142122/#bib18)\u00a0framework to explicitly estimate the posterior probability that the same variant is causal in 2 studies while accounting for the uncertainty of LD. eCAVIAR computes the colocalization posterior probability (**CLPP**) by utilizing the marginal posterior probabilities. This framework allows for **multiple variants to be causal** in a single locus.\n    \"\"\"\n\n    METHOD_NAME: str = \"eCAVIAR\"\n    METHOD_METRIC: str = \"clpp\"\n\n    @staticmethod\n    def _get_clpp(left_pp: Column, right_pp: Column) -&gt; Column:\n        \"\"\"Calculate the colocalisation posterior probability (CLPP).\n\n        If the fact that the same variant is found causal for two studies are independent events,\n        CLPP is defined as the product of posterior porbabilities that a variant is causal in both studies.\n\n        Args:\n            left_pp (Column): left posterior probability\n            right_pp (Column): right posterior probability\n\n        Returns:\n            Column: CLPP\n\n        Examples:\n            &gt;&gt;&gt; d = [{\"left_pp\": 0.5, \"right_pp\": 0.5}, {\"left_pp\": 0.25, \"right_pp\": 0.75}]\n            &gt;&gt;&gt; df = spark.createDataFrame(d)\n            &gt;&gt;&gt; df.withColumn(\"clpp\", ECaviar._get_clpp(f.col(\"left_pp\"), f.col(\"right_pp\"))).show()\n            +-------+--------+------+\n            |left_pp|right_pp|  clpp|\n            +-------+--------+------+\n            |    0.5|     0.5|  0.25|\n            |   0.25|    0.75|0.1875|\n            +-------+--------+------+\n            &lt;BLANKLINE&gt;\n\n        \"\"\"\n        return left_pp * right_pp\n\n    @classmethod\n    def colocalise(\n        cls: type[ECaviar], overlapping_signals: StudyLocusOverlap\n    ) -&gt; Colocalisation:\n        \"\"\"Calculate bayesian colocalisation based on overlapping signals.\n\n        Args:\n            overlapping_signals (StudyLocusOverlap): overlapping signals.\n\n        Returns:\n            Colocalisation: colocalisation results based on eCAVIAR.\n        \"\"\"\n        return Colocalisation(\n            _df=(\n                overlapping_signals.df.withColumn(\n                    \"clpp\",\n                    ECaviar._get_clpp(\n                        f.col(\"statistics.left_posteriorProbability\"),\n                        f.col(\"statistics.right_posteriorProbability\"),\n                    ),\n                )\n                .groupBy(\"leftStudyLocusId\", \"rightStudyLocusId\", \"chromosome\")\n                .agg(\n                    f.count(\"*\").alias(\"numberColocalisingVariants\"),\n                    f.sum(f.col(\"clpp\")).alias(\"clpp\"),\n                )\n                .withColumn(\"colocalisationMethod\", f.lit(cls.METHOD_NAME))\n            ),\n            _schema=Colocalisation.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/methods/ecaviar/#gentropy.method.colocalisation.ECaviar.colocalise","title":"<code>colocalise(overlapping_signals: StudyLocusOverlap) -&gt; Colocalisation</code>  <code>classmethod</code>","text":"<p>Calculate bayesian colocalisation based on overlapping signals.</p> <p>Parameters:</p> Name Type Description Default <code>overlapping_signals</code> <code>StudyLocusOverlap</code> <p>overlapping signals.</p> required <p>Returns:</p> Name Type Description <code>Colocalisation</code> <code>Colocalisation</code> <p>colocalisation results based on eCAVIAR.</p> Source code in <code>src/gentropy/method/colocalisation.py</code> <pre><code>@classmethod\ndef colocalise(\n    cls: type[ECaviar], overlapping_signals: StudyLocusOverlap\n) -&gt; Colocalisation:\n    \"\"\"Calculate bayesian colocalisation based on overlapping signals.\n\n    Args:\n        overlapping_signals (StudyLocusOverlap): overlapping signals.\n\n    Returns:\n        Colocalisation: colocalisation results based on eCAVIAR.\n    \"\"\"\n    return Colocalisation(\n        _df=(\n            overlapping_signals.df.withColumn(\n                \"clpp\",\n                ECaviar._get_clpp(\n                    f.col(\"statistics.left_posteriorProbability\"),\n                    f.col(\"statistics.right_posteriorProbability\"),\n                ),\n            )\n            .groupBy(\"leftStudyLocusId\", \"rightStudyLocusId\", \"chromosome\")\n            .agg(\n                f.count(\"*\").alias(\"numberColocalisingVariants\"),\n                f.sum(f.col(\"clpp\")).alias(\"clpp\"),\n            )\n            .withColumn(\"colocalisationMethod\", f.lit(cls.METHOD_NAME))\n        ),\n        _schema=Colocalisation.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/methods/ld_annotator/","title":"LDAnnotator","text":""},{"location":"python_api/methods/ld_annotator/#gentropy.method.ld.LDAnnotator","title":"<code>gentropy.method.ld.LDAnnotator</code>","text":"<p>Class to annotate linkage disequilibrium (LD) operations from GnomAD.</p> Source code in <code>src/gentropy/method/ld.py</code> <pre><code>class LDAnnotator:\n    \"\"\"Class to annotate linkage disequilibrium (LD) operations from GnomAD.\"\"\"\n\n    @staticmethod\n    def _calculate_weighted_r_overall(ld_set: Column) -&gt; Column:\n        \"\"\"Aggregation of weighted R information using ancestry proportions.\n\n        Args:\n            ld_set (Column): LD set\n\n        Returns:\n            Column: LD set with added 'r2Overall' field\n        \"\"\"\n        return f.transform(\n            ld_set,\n            lambda x: f.struct(\n                x[\"tagVariantId\"].alias(\"tagVariantId\"),\n                # r2Overall is the accumulated sum of each r2 relative to the population size\n                f.aggregate(\n                    x[\"rValues\"],\n                    f.lit(0.0),\n                    lambda acc, y: acc\n                    + f.coalesce(\n                        f.pow(y[\"r\"], 2) * y[\"relativeSampleSize\"], f.lit(0.0)\n                    ),  # we use coalesce to avoid problems when r/relativeSampleSize is null\n                ).alias(\"r2Overall\"),\n            ),\n        )\n\n    @staticmethod\n    def _add_population_size(ld_set: Column, study_populations: Column) -&gt; Column:\n        \"\"\"Add population size to each rValues entry in the ldSet.\n\n        Args:\n            ld_set (Column): LD set\n            study_populations (Column): Study populations\n\n        Returns:\n            Column: LD set with added 'relativeSampleSize' field\n        \"\"\"\n        # Create a population to relativeSampleSize map from the struct\n        populations_map = f.map_from_arrays(\n            study_populations[\"ldPopulation\"],\n            study_populations[\"relativeSampleSize\"],\n        )\n        return f.transform(\n            ld_set,\n            lambda x: f.struct(\n                x[\"tagVariantId\"].alias(\"tagVariantId\"),\n                f.transform(\n                    x[\"rValues\"],\n                    lambda y: f.struct(\n                        y[\"population\"].alias(\"population\"),\n                        y[\"r\"].alias(\"r\"),\n                        populations_map[y[\"population\"]].alias(\"relativeSampleSize\"),\n                    ),\n                ).alias(\"rValues\"),\n            ),\n        )\n\n    @staticmethod\n    def _qc_unresolved_ld(ld_set: Column, quality_controls: Column) -&gt; Column:\n        \"\"\"Flag associations with unresolved LD.\n\n        Args:\n            ld_set (Column): LD set\n            quality_controls (Column): Quality controls\n\n        Returns:\n            Column: Quality controls with added 'UNRESOLVED_LD' field\n        \"\"\"\n        return StudyLocus.update_quality_flag(\n            quality_controls,\n            ld_set.isNull(),\n            StudyLocusQualityCheck.UNRESOLVED_LD,\n        )\n\n    @staticmethod\n    def _rescue_lead_variant(ld_set: Column, variant_id: Column) -&gt; Column:\n        \"\"\"Rescue lead variant.\n\n        In cases in which no LD information is available but a lead variant is available, we include the lead as the only variant in the ldSet.\n\n        Args:\n            ld_set (Column): LD set\n            variant_id (Column): Variant ID\n\n        Returns:\n            Column: LD set with added 'tagVariantId' field\n        \"\"\"\n        return f.when(\n            ((ld_set.isNull() | (f.size(ld_set) == 0)) &amp; variant_id.isNotNull()),\n            f.array(\n                f.struct(\n                    variant_id.alias(\"tagVariantId\"),\n                    f.lit(1).alias(\"r2Overall\"),\n                )\n            ),\n        ).otherwise(ld_set)\n\n    @classmethod\n    def ld_annotate(\n        cls: type[LDAnnotator],\n        associations: StudyLocus,\n        studies: StudyIndex,\n        ld_index: LDIndex,\n    ) -&gt; StudyLocus:\n        \"\"\"Annotate linkage disequilibrium (LD) information to a set of studyLocus.\n\n        This function:\n            1. Annotates study locus with population structure information from the study index\n            2. Joins the LD index to the StudyLocus\n            3. Adds the population size of the study to each rValues entry in the ldSet\n            4. Calculates the overall R weighted by the ancestry proportions in every given study.\n            5. Flags associations with variants that are not found in the LD reference\n            6. Rescues lead variant when no LD information is available but lead variant is available\n\n        Args:\n            associations (StudyLocus): Dataset to be LD annotated\n            studies (StudyIndex): Dataset with study information\n            ld_index (LDIndex): Dataset with LD information for every variant present in LD matrix\n\n        Returns:\n            StudyLocus: including additional column with LD information.\n        \"\"\"\n        return StudyLocus(\n            _df=(\n                associations.df\n                # Drop ldSet column if already available\n                .select(*[col for col in associations.df.columns if col != \"ldSet\"])\n                # Annotate study locus with population structure from study index\n                .join(\n                    studies.df.select(\"studyId\", \"ldPopulationStructure\"),\n                    on=\"studyId\",\n                    how=\"left\",\n                )\n                # Bring LD information from LD Index\n                .join(\n                    ld_index.df,\n                    on=[\"variantId\", \"chromosome\"],\n                    how=\"left\",\n                )\n                # Add population size to each rValues entry in the ldSet if population structure available:\n                .withColumn(\n                    \"ldSet\",\n                    f.when(\n                        f.col(\"ldPopulationStructure\").isNotNull(),\n                        cls._add_population_size(\n                            f.col(\"ldSet\"), f.col(\"ldPopulationStructure\")\n                        ),\n                    ),\n                )\n                # Aggregate weighted R information using ancestry proportions\n                .withColumn(\n                    \"ldSet\",\n                    f.when(\n                        f.col(\"ldPopulationStructure\").isNotNull(),\n                        cls._calculate_weighted_r_overall(f.col(\"ldSet\")),\n                    ),\n                )\n                .drop(\"ldPopulationStructure\")\n                # QC: Flag associations with variants that are not found in the LD reference\n                .withColumn(\n                    \"qualityControls\",\n                    cls._qc_unresolved_ld(f.col(\"ldSet\"), f.col(\"qualityControls\")),\n                )\n                # Add lead variant to empty ldSet when no LD information is available but lead variant is available\n                .withColumn(\n                    \"ldSet\",\n                    cls._rescue_lead_variant(f.col(\"ldSet\"), f.col(\"variantId\")),\n                )\n            ),\n            _schema=StudyLocus.get_schema(),\n        )._qc_no_population()\n</code></pre>"},{"location":"python_api/methods/ld_annotator/#gentropy.method.ld.LDAnnotator.ld_annotate","title":"<code>ld_annotate(associations: StudyLocus, studies: StudyIndex, ld_index: LDIndex) -&gt; StudyLocus</code>  <code>classmethod</code>","text":"<p>Annotate linkage disequilibrium (LD) information to a set of studyLocus.</p> This function <ol> <li>Annotates study locus with population structure information from the study index</li> <li>Joins the LD index to the StudyLocus</li> <li>Adds the population size of the study to each rValues entry in the ldSet</li> <li>Calculates the overall R weighted by the ancestry proportions in every given study.</li> <li>Flags associations with variants that are not found in the LD reference</li> <li>Rescues lead variant when no LD information is available but lead variant is available</li> </ol> <p>Parameters:</p> Name Type Description Default <code>associations</code> <code>StudyLocus</code> <p>Dataset to be LD annotated</p> required <code>studies</code> <code>StudyIndex</code> <p>Dataset with study information</p> required <code>ld_index</code> <code>LDIndex</code> <p>Dataset with LD information for every variant present in LD matrix</p> required <p>Returns:</p> Name Type Description <code>StudyLocus</code> <code>StudyLocus</code> <p>including additional column with LD information.</p> Source code in <code>src/gentropy/method/ld.py</code> <pre><code>@classmethod\ndef ld_annotate(\n    cls: type[LDAnnotator],\n    associations: StudyLocus,\n    studies: StudyIndex,\n    ld_index: LDIndex,\n) -&gt; StudyLocus:\n    \"\"\"Annotate linkage disequilibrium (LD) information to a set of studyLocus.\n\n    This function:\n        1. Annotates study locus with population structure information from the study index\n        2. Joins the LD index to the StudyLocus\n        3. Adds the population size of the study to each rValues entry in the ldSet\n        4. Calculates the overall R weighted by the ancestry proportions in every given study.\n        5. Flags associations with variants that are not found in the LD reference\n        6. Rescues lead variant when no LD information is available but lead variant is available\n\n    Args:\n        associations (StudyLocus): Dataset to be LD annotated\n        studies (StudyIndex): Dataset with study information\n        ld_index (LDIndex): Dataset with LD information for every variant present in LD matrix\n\n    Returns:\n        StudyLocus: including additional column with LD information.\n    \"\"\"\n    return StudyLocus(\n        _df=(\n            associations.df\n            # Drop ldSet column if already available\n            .select(*[col for col in associations.df.columns if col != \"ldSet\"])\n            # Annotate study locus with population structure from study index\n            .join(\n                studies.df.select(\"studyId\", \"ldPopulationStructure\"),\n                on=\"studyId\",\n                how=\"left\",\n            )\n            # Bring LD information from LD Index\n            .join(\n                ld_index.df,\n                on=[\"variantId\", \"chromosome\"],\n                how=\"left\",\n            )\n            # Add population size to each rValues entry in the ldSet if population structure available:\n            .withColumn(\n                \"ldSet\",\n                f.when(\n                    f.col(\"ldPopulationStructure\").isNotNull(),\n                    cls._add_population_size(\n                        f.col(\"ldSet\"), f.col(\"ldPopulationStructure\")\n                    ),\n                ),\n            )\n            # Aggregate weighted R information using ancestry proportions\n            .withColumn(\n                \"ldSet\",\n                f.when(\n                    f.col(\"ldPopulationStructure\").isNotNull(),\n                    cls._calculate_weighted_r_overall(f.col(\"ldSet\")),\n                ),\n            )\n            .drop(\"ldPopulationStructure\")\n            # QC: Flag associations with variants that are not found in the LD reference\n            .withColumn(\n                \"qualityControls\",\n                cls._qc_unresolved_ld(f.col(\"ldSet\"), f.col(\"qualityControls\")),\n            )\n            # Add lead variant to empty ldSet when no LD information is available but lead variant is available\n            .withColumn(\n                \"ldSet\",\n                cls._rescue_lead_variant(f.col(\"ldSet\"), f.col(\"variantId\")),\n            )\n        ),\n        _schema=StudyLocus.get_schema(),\n    )._qc_no_population()\n</code></pre>"},{"location":"python_api/methods/pics/","title":"PICS","text":"<p>PICS Overview:</p> <p>PICS is a fine-mapping method designed to identify the most likely causal SNPs associated with a trait or disease within a genomic region. It leverages both haplotype information and the observed association patterns from genome-wide association studies (GWAS).</p> <p>Please refer to the original publication for in-depth details: PICS Publication.</p> <p>We use PICS for both GWAS clumping results and GWAS curated studies.</p>"},{"location":"python_api/methods/pics/#gentropy.method.pics.PICS","title":"<code>gentropy.method.pics.PICS</code>","text":"<p>Probabilistic Identification of Causal SNPs (PICS), an algorithm estimating the probability that an individual variant is causal considering the haplotype structure and observed pattern of association at the genetic locus.</p> Source code in <code>src/gentropy/method/pics.py</code> <pre><code>class PICS:\n    \"\"\"Probabilistic Identification of Causal SNPs (PICS), an algorithm estimating the probability that an individual variant is causal considering the haplotype structure and observed pattern of association at the genetic locus.\"\"\"\n\n    @staticmethod\n    def _pics_relative_posterior_probability(\n        neglog_p: float, pics_snp_mu: float, pics_snp_std: float\n    ) -&gt; float:\n        \"\"\"Compute the PICS posterior probability for a given SNP.\n\n        !!! info \"This probability needs to be scaled to take into account the probabilities of the other variants in the locus.\"\n\n        Args:\n            neglog_p (float): Negative log p-value of the lead variant\n            pics_snp_mu (float): Mean P value of the association between a SNP and a trait\n            pics_snp_std (float): Standard deviation for the P value of the association between a SNP and a trait\n\n        Returns:\n            float: Posterior probability of the association between a SNP and a trait\n\n        Examples:\n            &gt;&gt;&gt; rel_prob = PICS._pics_relative_posterior_probability(neglog_p=10.0, pics_snp_mu=1.0, pics_snp_std=10.0)\n            &gt;&gt;&gt; round(rel_prob, 3)\n            0.368\n        \"\"\"\n        return float(norm(pics_snp_mu, pics_snp_std).sf(neglog_p) * 2)\n\n    @staticmethod\n    def _pics_standard_deviation(neglog_p: float, r2: float, k: float) -&gt; float | None:\n        \"\"\"Compute the PICS standard deviation.\n\n        This distribution is obtained after a series of permutation tests described in the PICS method, and it is only\n        valid when the SNP is highly linked with the lead (r2 &gt; 0.5).\n\n        Args:\n            neglog_p (float): Negative log p-value of the lead variant\n            r2 (float): LD score between a given SNP and the lead variant\n            k (float): Empiric constant that can be adjusted to fit the curve, 6.4 recommended.\n\n        Returns:\n            float | None: Standard deviation for the P value of the association between a SNP and a trait\n\n        Examples:\n            &gt;&gt;&gt; PICS._pics_standard_deviation(neglog_p=1.0, r2=1.0, k=6.4)\n            0.0\n            &gt;&gt;&gt; round(PICS._pics_standard_deviation(neglog_p=10.0, r2=0.5, k=6.4), 3)\n            1.493\n            &gt;&gt;&gt; print(PICS._pics_standard_deviation(neglog_p=1.0, r2=0.0, k=6.4))\n            None\n        \"\"\"\n        return (\n            abs(((1 - (r2**0.5) ** k) ** 0.5) * (neglog_p**0.5) / 2)\n            if r2 &gt;= 0.5\n            else None\n        )\n\n    @staticmethod\n    def _pics_mu(neglog_p: float, r2: float) -&gt; float | None:\n        \"\"\"Compute the PICS mu that estimates the probability of association between a given SNP and the trait.\n\n        This distribution is obtained after a series of permutation tests described in the PICS method, and it is only\n        valid when the SNP is highly linked with the lead (r2 &gt; 0.5).\n\n        Args:\n            neglog_p (float): Negative log p-value of the lead variant\n            r2 (float): LD score between a given SNP and the lead variant\n\n        Returns:\n            float | None: Mean P value of the association between a SNP and a trait\n\n        Examples:\n            &gt;&gt;&gt; PICS._pics_mu(neglog_p=1.0, r2=1.0)\n            1.0\n            &gt;&gt;&gt; PICS._pics_mu(neglog_p=10.0, r2=0.5)\n            5.0\n            &gt;&gt;&gt; print(PICS._pics_mu(neglog_p=10.0, r2=0.3))\n            None\n        \"\"\"\n        return neglog_p * r2 if r2 &gt;= 0.5 else None\n\n    @staticmethod\n    def _finemap(\n        ld_set: list[Row], lead_neglog_p: float, k: float\n    ) -&gt; list[dict[str, Any]] | None:\n        \"\"\"Calculates the probability of a variant being causal in a study-locus context by applying the PICS method.\n\n        It is intended to be applied as an UDF in `PICS.finemap`, where each row is a StudyLocus association.\n        The function iterates over every SNP in the `ldSet` array, and it returns an updated locus with\n        its association signal and causality probability as of PICS.\n\n        Args:\n            ld_set (list[Row]): list of tagging variants after expanding the locus\n            lead_neglog_p (float): P value of the association signal between the lead variant and the study in the form of -log10.\n            k (float): Empiric constant that can be adjusted to fit the curve, 6.4 recommended.\n\n        Returns:\n            list[dict[str, Any]] | None: List of tagging variants with an estimation of the association signal and their posterior probability as of PICS.\n\n        Examples:\n            &gt;&gt;&gt; from pyspark.sql import Row\n            &gt;&gt;&gt; ld_set = [\n            ...     Row(variantId=\"var1\", r2Overall=0.8),\n            ...     Row(variantId=\"var2\", r2Overall=1),\n            ... ]\n            &gt;&gt;&gt; PICS._finemap(ld_set, lead_neglog_p=10.0, k=6.4)\n            [{'variantId': 'var1', 'r2Overall': 0.8, 'standardError': 0.07420896512708416, 'posteriorProbability': 0.07116959886882368}, {'variantId': 'var2', 'r2Overall': 1, 'standardError': 0.9977000638225533, 'posteriorProbability': 0.9288304011311763}]\n            &gt;&gt;&gt; empty_ld_set = []\n            &gt;&gt;&gt; PICS._finemap(empty_ld_set, lead_neglog_p=10.0, k=6.4)\n            []\n            &gt;&gt;&gt; ld_set_with_no_r2 = [\n            ...     Row(variantId=\"var1\", r2Overall=None),\n            ...     Row(variantId=\"var2\", r2Overall=None),\n            ... ]\n            &gt;&gt;&gt; PICS._finemap(ld_set_with_no_r2, lead_neglog_p=10.0, k=6.4)\n            []\n        \"\"\"\n        if ld_set is None:\n            return None\n        elif not ld_set:\n            return []\n        tmp_credible_set = []\n        new_credible_set = []\n        # First iteration: calculation of mu, standard deviation, and the relative posterior probability\n        for tag_struct in ld_set:\n            tag_dict = (\n                tag_struct.asDict()\n            )  # tag_struct is of type pyspark.Row, we'll represent it as a dict\n            if (\n                not tag_dict[\"r2Overall\"]\n                or tag_dict[\"r2Overall\"] &lt; 0.5\n                or not lead_neglog_p\n            ):\n                # If PICS cannot be calculated, we drop the variant from the credible set\n                continue\n\n            pics_snp_mu = PICS._pics_mu(lead_neglog_p, tag_dict[\"r2Overall\"])\n            pics_snp_std = PICS._pics_standard_deviation(\n                lead_neglog_p, tag_dict[\"r2Overall\"], k\n            )\n            pics_snp_std = 0.001 if pics_snp_std == 0 else pics_snp_std\n            if pics_snp_mu is not None and pics_snp_std is not None:\n                posterior_probability = PICS._pics_relative_posterior_probability(\n                    lead_neglog_p, pics_snp_mu, pics_snp_std\n                )\n                tag_dict[\"standardError\"] = 10**-pics_snp_std\n                tag_dict[\"relativePosteriorProbability\"] = posterior_probability\n\n                tmp_credible_set.append(tag_dict)\n\n        # Second iteration: calculation of the sum of all the posteriors in each study-locus, so that we scale them between 0-1\n        total_posteriors = sum(\n            tag_dict.get(\"relativePosteriorProbability\", 0)\n            for tag_dict in tmp_credible_set\n        )\n\n        # Third iteration: calculation of the final posteriorProbability\n        for tag_dict in tmp_credible_set:\n            if total_posteriors != 0:\n                tag_dict[\"posteriorProbability\"] = float(\n                    tag_dict.get(\"relativePosteriorProbability\", 0) / total_posteriors\n                )\n            tag_dict.pop(\"relativePosteriorProbability\")\n            new_credible_set.append(tag_dict)\n        return new_credible_set\n\n    @classmethod\n    def finemap(\n        cls: type[PICS], associations: StudyLocus, k: float = 6.4\n    ) -&gt; StudyLocus:\n        \"\"\"Run PICS on a study locus.\n\n        !!! info \"Study locus needs to be LD annotated\"\n\n            The study locus needs to be LD annotated before PICS can be calculated.\n\n        Args:\n            associations (StudyLocus): Study locus to finemap using PICS\n            k (float): Empiric constant that can be adjusted to fit the curve, 6.4 recommended.\n\n        Returns:\n            StudyLocus: Study locus with PICS results\n        \"\"\"\n        # Register UDF by defining the structure of the output locus array of structs\n        # it also renames tagVariantId to variantId\n\n        picsed_ldset_schema = t.ArrayType(\n            t.StructType(\n                [\n                    t.StructField(\"tagVariantId\", t.StringType(), True),\n                    t.StructField(\"r2Overall\", t.DoubleType(), True),\n                    t.StructField(\"posteriorProbability\", t.DoubleType(), True),\n                    t.StructField(\"standardError\", t.DoubleType(), True),\n                ]\n            )\n        )\n        picsed_study_locus_schema = t.ArrayType(\n            t.StructType(\n                [\n                    t.StructField(\"variantId\", t.StringType(), True),\n                    t.StructField(\"r2Overall\", t.DoubleType(), True),\n                    t.StructField(\"posteriorProbability\", t.DoubleType(), True),\n                    t.StructField(\"standardError\", t.DoubleType(), True),\n                ]\n            )\n        )\n        _finemap_udf = f.udf(\n            lambda locus, neglog_p: PICS._finemap(locus, neglog_p, k),\n            picsed_ldset_schema,\n        )\n        non_picsable_expr = (\n            f.size(f.filter(f.col(\"ldSet\"), lambda x: x.r2Overall &gt;= 0.5)) == 0\n        )\n        return StudyLocus(\n            _df=(\n                associations.df\n                # Old locus column will be dropped if available\n                .select(*[col for col in associations.df.columns if col != \"locus\"])\n                # Estimate neglog_pvalue for the lead variant\n                .withColumn(\"neglog_pvalue\", associations.neglog_pvalue())\n                # New locus containing the PICS results\n                .withColumn(\n                    \"locus\",\n                    f.when(\n                        f.col(\"ldSet\").isNotNull(),\n                        _finemap_udf(f.col(\"ldSet\"), f.col(\"neglog_pvalue\")).cast(\n                            picsed_study_locus_schema\n                        ),\n                    ),\n                )\n                .withColumn(\n                    \"qualityControls\",\n                    StudyLocus.update_quality_flag(\n                        f.col(\"qualityControls\"),\n                        non_picsable_expr,\n                        StudyLocusQualityCheck.NOT_QUALIFYING_LD_BLOCK,\n                    ),\n                )\n                .withColumn(\n                    \"finemappingMethod\",\n                    f.coalesce(f.col(\"finemappingMethod\"), f.lit(\"pics\")),\n                )\n                .drop(\"neglog_pvalue\")\n            ),\n            _schema=StudyLocus.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/methods/pics/#gentropy.method.pics.PICS.finemap","title":"<code>finemap(associations: StudyLocus, k: float = 6.4) -&gt; StudyLocus</code>  <code>classmethod</code>","text":"<p>Run PICS on a study locus.</p> <p>Study locus needs to be LD annotated</p> <p>The study locus needs to be LD annotated before PICS can be calculated.</p> <p>Parameters:</p> Name Type Description Default <code>associations</code> <code>StudyLocus</code> <p>Study locus to finemap using PICS</p> required <code>k</code> <code>float</code> <p>Empiric constant that can be adjusted to fit the curve, 6.4 recommended.</p> <code>6.4</code> <p>Returns:</p> Name Type Description <code>StudyLocus</code> <code>StudyLocus</code> <p>Study locus with PICS results</p> Source code in <code>src/gentropy/method/pics.py</code> <pre><code>@classmethod\ndef finemap(\n    cls: type[PICS], associations: StudyLocus, k: float = 6.4\n) -&gt; StudyLocus:\n    \"\"\"Run PICS on a study locus.\n\n    !!! info \"Study locus needs to be LD annotated\"\n\n        The study locus needs to be LD annotated before PICS can be calculated.\n\n    Args:\n        associations (StudyLocus): Study locus to finemap using PICS\n        k (float): Empiric constant that can be adjusted to fit the curve, 6.4 recommended.\n\n    Returns:\n        StudyLocus: Study locus with PICS results\n    \"\"\"\n    # Register UDF by defining the structure of the output locus array of structs\n    # it also renames tagVariantId to variantId\n\n    picsed_ldset_schema = t.ArrayType(\n        t.StructType(\n            [\n                t.StructField(\"tagVariantId\", t.StringType(), True),\n                t.StructField(\"r2Overall\", t.DoubleType(), True),\n                t.StructField(\"posteriorProbability\", t.DoubleType(), True),\n                t.StructField(\"standardError\", t.DoubleType(), True),\n            ]\n        )\n    )\n    picsed_study_locus_schema = t.ArrayType(\n        t.StructType(\n            [\n                t.StructField(\"variantId\", t.StringType(), True),\n                t.StructField(\"r2Overall\", t.DoubleType(), True),\n                t.StructField(\"posteriorProbability\", t.DoubleType(), True),\n                t.StructField(\"standardError\", t.DoubleType(), True),\n            ]\n        )\n    )\n    _finemap_udf = f.udf(\n        lambda locus, neglog_p: PICS._finemap(locus, neglog_p, k),\n        picsed_ldset_schema,\n    )\n    non_picsable_expr = (\n        f.size(f.filter(f.col(\"ldSet\"), lambda x: x.r2Overall &gt;= 0.5)) == 0\n    )\n    return StudyLocus(\n        _df=(\n            associations.df\n            # Old locus column will be dropped if available\n            .select(*[col for col in associations.df.columns if col != \"locus\"])\n            # Estimate neglog_pvalue for the lead variant\n            .withColumn(\"neglog_pvalue\", associations.neglog_pvalue())\n            # New locus containing the PICS results\n            .withColumn(\n                \"locus\",\n                f.when(\n                    f.col(\"ldSet\").isNotNull(),\n                    _finemap_udf(f.col(\"ldSet\"), f.col(\"neglog_pvalue\")).cast(\n                        picsed_study_locus_schema\n                    ),\n                ),\n            )\n            .withColumn(\n                \"qualityControls\",\n                StudyLocus.update_quality_flag(\n                    f.col(\"qualityControls\"),\n                    non_picsable_expr,\n                    StudyLocusQualityCheck.NOT_QUALIFYING_LD_BLOCK,\n                ),\n            )\n            .withColumn(\n                \"finemappingMethod\",\n                f.coalesce(f.col(\"finemappingMethod\"), f.lit(\"pics\")),\n            )\n            .drop(\"neglog_pvalue\")\n        ),\n        _schema=StudyLocus.get_schema(),\n    )\n</code></pre>"},{"location":"python_api/methods/sumstat_imputation/","title":"Summary Statistics Imputation","text":"<p>Summary statistics imputation leverages linkage disequilibrium (LD) information to compute Z-scores of missing SNPs from neighbouring observed SNPs SNPs by taking advantage of the Linkage Disequilibrium.</p> <p>We implemented the basic model from RAISS (Robust and Accurate Imputation from Summary Statistics) package (see the original paper).</p> <p>The full repository for the RAISS package can be found here.</p> <p>The original model was suggested in 2014 by Bogdan Pasaniuc et al. here.</p> <p>It represents the following formula:</p> <p>E(zi|z_t) = M{i,t} \\cdot (M_{t,t})^{-1} \\cdot z_t</p> <p>Where:</p> <ul> <li> <p>E(z_i|z_t) represents the expected z-score of SNP 'i' given the observed z-scores at known SNP indexes 't'.</p> </li> <li> <p>M_{i,t} represents the LD (Linkage Disequilibrium) matrix between SNP 'i' and the known SNPs at indexes 't'.</p> </li> <li> <p>(M_{t,t})^{-1} represents the inverse of the LD matrix of the known SNPs at indexes 't'.</p> </li> <li> <p>z_t represents the vector of observed z-scores at the known SNP indexes 't'.</p> </li> </ul>"},{"location":"python_api/methods/sumstat_imputation/#gentropy.method.sumstat_imputation.SummaryStatisticsImputation","title":"<code>gentropy.method.sumstat_imputation.SummaryStatisticsImputation</code>","text":"<p>Implementation of RAISS summary statstics imputation model.</p> Source code in <code>src/gentropy/method/sumstat_imputation.py</code> <pre><code>class SummaryStatisticsImputation:\n    \"\"\"Implementation of RAISS summary statstics imputation model.\"\"\"\n\n    @staticmethod\n    def raiss_model(\n        z_scores_known: np.ndarray,\n        ld_matrix_known: np.ndarray,\n        ld_matrix_known_missing: np.ndarray,\n        lamb: float = 0.01,\n        rtol: float = 0.01,\n    ) -&gt; dict[str, Any]:\n        \"\"\"Compute the imputation of the z-score using the RAISS model.\n\n        Args:\n            z_scores_known (np.ndarray): the vector of known Z scores\n            ld_matrix_known (np.ndarray) : the matrix of known LD correlations\n            ld_matrix_known_missing (np.ndarray): LD matrix of known SNPs with other unknown SNPs in large matrix (similar to ld[unknowns, :][:,known])\n            lamb (float): size of the small value added to the diagonal of the covariance matrix before inversion. Defaults to 0.01.\n            rtol (float): threshold to filter eigenvectos by its eigenvalue. It makes an inversion biased but much more numerically robust. Default to 0.01.\n\n        Returns:\n            dict[str, Any]:\n                - var (np.ndarray): variance of the imputed SNPs\n                - mu (np.ndarray): the estimation of the zscore of the imputed SNPs\n                - ld_score (np.ndarray): the linkage disequilibrium score of the imputed SNPs\n                - condition_number (np.ndarray): the condition number of the correlation matrix\n                - correct_inversion (np.ndarray): a boolean array indicating if the inversion was successful\n                - imputation_r2 (np.ndarray): the R2 of the imputation\n        \"\"\"\n        sig_t_inv = SummaryStatisticsImputation._invert_sig_t(\n            ld_matrix_known, lamb, rtol\n        )\n        if sig_t_inv is None:\n            return {\n                \"var\": None,\n                \"mu\": None,\n                \"ld_score\": None,\n                \"condition_number\": None,\n                \"correct_inversion\": None,\n                \"imputation_r2\": None,\n            }\n        else:\n            condition_number = np.array(\n                [np.linalg.cond(ld_matrix_known)] * ld_matrix_known_missing.shape[0]\n            )\n            correct_inversion = np.array(\n                [\n                    SummaryStatisticsImputation._check_inversion(\n                        ld_matrix_known, sig_t_inv\n                    )\n                ]\n                * ld_matrix_known_missing.shape[0]\n            )\n\n            var, ld_score = SummaryStatisticsImputation._compute_var(\n                ld_matrix_known_missing, sig_t_inv, lamb\n            )\n\n            mu = SummaryStatisticsImputation._compute_mu(\n                ld_matrix_known_missing, sig_t_inv, z_scores_known\n            )\n            var_norm = SummaryStatisticsImputation._var_in_boundaries(var, lamb)\n\n            R2 = (1 + lamb) - var_norm\n\n            mu = mu / np.sqrt(R2)\n            return {\n                \"var\": var,\n                \"mu\": mu,\n                \"ld_score\": ld_score,\n                \"condition_number\": condition_number,\n                \"correct_inversion\": correct_inversion,\n                \"imputation_r2\": 1 - var,\n            }\n\n    @staticmethod\n    def _compute_mu(\n        sig_i_t: np.ndarray, sig_t_inv: np.ndarray, zt: np.ndarray\n    ) -&gt; np.ndarray:\n        \"\"\"Compute the estimation of z-score from neighborring snp.\n\n        Args:\n            sig_i_t (np.ndarray) : correlation matrix with line corresponding to unknown Snp (snp to impute) and column to known SNPs\n            sig_t_inv (np.ndarray): inverse of the correlation matrix of known matrix\n            zt (np.ndarray): Zscores of known snp\n        Returns:\n            np.ndarray: a vector of length i containing the estimate of zscore\n\n        \"\"\"\n        return np.dot(sig_i_t, np.dot(sig_t_inv, zt))\n\n    @staticmethod\n    def _compute_var(\n        sig_i_t: np.ndarray, sig_t_inv: np.ndarray, lamb: float\n    ) -&gt; tuple[np.ndarray, np.ndarray]:\n        \"\"\"Compute the expected variance of the imputed SNPs.\n\n        Args:\n            sig_i_t (np.ndarray) : correlation matrix with line corresponding to unknown Snp (snp to impute) and column to known SNPs\n            sig_t_inv (np.ndarray): inverse of the correlation matrix of known matrix\n            lamb (float): regularization term added to matrix\n\n        Returns:\n            tuple[np.ndarray, np.ndarray]: a tuple containing the variance and the ld score\n        \"\"\"\n        var = (1 + lamb) - np.einsum(\n            \"ij,jk,ki-&gt;i\", sig_i_t, sig_t_inv, sig_i_t.transpose()\n        )\n        ld_score = (sig_i_t**2).sum(1)\n\n        return var, ld_score\n\n    @staticmethod\n    def _check_inversion(sig_t: np.ndarray, sig_t_inv: np.ndarray) -&gt; bool:\n        \"\"\"Check if the inversion is correct.\n\n        Args:\n            sig_t (np.ndarray): the correlation matrix\n            sig_t_inv (np.ndarray): the inverse of the correlation matrix\n        Returns:\n            bool: True if the inversion is correct, False otherwise\n        \"\"\"\n        return np.allclose(sig_t, np.dot(sig_t, np.dot(sig_t_inv, sig_t)))\n\n    @staticmethod\n    def _var_in_boundaries(var: np.ndarray, lamb: float) -&gt; np.ndarray:\n        \"\"\"Forces the variance to be in the 0 to 1+lambda boundary. Theoritically we shouldn't have to do that.\n\n        Args:\n            var (np.ndarray): the variance of the imputed SNPs\n            lamb (float): regularization term added to the diagonal of the sig_t matrix\n\n        Returns:\n            np.ndarray: the variance of the imputed SNPs\n        \"\"\"\n        id_neg = np.where(var &lt; 0)\n        var[id_neg] = 0\n        id_inf = np.where(var &gt; (0.99999 + lamb))\n        var[id_inf] = 1\n\n        return var\n\n    @staticmethod\n    def _invert_sig_t(sig_t: np.ndarray, lamb: float, rtol: float) -&gt; np.ndarray:\n        \"\"\"Invert the correlation matrix. If the provided regularization values are not enough to stabilize the inversion process for the given matrix, the function calls itself recursively, increasing lamb and rtol by 10%.\n\n        Args:\n            sig_t (np.ndarray): the correlation matrix\n            lamb (float): regularization term added to the diagonal of the sig_t matrix\n            rtol (float): threshold to filter eigenvector with a eigenvalue under rtol make inversion biased but much more numerically robust\n\n        Returns:\n            np.ndarray: the inverse of the correlation matrix\n        \"\"\"\n        try:\n            np.fill_diagonal(sig_t, (1 + lamb))\n            sig_t_inv = scipy.linalg.pinv(sig_t, rtol=rtol, atol=0)\n            return sig_t_inv\n        except np.linalg.LinAlgError:\n            return SummaryStatisticsImputation._invert_sig_t(\n                sig_t, lamb * 1.1, rtol * 1.1\n            )\n</code></pre>"},{"location":"python_api/methods/sumstat_imputation/#gentropy.method.sumstat_imputation.SummaryStatisticsImputation.raiss_model","title":"<code>raiss_model(z_scores_known: np.ndarray, ld_matrix_known: np.ndarray, ld_matrix_known_missing: np.ndarray, lamb: float = 0.01, rtol: float = 0.01) -&gt; dict[str, Any]</code>  <code>staticmethod</code>","text":"<p>Compute the imputation of the z-score using the RAISS model.</p> <p>Parameters:</p> Name Type Description Default <code>z_scores_known</code> <code>ndarray</code> <p>the vector of known Z scores</p> required <code>ld_matrix_known</code> <code>np.ndarray) </code> <p>the matrix of known LD correlations</p> required <code>ld_matrix_known_missing</code> <code>ndarray</code> <p>LD matrix of known SNPs with other unknown SNPs in large matrix (similar to ldunknowns, :)</p> required <code>lamb</code> <code>float</code> <p>size of the small value added to the diagonal of the covariance matrix before inversion. Defaults to 0.01.</p> <code>0.01</code> <code>rtol</code> <code>float</code> <p>threshold to filter eigenvectos by its eigenvalue. It makes an inversion biased but much more numerically robust. Default to 0.01.</p> <code>0.01</code> <p>Returns:</p> Type Description <code>dict[str, Any]</code> <p>dict[str, Any]: - var (np.ndarray): variance of the imputed SNPs - mu (np.ndarray): the estimation of the zscore of the imputed SNPs - ld_score (np.ndarray): the linkage disequilibrium score of the imputed SNPs - condition_number (np.ndarray): the condition number of the correlation matrix - correct_inversion (np.ndarray): a boolean array indicating if the inversion was successful - imputation_r2 (np.ndarray): the R2 of the imputation</p> Source code in <code>src/gentropy/method/sumstat_imputation.py</code> <pre><code>@staticmethod\ndef raiss_model(\n    z_scores_known: np.ndarray,\n    ld_matrix_known: np.ndarray,\n    ld_matrix_known_missing: np.ndarray,\n    lamb: float = 0.01,\n    rtol: float = 0.01,\n) -&gt; dict[str, Any]:\n    \"\"\"Compute the imputation of the z-score using the RAISS model.\n\n    Args:\n        z_scores_known (np.ndarray): the vector of known Z scores\n        ld_matrix_known (np.ndarray) : the matrix of known LD correlations\n        ld_matrix_known_missing (np.ndarray): LD matrix of known SNPs with other unknown SNPs in large matrix (similar to ld[unknowns, :][:,known])\n        lamb (float): size of the small value added to the diagonal of the covariance matrix before inversion. Defaults to 0.01.\n        rtol (float): threshold to filter eigenvectos by its eigenvalue. It makes an inversion biased but much more numerically robust. Default to 0.01.\n\n    Returns:\n        dict[str, Any]:\n            - var (np.ndarray): variance of the imputed SNPs\n            - mu (np.ndarray): the estimation of the zscore of the imputed SNPs\n            - ld_score (np.ndarray): the linkage disequilibrium score of the imputed SNPs\n            - condition_number (np.ndarray): the condition number of the correlation matrix\n            - correct_inversion (np.ndarray): a boolean array indicating if the inversion was successful\n            - imputation_r2 (np.ndarray): the R2 of the imputation\n    \"\"\"\n    sig_t_inv = SummaryStatisticsImputation._invert_sig_t(\n        ld_matrix_known, lamb, rtol\n    )\n    if sig_t_inv is None:\n        return {\n            \"var\": None,\n            \"mu\": None,\n            \"ld_score\": None,\n            \"condition_number\": None,\n            \"correct_inversion\": None,\n            \"imputation_r2\": None,\n        }\n    else:\n        condition_number = np.array(\n            [np.linalg.cond(ld_matrix_known)] * ld_matrix_known_missing.shape[0]\n        )\n        correct_inversion = np.array(\n            [\n                SummaryStatisticsImputation._check_inversion(\n                    ld_matrix_known, sig_t_inv\n                )\n            ]\n            * ld_matrix_known_missing.shape[0]\n        )\n\n        var, ld_score = SummaryStatisticsImputation._compute_var(\n            ld_matrix_known_missing, sig_t_inv, lamb\n        )\n\n        mu = SummaryStatisticsImputation._compute_mu(\n            ld_matrix_known_missing, sig_t_inv, z_scores_known\n        )\n        var_norm = SummaryStatisticsImputation._var_in_boundaries(var, lamb)\n\n        R2 = (1 + lamb) - var_norm\n\n        mu = mu / np.sqrt(R2)\n        return {\n            \"var\": var,\n            \"mu\": mu,\n            \"ld_score\": ld_score,\n            \"condition_number\": condition_number,\n            \"correct_inversion\": correct_inversion,\n            \"imputation_r2\": 1 - var,\n        }\n</code></pre>"},{"location":"python_api/methods/sumstat_quality_controls/","title":"QC of GWAS Summary Statistics","text":"<p>This class consists of several general quality control checks for GWAS with full summary statistics. There are several checks included:</p> <ol> <li> <p>Genomic control lambda (median of the distribution of Chi2 statistics divided by expected for Chi2 with df=1). Lambda should be reasonably close to 1. Ideally not bigger than 2.</p> </li> <li> <p>P-Z check: the linear regression between log10 of reported p-values and log10 of p-values inferred from betas and standard errors. Intercept of the regression should be close to 0, slope close to 1.</p> </li> <li> <p>Mean beta check: mean of beta. Should be close to 0.</p> </li> <li> <p>The N_eff check: It estimates the ratio between effective sample size and the expected one and checks its distribution. It is possible to conduct only if the effective allele frequency is provided in the study. The median ratio is always close to 1, standard error should be close to 0.</p> </li> <li> <p>Number of SNPs and number of significant SNPs.</p> </li> </ol>"},{"location":"python_api/methods/sumstat_quality_controls/#gentropy.method.sumstat_quality_controls.SummaryStatisticsQC","title":"<code>gentropy.method.sumstat_quality_controls.SummaryStatisticsQC</code>","text":"<p>Summary statistics QC methods.</p> <p>This module contains methods for quality control of GWAS summary statistics. The list of methods includes:</p> <pre><code>- sumstat_qc_beta_check: This is the mean beta check. The mean beta should be close to 0.\n\n- sumstat_qc_pz_check: This is the PZ check. It runs a linear regression between reported p-values and p-values inferred from z-scores.\n\n- sumstat_n_eff_check: This is the effective sample size check. It estimates the ratio between the effective sample size and the expected one and checks its distribution.\n\n- gc_lambda_check: This is the genomic control lambda check.\n\n- number_of_snps: This function calculates the number of SNPs and the number of SNPs with a p-value less than 5e-8.\n</code></pre> Source code in <code>src/gentropy/method/sumstat_quality_controls.py</code> <pre><code>class SummaryStatisticsQC:\n    \"\"\"Summary statistics QC methods.\n\n    This module contains methods for quality control of GWAS summary statistics.\n    The list of methods includes:\n\n        - sumstat_qc_beta_check: This is the mean beta check. The mean beta should be close to 0.\n\n        - sumstat_qc_pz_check: This is the PZ check. It runs a linear regression between reported p-values and p-values inferred from z-scores.\n\n        - sumstat_n_eff_check: This is the effective sample size check. It estimates the ratio between the effective sample size and the expected one and checks its distribution.\n\n        - gc_lambda_check: This is the genomic control lambda check.\n\n        - number_of_snps: This function calculates the number of SNPs and the number of SNPs with a p-value less than 5e-8.\n    \"\"\"\n\n    @staticmethod\n    def sumstat_qc_beta_check(\n        gwas_for_qc: SummaryStatistics,\n    ) -&gt; DataFrame:\n        \"\"\"The mean beta check for QC of GWAS summary statstics.\n\n        Args:\n            gwas_for_qc (SummaryStatistics): The instance of the SummaryStatistics class.\n\n        Returns:\n            DataFrame: PySpark DataFrame with the mean beta for each study.\n        \"\"\"\n        gwas_df = gwas_for_qc._df\n        qc_c = gwas_df.groupBy(\"studyId\").agg(\n            f.mean(\"beta\").alias(\"mean_beta\"),\n        )\n        return qc_c\n\n    @staticmethod\n    def _calculate_logpval(z2: float) -&gt; float:\n        \"\"\"Calculate negative log10-pval from Z-score.\n\n        Args:\n            z2 (float): Z-score squared.\n\n        Returns:\n            float: log10-pval.\n\n        Examples:\n            &gt;&gt;&gt; SummaryStatisticsQC._calculate_logpval(1.0)\n            0.49851554582799334\n        \"\"\"\n        logpval = -np.log10(sc.stats.chi2.sf((z2), 1))\n        return float(logpval)\n\n    @staticmethod\n    def sumstat_qc_pz_check(\n        gwas_for_qc: SummaryStatistics,\n        limit: int = 10_000_000,\n    ) -&gt; DataFrame:\n        \"\"\"The PZ check for QC of GWAS summary statstics. It runs linear regression between reported p-values and p-values infered from z-scores.\n\n        Args:\n            gwas_for_qc (SummaryStatistics): The instance of the SummaryStatistics class.\n            limit (int): The limit for the number of variants to be used for the estimation.\n\n        Returns:\n            DataFrame: PySpark DataFrame with the results of the linear regression for each study.\n        \"\"\"\n        gwas_df = gwas_for_qc._df\n\n        calculate_logpval_udf = f.udf(\n            SummaryStatisticsQC._calculate_logpval, t.DoubleType()\n        )\n\n        window = Window.partitionBy(\"studyId\").orderBy(\"studyId\")\n\n        gwas_df = (\n            gwas_df.withColumn(\"row_num\", row_number().over(window))\n            .filter(f.col(\"row_num\") &lt;= limit)\n            .drop(\"row_num\")\n        )\n\n        qc_c = (\n            gwas_df.withColumn(\"zscore\", f.col(\"beta\") / f.col(\"standardError\"))\n            .withColumn(\"new_logpval\", calculate_logpval_udf(f.col(\"zscore\") ** 2))\n            .withColumn(\"log_mantissa\", log10(\"pValueMantissa\"))\n            .withColumn(\n                \"diffpval\",\n                -f.col(\"log_mantissa\") - f.col(\"pValueExponent\") - f.col(\"new_logpval\"),\n            )\n            .groupBy(\"studyId\")\n            .agg(\n                f.mean(\"diffpval\").alias(\"mean_diff_pz\"),\n                f.stddev(\"diffpval\").alias(\"se_diff_pz\"),\n            )\n            .select(\"studyId\", \"mean_diff_pz\", \"se_diff_pz\")\n        )\n\n        return qc_c\n\n    @staticmethod\n    def sumstat_n_eff_check(\n        gwas_for_qc: SummaryStatistics,\n        n_total: int = 100_000,\n        limit: int = 10_000_000,\n        min_count: int = 100,\n    ) -&gt; DataFrame:\n        \"\"\"The effective sample size check for QC of GWAS summary statstics.\n\n        It estiamtes the ratio between effective sample size and the expected one and checks it's distribution.\n        It is possible to conduct only if the effective allele frequency is provided in the study.\n        The median rartio is always close to 1, but standard error could be inflated.\n\n        Args:\n            gwas_for_qc (SummaryStatistics): The instance of the SummaryStatistics class.\n            n_total (int): The reported sample size of the study. The QC metrics is robust toward the sample size.\n            limit (int): The limit for the number of variants to be used for the estimation.\n            min_count (int): The minimum number of variants to be used for the estimation.\n\n        Returns:\n            DataFrame: PySpark DataFrame with the effective sample size ratio for each study.\n        \"\"\"\n        gwas_df = gwas_for_qc._df\n\n        gwas_df = gwas_df.dropna(subset=[\"effectAlleleFrequencyFromSource\"])\n\n        counts_df = gwas_df.groupBy(\"studyId\").count()\n\n        # Join the original DataFrame with the counts DataFrame\n        df_with_counts = gwas_df.join(counts_df, on=\"studyId\")\n\n        # Filter the DataFrame to keep only the groups with count greater than or equal to min_count\n        filtered_df = df_with_counts.filter(f.col(\"count\") &gt;= min_count).drop(\"count\")\n\n        window = Window.partitionBy(\"studyId\").orderBy(\"studyId\")\n        gwas_df = (\n            filtered_df.withColumn(\"row_num\", row_number().over(window))\n            .filter(f.col(\"row_num\") &lt;= limit)\n            .drop(\"row_num\")\n        )\n\n        gwas_df = gwas_df.withColumn(\n            \"var_af\",\n            2\n            * (\n                f.col(\"effectAlleleFrequencyFromSource\")\n                * (1 - f.col(\"effectAlleleFrequencyFromSource\"))\n            ),\n        ).withColumn(\n            \"pheno_var\",\n            ((f.col(\"standardError\") ** 2) * n_total * f.col(\"var_af\"))\n            + ((f.col(\"beta\") ** 2) * f.col(\"var_af\")),\n        )\n\n        window = Window.partitionBy(\"studyId\").orderBy(\"studyId\")\n\n        # Calculate the median of 'pheno_var' for each 'studyId' and add it as a new column\n        gwas_df = gwas_df.withColumn(\n            \"pheno_median\", expr(\"percentile_approx(pheno_var, 0.5)\").over(window)\n        )\n\n        gwas_df = gwas_df.withColumn(\n            \"N_hat_ratio\",\n            (\n                (f.col(\"pheno_median\") - ((f.col(\"beta\") ** 2) * f.col(\"var_af\")))\n                / ((f.col(\"standardError\") ** 2) * f.col(\"var_af\") * n_total)\n            ),\n        )\n\n        qc_c = (\n            gwas_df.groupBy(\"studyId\")\n            .agg(\n                f.stddev(\"N_hat_ratio\").alias(\"se_N\"),\n            )\n            .select(\"studyId\", \"se_N\")\n        )\n\n        return qc_c\n\n    @staticmethod\n    def gc_lambda_check(\n        gwas_for_qc: SummaryStatistics,\n        limit: int = 10_000_000,\n    ) -&gt; DataFrame:\n        \"\"\"The genomic control lambda check for QC of GWAS summary statstics.\n\n        Args:\n            gwas_for_qc (SummaryStatistics): The instance of the SummaryStatistics class.\n            limit (int): The limit for the number of variants to be used for the estimation.\n\n        Returns:\n            DataFrame: PySpark DataFrame with the genomic control lambda for each study.\n        \"\"\"\n        gwas_df = gwas_for_qc._df\n        window = Window.partitionBy(\"studyId\").orderBy(\"studyId\")\n        gwas_df = (\n            gwas_df.withColumn(\"row_num\", row_number().over(window))\n            .filter(f.col(\"row_num\") &lt;= limit)\n            .drop(\"row_num\")\n        )\n\n        qc_c = (\n            gwas_df.select(\"studyId\", \"beta\", \"standardError\")\n            .withColumn(\"Z2\", (f.col(\"beta\") / f.col(\"standardError\")) ** 2)\n            .groupBy(\"studyId\")\n            .agg(f.expr(\"percentile_approx(Z2, 0.5)\").alias(\"gc_lambda\"))\n            .withColumn(\"gc_lambda\", f.col(\"gc_lambda\") / chi2.ppf(0.5, df=1))\n            .select(\"studyId\", \"gc_lambda\")\n        )\n\n        return qc_c\n\n    @staticmethod\n    def number_of_snps(\n        gwas_for_qc: SummaryStatistics, pval_threhod: float = 5e-8\n    ) -&gt; DataFrame:\n        \"\"\"The function caluates number of SNPs and number of SNPs with p-value less than 5e-8.\n\n        Args:\n            gwas_for_qc (SummaryStatistics): The instance of the SummaryStatistics class.\n            pval_threhod (float): The threshold for the p-value.\n\n        Returns:\n            DataFrame: PySpark DataFrame with the number of SNPs and number of SNPs with p-value less than threshold.\n        \"\"\"\n        gwas_df = gwas_for_qc._df\n\n        snp_counts = gwas_df.groupBy(\"studyId\").agg(\n            f.count(\"*\").alias(\"n_variants\"),\n            f.sum(\n                (\n                    f.log10(f.col(\"pValueMantissa\")) + f.col(\"pValueExponent\")\n                    &lt;= np.log10(pval_threhod)\n                ).cast(\"int\")\n            ).alias(\"n_variants_sig\"),\n        )\n\n        return snp_counts\n\n    @staticmethod\n    def get_quality_control_metrics(\n        gwas: SummaryStatistics,\n        limit: int = 100_000_000,\n        min_count: int = 100_000,\n        n_total: int = 100_000,\n    ) -&gt; DataFrame:\n        \"\"\"The function calculates the quality control metrics for the summary statistics.\n\n        Args:\n            gwas (SummaryStatistics): The instance of the SummaryStatistics class.\n            limit (int): The limit for the number of variants to be used for the estimation.\n            min_count (int): The minimum number of variants to be used for the estimation.\n            n_total (int): The total sample size.\n\n        Returns:\n            DataFrame: PySpark DataFrame with the quality control metrics for the summary statistics.\n        \"\"\"\n        qc1 = SummaryStatisticsQC.sumstat_qc_beta_check(gwas_for_qc=gwas)\n        qc2 = SummaryStatisticsQC.sumstat_qc_pz_check(gwas_for_qc=gwas, limit=limit)\n        qc3 = SummaryStatisticsQC.sumstat_n_eff_check(\n            gwas_for_qc=gwas, n_total=n_total, limit=limit, min_count=min_count\n        )\n        qc4 = SummaryStatisticsQC.gc_lambda_check(gwas_for_qc=gwas, limit=limit)\n        qc5 = SummaryStatisticsQC.number_of_snps(gwas_for_qc=gwas)\n        df = (\n            qc1.join(qc2, on=\"studyId\", how=\"outer\")\n            .join(qc3, on=\"studyId\", how=\"outer\")\n            .join(qc4, on=\"studyId\", how=\"outer\")\n            .join(qc5, on=\"studyId\", how=\"outer\")\n        )\n\n        return df\n</code></pre>"},{"location":"python_api/methods/sumstat_quality_controls/#gentropy.method.sumstat_quality_controls.SummaryStatisticsQC.gc_lambda_check","title":"<code>gc_lambda_check(gwas_for_qc: SummaryStatistics, limit: int = 10000000) -&gt; DataFrame</code>  <code>staticmethod</code>","text":"<p>The genomic control lambda check for QC of GWAS summary statstics.</p> <p>Parameters:</p> Name Type Description Default <code>gwas_for_qc</code> <code>SummaryStatistics</code> <p>The instance of the SummaryStatistics class.</p> required <code>limit</code> <code>int</code> <p>The limit for the number of variants to be used for the estimation.</p> <code>10000000</code> <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>PySpark DataFrame with the genomic control lambda for each study.</p> Source code in <code>src/gentropy/method/sumstat_quality_controls.py</code> <pre><code>@staticmethod\ndef gc_lambda_check(\n    gwas_for_qc: SummaryStatistics,\n    limit: int = 10_000_000,\n) -&gt; DataFrame:\n    \"\"\"The genomic control lambda check for QC of GWAS summary statstics.\n\n    Args:\n        gwas_for_qc (SummaryStatistics): The instance of the SummaryStatistics class.\n        limit (int): The limit for the number of variants to be used for the estimation.\n\n    Returns:\n        DataFrame: PySpark DataFrame with the genomic control lambda for each study.\n    \"\"\"\n    gwas_df = gwas_for_qc._df\n    window = Window.partitionBy(\"studyId\").orderBy(\"studyId\")\n    gwas_df = (\n        gwas_df.withColumn(\"row_num\", row_number().over(window))\n        .filter(f.col(\"row_num\") &lt;= limit)\n        .drop(\"row_num\")\n    )\n\n    qc_c = (\n        gwas_df.select(\"studyId\", \"beta\", \"standardError\")\n        .withColumn(\"Z2\", (f.col(\"beta\") / f.col(\"standardError\")) ** 2)\n        .groupBy(\"studyId\")\n        .agg(f.expr(\"percentile_approx(Z2, 0.5)\").alias(\"gc_lambda\"))\n        .withColumn(\"gc_lambda\", f.col(\"gc_lambda\") / chi2.ppf(0.5, df=1))\n        .select(\"studyId\", \"gc_lambda\")\n    )\n\n    return qc_c\n</code></pre>"},{"location":"python_api/methods/sumstat_quality_controls/#gentropy.method.sumstat_quality_controls.SummaryStatisticsQC.get_quality_control_metrics","title":"<code>get_quality_control_metrics(gwas: SummaryStatistics, limit: int = 100000000, min_count: int = 100000, n_total: int = 100000) -&gt; DataFrame</code>  <code>staticmethod</code>","text":"<p>The function calculates the quality control metrics for the summary statistics.</p> <p>Parameters:</p> Name Type Description Default <code>gwas</code> <code>SummaryStatistics</code> <p>The instance of the SummaryStatistics class.</p> required <code>limit</code> <code>int</code> <p>The limit for the number of variants to be used for the estimation.</p> <code>100000000</code> <code>min_count</code> <code>int</code> <p>The minimum number of variants to be used for the estimation.</p> <code>100000</code> <code>n_total</code> <code>int</code> <p>The total sample size.</p> <code>100000</code> <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>PySpark DataFrame with the quality control metrics for the summary statistics.</p> Source code in <code>src/gentropy/method/sumstat_quality_controls.py</code> <pre><code>@staticmethod\ndef get_quality_control_metrics(\n    gwas: SummaryStatistics,\n    limit: int = 100_000_000,\n    min_count: int = 100_000,\n    n_total: int = 100_000,\n) -&gt; DataFrame:\n    \"\"\"The function calculates the quality control metrics for the summary statistics.\n\n    Args:\n        gwas (SummaryStatistics): The instance of the SummaryStatistics class.\n        limit (int): The limit for the number of variants to be used for the estimation.\n        min_count (int): The minimum number of variants to be used for the estimation.\n        n_total (int): The total sample size.\n\n    Returns:\n        DataFrame: PySpark DataFrame with the quality control metrics for the summary statistics.\n    \"\"\"\n    qc1 = SummaryStatisticsQC.sumstat_qc_beta_check(gwas_for_qc=gwas)\n    qc2 = SummaryStatisticsQC.sumstat_qc_pz_check(gwas_for_qc=gwas, limit=limit)\n    qc3 = SummaryStatisticsQC.sumstat_n_eff_check(\n        gwas_for_qc=gwas, n_total=n_total, limit=limit, min_count=min_count\n    )\n    qc4 = SummaryStatisticsQC.gc_lambda_check(gwas_for_qc=gwas, limit=limit)\n    qc5 = SummaryStatisticsQC.number_of_snps(gwas_for_qc=gwas)\n    df = (\n        qc1.join(qc2, on=\"studyId\", how=\"outer\")\n        .join(qc3, on=\"studyId\", how=\"outer\")\n        .join(qc4, on=\"studyId\", how=\"outer\")\n        .join(qc5, on=\"studyId\", how=\"outer\")\n    )\n\n    return df\n</code></pre>"},{"location":"python_api/methods/sumstat_quality_controls/#gentropy.method.sumstat_quality_controls.SummaryStatisticsQC.number_of_snps","title":"<code>number_of_snps(gwas_for_qc: SummaryStatistics, pval_threhod: float = 5e-08) -&gt; DataFrame</code>  <code>staticmethod</code>","text":"<p>The function caluates number of SNPs and number of SNPs with p-value less than 5e-8.</p> <p>Parameters:</p> Name Type Description Default <code>gwas_for_qc</code> <code>SummaryStatistics</code> <p>The instance of the SummaryStatistics class.</p> required <code>pval_threhod</code> <code>float</code> <p>The threshold for the p-value.</p> <code>5e-08</code> <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>PySpark DataFrame with the number of SNPs and number of SNPs with p-value less than threshold.</p> Source code in <code>src/gentropy/method/sumstat_quality_controls.py</code> <pre><code>@staticmethod\ndef number_of_snps(\n    gwas_for_qc: SummaryStatistics, pval_threhod: float = 5e-8\n) -&gt; DataFrame:\n    \"\"\"The function caluates number of SNPs and number of SNPs with p-value less than 5e-8.\n\n    Args:\n        gwas_for_qc (SummaryStatistics): The instance of the SummaryStatistics class.\n        pval_threhod (float): The threshold for the p-value.\n\n    Returns:\n        DataFrame: PySpark DataFrame with the number of SNPs and number of SNPs with p-value less than threshold.\n    \"\"\"\n    gwas_df = gwas_for_qc._df\n\n    snp_counts = gwas_df.groupBy(\"studyId\").agg(\n        f.count(\"*\").alias(\"n_variants\"),\n        f.sum(\n            (\n                f.log10(f.col(\"pValueMantissa\")) + f.col(\"pValueExponent\")\n                &lt;= np.log10(pval_threhod)\n            ).cast(\"int\")\n        ).alias(\"n_variants_sig\"),\n    )\n\n    return snp_counts\n</code></pre>"},{"location":"python_api/methods/sumstat_quality_controls/#gentropy.method.sumstat_quality_controls.SummaryStatisticsQC.sumstat_n_eff_check","title":"<code>sumstat_n_eff_check(gwas_for_qc: SummaryStatistics, n_total: int = 100000, limit: int = 10000000, min_count: int = 100) -&gt; DataFrame</code>  <code>staticmethod</code>","text":"<p>The effective sample size check for QC of GWAS summary statstics.</p> <p>It estiamtes the ratio between effective sample size and the expected one and checks it's distribution. It is possible to conduct only if the effective allele frequency is provided in the study. The median rartio is always close to 1, but standard error could be inflated.</p> <p>Parameters:</p> Name Type Description Default <code>gwas_for_qc</code> <code>SummaryStatistics</code> <p>The instance of the SummaryStatistics class.</p> required <code>n_total</code> <code>int</code> <p>The reported sample size of the study. The QC metrics is robust toward the sample size.</p> <code>100000</code> <code>limit</code> <code>int</code> <p>The limit for the number of variants to be used for the estimation.</p> <code>10000000</code> <code>min_count</code> <code>int</code> <p>The minimum number of variants to be used for the estimation.</p> <code>100</code> <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>PySpark DataFrame with the effective sample size ratio for each study.</p> Source code in <code>src/gentropy/method/sumstat_quality_controls.py</code> <pre><code>@staticmethod\ndef sumstat_n_eff_check(\n    gwas_for_qc: SummaryStatistics,\n    n_total: int = 100_000,\n    limit: int = 10_000_000,\n    min_count: int = 100,\n) -&gt; DataFrame:\n    \"\"\"The effective sample size check for QC of GWAS summary statstics.\n\n    It estiamtes the ratio between effective sample size and the expected one and checks it's distribution.\n    It is possible to conduct only if the effective allele frequency is provided in the study.\n    The median rartio is always close to 1, but standard error could be inflated.\n\n    Args:\n        gwas_for_qc (SummaryStatistics): The instance of the SummaryStatistics class.\n        n_total (int): The reported sample size of the study. The QC metrics is robust toward the sample size.\n        limit (int): The limit for the number of variants to be used for the estimation.\n        min_count (int): The minimum number of variants to be used for the estimation.\n\n    Returns:\n        DataFrame: PySpark DataFrame with the effective sample size ratio for each study.\n    \"\"\"\n    gwas_df = gwas_for_qc._df\n\n    gwas_df = gwas_df.dropna(subset=[\"effectAlleleFrequencyFromSource\"])\n\n    counts_df = gwas_df.groupBy(\"studyId\").count()\n\n    # Join the original DataFrame with the counts DataFrame\n    df_with_counts = gwas_df.join(counts_df, on=\"studyId\")\n\n    # Filter the DataFrame to keep only the groups with count greater than or equal to min_count\n    filtered_df = df_with_counts.filter(f.col(\"count\") &gt;= min_count).drop(\"count\")\n\n    window = Window.partitionBy(\"studyId\").orderBy(\"studyId\")\n    gwas_df = (\n        filtered_df.withColumn(\"row_num\", row_number().over(window))\n        .filter(f.col(\"row_num\") &lt;= limit)\n        .drop(\"row_num\")\n    )\n\n    gwas_df = gwas_df.withColumn(\n        \"var_af\",\n        2\n        * (\n            f.col(\"effectAlleleFrequencyFromSource\")\n            * (1 - f.col(\"effectAlleleFrequencyFromSource\"))\n        ),\n    ).withColumn(\n        \"pheno_var\",\n        ((f.col(\"standardError\") ** 2) * n_total * f.col(\"var_af\"))\n        + ((f.col(\"beta\") ** 2) * f.col(\"var_af\")),\n    )\n\n    window = Window.partitionBy(\"studyId\").orderBy(\"studyId\")\n\n    # Calculate the median of 'pheno_var' for each 'studyId' and add it as a new column\n    gwas_df = gwas_df.withColumn(\n        \"pheno_median\", expr(\"percentile_approx(pheno_var, 0.5)\").over(window)\n    )\n\n    gwas_df = gwas_df.withColumn(\n        \"N_hat_ratio\",\n        (\n            (f.col(\"pheno_median\") - ((f.col(\"beta\") ** 2) * f.col(\"var_af\")))\n            / ((f.col(\"standardError\") ** 2) * f.col(\"var_af\") * n_total)\n        ),\n    )\n\n    qc_c = (\n        gwas_df.groupBy(\"studyId\")\n        .agg(\n            f.stddev(\"N_hat_ratio\").alias(\"se_N\"),\n        )\n        .select(\"studyId\", \"se_N\")\n    )\n\n    return qc_c\n</code></pre>"},{"location":"python_api/methods/sumstat_quality_controls/#gentropy.method.sumstat_quality_controls.SummaryStatisticsQC.sumstat_qc_beta_check","title":"<code>sumstat_qc_beta_check(gwas_for_qc: SummaryStatistics) -&gt; DataFrame</code>  <code>staticmethod</code>","text":"<p>The mean beta check for QC of GWAS summary statstics.</p> <p>Parameters:</p> Name Type Description Default <code>gwas_for_qc</code> <code>SummaryStatistics</code> <p>The instance of the SummaryStatistics class.</p> required <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>PySpark DataFrame with the mean beta for each study.</p> Source code in <code>src/gentropy/method/sumstat_quality_controls.py</code> <pre><code>@staticmethod\ndef sumstat_qc_beta_check(\n    gwas_for_qc: SummaryStatistics,\n) -&gt; DataFrame:\n    \"\"\"The mean beta check for QC of GWAS summary statstics.\n\n    Args:\n        gwas_for_qc (SummaryStatistics): The instance of the SummaryStatistics class.\n\n    Returns:\n        DataFrame: PySpark DataFrame with the mean beta for each study.\n    \"\"\"\n    gwas_df = gwas_for_qc._df\n    qc_c = gwas_df.groupBy(\"studyId\").agg(\n        f.mean(\"beta\").alias(\"mean_beta\"),\n    )\n    return qc_c\n</code></pre>"},{"location":"python_api/methods/sumstat_quality_controls/#gentropy.method.sumstat_quality_controls.SummaryStatisticsQC.sumstat_qc_pz_check","title":"<code>sumstat_qc_pz_check(gwas_for_qc: SummaryStatistics, limit: int = 10000000) -&gt; DataFrame</code>  <code>staticmethod</code>","text":"<p>The PZ check for QC of GWAS summary statstics. It runs linear regression between reported p-values and p-values infered from z-scores.</p> <p>Parameters:</p> Name Type Description Default <code>gwas_for_qc</code> <code>SummaryStatistics</code> <p>The instance of the SummaryStatistics class.</p> required <code>limit</code> <code>int</code> <p>The limit for the number of variants to be used for the estimation.</p> <code>10000000</code> <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>PySpark DataFrame with the results of the linear regression for each study.</p> Source code in <code>src/gentropy/method/sumstat_quality_controls.py</code> <pre><code>@staticmethod\ndef sumstat_qc_pz_check(\n    gwas_for_qc: SummaryStatistics,\n    limit: int = 10_000_000,\n) -&gt; DataFrame:\n    \"\"\"The PZ check for QC of GWAS summary statstics. It runs linear regression between reported p-values and p-values infered from z-scores.\n\n    Args:\n        gwas_for_qc (SummaryStatistics): The instance of the SummaryStatistics class.\n        limit (int): The limit for the number of variants to be used for the estimation.\n\n    Returns:\n        DataFrame: PySpark DataFrame with the results of the linear regression for each study.\n    \"\"\"\n    gwas_df = gwas_for_qc._df\n\n    calculate_logpval_udf = f.udf(\n        SummaryStatisticsQC._calculate_logpval, t.DoubleType()\n    )\n\n    window = Window.partitionBy(\"studyId\").orderBy(\"studyId\")\n\n    gwas_df = (\n        gwas_df.withColumn(\"row_num\", row_number().over(window))\n        .filter(f.col(\"row_num\") &lt;= limit)\n        .drop(\"row_num\")\n    )\n\n    qc_c = (\n        gwas_df.withColumn(\"zscore\", f.col(\"beta\") / f.col(\"standardError\"))\n        .withColumn(\"new_logpval\", calculate_logpval_udf(f.col(\"zscore\") ** 2))\n        .withColumn(\"log_mantissa\", log10(\"pValueMantissa\"))\n        .withColumn(\n            \"diffpval\",\n            -f.col(\"log_mantissa\") - f.col(\"pValueExponent\") - f.col(\"new_logpval\"),\n        )\n        .groupBy(\"studyId\")\n        .agg(\n            f.mean(\"diffpval\").alias(\"mean_diff_pz\"),\n            f.stddev(\"diffpval\").alias(\"se_diff_pz\"),\n        )\n        .select(\"studyId\", \"mean_diff_pz\", \"se_diff_pz\")\n    )\n\n    return qc_c\n</code></pre>"},{"location":"python_api/methods/susie_inf/","title":"SuSiE-inf - Fine-mapping with infinitesimal effects v1.1","text":"<p>This is an implementation of the SuSiE-inf method found here: https://github.com/FinucaneLab/fine-mapping-inf https://www.nature.com/articles/s41588-023-01597-3</p> <p>This fine-mapping approach has two approaches for updating estimates of the variance components - Method of Moments and Maximum Likelihood Estimator ('MoM' / 'MLE') The function takes an array of Z-scores and a numpy array matrix of variant LD to perform finemapping.</p>"},{"location":"python_api/methods/susie_inf/#gentropy.method.susie_inf.SUSIE_inf","title":"<code>gentropy.method.susie_inf.SUSIE_inf</code>  <code>dataclass</code>","text":"<p>SuSiE fine-mapping of a study locus from fine-mapping-inf package.</p> <p>Note: code copied from fine-mapping-inf package as a placeholder https://github.com/FinucaneLab/fine-mapping-inf</p> Source code in <code>src/gentropy/method/susie_inf.py</code> <pre><code>@dataclass\nclass SUSIE_inf:\n    \"\"\"SuSiE fine-mapping of a study locus from fine-mapping-inf package.\n\n    Note: code copied from fine-mapping-inf package as a placeholder\n    https://github.com/FinucaneLab/fine-mapping-inf\n    \"\"\"\n\n    @staticmethod\n    def susie_inf(  # noqa: C901\n        z: np.ndarray,\n        meansq: float = 1,\n        n: int = 100000,\n        L: int = 10,\n        LD: np.ndarray | None = None,\n        V: np.ndarray | None = None,\n        Dsq: np.ndarray | None = None,\n        est_ssq: bool = True,\n        ssq: np.ndarray | None = None,\n        ssq_range: tuple[float, float] = (0, 1),\n        pi0: np.ndarray | None = None,\n        est_sigmasq: bool = True,\n        est_tausq: bool = False,\n        sigmasq: float = 1,\n        tausq: float = 0,\n        sigmasq_range: tuple[float, float] | None = None,\n        tausq_range: tuple[float, float] | None = None,\n        PIP: np.ndarray | None = None,\n        mu: np.ndarray | None = None,\n        method: str = \"moments\",\n        maxiter: int = 100,\n        PIP_tol: float = 0.001,\n    ) -&gt; dict[str, Any]:\n        \"\"\"Susie with random effects.\n\n        Args:\n            z (np.ndarray): vector of z-scores (equal to X'y/sqrt(n))\n            meansq (float): average squared magnitude of y (equal to ||y||^2/n)\n            n (int): sample size\n            L (int): number of modeled causal effects\n            LD (np.ndarray | None): LD matrix (equal to X'X/n)\n            V (np.ndarray | None): precomputed p x p matrix of eigenvectors of X'X\n            Dsq (np.ndarray | None): precomputed length-p vector of eigenvalues of X'X\n            est_ssq (bool): estimate prior effect size variances s^2 using MLE\n            ssq (np.ndarray | None): length-L initialization s^2 for each effect\n            ssq_range (tuple[float, float]): lower and upper bounds for each s^2, if estimated\n            pi0 (np.ndarray | None): length-p vector of prior causal probability for each SNP; must sum to 1\n            est_sigmasq (bool): estimate variance sigma^2\n            est_tausq (bool): estimate both variances sigma^2 and tau^2\n            sigmasq (float): initial value for sigma^2\n            tausq (float): initial value for tau^2\n            sigmasq_range (tuple[float, float] | None): lower and upper bounds for sigma^2, if estimated using MLE\n            tausq_range (tuple[float, float] | None): lower and upper bounds for tau^2, if estimated using MLE\n            PIP (np.ndarray | None): p x L initializations of PIPs\n            mu (np.ndarray | None): p x L initializations of mu\n            method (str): one of {'moments','MLE'}\n            maxiter (int): maximum number of SuSiE iterations\n            PIP_tol (float): convergence threshold for PIP difference between iterations\n\n        Returns:\n            dict[str, Any]: Dictionary with keys:\n                PIP -- p x L matrix of PIPs, individually for each effect\n                mu -- p x L matrix of posterior means conditional on causal\n                omega -- p x L matrix of posterior precisions conditional on causal\n                lbf_variable -- p x L matrix of log-Bayes-factors, for each effect\n                ssq -- length-L array of final effect size variances s^2\n                sigmasq -- final value of sigma^2\n                tausq -- final value of tau^2\n                alpha -- length-p array of posterior means of infinitesimal effects\n                lbf -- length-p array of log-Bayes-factors for each CS\n\n        Raises:\n            RuntimeError: if missing LD\n            RuntimeError: if unsupported variance estimation method\n        \"\"\"\n        p = len(z)\n        # Precompute V,D^2 in the SVD X=UDV', and V'X'y and y'y\n        if (V is None or Dsq is None) and LD is None:\n            raise RuntimeError(\"Missing LD\")\n        elif V is None or Dsq is None:\n            eigvals, V = scipy.linalg.eigh(LD)\n            Dsq = np.maximum(n * eigvals, 0)\n        else:\n            Dsq = np.maximum(Dsq, 0)\n        Xty = np.sqrt(n) * z\n        VtXty = V.T.dot(Xty)\n        yty = n * meansq\n        # Initialize diagonal variances, diag(X' Omega X), X' Omega y\n        var = tausq * Dsq + sigmasq\n        diagXtOmegaX = np.sum(V**2 * (Dsq / var), axis=1)\n        XtOmegay = V.dot(VtXty / var)\n        # Initialize s_l^2, PIP_j, mu_j, omega_j\n        if ssq is None:\n            ssq = np.ones(L) * 0.2\n        if PIP is None:\n            PIP = np.ones((p, L)) / p\n        if mu is None:\n            mu = np.zeros((p, L))\n        lbf_variable = np.zeros((p, L))\n        omega = diagXtOmegaX[:, np.newaxis] + 1 / ssq\n        # Initialize prior causal probabilities\n        if pi0 is None:\n            logpi0 = np.ones(p) * np.log(1.0 / p)\n        else:\n            logpi0 = -np.ones(p) * np.inf\n            inds = np.nonzero(pi0 &gt; 0)[0]\n            logpi0[inds] = np.log(pi0[inds])\n\n        ####### Main SuSiE iteration loop ######\n        def f(x: float) -&gt; float:\n            \"\"\"Negative ELBO as function of x = sigma_e^2.\n\n            Args:\n                x (float): sigma_e^2\n\n            Returns:\n                float: negative ELBO as function of x = sigma_e^2\n            \"\"\"\n            return -scipy.special.logsumexp(\n                -0.5 * np.log(1 + x * diagXtOmegaX)\n                + x * XtOmegar**2 / (2 * (1 + x * diagXtOmegaX))\n                + logpi0\n            )\n\n        for it in range(maxiter):\n            PIP_prev = PIP.copy()\n            # Single effect regression for each effect l = 1,...,L\n            for _l in range(L):\n                # Compute X' Omega r_l for residual r_l\n                b = np.sum(mu * PIP, axis=1) - mu[:, _l] * PIP[:, _l]\n                XtOmegaXb = V.dot(V.T.dot(b) * Dsq / var)\n                XtOmegar = XtOmegay - XtOmegaXb\n                if est_ssq:\n                    # Update prior variance ssq[l]\n                    res = minimize_scalar(f, bounds=ssq_range, method=\"bounded\")\n                    if res.success:\n                        ssq[_l] = res.x\n                # Update omega, mu, and PIP\n                omega[:, _l] = diagXtOmegaX + 1 / ssq[_l]\n                mu[:, _l] = XtOmegar / omega[:, _l]\n                lbf_variable[:, _l] = XtOmegar**2 / (2 * omega[:, _l]) - 0.5 * np.log(\n                    omega[:, _l] * ssq[_l]\n                )\n                logPIP = lbf_variable[:, _l] + logpi0\n                PIP[:, _l] = np.exp(logPIP - scipy.special.logsumexp(logPIP))\n            # Update variance components\n            if est_sigmasq or est_tausq:\n                if method == \"moments\":\n                    (sigmasq, tausq) = SUSIE_inf._MoM(\n                        PIP,\n                        mu,\n                        omega,\n                        sigmasq,\n                        tausq,\n                        n,\n                        V,\n                        Dsq,\n                        VtXty,\n                        Xty,\n                        yty,\n                        est_sigmasq,\n                        est_tausq,\n                    )\n                elif method == \"MLE\":\n                    (sigmasq, tausq) = SUSIE_inf._MLE(\n                        PIP,\n                        mu,\n                        omega,\n                        sigmasq,\n                        tausq,\n                        n,\n                        V,\n                        Dsq,\n                        VtXty,\n                        yty,\n                        est_sigmasq,\n                        est_tausq,\n                        it,\n                        sigmasq_range,\n                        tausq_range,\n                    )\n                else:\n                    raise RuntimeError(\"Unsupported variance estimation method\")\n                # Update X' Omega X, X' Omega y\n                var = tausq * Dsq + sigmasq\n                diagXtOmegaX = np.sum(V**2 * (Dsq / var), axis=1)\n                XtOmegay = V.dot(VtXty / var)\n            # Determine convergence from PIP differences\n            PIP_diff = np.max(np.abs(PIP_prev - PIP))\n            if PIP_diff &lt; PIP_tol:\n                break\n        # Compute posterior means of b and alpha\n        b = np.sum(mu * PIP, axis=1)\n        XtOmegaXb = V.dot(V.T.dot(b) * Dsq / var)\n        XtOmegar = XtOmegay - XtOmegaXb\n        alpha = tausq * XtOmegar\n\n        priors = np.log(np.repeat(1 / p, p))\n        lbf_cs = np.apply_along_axis(\n            lambda x: logsumexp(x + priors), axis=0, arr=lbf_variable\n        )\n        return {\n            \"PIP\": PIP,\n            \"mu\": mu,\n            \"omega\": omega,\n            \"lbf_variable\": lbf_variable,\n            \"ssq\": ssq,\n            \"sigmasq\": sigmasq,\n            \"tausq\": tausq,\n            \"alpha\": alpha,\n            \"lbf\": lbf_cs,\n        }\n\n    @staticmethod\n    def _MoM(\n        PIP: np.ndarray,\n        mu: np.ndarray,\n        omega: np.ndarray,\n        sigmasq: float,\n        tausq: float,\n        n: int,\n        V: np.ndarray,\n        Dsq: np.ndarray,\n        VtXty: np.ndarray,\n        Xty: np.ndarray,\n        yty: float,\n        est_sigmasq: bool,\n        est_tausq: bool,\n    ) -&gt; tuple[float, float]:\n        \"\"\"Subroutine to estimate sigma^2, tau^2 using method-of-moments.\n\n        Args:\n            PIP (np.ndarray): p x L matrix of PIPs\n            mu (np.ndarray): p x L matrix of posterior means conditional on causal\n            omega (np.ndarray): p x L matrix of posterior precisions conditional on causal\n            sigmasq (float): initial value for sigma^2\n            tausq (float): initial value for tau^2\n            n (int): sample size\n            V (np.ndarray): precomputed p x p matrix of eigenvectors of X'X\n            Dsq (np.ndarray): precomputed length-p vector of eigenvalues of X'X\n            VtXty (np.ndarray): precomputed length-p vector V'X'y\n            Xty (np.ndarray): precomputed length-p vector X'y\n            yty (float): precomputed y'y\n            est_sigmasq (bool): estimate variance sigma^2\n            est_tausq (bool): estimate both variances sigma^2 and tau^2\n\n        Returns:\n            tuple[float, float]: (sigmasq,tausq) tuple of updated variances\n        \"\"\"\n        (p, L) = mu.shape\n        # Compute A\n        A = np.array([[n, sum(Dsq)], [0, sum(Dsq**2)]])\n        A[1, 0] = A[0, 1]\n        # Compute diag(V'MV)\n        b = np.sum(mu * PIP, axis=1)\n        Vtb = V.T.dot(b)\n        diagVtMV = Vtb**2\n        tmpD = np.zeros(p)\n        for _l in range(L):\n            bl = mu[:, _l] * PIP[:, _l]\n            Vtbl = V.T.dot(bl)\n            diagVtMV -= Vtbl**2\n            tmpD += PIP[:, _l] * (mu[:, _l] ** 2 + 1 / omega[:, _l])\n        diagVtMV += np.sum((V.T) ** 2 * tmpD, axis=1)\n        # Compute x\n        x = np.zeros(2)\n        x[0] = yty - 2 * sum(b * Xty) + sum(Dsq * diagVtMV)\n        x[1] = sum(Xty**2) - 2 * sum(Vtb * VtXty * Dsq) + sum(Dsq**2 * diagVtMV)\n        if est_tausq:\n            sol = scipy.linalg.solve(A, x)\n            if sol[0] &gt; 0 and sol[1] &gt; 0:\n                (sigmasq, tausq) = sol\n            else:\n                (sigmasq, tausq) = (x[0] / n, 0)\n        elif est_sigmasq:\n            sigmasq = (x[0] - A[0, 1] * tausq) / n\n        return sigmasq, tausq\n\n    @staticmethod\n    def _MLE(\n        PIP: np.ndarray,\n        mu: np.ndarray,\n        omega: np.ndarray,\n        sigmasq: float,\n        tausq: float,\n        n: int,\n        V: np.ndarray,\n        Dsq: np.ndarray,\n        VtXty: np.ndarray,\n        yty: float,\n        est_sigmasq: bool,\n        est_tausq: bool,\n        it: int,\n        sigmasq_range: tuple[float, float] | None = None,\n        tausq_range: tuple[float, float] | None = None,\n    ) -&gt; tuple[float, float]:\n        \"\"\"Subroutine to estimate sigma^2, tau^2 using MLE.\n\n        Args:\n            PIP (np.ndarray): p x L matrix of PIPs\n            mu (np.ndarray): p x L matrix of posterior means conditional on causal\n            omega (np.ndarray): p x L matrix of posterior precisions conditional on causal\n            sigmasq (float): initial value for sigma^2\n            tausq (float): initial value for tau^2\n            n (int): sample size\n            V (np.ndarray): precomputed p x p matrix of eigenvectors of X'X\n            Dsq (np.ndarray): precomputed length-p vector of eigenvalues of X'X\n            VtXty (np.ndarray): precomputed length-p vector V'X'y\n            yty (float): precomputed y'y\n            est_sigmasq (bool): estimate variance sigma^2\n            est_tausq (bool): estimate both variances sigma^2 and tau^2\n            it (int): iteration number\n            sigmasq_range (tuple[float, float] | None): lower and upper bounds for sigma^2, if estimated using MLE\n            tausq_range (tuple[float, float] | None): lower and upper bounds for tau^2, if estimated using MLE\n\n        Returns:\n            tuple[float, float]: (sigmasq,tausq) tuple of updated variances\n        \"\"\"\n        (p, L) = mu.shape\n        if sigmasq_range is None:\n            sigmasq_range = (0.2 * yty / n, 1.2 * yty / n)\n        if tausq_range is None:\n            tausq_range = (1e-12, 1.2 * yty / (n * p))\n        # Compute diag(V'MV)\n        b = np.sum(mu * PIP, axis=1)\n        Vtb = V.T.dot(b)\n        diagVtMV = Vtb**2\n        tmpD = np.zeros(p)\n        for _l in range(L):\n            bl = mu[:, _l] * PIP[:, _l]\n            Vtbl = V.T.dot(bl)\n            diagVtMV -= Vtbl**2\n            tmpD += PIP[:, _l] * (mu[:, _l] ** 2 + 1 / omega[:, _l])\n        diagVtMV += np.sum((V.T) ** 2 * tmpD, axis=1)\n\n        # negative ELBO as function of x = (sigma_e^2,sigma_g^2)\n        def f(x: tuple[float, float]) -&gt; float:\n            \"\"\"Negative ELBO as function of x = (sigma_e^2,sigma_g^2).\n\n            Args:\n                x (tuple[float, float]): (sigma_e^2,sigma_g^2)\n\n            Returns:\n                float: negative ELBO as function of x = (sigma_e^2,sigma_g^2)\n            \"\"\"\n            return (\n                0.5 * (n - p) * np.log(x[0])\n                + 0.5 / x[0] * yty\n                + np.sum(\n                    0.5 * np.log(x[1] * Dsq + x[0])\n                    - 0.5 * x[1] / x[0] * VtXty**2 / (x[1] * Dsq + x[0])\n                    - Vtb * VtXty / (x[1] * Dsq + x[0])\n                    + 0.5 * Dsq / (x[1] * Dsq + x[0]) * diagVtMV\n                )\n            )\n\n        if est_tausq:\n            res = minimize(\n                f,\n                (sigmasq, tausq),\n                method=\"L-BFGS-B\",\n                bounds=(sigmasq_range, tausq_range),\n            )\n            if res.success:\n                sigmasq, tausq = res.x\n        elif est_sigmasq:\n\n            def g(x: float) -&gt; float:\n                \"\"\"Negative ELBO as function of x = sigma_e^2.\n\n                Args:\n                    x (float): sigma_e^2\n\n                Returns:\n                    float: negative ELBO as function of x = sigma_e^2\n                \"\"\"\n                return f((x, tausq))\n\n            res = minimize(g, sigmasq, method=\"L-BFGS-B\", bounds=(sigmasq_range,))\n            if res.success:\n                sigmasq = res.x\n        return sigmasq, tausq\n\n    @staticmethod\n    def cred_inf(\n        PIP: np.ndarray,\n        n: int = 100_000,\n        coverage: float = 0.99,\n        purity: float = 0.5,\n        LD: np.ndarray | None = None,\n        V: np.ndarray | None = None,\n        Dsq: np.ndarray | None = None,\n        dedup: bool = True,\n    ) -&gt; list[Any]:\n        \"\"\"Compute credible sets from single-effect PIPs.\n\n        Args:\n            PIP (np.ndarray): p x L matrix of PIPs\n            n (int): sample size\n            coverage (float): coverage of credible sets\n            purity (float): purity of credible sets\n            LD (np.ndarray | None): LD matrix (equal to X'X/n)\n            V (np.ndarray | None): precomputed p x p matrix of eigenvectors of X'X\n            Dsq (np.ndarray | None): precomputed length-p vector of eigenvalues of X'X\n            dedup (bool): whether to deduplicate credible sets\n\n        Returns:\n            list[Any]: list of L lists of SNP indices in each credible set\n\n        Raises:\n            RuntimeError: if missing inputs for purity filtering\n        \"\"\"\n        if (V is None or Dsq is None or n is None) and LD is None:\n            raise RuntimeError(\"Missing inputs for purity filtering\")\n        # Compute credible sets\n        cred = []\n        for i in range(PIP.shape[1]):\n            sortinds = np.argsort(PIP[:, i])[::-1]\n            ind = min(np.nonzero(np.cumsum(PIP[sortinds, i]) &gt;= coverage)[0])\n            credset = sortinds[: (ind + 1)]\n            # Filter by purity\n            if len(credset) == 1:\n                cred.append(list(credset))\n                continue\n            if len(credset) &lt; 100:\n                rows = credset\n            else:\n                np.random.seed(123)\n                rows = np.random.choice(credset, size=100, replace=False)\n            if LD is not None:\n                LDloc = LD[np.ix_(rows, rows)]\n            elif V is not None and Dsq is not None:\n                LDloc = (V[rows, :] * Dsq).dot(V[rows, :].T) / n\n            else:\n                raise ValueError(\"Both LD and V, Dsq cannot be None\")\n            if np.min(np.abs(LDloc)) &gt; purity:\n                cred.append(sorted(credset))\n        if dedup:\n            cred = list(\n                map(\n                    list,\n                    sorted(set(map(tuple, cred)), key=list(map(tuple, cred)).index),\n                )\n            )\n        return cred\n</code></pre>"},{"location":"python_api/methods/susie_inf/#gentropy.method.susie_inf.SUSIE_inf.cred_inf","title":"<code>cred_inf(PIP: np.ndarray, n: int = 100000, coverage: float = 0.99, purity: float = 0.5, LD: np.ndarray | None = None, V: np.ndarray | None = None, Dsq: np.ndarray | None = None, dedup: bool = True) -&gt; list[Any]</code>  <code>staticmethod</code>","text":"<p>Compute credible sets from single-effect PIPs.</p> <p>Parameters:</p> Name Type Description Default <code>PIP</code> <code>ndarray</code> <p>p x L matrix of PIPs</p> required <code>n</code> <code>int</code> <p>sample size</p> <code>100000</code> <code>coverage</code> <code>float</code> <p>coverage of credible sets</p> <code>0.99</code> <code>purity</code> <code>float</code> <p>purity of credible sets</p> <code>0.5</code> <code>LD</code> <code>ndarray | None</code> <p>LD matrix (equal to X'X/n)</p> <code>None</code> <code>V</code> <code>ndarray | None</code> <p>precomputed p x p matrix of eigenvectors of X'X</p> <code>None</code> <code>Dsq</code> <code>ndarray | None</code> <p>precomputed length-p vector of eigenvalues of X'X</p> <code>None</code> <code>dedup</code> <code>bool</code> <p>whether to deduplicate credible sets</p> <code>True</code> <p>Returns:</p> Type Description <code>list[Any]</code> <p>list[Any]: list of L lists of SNP indices in each credible set</p> <p>Raises:</p> Type Description <code>RuntimeError</code> <p>if missing inputs for purity filtering</p> Source code in <code>src/gentropy/method/susie_inf.py</code> <pre><code>@staticmethod\ndef cred_inf(\n    PIP: np.ndarray,\n    n: int = 100_000,\n    coverage: float = 0.99,\n    purity: float = 0.5,\n    LD: np.ndarray | None = None,\n    V: np.ndarray | None = None,\n    Dsq: np.ndarray | None = None,\n    dedup: bool = True,\n) -&gt; list[Any]:\n    \"\"\"Compute credible sets from single-effect PIPs.\n\n    Args:\n        PIP (np.ndarray): p x L matrix of PIPs\n        n (int): sample size\n        coverage (float): coverage of credible sets\n        purity (float): purity of credible sets\n        LD (np.ndarray | None): LD matrix (equal to X'X/n)\n        V (np.ndarray | None): precomputed p x p matrix of eigenvectors of X'X\n        Dsq (np.ndarray | None): precomputed length-p vector of eigenvalues of X'X\n        dedup (bool): whether to deduplicate credible sets\n\n    Returns:\n        list[Any]: list of L lists of SNP indices in each credible set\n\n    Raises:\n        RuntimeError: if missing inputs for purity filtering\n    \"\"\"\n    if (V is None or Dsq is None or n is None) and LD is None:\n        raise RuntimeError(\"Missing inputs for purity filtering\")\n    # Compute credible sets\n    cred = []\n    for i in range(PIP.shape[1]):\n        sortinds = np.argsort(PIP[:, i])[::-1]\n        ind = min(np.nonzero(np.cumsum(PIP[sortinds, i]) &gt;= coverage)[0])\n        credset = sortinds[: (ind + 1)]\n        # Filter by purity\n        if len(credset) == 1:\n            cred.append(list(credset))\n            continue\n        if len(credset) &lt; 100:\n            rows = credset\n        else:\n            np.random.seed(123)\n            rows = np.random.choice(credset, size=100, replace=False)\n        if LD is not None:\n            LDloc = LD[np.ix_(rows, rows)]\n        elif V is not None and Dsq is not None:\n            LDloc = (V[rows, :] * Dsq).dot(V[rows, :].T) / n\n        else:\n            raise ValueError(\"Both LD and V, Dsq cannot be None\")\n        if np.min(np.abs(LDloc)) &gt; purity:\n            cred.append(sorted(credset))\n    if dedup:\n        cred = list(\n            map(\n                list,\n                sorted(set(map(tuple, cred)), key=list(map(tuple, cred)).index),\n            )\n        )\n    return cred\n</code></pre>"},{"location":"python_api/methods/susie_inf/#gentropy.method.susie_inf.SUSIE_inf.susie_inf","title":"<code>susie_inf(z: np.ndarray, meansq: float = 1, n: int = 100000, L: int = 10, LD: np.ndarray | None = None, V: np.ndarray | None = None, Dsq: np.ndarray | None = None, est_ssq: bool = True, ssq: np.ndarray | None = None, ssq_range: tuple[float, float] = (0, 1), pi0: np.ndarray | None = None, est_sigmasq: bool = True, est_tausq: bool = False, sigmasq: float = 1, tausq: float = 0, sigmasq_range: tuple[float, float] | None = None, tausq_range: tuple[float, float] | None = None, PIP: np.ndarray | None = None, mu: np.ndarray | None = None, method: str = 'moments', maxiter: int = 100, PIP_tol: float = 0.001) -&gt; dict[str, Any]</code>  <code>staticmethod</code>","text":"<p>Susie with random effects.</p> <p>Parameters:</p> Name Type Description Default <code>z</code> <code>ndarray</code> <p>vector of z-scores (equal to X'y/sqrt(n))</p> required <code>meansq</code> <code>float</code> <p>average squared magnitude of y (equal to ||y||^2/n)</p> <code>1</code> <code>n</code> <code>int</code> <p>sample size</p> <code>100000</code> <code>L</code> <code>int</code> <p>number of modeled causal effects</p> <code>10</code> <code>LD</code> <code>ndarray | None</code> <p>LD matrix (equal to X'X/n)</p> <code>None</code> <code>V</code> <code>ndarray | None</code> <p>precomputed p x p matrix of eigenvectors of X'X</p> <code>None</code> <code>Dsq</code> <code>ndarray | None</code> <p>precomputed length-p vector of eigenvalues of X'X</p> <code>None</code> <code>est_ssq</code> <code>bool</code> <p>estimate prior effect size variances s^2 using MLE</p> <code>True</code> <code>ssq</code> <code>ndarray | None</code> <p>length-L initialization s^2 for each effect</p> <code>None</code> <code>ssq_range</code> <code>tuple[float, float]</code> <p>lower and upper bounds for each s^2, if estimated</p> <code>(0, 1)</code> <code>pi0</code> <code>ndarray | None</code> <p>length-p vector of prior causal probability for each SNP; must sum to 1</p> <code>None</code> <code>est_sigmasq</code> <code>bool</code> <p>estimate variance sigma^2</p> <code>True</code> <code>est_tausq</code> <code>bool</code> <p>estimate both variances sigma^2 and tau^2</p> <code>False</code> <code>sigmasq</code> <code>float</code> <p>initial value for sigma^2</p> <code>1</code> <code>tausq</code> <code>float</code> <p>initial value for tau^2</p> <code>0</code> <code>sigmasq_range</code> <code>tuple[float, float] | None</code> <p>lower and upper bounds for sigma^2, if estimated using MLE</p> <code>None</code> <code>tausq_range</code> <code>tuple[float, float] | None</code> <p>lower and upper bounds for tau^2, if estimated using MLE</p> <code>None</code> <code>PIP</code> <code>ndarray | None</code> <p>p x L initializations of PIPs</p> <code>None</code> <code>mu</code> <code>ndarray | None</code> <p>p x L initializations of mu</p> <code>None</code> <code>method</code> <code>str</code> <p>one of {'moments','MLE'}</p> <code>'moments'</code> <code>maxiter</code> <code>int</code> <p>maximum number of SuSiE iterations</p> <code>100</code> <code>PIP_tol</code> <code>float</code> <p>convergence threshold for PIP difference between iterations</p> <code>0.001</code> <p>Returns:</p> Type Description <code>dict[str, Any]</code> <p>dict[str, Any]: Dictionary with keys: PIP -- p x L matrix of PIPs, individually for each effect mu -- p x L matrix of posterior means conditional on causal omega -- p x L matrix of posterior precisions conditional on causal lbf_variable -- p x L matrix of log-Bayes-factors, for each effect ssq -- length-L array of final effect size variances s^2 sigmasq -- final value of sigma^2 tausq -- final value of tau^2 alpha -- length-p array of posterior means of infinitesimal effects lbf -- length-p array of log-Bayes-factors for each CS</p> <p>Raises:</p> Type Description <code>RuntimeError</code> <p>if missing LD</p> <code>RuntimeError</code> <p>if unsupported variance estimation method</p> Source code in <code>src/gentropy/method/susie_inf.py</code> <pre><code>@staticmethod\ndef susie_inf(  # noqa: C901\n    z: np.ndarray,\n    meansq: float = 1,\n    n: int = 100000,\n    L: int = 10,\n    LD: np.ndarray | None = None,\n    V: np.ndarray | None = None,\n    Dsq: np.ndarray | None = None,\n    est_ssq: bool = True,\n    ssq: np.ndarray | None = None,\n    ssq_range: tuple[float, float] = (0, 1),\n    pi0: np.ndarray | None = None,\n    est_sigmasq: bool = True,\n    est_tausq: bool = False,\n    sigmasq: float = 1,\n    tausq: float = 0,\n    sigmasq_range: tuple[float, float] | None = None,\n    tausq_range: tuple[float, float] | None = None,\n    PIP: np.ndarray | None = None,\n    mu: np.ndarray | None = None,\n    method: str = \"moments\",\n    maxiter: int = 100,\n    PIP_tol: float = 0.001,\n) -&gt; dict[str, Any]:\n    \"\"\"Susie with random effects.\n\n    Args:\n        z (np.ndarray): vector of z-scores (equal to X'y/sqrt(n))\n        meansq (float): average squared magnitude of y (equal to ||y||^2/n)\n        n (int): sample size\n        L (int): number of modeled causal effects\n        LD (np.ndarray | None): LD matrix (equal to X'X/n)\n        V (np.ndarray | None): precomputed p x p matrix of eigenvectors of X'X\n        Dsq (np.ndarray | None): precomputed length-p vector of eigenvalues of X'X\n        est_ssq (bool): estimate prior effect size variances s^2 using MLE\n        ssq (np.ndarray | None): length-L initialization s^2 for each effect\n        ssq_range (tuple[float, float]): lower and upper bounds for each s^2, if estimated\n        pi0 (np.ndarray | None): length-p vector of prior causal probability for each SNP; must sum to 1\n        est_sigmasq (bool): estimate variance sigma^2\n        est_tausq (bool): estimate both variances sigma^2 and tau^2\n        sigmasq (float): initial value for sigma^2\n        tausq (float): initial value for tau^2\n        sigmasq_range (tuple[float, float] | None): lower and upper bounds for sigma^2, if estimated using MLE\n        tausq_range (tuple[float, float] | None): lower and upper bounds for tau^2, if estimated using MLE\n        PIP (np.ndarray | None): p x L initializations of PIPs\n        mu (np.ndarray | None): p x L initializations of mu\n        method (str): one of {'moments','MLE'}\n        maxiter (int): maximum number of SuSiE iterations\n        PIP_tol (float): convergence threshold for PIP difference between iterations\n\n    Returns:\n        dict[str, Any]: Dictionary with keys:\n            PIP -- p x L matrix of PIPs, individually for each effect\n            mu -- p x L matrix of posterior means conditional on causal\n            omega -- p x L matrix of posterior precisions conditional on causal\n            lbf_variable -- p x L matrix of log-Bayes-factors, for each effect\n            ssq -- length-L array of final effect size variances s^2\n            sigmasq -- final value of sigma^2\n            tausq -- final value of tau^2\n            alpha -- length-p array of posterior means of infinitesimal effects\n            lbf -- length-p array of log-Bayes-factors for each CS\n\n    Raises:\n        RuntimeError: if missing LD\n        RuntimeError: if unsupported variance estimation method\n    \"\"\"\n    p = len(z)\n    # Precompute V,D^2 in the SVD X=UDV', and V'X'y and y'y\n    if (V is None or Dsq is None) and LD is None:\n        raise RuntimeError(\"Missing LD\")\n    elif V is None or Dsq is None:\n        eigvals, V = scipy.linalg.eigh(LD)\n        Dsq = np.maximum(n * eigvals, 0)\n    else:\n        Dsq = np.maximum(Dsq, 0)\n    Xty = np.sqrt(n) * z\n    VtXty = V.T.dot(Xty)\n    yty = n * meansq\n    # Initialize diagonal variances, diag(X' Omega X), X' Omega y\n    var = tausq * Dsq + sigmasq\n    diagXtOmegaX = np.sum(V**2 * (Dsq / var), axis=1)\n    XtOmegay = V.dot(VtXty / var)\n    # Initialize s_l^2, PIP_j, mu_j, omega_j\n    if ssq is None:\n        ssq = np.ones(L) * 0.2\n    if PIP is None:\n        PIP = np.ones((p, L)) / p\n    if mu is None:\n        mu = np.zeros((p, L))\n    lbf_variable = np.zeros((p, L))\n    omega = diagXtOmegaX[:, np.newaxis] + 1 / ssq\n    # Initialize prior causal probabilities\n    if pi0 is None:\n        logpi0 = np.ones(p) * np.log(1.0 / p)\n    else:\n        logpi0 = -np.ones(p) * np.inf\n        inds = np.nonzero(pi0 &gt; 0)[0]\n        logpi0[inds] = np.log(pi0[inds])\n\n    ####### Main SuSiE iteration loop ######\n    def f(x: float) -&gt; float:\n        \"\"\"Negative ELBO as function of x = sigma_e^2.\n\n        Args:\n            x (float): sigma_e^2\n\n        Returns:\n            float: negative ELBO as function of x = sigma_e^2\n        \"\"\"\n        return -scipy.special.logsumexp(\n            -0.5 * np.log(1 + x * diagXtOmegaX)\n            + x * XtOmegar**2 / (2 * (1 + x * diagXtOmegaX))\n            + logpi0\n        )\n\n    for it in range(maxiter):\n        PIP_prev = PIP.copy()\n        # Single effect regression for each effect l = 1,...,L\n        for _l in range(L):\n            # Compute X' Omega r_l for residual r_l\n            b = np.sum(mu * PIP, axis=1) - mu[:, _l] * PIP[:, _l]\n            XtOmegaXb = V.dot(V.T.dot(b) * Dsq / var)\n            XtOmegar = XtOmegay - XtOmegaXb\n            if est_ssq:\n                # Update prior variance ssq[l]\n                res = minimize_scalar(f, bounds=ssq_range, method=\"bounded\")\n                if res.success:\n                    ssq[_l] = res.x\n            # Update omega, mu, and PIP\n            omega[:, _l] = diagXtOmegaX + 1 / ssq[_l]\n            mu[:, _l] = XtOmegar / omega[:, _l]\n            lbf_variable[:, _l] = XtOmegar**2 / (2 * omega[:, _l]) - 0.5 * np.log(\n                omega[:, _l] * ssq[_l]\n            )\n            logPIP = lbf_variable[:, _l] + logpi0\n            PIP[:, _l] = np.exp(logPIP - scipy.special.logsumexp(logPIP))\n        # Update variance components\n        if est_sigmasq or est_tausq:\n            if method == \"moments\":\n                (sigmasq, tausq) = SUSIE_inf._MoM(\n                    PIP,\n                    mu,\n                    omega,\n                    sigmasq,\n                    tausq,\n                    n,\n                    V,\n                    Dsq,\n                    VtXty,\n                    Xty,\n                    yty,\n                    est_sigmasq,\n                    est_tausq,\n                )\n            elif method == \"MLE\":\n                (sigmasq, tausq) = SUSIE_inf._MLE(\n                    PIP,\n                    mu,\n                    omega,\n                    sigmasq,\n                    tausq,\n                    n,\n                    V,\n                    Dsq,\n                    VtXty,\n                    yty,\n                    est_sigmasq,\n                    est_tausq,\n                    it,\n                    sigmasq_range,\n                    tausq_range,\n                )\n            else:\n                raise RuntimeError(\"Unsupported variance estimation method\")\n            # Update X' Omega X, X' Omega y\n            var = tausq * Dsq + sigmasq\n            diagXtOmegaX = np.sum(V**2 * (Dsq / var), axis=1)\n            XtOmegay = V.dot(VtXty / var)\n        # Determine convergence from PIP differences\n        PIP_diff = np.max(np.abs(PIP_prev - PIP))\n        if PIP_diff &lt; PIP_tol:\n            break\n    # Compute posterior means of b and alpha\n    b = np.sum(mu * PIP, axis=1)\n    XtOmegaXb = V.dot(V.T.dot(b) * Dsq / var)\n    XtOmegar = XtOmegay - XtOmegaXb\n    alpha = tausq * XtOmegar\n\n    priors = np.log(np.repeat(1 / p, p))\n    lbf_cs = np.apply_along_axis(\n        lambda x: logsumexp(x + priors), axis=0, arr=lbf_variable\n    )\n    return {\n        \"PIP\": PIP,\n        \"mu\": mu,\n        \"omega\": omega,\n        \"lbf_variable\": lbf_variable,\n        \"ssq\": ssq,\n        \"sigmasq\": sigmasq,\n        \"tausq\": tausq,\n        \"alpha\": alpha,\n        \"lbf\": lbf_cs,\n    }\n</code></pre>"},{"location":"python_api/methods/l2g/_l2g/","title":"Locus to Gene (L2G) model","text":"<p>The \u201clocus-to-gene\u201d (L2G) model derives features to prioritize likely causal genes at each GWAS locus based on genetic and functional genomics features. The main categories of predictive features are:</p> <ul> <li>Distance: (from credible set variants to gene)</li> <li>Molecular QTL Colocalization</li> <li>Chromatin Interaction: (e.g., promoter-capture Hi-C)</li> <li>Variant Pathogenicity: (from VEP)</li> </ul> <p>The L2G model is distinct from the variant-to-gene (V2G) pipeline in that it:</p> <ul> <li>Uses a machine-learning model to learn the weights of each evidence source based on a gold standard of previously identified causal genes.</li> <li>Relies upon fine-mapping and colocalization data.</li> </ul> <p>Some of the predictive features weight variant-to-gene (or genomic region-to-gene) evidence based on the posterior probability that the variant is causal, determined through fine-mapping of the GWAS association.</p> <p>Details of the L2G model are provided in our Nature Genetics publication (ref - Nature Genetics Publication):</p> <ul> <li>Title: An open approach to systematically prioritize causal variants and genes at all published human GWAS trait-associated loci.</li> <li>Authors: Mountjoy, E., Schmidt, E.M., Carmona, M. et al.</li> <li>Journal: Nat Genet 53, 1527\u20131533 (2021).</li> <li>DOI: 10.1038/s41588-021-00945-5</li> </ul>"},{"location":"python_api/methods/l2g/evaluator/","title":"W&B evaluator","text":""},{"location":"python_api/methods/l2g/evaluator/#gentropy.method.l2g.evaluator.WandbEvaluator","title":"<code>gentropy.method.l2g.evaluator.WandbEvaluator</code>","text":"<p>             Bases: <code>Evaluator</code></p> <p>Wrapper for pyspark Evaluators. It is expected that the user will provide an Evaluators, and this wrapper will log metrics from said evaluator to W&amp;B.</p> Source code in <code>src/gentropy/method/l2g/evaluator.py</code> <pre><code>class WandbEvaluator(Evaluator):\n    \"\"\"Wrapper for pyspark Evaluators. It is expected that the user will provide an Evaluators, and this wrapper will log metrics from said evaluator to W&amp;B.\"\"\"\n\n    spark_ml_evaluator: Param[Evaluator] = Param(\n        Params._dummy(), \"spark_ml_evaluator\", \"evaluator from pyspark.ml.evaluation\"\n    )\n\n    wandb_run: Param[Run] = Param(\n        Params._dummy(),\n        \"wandb_run\",\n        \"wandb run.  Expects an already initialized run.  You should set this, or wandb_run_kwargs, NOT BOTH\",\n    )\n\n    wandb_run_kwargs: Param[Any] = Param(\n        Params._dummy(),\n        \"wandb_run_kwargs\",\n        \"kwargs to be passed to wandb.init.  You should set this, or wandb_runId, NOT BOTH.  Setting this is useful when using with WandbCrossValdidator\",\n    )\n\n    wandb_runId: Param[str] = Param(  # noqa: N815\n        Params._dummy(),\n        \"wandb_runId\",\n        \"wandb run id.  if not providing an intialized run to wandb_run, a run with id wandb_runId will be resumed\",\n    )\n\n    wandb_project_name: Param[str] = Param(\n        Params._dummy(),\n        \"wandb_project_name\",\n        \"name of W&amp;B project\",\n        typeConverter=TypeConverters.toString,\n    )\n\n    label_values: Param[list[str]] = Param(\n        Params._dummy(),\n        \"label_values\",\n        \"for classification and multiclass classification, this is a list of values the label can assume\\nIf provided Multiclass or Multilabel evaluator without label_values, we'll figure it out from dataset passed through to evaluate.\",\n    )\n\n    _input_kwargs: Dict[str, Any]\n\n    @keyword_only\n    def __init__(\n        self: WandbEvaluator,\n        label_values: list[str] | None = None,\n        **kwargs: BinaryClassificationEvaluator\n        | MulticlassClassificationEvaluator\n        | Run,\n    ) -&gt; None:\n        \"\"\"Initialize a WandbEvaluator.\n\n        Args:\n            label_values (list[str] | None): List of label values.\n            **kwargs (BinaryClassificationEvaluator | MulticlassClassificationEvaluator | Run): Keyword arguments.\n        \"\"\"\n        if label_values is None:\n            label_values = []\n        super(Evaluator, self).__init__()\n\n        self.metrics = {\n            MulticlassClassificationEvaluator: [\n                \"f1\",\n                \"accuracy\",\n                \"weightedPrecision\",\n                \"weightedRecall\",\n                \"weightedTruePositiveRate\",\n                \"weightedFalsePositiveRate\",\n                \"weightedFMeasure\",\n                \"truePositiveRateByLabel\",\n                \"falsePositiveRateByLabel\",\n                \"precisionByLabel\",\n                \"recallByLabel\",\n                \"fMeasureByLabel\",\n                \"logLoss\",\n                \"hammingLoss\",\n            ],\n            BinaryClassificationEvaluator: [\"areaUnderROC\", \"areaUnderPR\"],\n        }\n\n        self._setDefault(label_values=[])\n        kwargs = self._input_kwargs\n        self._set(**kwargs)\n\n    def setspark_ml_evaluator(self: WandbEvaluator, value: Evaluator) -&gt; None:\n        \"\"\"Set the spark_ml_evaluator parameter.\n\n        Args:\n            value (Evaluator): Spark ML evaluator.\n        \"\"\"\n        self._set(spark_ml_evaluator=value)\n\n    def setlabel_values(self: WandbEvaluator, value: list[str]) -&gt; None:\n        \"\"\"Set the label_values parameter.\n\n        Args:\n            value (list[str]): List of label values.\n        \"\"\"\n        self._set(label_values=value)\n\n    def getspark_ml_evaluator(self: WandbEvaluator) -&gt; Evaluator:\n        \"\"\"Get the spark_ml_evaluator parameter.\n\n        Returns:\n            Evaluator: Spark ML evaluator.\n        \"\"\"\n        return self.getOrDefault(self.spark_ml_evaluator)\n\n    def getwandb_run(self: WandbEvaluator) -&gt; Run:\n        \"\"\"Get the wandb_run parameter.\n\n        Returns:\n            Run: Wandb run object.\n        \"\"\"\n        return self.getOrDefault(self.wandb_run)\n\n    def getwandb_project_name(self: WandbEvaluator) -&gt; Any:\n        \"\"\"Get the wandb_project_name parameter.\n\n        Returns:\n            Any: Name of the W&amp;B project.\n        \"\"\"\n        return self.getOrDefault(self.wandb_project_name)\n\n    def getlabel_values(self: WandbEvaluator) -&gt; list[str]:\n        \"\"\"Get the label_values parameter.\n\n        Returns:\n            list[str]: List of label values.\n        \"\"\"\n        return self.getOrDefault(self.label_values)\n\n    def _evaluate(self: WandbEvaluator, dataset: DataFrame) -&gt; float:\n        \"\"\"Evaluate the model on the given dataset.\n\n        Args:\n            dataset (DataFrame): Dataset to evaluate the model on.\n\n        Returns:\n            float: Metric value.\n        \"\"\"\n        dataset.persist()\n        metric_values: list[tuple[str, Any]] = []\n        label_values = self.getlabel_values()\n        spark_ml_evaluator: BinaryClassificationEvaluator | MulticlassClassificationEvaluator = (\n            self.getspark_ml_evaluator()  # type: ignore[assignment, unused-ignore]\n        )\n        run = self.getwandb_run()\n        evaluator_type = type(spark_ml_evaluator)\n        for metric in self.metrics[evaluator_type]:\n            if \"ByLabel\" in metric and label_values == []:\n                print(  # noqa: T201\n                    \"no label_values for the target have been provided and will be determined by the dataset.  This could take some time\"\n                )\n                label_values = [\n                    r[spark_ml_evaluator.getLabelCol()]\n                    for r in dataset.select(spark_ml_evaluator.getLabelCol())\n                    .distinct()\n                    .collect()\n                ]\n                if isinstance(label_values[0], list):\n                    merged = list(itertools.chain(*label_values))\n                    label_values = list(dict.fromkeys(merged).keys())\n                    self.setlabel_values(label_values)\n            for label in label_values:\n                out = spark_ml_evaluator.evaluate(\n                    dataset,\n                    {\n                        spark_ml_evaluator.metricLabel: label,  # type: ignore[assignment, unused-ignore]\n                        spark_ml_evaluator.metricName: metric,\n                    },\n                )\n                metric_values.append((f\"{metric}:{label}\", out))\n            out = spark_ml_evaluator.evaluate(\n                dataset, {spark_ml_evaluator.metricName: metric}\n            )\n            metric_values.append((f\"{metric}\", out))\n        run.log(dict(metric_values))\n        config = [\n            (f\"{k.parent.split('_')[0]}.{k.name}\", v)\n            for k, v in spark_ml_evaluator.extractParamMap().items()\n            if \"metric\" not in k.name\n        ]\n        run.config.update(dict(config))\n        return_metric = spark_ml_evaluator.evaluate(dataset)\n        dataset.unpersist()\n        return return_metric\n</code></pre>"},{"location":"python_api/methods/l2g/evaluator/#gentropy.method.l2g.evaluator.WandbEvaluator.__init__","title":"<code>__init__(label_values: list[str] | None = None, **kwargs: BinaryClassificationEvaluator | MulticlassClassificationEvaluator | Run) -&gt; None</code>","text":"<p>Initialize a WandbEvaluator.</p> <p>Parameters:</p> Name Type Description Default <code>label_values</code> <code>list[str] | None</code> <p>List of label values.</p> <code>None</code> <code>**kwargs</code> <code>BinaryClassificationEvaluator | MulticlassClassificationEvaluator | Run</code> <p>Keyword arguments.</p> <code>{}</code> Source code in <code>src/gentropy/method/l2g/evaluator.py</code> <pre><code>@keyword_only\ndef __init__(\n    self: WandbEvaluator,\n    label_values: list[str] | None = None,\n    **kwargs: BinaryClassificationEvaluator\n    | MulticlassClassificationEvaluator\n    | Run,\n) -&gt; None:\n    \"\"\"Initialize a WandbEvaluator.\n\n    Args:\n        label_values (list[str] | None): List of label values.\n        **kwargs (BinaryClassificationEvaluator | MulticlassClassificationEvaluator | Run): Keyword arguments.\n    \"\"\"\n    if label_values is None:\n        label_values = []\n    super(Evaluator, self).__init__()\n\n    self.metrics = {\n        MulticlassClassificationEvaluator: [\n            \"f1\",\n            \"accuracy\",\n            \"weightedPrecision\",\n            \"weightedRecall\",\n            \"weightedTruePositiveRate\",\n            \"weightedFalsePositiveRate\",\n            \"weightedFMeasure\",\n            \"truePositiveRateByLabel\",\n            \"falsePositiveRateByLabel\",\n            \"precisionByLabel\",\n            \"recallByLabel\",\n            \"fMeasureByLabel\",\n            \"logLoss\",\n            \"hammingLoss\",\n        ],\n        BinaryClassificationEvaluator: [\"areaUnderROC\", \"areaUnderPR\"],\n    }\n\n    self._setDefault(label_values=[])\n    kwargs = self._input_kwargs\n    self._set(**kwargs)\n</code></pre>"},{"location":"python_api/methods/l2g/evaluator/#gentropy.method.l2g.evaluator.WandbEvaluator.getlabel_values","title":"<code>getlabel_values() -&gt; list[str]</code>","text":"<p>Get the label_values parameter.</p> <p>Returns:</p> Type Description <code>list[str]</code> <p>list[str]: List of label values.</p> Source code in <code>src/gentropy/method/l2g/evaluator.py</code> <pre><code>def getlabel_values(self: WandbEvaluator) -&gt; list[str]:\n    \"\"\"Get the label_values parameter.\n\n    Returns:\n        list[str]: List of label values.\n    \"\"\"\n    return self.getOrDefault(self.label_values)\n</code></pre>"},{"location":"python_api/methods/l2g/evaluator/#gentropy.method.l2g.evaluator.WandbEvaluator.getspark_ml_evaluator","title":"<code>getspark_ml_evaluator() -&gt; Evaluator</code>","text":"<p>Get the spark_ml_evaluator parameter.</p> <p>Returns:</p> Name Type Description <code>Evaluator</code> <code>Evaluator</code> <p>Spark ML evaluator.</p> Source code in <code>src/gentropy/method/l2g/evaluator.py</code> <pre><code>def getspark_ml_evaluator(self: WandbEvaluator) -&gt; Evaluator:\n    \"\"\"Get the spark_ml_evaluator parameter.\n\n    Returns:\n        Evaluator: Spark ML evaluator.\n    \"\"\"\n    return self.getOrDefault(self.spark_ml_evaluator)\n</code></pre>"},{"location":"python_api/methods/l2g/evaluator/#gentropy.method.l2g.evaluator.WandbEvaluator.getwandb_project_name","title":"<code>getwandb_project_name() -&gt; Any</code>","text":"<p>Get the wandb_project_name parameter.</p> <p>Returns:</p> Name Type Description <code>Any</code> <code>Any</code> <p>Name of the W&amp;B project.</p> Source code in <code>src/gentropy/method/l2g/evaluator.py</code> <pre><code>def getwandb_project_name(self: WandbEvaluator) -&gt; Any:\n    \"\"\"Get the wandb_project_name parameter.\n\n    Returns:\n        Any: Name of the W&amp;B project.\n    \"\"\"\n    return self.getOrDefault(self.wandb_project_name)\n</code></pre>"},{"location":"python_api/methods/l2g/evaluator/#gentropy.method.l2g.evaluator.WandbEvaluator.getwandb_run","title":"<code>getwandb_run() -&gt; Run</code>","text":"<p>Get the wandb_run parameter.</p> <p>Returns:</p> Name Type Description <code>Run</code> <code>Run</code> <p>Wandb run object.</p> Source code in <code>src/gentropy/method/l2g/evaluator.py</code> <pre><code>def getwandb_run(self: WandbEvaluator) -&gt; Run:\n    \"\"\"Get the wandb_run parameter.\n\n    Returns:\n        Run: Wandb run object.\n    \"\"\"\n    return self.getOrDefault(self.wandb_run)\n</code></pre>"},{"location":"python_api/methods/l2g/evaluator/#gentropy.method.l2g.evaluator.WandbEvaluator.setlabel_values","title":"<code>setlabel_values(value: list[str]) -&gt; None</code>","text":"<p>Set the label_values parameter.</p> <p>Parameters:</p> Name Type Description Default <code>value</code> <code>list[str]</code> <p>List of label values.</p> required Source code in <code>src/gentropy/method/l2g/evaluator.py</code> <pre><code>def setlabel_values(self: WandbEvaluator, value: list[str]) -&gt; None:\n    \"\"\"Set the label_values parameter.\n\n    Args:\n        value (list[str]): List of label values.\n    \"\"\"\n    self._set(label_values=value)\n</code></pre>"},{"location":"python_api/methods/l2g/evaluator/#gentropy.method.l2g.evaluator.WandbEvaluator.setspark_ml_evaluator","title":"<code>setspark_ml_evaluator(value: Evaluator) -&gt; None</code>","text":"<p>Set the spark_ml_evaluator parameter.</p> <p>Parameters:</p> Name Type Description Default <code>value</code> <code>Evaluator</code> <p>Spark ML evaluator.</p> required Source code in <code>src/gentropy/method/l2g/evaluator.py</code> <pre><code>def setspark_ml_evaluator(self: WandbEvaluator, value: Evaluator) -&gt; None:\n    \"\"\"Set the spark_ml_evaluator parameter.\n\n    Args:\n        value (Evaluator): Spark ML evaluator.\n    \"\"\"\n    self._set(spark_ml_evaluator=value)\n</code></pre>"},{"location":"python_api/methods/l2g/feature_factory/","title":"L2G Feature Factory","text":""},{"location":"python_api/methods/l2g/feature_factory/#gentropy.method.l2g.feature_factory.ColocalisationFactory","title":"<code>gentropy.method.l2g.feature_factory.ColocalisationFactory</code>","text":"<p>Feature extraction in colocalisation.</p> Source code in <code>src/gentropy/method/l2g/feature_factory.py</code> <pre><code>class ColocalisationFactory:\n    \"\"\"Feature extraction in colocalisation.\"\"\"\n\n    @classmethod\n    def _add_colocalisation_metric(cls: type[ColocalisationFactory]) -&gt; Column:\n        \"\"\"Expression that adds a `colocalisationMetric` column to the colocalisation dataframe in preparation for feature extraction.\n\n        Returns:\n            Column: The expression that adds a `colocalisationMetric` column with the derived metric\n        \"\"\"\n        method_metric_map = {\n            ECaviar.METHOD_NAME: ECaviar.METHOD_METRIC,\n            Coloc.METHOD_NAME: Coloc.METHOD_METRIC,\n        }\n        map_expr = f.create_map(*[f.lit(x) for x in chain(*method_metric_map.items())])\n        return map_expr[f.col(\"colocalisationMethod\")].alias(\"colocalisationMetric\")\n\n    @staticmethod\n    def _get_max_coloc_per_credible_set(\n        colocalisation: Colocalisation,\n        credible_set: StudyLocus,\n        studies: StudyIndex,\n    ) -&gt; L2GFeature:\n        \"\"\"Get the maximum colocalisation posterior probability for each pair of overlapping study-locus per type of colocalisation method and QTL type.\n\n        Args:\n            colocalisation (Colocalisation): Colocalisation dataset\n            credible_set (StudyLocus): Study locus dataset\n            studies (StudyIndex): Study index dataset\n\n        Returns:\n            L2GFeature: Stores the features with the max coloc probabilities for each pair of study-locus\n        \"\"\"\n        colocalisation_df = colocalisation.df.select(\n            f.col(\"leftStudyLocusId\").alias(\"studyLocusId\"),\n            \"rightStudyLocusId\",\n            f.coalesce(\"log2h4h3\", \"clpp\").alias(\"score\"),\n            ColocalisationFactory._add_colocalisation_metric(),\n        )\n\n        colocalising_credible_sets = (\n            credible_set.df.select(\"studyLocusId\", \"studyId\")\n            # annotate studyLoci with overlapping IDs on the left - to just keep GWAS associations\n            .join(\n                colocalisation_df,\n                on=\"studyLocusId\",\n                how=\"inner\",\n            )\n            # bring study metadata to just keep QTL studies on the right\n            .join(\n                credible_set.df.join(\n                    studies.df.select(\"studyId\", \"studyType\", \"geneId\"), \"studyId\"\n                ).selectExpr(\n                    \"studyLocusId as rightStudyLocusId\",\n                    \"studyType as right_studyType\",\n                    \"geneId\",\n                ),\n                on=\"rightStudyLocusId\",\n                how=\"inner\",\n            )\n            .filter(f.col(\"right_studyType\") != \"gwas\")\n            .select(\n                \"studyLocusId\",\n                \"right_studyType\",\n                \"geneId\",\n                \"score\",\n                \"colocalisationMetric\",\n            )\n        )\n\n        # Max PP calculation per credible set AND type of QTL AND colocalisation method\n        local_max = (\n            get_record_with_maximum_value(\n                colocalising_credible_sets,\n                [\"studyLocusId\", \"right_studyType\", \"geneId\", \"colocalisationMetric\"],\n                \"score\",\n            )\n            .select(\n                \"*\",\n                f.col(\"score\").alias(\"max_score\"),\n                f.lit(\"Local\").alias(\"score_type\"),\n            )\n            .drop(\"score\")\n        )\n\n        neighbourhood_max = (\n            local_max.selectExpr(\n                \"studyLocusId\", \"max_score as local_max_score\", \"geneId\"\n            )\n            .join(\n                # Add maximum in the neighborhood\n                get_record_with_maximum_value(\n                    colocalising_credible_sets.withColumnRenamed(\n                        \"score\", \"tmp_nbh_max_score\"\n                    ),\n                    [\"studyLocusId\", \"right_studyType\", \"colocalisationMetric\"],\n                    \"tmp_nbh_max_score\",\n                ).drop(\"geneId\"),\n                on=\"studyLocusId\",\n            )\n            .withColumn(\"score_type\", f.lit(\"Neighborhood\"))\n            .withColumn(\n                \"max_score\",\n                f.log10(\n                    f.abs(\n                        f.col(\"local_max_score\")\n                        - f.col(\"tmp_nbh_max_score\")\n                        + f.lit(0.0001)  # intercept\n                    )\n                ),\n            )\n        ).drop(\"tmp_nbh_max_score\", \"local_max_score\")\n\n        return L2GFeature(\n            _df=(\n                # Combine local and neighborhood metrics\n                local_max.unionByName(\n                    neighbourhood_max, allowMissingColumns=True\n                ).select(\n                    \"studyLocusId\",\n                    \"geneId\",\n                    # Feature name is a concatenation of the QTL type, colocalisation metric and if it's local or in the vicinity\n                    f.concat_ws(\n                        \"\",\n                        f.col(\"right_studyType\"),\n                        f.lit(\"Coloc\"),\n                        f.initcap(f.col(\"colocalisationMetric\")),\n                        f.lit(\"Maximum\"),\n                        f.regexp_replace(f.col(\"score_type\"), \"Local\", \"\"),\n                    ).alias(\"featureName\"),\n                    f.col(\"max_score\").cast(\"float\").alias(\"featureValue\"),\n                )\n            ),\n            _schema=L2GFeature.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/methods/l2g/feature_factory/#gentropy.method.l2g.feature_factory.StudyLocusFactory","title":"<code>gentropy.method.l2g.feature_factory.StudyLocusFactory</code>  <code>dataclass</code>","text":"<p>             Bases: <code>StudyLocus</code></p> <p>Feature extraction in study locus.</p> Source code in <code>src/gentropy/method/l2g/feature_factory.py</code> <pre><code>class StudyLocusFactory(StudyLocus):\n    \"\"\"Feature extraction in study locus.\"\"\"\n\n    @staticmethod\n    def _get_tss_distance_features(credible_set: StudyLocus, v2g: V2G) -&gt; L2GFeature:\n        \"\"\"Joins StudyLocus with the V2G to extract a score that is based on the distance to a gene TSS of any variant weighted by its posterior probability in a credible set.\n\n        Args:\n            credible_set (StudyLocus): Credible set dataset\n            v2g (V2G): Dataframe containing the distances of all variants to all genes TSS within a region\n\n        Returns:\n            L2GFeature: Stores the features with the score of weighting the distance to the TSS by the posterior probability of the variant\n\n        \"\"\"\n        wide_df = (\n            credible_set.filter_credible_set(CredibleInterval.IS95)\n            .df.withColumn(\"variantInLocus\", f.explode_outer(\"locus\"))\n            .select(\n                \"studyLocusId\",\n                \"variantId\",\n                f.col(\"variantInLocus.variantId\").alias(\"variantInLocusId\"),\n                f.col(\"variantInLocus.posteriorProbability\").alias(\n                    \"variantInLocusPosteriorProbability\"\n                ),\n            )\n            .join(\n                v2g.df.filter(f.col(\"datasourceId\") == \"canonical_tss\").selectExpr(\n                    \"variantId as variantInLocusId\", \"geneId\", \"score\"\n                ),\n                on=\"variantInLocusId\",\n                how=\"inner\",\n            )\n            .withColumn(\n                \"weightedScore\",\n                f.col(\"score\") * f.col(\"variantInLocusPosteriorProbability\"),\n            )\n            .groupBy(\"studyLocusId\", \"geneId\")\n            .agg(\n                f.min(\"weightedScore\").alias(\"distanceTssMinimum\"),\n                f.mean(\"weightedScore\").alias(\"distanceTssMean\"),\n            )\n        )\n\n        return L2GFeature(\n            _df=convert_from_wide_to_long(\n                wide_df,\n                id_vars=(\"studyLocusId\", \"geneId\"),\n                var_name=\"featureName\",\n                value_name=\"featureValue\",\n            ),\n            _schema=L2GFeature.get_schema(),\n        )\n\n    @staticmethod\n    def _get_vep_features(\n        credible_set: StudyLocus,\n        v2g: V2G,\n    ) -&gt; L2GFeature:\n        \"\"\"Get the maximum VEP score for all variants in a locus's 95% credible set.\n\n        This informs about functional impact of the variants in the locus. For more information on variant consequences, see: https://www.ensembl.org/info/genome/variation/prediction/predicted_data.html\n        Two metrics: max VEP score per study locus and gene, and max VEP score per study locus.\n\n\n        Args:\n            credible_set (StudyLocus): Study locus dataset with the associations to be annotated\n            v2g (V2G): V2G dataset with the variant/gene relationships and their consequences\n\n        Returns:\n            L2GFeature: Stores the features with the max VEP score.\n        \"\"\"\n\n        def _aggregate_vep_feature(\n            df: DataFrame,\n            aggregation_expr: Column,\n            aggregation_cols: list[str],\n            feature_name: str,\n        ) -&gt; DataFrame:\n            \"\"\"Extracts the maximum or average VEP score after grouping by the given columns. Different aggregations return different predictive annotations.\n\n            If the group_cols include \"geneId\", the maximum/mean VEP score per gene is returned.\n            Otherwise, the maximum/mean VEP score for all genes in the neighborhood of the locus is returned.\n\n            Args:\n                df (DataFrame): DataFrame with the VEP scores for each variant in a studyLocus\n                aggregation_expr (Column): Aggregation expression to apply\n                aggregation_cols (list[str]): Columns to group by\n                feature_name (str): Name of the feature to be returned\n\n            Returns:\n                DataFrame: DataFrame with the maximum VEP score per locus or per locus/gene\n            \"\"\"\n            if \"geneId\" in aggregation_cols:\n                return df.groupBy(aggregation_cols).agg(\n                    aggregation_expr.alias(feature_name)\n                )\n            return (\n                df.groupBy(aggregation_cols)\n                .agg(\n                    aggregation_expr.alias(feature_name),\n                    f.collect_set(\"geneId\").alias(\"geneId\"),\n                )\n                .withColumn(\"geneId\", f.explode(\"geneId\"))\n            )\n\n        credible_set_w_variant_consequences = (\n            credible_set.filter_credible_set(CredibleInterval.IS95)\n            .df.withColumn(\"variantInLocus\", f.explode_outer(\"locus\"))\n            .select(\n                f.col(\"studyLocusId\"),\n                f.col(\"variantId\"),\n                f.col(\"studyId\"),\n                f.col(\"variantInLocus.variantId\").alias(\"variantInLocusId\"),\n                f.col(\"variantInLocus.posteriorProbability\").alias(\n                    \"variantInLocusPosteriorProbability\"\n                ),\n            )\n            .join(\n                # Join with V2G to get variant consequences\n                v2g.df.filter(f.col(\"datasourceId\") == \"variantConsequence\").selectExpr(\n                    \"variantId as variantInLocusId\", \"geneId\", \"score\"\n                ),\n                on=\"variantInLocusId\",\n            )\n            .select(\n                \"studyLocusId\",\n                \"variantId\",\n                \"studyId\",\n                \"geneId\",\n                (f.col(\"score\") * f.col(\"variantInLocusPosteriorProbability\")).alias(\n                    \"weightedScore\"\n                ),\n            )\n            .distinct()\n        )\n\n        return L2GFeature(\n            _df=convert_from_wide_to_long(\n                reduce(\n                    lambda x, y: x.unionByName(y, allowMissingColumns=True),\n                    [\n                        # Calculate overall max VEP score for all genes in the vicinity\n                        credible_set_w_variant_consequences.transform(\n                            _aggregate_vep_feature,\n                            f.max(\"weightedScore\"),\n                            [\"studyLocusId\"],\n                            \"vepMaximumNeighborhood\",\n                        ),\n                        # Calculate overall max VEP score per gene\n                        credible_set_w_variant_consequences.transform(\n                            _aggregate_vep_feature,\n                            f.max(\"weightedScore\"),\n                            [\"studyLocusId\", \"geneId\"],\n                            \"vepMaximum\",\n                        ),\n                        # Calculate mean VEP score for all genes in the vicinity\n                        credible_set_w_variant_consequences.transform(\n                            _aggregate_vep_feature,\n                            f.mean(\"weightedScore\"),\n                            [\"studyLocusId\"],\n                            \"vepMeanNeighborhood\",\n                        ),\n                        # Calculate mean VEP score per gene\n                        credible_set_w_variant_consequences.transform(\n                            _aggregate_vep_feature,\n                            f.mean(\"weightedScore\"),\n                            [\"studyLocusId\", \"geneId\"],\n                            \"vepMean\",\n                        ),\n                    ],\n                ),\n                id_vars=(\"studyLocusId\", \"geneId\"),\n                var_name=\"featureName\",\n                value_name=\"featureValue\",\n            ).filter(f.col(\"featureValue\").isNotNull()),\n            _schema=L2GFeature.get_schema(),\n        )\n</code></pre>"},{"location":"python_api/methods/l2g/model/","title":"L2G Model","text":""},{"location":"python_api/methods/l2g/model/#gentropy.method.l2g.model.LocusToGeneModel","title":"<code>gentropy.method.l2g.model.LocusToGeneModel</code>  <code>dataclass</code>","text":"<p>Wrapper for the Locus to Gene classifier.</p> Source code in <code>src/gentropy/method/l2g/model.py</code> <pre><code>@dataclass\nclass LocusToGeneModel:\n    \"\"\"Wrapper for the Locus to Gene classifier.\"\"\"\n\n    features_list: list[str]\n    estimator: Any = None\n    pipeline: Pipeline = Pipeline(stages=[])\n    model: PipelineModel | None = None\n    wandb_l2g_project_name: str = \"otg_l2g\"\n\n    def __post_init__(self: LocusToGeneModel) -&gt; None:\n        \"\"\"Post init that adds the model to the ML pipeline.\"\"\"\n        label_indexer = StringIndexer(\n            inputCol=\"goldStandardSet\", outputCol=\"label\", handleInvalid=\"keep\"\n        )\n        vector_assembler = LocusToGeneModel.features_vector_assembler(\n            self.features_list\n        )\n\n        self.pipeline = Pipeline(\n            stages=[\n                label_indexer,\n                vector_assembler,\n            ]\n        )\n\n    def save(self: LocusToGeneModel, path: str) -&gt; None:\n        \"\"\"Saves fitted pipeline model to disk.\n\n        Args:\n            path (str): Path to save the model to\n\n        Raises:\n            ValueError: If the model has not been fitted yet\n        \"\"\"\n        if self.model is None:\n            raise ValueError(\"Model has not been fitted yet.\")\n        self.model.write().overwrite().save(path)\n\n    @property\n    def classifier(self: LocusToGeneModel) -&gt; Any:\n        \"\"\"Return the model.\n\n        Returns:\n            Any: An estimator object from Spark ML\n        \"\"\"\n        return self.estimator\n\n    @staticmethod\n    def features_vector_assembler(features_cols: list[str]) -&gt; VectorAssembler:\n        \"\"\"Spark transformer to assemble the feature columns into a vector.\n\n        Args:\n            features_cols (list[str]): List of feature columns to assemble\n\n        Returns:\n            VectorAssembler: Spark transformer to assemble the feature columns into a vector\n\n        Examples:\n            &gt;&gt;&gt; from pyspark.ml.feature import VectorAssembler\n            &gt;&gt;&gt; df = spark.createDataFrame([(5.2, 3.5)], schema=\"feature_1 FLOAT, feature_2 FLOAT\")\n            &gt;&gt;&gt; assembler = LocusToGeneModel.features_vector_assembler([\"feature_1\", \"feature_2\"])\n            &gt;&gt;&gt; assembler.transform(df).show()\n            +---------+---------+--------------------+\n            |feature_1|feature_2|            features|\n            +---------+---------+--------------------+\n            |      5.2|      3.5|[5.19999980926513...|\n            +---------+---------+--------------------+\n            &lt;BLANKLINE&gt;\n        \"\"\"\n        return (\n            VectorAssembler(handleInvalid=\"error\")\n            .setInputCols(features_cols)\n            .setOutputCol(\"features\")\n        )\n\n    def log_to_wandb(\n        self: LocusToGeneModel,\n        results: DataFrame,\n        training_data: L2GFeatureMatrix,\n        evaluators: list[\n            BinaryClassificationEvaluator | MulticlassClassificationEvaluator\n        ],\n        wandb_run: Run,\n    ) -&gt; None:\n        \"\"\"Log evaluation results and feature importance to W&amp;B.\n\n        Args:\n            results (DataFrame): Dataframe containing the predictions\n            training_data (L2GFeatureMatrix): Training data used for the model. If provided, the table and the number of positive and negative labels will be logged to W&amp;B\n            evaluators (list[BinaryClassificationEvaluator | MulticlassClassificationEvaluator]): List of Spark ML evaluators to use for evaluation\n            wandb_run (Run): W&amp;B run to log the results to\n        \"\"\"\n        ## Track evaluation metrics\n        for evaluator in evaluators:\n            wandb_evaluator = WandbEvaluator(\n                spark_ml_evaluator=evaluator, wandb_run=wandb_run\n            )\n            wandb_evaluator.evaluate(results)\n        ## Track feature importance\n        wandb_run.log({\"importances\": self.get_feature_importance()})\n        ## Track training set\n        training_table = Table(dataframe=training_data.df.toPandas())\n        wandb_run.log({\"trainingSet\": training_table})\n        # Count number of positive and negative labels\n        gs_counts_dict = {\n            \"goldStandard\" + row[\"goldStandardSet\"].capitalize(): row[\"count\"]\n            for row in training_data.df.groupBy(\"goldStandardSet\").count().collect()\n        }\n        wandb_run.log(gs_counts_dict)\n        # Missingness rates\n        wandb_run.log(\n            {\"missingnessRates\": training_data.calculate_feature_missingness_rate()}\n        )\n\n    @classmethod\n    def load_from_disk(\n        cls: Type[LocusToGeneModel], path: str, features_list: list[str]\n    ) -&gt; LocusToGeneModel:\n        \"\"\"Load a fitted pipeline model from disk.\n\n        Args:\n            path (str): Path to the model\n            features_list (list[str]): List of features used for the model\n\n        Returns:\n            LocusToGeneModel: L2G model loaded from disk\n        \"\"\"\n        return cls(model=PipelineModel.load(path), features_list=features_list)\n\n    @classifier.setter  # type: ignore\n    def classifier(self: LocusToGeneModel, new_estimator: Any) -&gt; None:\n        \"\"\"Set the model.\n\n        Args:\n            new_estimator (Any): An estimator object from Spark ML\n        \"\"\"\n        self.estimator = new_estimator\n\n    def get_param_grid(self: LocusToGeneModel) -&gt; list[Any]:\n        \"\"\"Return the parameter grid for the model.\n\n        Returns:\n            list[Any]: List of parameter maps to use for cross validation\n        \"\"\"\n        return (\n            ParamGridBuilder()\n            .addGrid(self.estimator.max_depth, [3, 5, 7])\n            .addGrid(self.estimator.learning_rate, [0.01, 0.1, 1.0])\n            .build()\n        )\n\n    def add_pipeline_stage(\n        self: LocusToGeneModel, transformer: Transformer\n    ) -&gt; LocusToGeneModel:\n        \"\"\"Adds a stage to the L2G pipeline.\n\n        Args:\n            transformer (Transformer): Spark transformer to add to the pipeline\n\n        Returns:\n            LocusToGeneModel: L2G model with the new transformer\n\n        Examples:\n            &gt;&gt;&gt; from pyspark.ml.regression import LinearRegression\n            &gt;&gt;&gt; estimator = LinearRegression()\n            &gt;&gt;&gt; test_model = LocusToGeneModel(features_list=[\"a\", \"b\"])\n            &gt;&gt;&gt; print(len(test_model.pipeline.getStages()))\n            2\n            &gt;&gt;&gt; print(len(test_model.add_pipeline_stage(estimator).pipeline.getStages()))\n            3\n        \"\"\"\n        pipeline_stages = self.pipeline.getStages()\n        new_stages = pipeline_stages + [transformer]\n        self.pipeline = Pipeline(stages=new_stages)\n        return self\n\n    def evaluate(\n        self: LocusToGeneModel,\n        results: DataFrame,\n        hyperparameters: dict[str, Any],\n        wandb_run_name: str | None,\n        gold_standard_data: L2GFeatureMatrix | None = None,\n    ) -&gt; None:\n        \"\"\"Perform evaluation of the model predictions for the test set and track the results with W&amp;B.\n\n        Args:\n            results (DataFrame): Dataframe containing the predictions\n            hyperparameters (dict[str, Any]): Hyperparameters used for the model\n            wandb_run_name (str | None): Descriptive name for the run to be tracked with W&amp;B\n            gold_standard_data (L2GFeatureMatrix | None): Feature matrix for the associations in the gold standard. If provided, the ratio of positive to negative labels will be logged to W&amp;B\n        \"\"\"\n        binary_evaluator = BinaryClassificationEvaluator(\n            rawPredictionCol=\"rawPrediction\", labelCol=\"label\"\n        )\n        multi_evaluator = MulticlassClassificationEvaluator(\n            labelCol=\"label\", predictionCol=\"prediction\"\n        )\n\n        if wandb_run_name and gold_standard_data:\n            run = wandb_init(\n                project=self.wandb_l2g_project_name,\n                config=hyperparameters,\n                name=wandb_run_name,\n            )\n            if isinstance(run, Run):\n                self.log_to_wandb(\n                    results,\n                    gold_standard_data,\n                    [binary_evaluator, multi_evaluator],\n                    run,\n                )\n                run.finish()\n\n    @property\n    def feature_name_map(self: LocusToGeneModel) -&gt; dict[str, str]:\n        \"\"\"Return a dictionary mapping encoded feature names to the original names.\n\n        Returns:\n            dict[str, str]: Feature name map of the model\n\n        Raises:\n            ValueError: If the model has not been fitted yet\n        \"\"\"\n        if not self.model:\n            raise ValueError(\"Model not fitted yet. `fit()` has to be called first.\")\n        elif isinstance(self.model.stages[1], VectorAssembler):\n            feature_names = self.model.stages[1].getInputCols()\n        return {f\"f{i}\": feature_name for i, feature_name in enumerate(feature_names)}\n\n    def get_feature_importance(self: LocusToGeneModel) -&gt; dict[str, float]:\n        \"\"\"Return dictionary with relative importances of every feature in the model. Feature names are encoded and have to be mapped back to their original names.\n\n        Returns:\n            dict[str, float]: Dictionary mapping feature names to their importance\n\n        Raises:\n            ValueError: If the model has not been fitted yet or is not an XGBoost model\n        \"\"\"\n        if not self.model or not isinstance(\n            self.model.stages[-1], SparkXGBClassifierModel\n        ):\n            raise ValueError(\n                f\"Model type {type(self.model)} not supported for feature importance.\"\n            )\n        importance_map = self.model.stages[-1].get_feature_importances()\n        return {self.feature_name_map[k]: v for k, v in importance_map.items()}\n\n    def fit(\n        self: LocusToGeneModel,\n        feature_matrix: L2GFeatureMatrix,\n    ) -&gt; LocusToGeneModel:\n        \"\"\"Fit the pipeline to the feature matrix dataframe.\n\n        Args:\n            feature_matrix (L2GFeatureMatrix): Feature matrix dataframe to fit the model to\n\n        Returns:\n            LocusToGeneModel: Fitted model\n        \"\"\"\n        self.model = self.pipeline.fit(feature_matrix.df)\n        return self\n\n    def predict(\n        self: LocusToGeneModel,\n        feature_matrix: L2GFeatureMatrix,\n    ) -&gt; DataFrame:\n        \"\"\"Apply the model to a given feature matrix dataframe. The feature matrix needs to be preprocessed first.\n\n        Args:\n            feature_matrix (L2GFeatureMatrix): Feature matrix dataframe to apply the model to\n\n        Returns:\n            DataFrame: Dataframe with predictions\n\n        Raises:\n            ValueError: If the model has not been fitted yet\n        \"\"\"\n        if not self.model:\n            raise ValueError(\"Model not fitted yet. `fit()` has to be called first.\")\n        return self.model.transform(feature_matrix.df)\n</code></pre>"},{"location":"python_api/methods/l2g/model/#gentropy.method.l2g.model.LocusToGeneModel.classifier","title":"<code>classifier: Any</code>  <code>property</code> <code>writable</code>","text":"<p>Return the model.</p> <p>Returns:</p> Name Type Description <code>Any</code> <code>Any</code> <p>An estimator object from Spark ML</p>"},{"location":"python_api/methods/l2g/model/#gentropy.method.l2g.model.LocusToGeneModel.feature_name_map","title":"<code>feature_name_map: dict[str, str]</code>  <code>property</code>","text":"<p>Return a dictionary mapping encoded feature names to the original names.</p> <p>Returns:</p> Type Description <code>dict[str, str]</code> <p>dict[str, str]: Feature name map of the model</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>If the model has not been fitted yet</p>"},{"location":"python_api/methods/l2g/model/#gentropy.method.l2g.model.LocusToGeneModel.add_pipeline_stage","title":"<code>add_pipeline_stage(transformer: Transformer) -&gt; LocusToGeneModel</code>","text":"<p>Adds a stage to the L2G pipeline.</p> <p>Parameters:</p> Name Type Description Default <code>transformer</code> <code>Transformer</code> <p>Spark transformer to add to the pipeline</p> required <p>Returns:</p> Name Type Description <code>LocusToGeneModel</code> <code>LocusToGeneModel</code> <p>L2G model with the new transformer</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pyspark.ml.regression import LinearRegression\n&gt;&gt;&gt; estimator = LinearRegression()\n&gt;&gt;&gt; test_model = LocusToGeneModel(features_list=[\"a\", \"b\"])\n&gt;&gt;&gt; print(len(test_model.pipeline.getStages()))\n2\n&gt;&gt;&gt; print(len(test_model.add_pipeline_stage(estimator).pipeline.getStages()))\n3\n</code></pre> Source code in <code>src/gentropy/method/l2g/model.py</code> <pre><code>def add_pipeline_stage(\n    self: LocusToGeneModel, transformer: Transformer\n) -&gt; LocusToGeneModel:\n    \"\"\"Adds a stage to the L2G pipeline.\n\n    Args:\n        transformer (Transformer): Spark transformer to add to the pipeline\n\n    Returns:\n        LocusToGeneModel: L2G model with the new transformer\n\n    Examples:\n        &gt;&gt;&gt; from pyspark.ml.regression import LinearRegression\n        &gt;&gt;&gt; estimator = LinearRegression()\n        &gt;&gt;&gt; test_model = LocusToGeneModel(features_list=[\"a\", \"b\"])\n        &gt;&gt;&gt; print(len(test_model.pipeline.getStages()))\n        2\n        &gt;&gt;&gt; print(len(test_model.add_pipeline_stage(estimator).pipeline.getStages()))\n        3\n    \"\"\"\n    pipeline_stages = self.pipeline.getStages()\n    new_stages = pipeline_stages + [transformer]\n    self.pipeline = Pipeline(stages=new_stages)\n    return self\n</code></pre>"},{"location":"python_api/methods/l2g/model/#gentropy.method.l2g.model.LocusToGeneModel.evaluate","title":"<code>evaluate(results: DataFrame, hyperparameters: dict[str, Any], wandb_run_name: str | None, gold_standard_data: L2GFeatureMatrix | None = None) -&gt; None</code>","text":"<p>Perform evaluation of the model predictions for the test set and track the results with W&amp;B.</p> <p>Parameters:</p> Name Type Description Default <code>results</code> <code>DataFrame</code> <p>Dataframe containing the predictions</p> required <code>hyperparameters</code> <code>dict[str, Any]</code> <p>Hyperparameters used for the model</p> required <code>wandb_run_name</code> <code>str | None</code> <p>Descriptive name for the run to be tracked with W&amp;B</p> required <code>gold_standard_data</code> <code>L2GFeatureMatrix | None</code> <p>Feature matrix for the associations in the gold standard. If provided, the ratio of positive to negative labels will be logged to W&amp;B</p> <code>None</code> Source code in <code>src/gentropy/method/l2g/model.py</code> <pre><code>def evaluate(\n    self: LocusToGeneModel,\n    results: DataFrame,\n    hyperparameters: dict[str, Any],\n    wandb_run_name: str | None,\n    gold_standard_data: L2GFeatureMatrix | None = None,\n) -&gt; None:\n    \"\"\"Perform evaluation of the model predictions for the test set and track the results with W&amp;B.\n\n    Args:\n        results (DataFrame): Dataframe containing the predictions\n        hyperparameters (dict[str, Any]): Hyperparameters used for the model\n        wandb_run_name (str | None): Descriptive name for the run to be tracked with W&amp;B\n        gold_standard_data (L2GFeatureMatrix | None): Feature matrix for the associations in the gold standard. If provided, the ratio of positive to negative labels will be logged to W&amp;B\n    \"\"\"\n    binary_evaluator = BinaryClassificationEvaluator(\n        rawPredictionCol=\"rawPrediction\", labelCol=\"label\"\n    )\n    multi_evaluator = MulticlassClassificationEvaluator(\n        labelCol=\"label\", predictionCol=\"prediction\"\n    )\n\n    if wandb_run_name and gold_standard_data:\n        run = wandb_init(\n            project=self.wandb_l2g_project_name,\n            config=hyperparameters,\n            name=wandb_run_name,\n        )\n        if isinstance(run, Run):\n            self.log_to_wandb(\n                results,\n                gold_standard_data,\n                [binary_evaluator, multi_evaluator],\n                run,\n            )\n            run.finish()\n</code></pre>"},{"location":"python_api/methods/l2g/model/#gentropy.method.l2g.model.LocusToGeneModel.features_vector_assembler","title":"<code>features_vector_assembler(features_cols: list[str]) -&gt; VectorAssembler</code>  <code>staticmethod</code>","text":"<p>Spark transformer to assemble the feature columns into a vector.</p> <p>Parameters:</p> Name Type Description Default <code>features_cols</code> <code>list[str]</code> <p>List of feature columns to assemble</p> required <p>Returns:</p> Name Type Description <code>VectorAssembler</code> <code>VectorAssembler</code> <p>Spark transformer to assemble the feature columns into a vector</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pyspark.ml.feature import VectorAssembler\n&gt;&gt;&gt; df = spark.createDataFrame([(5.2, 3.5)], schema=\"feature_1 FLOAT, feature_2 FLOAT\")\n&gt;&gt;&gt; assembler = LocusToGeneModel.features_vector_assembler([\"feature_1\", \"feature_2\"])\n&gt;&gt;&gt; assembler.transform(df).show()\n+---------+---------+--------------------+\n|feature_1|feature_2|            features|\n+---------+---------+--------------------+\n|      5.2|      3.5|[5.19999980926513...|\n+---------+---------+--------------------+\n</code></pre> Source code in <code>src/gentropy/method/l2g/model.py</code> <pre><code>@staticmethod\ndef features_vector_assembler(features_cols: list[str]) -&gt; VectorAssembler:\n    \"\"\"Spark transformer to assemble the feature columns into a vector.\n\n    Args:\n        features_cols (list[str]): List of feature columns to assemble\n\n    Returns:\n        VectorAssembler: Spark transformer to assemble the feature columns into a vector\n\n    Examples:\n        &gt;&gt;&gt; from pyspark.ml.feature import VectorAssembler\n        &gt;&gt;&gt; df = spark.createDataFrame([(5.2, 3.5)], schema=\"feature_1 FLOAT, feature_2 FLOAT\")\n        &gt;&gt;&gt; assembler = LocusToGeneModel.features_vector_assembler([\"feature_1\", \"feature_2\"])\n        &gt;&gt;&gt; assembler.transform(df).show()\n        +---------+---------+--------------------+\n        |feature_1|feature_2|            features|\n        +---------+---------+--------------------+\n        |      5.2|      3.5|[5.19999980926513...|\n        +---------+---------+--------------------+\n        &lt;BLANKLINE&gt;\n    \"\"\"\n    return (\n        VectorAssembler(handleInvalid=\"error\")\n        .setInputCols(features_cols)\n        .setOutputCol(\"features\")\n    )\n</code></pre>"},{"location":"python_api/methods/l2g/model/#gentropy.method.l2g.model.LocusToGeneModel.fit","title":"<code>fit(feature_matrix: L2GFeatureMatrix) -&gt; LocusToGeneModel</code>","text":"<p>Fit the pipeline to the feature matrix dataframe.</p> <p>Parameters:</p> Name Type Description Default <code>feature_matrix</code> <code>L2GFeatureMatrix</code> <p>Feature matrix dataframe to fit the model to</p> required <p>Returns:</p> Name Type Description <code>LocusToGeneModel</code> <code>LocusToGeneModel</code> <p>Fitted model</p> Source code in <code>src/gentropy/method/l2g/model.py</code> <pre><code>def fit(\n    self: LocusToGeneModel,\n    feature_matrix: L2GFeatureMatrix,\n) -&gt; LocusToGeneModel:\n    \"\"\"Fit the pipeline to the feature matrix dataframe.\n\n    Args:\n        feature_matrix (L2GFeatureMatrix): Feature matrix dataframe to fit the model to\n\n    Returns:\n        LocusToGeneModel: Fitted model\n    \"\"\"\n    self.model = self.pipeline.fit(feature_matrix.df)\n    return self\n</code></pre>"},{"location":"python_api/methods/l2g/model/#gentropy.method.l2g.model.LocusToGeneModel.get_feature_importance","title":"<code>get_feature_importance() -&gt; dict[str, float]</code>","text":"<p>Return dictionary with relative importances of every feature in the model. Feature names are encoded and have to be mapped back to their original names.</p> <p>Returns:</p> Type Description <code>dict[str, float]</code> <p>dict[str, float]: Dictionary mapping feature names to their importance</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>If the model has not been fitted yet or is not an XGBoost model</p> Source code in <code>src/gentropy/method/l2g/model.py</code> <pre><code>def get_feature_importance(self: LocusToGeneModel) -&gt; dict[str, float]:\n    \"\"\"Return dictionary with relative importances of every feature in the model. Feature names are encoded and have to be mapped back to their original names.\n\n    Returns:\n        dict[str, float]: Dictionary mapping feature names to their importance\n\n    Raises:\n        ValueError: If the model has not been fitted yet or is not an XGBoost model\n    \"\"\"\n    if not self.model or not isinstance(\n        self.model.stages[-1], SparkXGBClassifierModel\n    ):\n        raise ValueError(\n            f\"Model type {type(self.model)} not supported for feature importance.\"\n        )\n    importance_map = self.model.stages[-1].get_feature_importances()\n    return {self.feature_name_map[k]: v for k, v in importance_map.items()}\n</code></pre>"},{"location":"python_api/methods/l2g/model/#gentropy.method.l2g.model.LocusToGeneModel.get_param_grid","title":"<code>get_param_grid() -&gt; list[Any]</code>","text":"<p>Return the parameter grid for the model.</p> <p>Returns:</p> Type Description <code>list[Any]</code> <p>list[Any]: List of parameter maps to use for cross validation</p> Source code in <code>src/gentropy/method/l2g/model.py</code> <pre><code>def get_param_grid(self: LocusToGeneModel) -&gt; list[Any]:\n    \"\"\"Return the parameter grid for the model.\n\n    Returns:\n        list[Any]: List of parameter maps to use for cross validation\n    \"\"\"\n    return (\n        ParamGridBuilder()\n        .addGrid(self.estimator.max_depth, [3, 5, 7])\n        .addGrid(self.estimator.learning_rate, [0.01, 0.1, 1.0])\n        .build()\n    )\n</code></pre>"},{"location":"python_api/methods/l2g/model/#gentropy.method.l2g.model.LocusToGeneModel.load_from_disk","title":"<code>load_from_disk(path: str, features_list: list[str]) -&gt; LocusToGeneModel</code>  <code>classmethod</code>","text":"<p>Load a fitted pipeline model from disk.</p> <p>Parameters:</p> Name Type Description Default <code>path</code> <code>str</code> <p>Path to the model</p> required <code>features_list</code> <code>list[str]</code> <p>List of features used for the model</p> required <p>Returns:</p> Name Type Description <code>LocusToGeneModel</code> <code>LocusToGeneModel</code> <p>L2G model loaded from disk</p> Source code in <code>src/gentropy/method/l2g/model.py</code> <pre><code>@classmethod\ndef load_from_disk(\n    cls: Type[LocusToGeneModel], path: str, features_list: list[str]\n) -&gt; LocusToGeneModel:\n    \"\"\"Load a fitted pipeline model from disk.\n\n    Args:\n        path (str): Path to the model\n        features_list (list[str]): List of features used for the model\n\n    Returns:\n        LocusToGeneModel: L2G model loaded from disk\n    \"\"\"\n    return cls(model=PipelineModel.load(path), features_list=features_list)\n</code></pre>"},{"location":"python_api/methods/l2g/model/#gentropy.method.l2g.model.LocusToGeneModel.log_to_wandb","title":"<code>log_to_wandb(results: DataFrame, training_data: L2GFeatureMatrix, evaluators: list[BinaryClassificationEvaluator | MulticlassClassificationEvaluator], wandb_run: Run) -&gt; None</code>","text":"<p>Log evaluation results and feature importance to W&amp;B.</p> <p>Parameters:</p> Name Type Description Default <code>results</code> <code>DataFrame</code> <p>Dataframe containing the predictions</p> required <code>training_data</code> <code>L2GFeatureMatrix</code> <p>Training data used for the model. If provided, the table and the number of positive and negative labels will be logged to W&amp;B</p> required <code>evaluators</code> <code>list[BinaryClassificationEvaluator | MulticlassClassificationEvaluator]</code> <p>List of Spark ML evaluators to use for evaluation</p> required <code>wandb_run</code> <code>Run</code> <p>W&amp;B run to log the results to</p> required Source code in <code>src/gentropy/method/l2g/model.py</code> <pre><code>def log_to_wandb(\n    self: LocusToGeneModel,\n    results: DataFrame,\n    training_data: L2GFeatureMatrix,\n    evaluators: list[\n        BinaryClassificationEvaluator | MulticlassClassificationEvaluator\n    ],\n    wandb_run: Run,\n) -&gt; None:\n    \"\"\"Log evaluation results and feature importance to W&amp;B.\n\n    Args:\n        results (DataFrame): Dataframe containing the predictions\n        training_data (L2GFeatureMatrix): Training data used for the model. If provided, the table and the number of positive and negative labels will be logged to W&amp;B\n        evaluators (list[BinaryClassificationEvaluator | MulticlassClassificationEvaluator]): List of Spark ML evaluators to use for evaluation\n        wandb_run (Run): W&amp;B run to log the results to\n    \"\"\"\n    ## Track evaluation metrics\n    for evaluator in evaluators:\n        wandb_evaluator = WandbEvaluator(\n            spark_ml_evaluator=evaluator, wandb_run=wandb_run\n        )\n        wandb_evaluator.evaluate(results)\n    ## Track feature importance\n    wandb_run.log({\"importances\": self.get_feature_importance()})\n    ## Track training set\n    training_table = Table(dataframe=training_data.df.toPandas())\n    wandb_run.log({\"trainingSet\": training_table})\n    # Count number of positive and negative labels\n    gs_counts_dict = {\n        \"goldStandard\" + row[\"goldStandardSet\"].capitalize(): row[\"count\"]\n        for row in training_data.df.groupBy(\"goldStandardSet\").count().collect()\n    }\n    wandb_run.log(gs_counts_dict)\n    # Missingness rates\n    wandb_run.log(\n        {\"missingnessRates\": training_data.calculate_feature_missingness_rate()}\n    )\n</code></pre>"},{"location":"python_api/methods/l2g/model/#gentropy.method.l2g.model.LocusToGeneModel.predict","title":"<code>predict(feature_matrix: L2GFeatureMatrix) -&gt; DataFrame</code>","text":"<p>Apply the model to a given feature matrix dataframe. The feature matrix needs to be preprocessed first.</p> <p>Parameters:</p> Name Type Description Default <code>feature_matrix</code> <code>L2GFeatureMatrix</code> <p>Feature matrix dataframe to apply the model to</p> required <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>Dataframe with predictions</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>If the model has not been fitted yet</p> Source code in <code>src/gentropy/method/l2g/model.py</code> <pre><code>def predict(\n    self: LocusToGeneModel,\n    feature_matrix: L2GFeatureMatrix,\n) -&gt; DataFrame:\n    \"\"\"Apply the model to a given feature matrix dataframe. The feature matrix needs to be preprocessed first.\n\n    Args:\n        feature_matrix (L2GFeatureMatrix): Feature matrix dataframe to apply the model to\n\n    Returns:\n        DataFrame: Dataframe with predictions\n\n    Raises:\n        ValueError: If the model has not been fitted yet\n    \"\"\"\n    if not self.model:\n        raise ValueError(\"Model not fitted yet. `fit()` has to be called first.\")\n    return self.model.transform(feature_matrix.df)\n</code></pre>"},{"location":"python_api/methods/l2g/model/#gentropy.method.l2g.model.LocusToGeneModel.save","title":"<code>save(path: str) -&gt; None</code>","text":"<p>Saves fitted pipeline model to disk.</p> <p>Parameters:</p> Name Type Description Default <code>path</code> <code>str</code> <p>Path to save the model to</p> required <p>Raises:</p> Type Description <code>ValueError</code> <p>If the model has not been fitted yet</p> Source code in <code>src/gentropy/method/l2g/model.py</code> <pre><code>def save(self: LocusToGeneModel, path: str) -&gt; None:\n    \"\"\"Saves fitted pipeline model to disk.\n\n    Args:\n        path (str): Path to save the model to\n\n    Raises:\n        ValueError: If the model has not been fitted yet\n    \"\"\"\n    if self.model is None:\n        raise ValueError(\"Model has not been fitted yet.\")\n    self.model.write().overwrite().save(path)\n</code></pre>"},{"location":"python_api/methods/l2g/trainer/","title":"L2G Trainer","text":""},{"location":"python_api/methods/l2g/trainer/#gentropy.method.l2g.trainer.LocusToGeneTrainer","title":"<code>gentropy.method.l2g.trainer.LocusToGeneTrainer</code>  <code>dataclass</code>","text":"<p>Modelling of what is the most likely causal gene associated with a given locus.</p> Source code in <code>src/gentropy/method/l2g/trainer.py</code> <pre><code>@dataclass\nclass LocusToGeneTrainer:\n    \"\"\"Modelling of what is the most likely causal gene associated with a given locus.\"\"\"\n\n    _model: LocusToGeneModel\n    train_set: L2GFeatureMatrix\n\n    @classmethod\n    def train(\n        cls: type[LocusToGeneTrainer],\n        gold_standard_data: L2GFeatureMatrix,\n        l2g_model: LocusToGeneModel,\n        evaluate: bool,\n        wandb_run_name: str | None = None,\n        model_path: str | None = None,\n        **hyperparams: dict[str, Any],\n    ) -&gt; LocusToGeneModel:\n        \"\"\"Train the Locus to Gene model.\n\n        Args:\n            gold_standard_data (L2GFeatureMatrix): Feature matrix for the associations in the gold standard\n            l2g_model (LocusToGeneModel): Model to fit to the data on\n            evaluate (bool): Whether to evaluate the model on a test set\n            wandb_run_name (str | None): Descriptive name for the run to be tracked with W&amp;B\n            model_path (str | None): Path to save the model to\n            **hyperparams (dict[str, Any]): Hyperparameters to use for the model\n\n        Returns:\n            LocusToGeneModel: Trained model\n        \"\"\"\n        train, test = gold_standard_data.train_test_split(fraction=0.8)\n\n        model = l2g_model.add_pipeline_stage(l2g_model.estimator).fit(train)\n\n        if evaluate:\n            l2g_model.evaluate(\n                results=model.predict(test),\n                hyperparameters=hyperparams,\n                wandb_run_name=wandb_run_name,\n                gold_standard_data=gold_standard_data,\n            )\n        if model_path:\n            l2g_model.save(model_path)\n        return l2g_model\n\n    @classmethod\n    def cross_validate(\n        cls: type[LocusToGeneTrainer],\n        l2g_model: LocusToGeneModel,\n        data: L2GFeatureMatrix,\n        num_folds: int,\n        param_grid: Optional[list] = None,  # type: ignore\n    ) -&gt; LocusToGeneModel:\n        \"\"\"Perform k-fold cross validation on the model.\n\n        By providing a model with a parameter grid, this method will perform k-fold cross validation on the model for each\n        combination of parameters and return the best model.\n\n        Args:\n            l2g_model (LocusToGeneModel): Model to fit to the data on\n            data (L2GFeatureMatrix): Data to perform cross validation on\n            num_folds (int): Number of folds to use for cross validation\n            param_grid (Optional[list]): List of parameter maps to use for cross validation\n\n        Returns:\n            LocusToGeneModel: Trained model fitted with the best hyperparameters\n\n        Raises:\n            ValueError: Parameter grid is empty. Cannot perform cross-validation.\n            ValueError: Unable to retrieve the best model.\n        \"\"\"\n        evaluator = MulticlassClassificationEvaluator()\n        params_grid = param_grid or l2g_model.get_param_grid()\n        if not param_grid:\n            raise ValueError(\n                \"Parameter grid is empty. Cannot perform cross-validation.\"\n            )\n        cv = CrossValidator(\n            numFolds=num_folds,\n            estimator=l2g_model.estimator,\n            estimatorParamMaps=params_grid,\n            evaluator=evaluator,\n            parallelism=2,\n            collectSubModels=False,\n            seed=42,\n        )\n\n        l2g_model.add_pipeline_stage(cv)  # type: ignore[assignment, unused-ignore]\n\n        # Integrate the best model from the last stage of the pipeline\n        if (full_pipeline_model := l2g_model.fit(data).model) is None or not hasattr(\n            full_pipeline_model, \"stages\"\n        ):\n            raise ValueError(\"Unable to retrieve the best model.\")\n        l2g_model.model = full_pipeline_model.stages[-1].bestModel  # type: ignore[assignment, unused-ignore]\n        return l2g_model\n</code></pre>"},{"location":"python_api/methods/l2g/trainer/#gentropy.method.l2g.trainer.LocusToGeneTrainer.cross_validate","title":"<code>cross_validate(l2g_model: LocusToGeneModel, data: L2GFeatureMatrix, num_folds: int, param_grid: Optional[list] = None) -&gt; LocusToGeneModel</code>  <code>classmethod</code>","text":"<p>Perform k-fold cross validation on the model.</p> <p>By providing a model with a parameter grid, this method will perform k-fold cross validation on the model for each combination of parameters and return the best model.</p> <p>Parameters:</p> Name Type Description Default <code>l2g_model</code> <code>LocusToGeneModel</code> <p>Model to fit to the data on</p> required <code>data</code> <code>L2GFeatureMatrix</code> <p>Data to perform cross validation on</p> required <code>num_folds</code> <code>int</code> <p>Number of folds to use for cross validation</p> required <code>param_grid</code> <code>Optional[list]</code> <p>List of parameter maps to use for cross validation</p> <code>None</code> <p>Returns:</p> Name Type Description <code>LocusToGeneModel</code> <code>LocusToGeneModel</code> <p>Trained model fitted with the best hyperparameters</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>Parameter grid is empty. Cannot perform cross-validation.</p> <code>ValueError</code> <p>Unable to retrieve the best model.</p> Source code in <code>src/gentropy/method/l2g/trainer.py</code> <pre><code>@classmethod\ndef cross_validate(\n    cls: type[LocusToGeneTrainer],\n    l2g_model: LocusToGeneModel,\n    data: L2GFeatureMatrix,\n    num_folds: int,\n    param_grid: Optional[list] = None,  # type: ignore\n) -&gt; LocusToGeneModel:\n    \"\"\"Perform k-fold cross validation on the model.\n\n    By providing a model with a parameter grid, this method will perform k-fold cross validation on the model for each\n    combination of parameters and return the best model.\n\n    Args:\n        l2g_model (LocusToGeneModel): Model to fit to the data on\n        data (L2GFeatureMatrix): Data to perform cross validation on\n        num_folds (int): Number of folds to use for cross validation\n        param_grid (Optional[list]): List of parameter maps to use for cross validation\n\n    Returns:\n        LocusToGeneModel: Trained model fitted with the best hyperparameters\n\n    Raises:\n        ValueError: Parameter grid is empty. Cannot perform cross-validation.\n        ValueError: Unable to retrieve the best model.\n    \"\"\"\n    evaluator = MulticlassClassificationEvaluator()\n    params_grid = param_grid or l2g_model.get_param_grid()\n    if not param_grid:\n        raise ValueError(\n            \"Parameter grid is empty. Cannot perform cross-validation.\"\n        )\n    cv = CrossValidator(\n        numFolds=num_folds,\n        estimator=l2g_model.estimator,\n        estimatorParamMaps=params_grid,\n        evaluator=evaluator,\n        parallelism=2,\n        collectSubModels=False,\n        seed=42,\n    )\n\n    l2g_model.add_pipeline_stage(cv)  # type: ignore[assignment, unused-ignore]\n\n    # Integrate the best model from the last stage of the pipeline\n    if (full_pipeline_model := l2g_model.fit(data).model) is None or not hasattr(\n        full_pipeline_model, \"stages\"\n    ):\n        raise ValueError(\"Unable to retrieve the best model.\")\n    l2g_model.model = full_pipeline_model.stages[-1].bestModel  # type: ignore[assignment, unused-ignore]\n    return l2g_model\n</code></pre>"},{"location":"python_api/methods/l2g/trainer/#gentropy.method.l2g.trainer.LocusToGeneTrainer.train","title":"<code>train(gold_standard_data: L2GFeatureMatrix, l2g_model: LocusToGeneModel, evaluate: bool, wandb_run_name: str | None = None, model_path: str | None = None, **hyperparams: dict[str, Any]) -&gt; LocusToGeneModel</code>  <code>classmethod</code>","text":"<p>Train the Locus to Gene model.</p> <p>Parameters:</p> Name Type Description Default <code>gold_standard_data</code> <code>L2GFeatureMatrix</code> <p>Feature matrix for the associations in the gold standard</p> required <code>l2g_model</code> <code>LocusToGeneModel</code> <p>Model to fit to the data on</p> required <code>evaluate</code> <code>bool</code> <p>Whether to evaluate the model on a test set</p> required <code>wandb_run_name</code> <code>str | None</code> <p>Descriptive name for the run to be tracked with W&amp;B</p> <code>None</code> <code>model_path</code> <code>str | None</code> <p>Path to save the model to</p> <code>None</code> <code>**hyperparams</code> <code>dict[str, Any]</code> <p>Hyperparameters to use for the model</p> <code>{}</code> <p>Returns:</p> Name Type Description <code>LocusToGeneModel</code> <code>LocusToGeneModel</code> <p>Trained model</p> Source code in <code>src/gentropy/method/l2g/trainer.py</code> <pre><code>@classmethod\ndef train(\n    cls: type[LocusToGeneTrainer],\n    gold_standard_data: L2GFeatureMatrix,\n    l2g_model: LocusToGeneModel,\n    evaluate: bool,\n    wandb_run_name: str | None = None,\n    model_path: str | None = None,\n    **hyperparams: dict[str, Any],\n) -&gt; LocusToGeneModel:\n    \"\"\"Train the Locus to Gene model.\n\n    Args:\n        gold_standard_data (L2GFeatureMatrix): Feature matrix for the associations in the gold standard\n        l2g_model (LocusToGeneModel): Model to fit to the data on\n        evaluate (bool): Whether to evaluate the model on a test set\n        wandb_run_name (str | None): Descriptive name for the run to be tracked with W&amp;B\n        model_path (str | None): Path to save the model to\n        **hyperparams (dict[str, Any]): Hyperparameters to use for the model\n\n    Returns:\n        LocusToGeneModel: Trained model\n    \"\"\"\n    train, test = gold_standard_data.train_test_split(fraction=0.8)\n\n    model = l2g_model.add_pipeline_stage(l2g_model.estimator).fit(train)\n\n    if evaluate:\n        l2g_model.evaluate(\n            results=model.predict(test),\n            hyperparameters=hyperparams,\n            wandb_run_name=wandb_run_name,\n            gold_standard_data=gold_standard_data,\n        )\n    if model_path:\n        l2g_model.save(model_path)\n    return l2g_model\n</code></pre>"},{"location":"python_api/steps/_steps/","title":"Step","text":"<p>This section provides description for the <code>Step</code> class. Each <code>Step</code> uses its own set of Methods and Datasets and implements the logic necessary to read a set of inputs, perform the transformation and write the outputs. All steps are available through the command line interface when running the <code>gentropy</code> command.</p>"},{"location":"python_api/steps/colocalisation/","title":"colocalisation","text":""},{"location":"python_api/steps/colocalisation/#gentropy.colocalisation.ColocalisationStep","title":"<code>gentropy.colocalisation.ColocalisationStep</code>","text":"<p>Colocalisation step.</p> <p>This workflow runs colocalisation analyses that assess the degree to which independent signals of the association share the same causal variant in a region of the genome, typically limited by linkage disequilibrium (LD).</p> Source code in <code>src/gentropy/colocalisation.py</code> <pre><code>class ColocalisationStep:\n    \"\"\"Colocalisation step.\n\n    This workflow runs colocalisation analyses that assess the degree to which independent signals of the association share the same causal variant in a region of the genome, typically limited by linkage disequilibrium (LD).\n    \"\"\"\n\n    def __init__(\n        self,\n        session: Session,\n        credible_set_path: str,\n        study_index_path: str,\n        coloc_path: str,\n        colocalisation_method: str,\n    ) -&gt; None:\n        \"\"\"Run Colocalisation step.\n\n        Args:\n            session (Session): Session object.\n            credible_set_path (str): Input credible sets path.\n            study_index_path (str): Input study index path.\n            coloc_path (str): Output Colocalisation path.\n            colocalisation_method (str): Colocalisation method.\n        \"\"\"\n        colocalisation_class = self._get_colocalisation_class(colocalisation_method)\n        # Extract\n        credible_set = (\n            StudyLocus.from_parquet(\n                session, credible_set_path, recursiveFileLookup=True\n            ).filter(col(\"finemappingMethod\") == \"SuSie\")\n            if colocalisation_class is Coloc\n            else StudyLocus.from_parquet(\n                session, credible_set_path, recursiveFileLookup=True\n            )\n        )\n        si = StudyIndex.from_parquet(\n            session, study_index_path, recursiveFileLookup=True\n        )\n\n        # Transform\n        overlaps = credible_set.filter_credible_set(\n            CredibleInterval.IS95\n        ).find_overlaps(si)\n        colocalisation_results = colocalisation_class.colocalise(overlaps)  # type: ignore\n\n        # Load\n        colocalisation_results.df.write.mode(session.write_mode).parquet(\n            f\"{coloc_path}/{colocalisation_method.lower()}\"\n        )\n\n    @classmethod\n    def _get_colocalisation_class(cls: type[ColocalisationStep], method: str) -&gt; type:\n        \"\"\"Get colocalisation class.\n\n        Args:\n            method (str): Colocalisation method.\n\n        Returns:\n            type: Colocalisation class.\n\n        Raises:\n            ValueError: if method not available.\n\n        Examples:\n            &gt;&gt;&gt; ColocalisationStep._get_colocalisation_class(\"ECaviar\")\n            &lt;class 'gentropy.method.colocalisation.ECaviar'&gt;\n        \"\"\"\n        module_name = \"gentropy.method.colocalisation\"\n        module = import_module(module_name)\n\n        available_methods = []\n        for class_name, class_obj in inspect.getmembers(module, inspect.isclass):\n            if class_obj.__module__ == module_name:\n                available_methods.append(class_name)\n                if class_name == method:\n                    return class_obj\n        raise ValueError(\n            f\"Method {method} is not supported. Available: {(', ').join(available_methods)}\"\n        )\n</code></pre>"},{"location":"python_api/steps/colocalisation/#gentropy.colocalisation.ColocalisationStep.__init__","title":"<code>__init__(session: Session, credible_set_path: str, study_index_path: str, coloc_path: str, colocalisation_method: str) -&gt; None</code>","text":"<p>Run Colocalisation step.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Session object.</p> required <code>credible_set_path</code> <code>str</code> <p>Input credible sets path.</p> required <code>study_index_path</code> <code>str</code> <p>Input study index path.</p> required <code>coloc_path</code> <code>str</code> <p>Output Colocalisation path.</p> required <code>colocalisation_method</code> <code>str</code> <p>Colocalisation method.</p> required Source code in <code>src/gentropy/colocalisation.py</code> <pre><code>def __init__(\n    self,\n    session: Session,\n    credible_set_path: str,\n    study_index_path: str,\n    coloc_path: str,\n    colocalisation_method: str,\n) -&gt; None:\n    \"\"\"Run Colocalisation step.\n\n    Args:\n        session (Session): Session object.\n        credible_set_path (str): Input credible sets path.\n        study_index_path (str): Input study index path.\n        coloc_path (str): Output Colocalisation path.\n        colocalisation_method (str): Colocalisation method.\n    \"\"\"\n    colocalisation_class = self._get_colocalisation_class(colocalisation_method)\n    # Extract\n    credible_set = (\n        StudyLocus.from_parquet(\n            session, credible_set_path, recursiveFileLookup=True\n        ).filter(col(\"finemappingMethod\") == \"SuSie\")\n        if colocalisation_class is Coloc\n        else StudyLocus.from_parquet(\n            session, credible_set_path, recursiveFileLookup=True\n        )\n    )\n    si = StudyIndex.from_parquet(\n        session, study_index_path, recursiveFileLookup=True\n    )\n\n    # Transform\n    overlaps = credible_set.filter_credible_set(\n        CredibleInterval.IS95\n    ).find_overlaps(si)\n    colocalisation_results = colocalisation_class.colocalise(overlaps)  # type: ignore\n\n    # Load\n    colocalisation_results.df.write.mode(session.write_mode).parquet(\n        f\"{coloc_path}/{colocalisation_method.lower()}\"\n    )\n</code></pre>"},{"location":"python_api/steps/eqtl_catalogue/","title":"eQTL Catalogue","text":""},{"location":"python_api/steps/eqtl_catalogue/#gentropy.eqtl_catalogue.EqtlCatalogueStep","title":"<code>gentropy.eqtl_catalogue.EqtlCatalogueStep</code>","text":"<p>eQTL Catalogue ingestion step.</p> <p>From SuSIE fine mapping results (available at their FTP ), we extract credible sets and study index datasets from gene expression QTL studies.</p> Source code in <code>src/gentropy/eqtl_catalogue.py</code> <pre><code>class EqtlCatalogueStep:\n    \"\"\"eQTL Catalogue ingestion step.\n\n    From SuSIE fine mapping results (available at [their FTP](https://ftp.ebi.ac.uk/pub/databases/spot/eQTL/susie/) ), we extract credible sets and study index datasets from gene expression QTL studies.\n    \"\"\"\n\n    def __init__(\n        self,\n        session: Session,\n        mqtl_quantification_methods_blacklist: list[str],\n        eqtl_catalogue_paths_imported: str,\n        eqtl_catalogue_study_index_out: str,\n        eqtl_catalogue_credible_sets_out: str,\n    ) -&gt; None:\n        \"\"\"Run eQTL Catalogue ingestion step.\n\n        Args:\n            session (Session): Session object.\n            mqtl_quantification_methods_blacklist (list[str]): Molecular trait quantification methods that we don't want to ingest. Available options in https://github.com/eQTL-Catalogue/eQTL-Catalogue-resources/blob/master/data_tables/dataset_metadata.tsv\n            eqtl_catalogue_paths_imported (str): Input eQTL Catalogue fine mapping results path.\n            eqtl_catalogue_study_index_out (str): Output eQTL Catalogue study index path.\n            eqtl_catalogue_credible_sets_out (str): Output eQTL Catalogue credible sets path.\n        \"\"\"\n        # Extract\n        studies_metadata = EqtlCatalogueStudyIndex.read_studies_from_source(\n            session, list(mqtl_quantification_methods_blacklist)\n        )\n\n        # Load raw data only for the studies we are interested in ingestion. This makes the proces much lighter.\n        studies_to_ingest = EqtlCatalogueStudyIndex.get_studies_of_interest(\n            studies_metadata\n        )\n        credible_sets_df = EqtlCatalogueFinemapping.read_credible_set_from_source(\n            session,\n            credible_set_path=[\n                f\"{eqtl_catalogue_paths_imported}/{qtd_id}.credible_sets.tsv\"\n                for qtd_id in studies_to_ingest\n            ],\n        )\n        lbf_df = EqtlCatalogueFinemapping.read_lbf_from_source(\n            session,\n            lbf_path=[\n                f\"{eqtl_catalogue_paths_imported}/{qtd_id}.lbf_variable.txt\"\n                for qtd_id in studies_to_ingest\n            ],\n        )\n\n        # Transform\n        processed_susie_df = EqtlCatalogueFinemapping.parse_susie_results(\n            credible_sets_df, lbf_df, studies_metadata\n        )\n        credible_sets = EqtlCatalogueFinemapping.from_susie_results(processed_susie_df)\n        study_index = EqtlCatalogueStudyIndex.from_susie_results(processed_susie_df)\n\n        # Load\n        study_index.df.write.mode(session.write_mode).parquet(\n            eqtl_catalogue_study_index_out\n        )\n        credible_sets.df.write.mode(session.write_mode).parquet(\n            eqtl_catalogue_credible_sets_out\n        )\n</code></pre>"},{"location":"python_api/steps/eqtl_catalogue/#gentropy.eqtl_catalogue.EqtlCatalogueStep.__init__","title":"<code>__init__(session: Session, mqtl_quantification_methods_blacklist: list[str], eqtl_catalogue_paths_imported: str, eqtl_catalogue_study_index_out: str, eqtl_catalogue_credible_sets_out: str) -&gt; None</code>","text":"<p>Run eQTL Catalogue ingestion step.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Session object.</p> required <code>mqtl_quantification_methods_blacklist</code> <code>list[str]</code> <p>Molecular trait quantification methods that we don't want to ingest. Available options in https://github.com/eQTL-Catalogue/eQTL-Catalogue-resources/blob/master/data_tables/dataset_metadata.tsv</p> required <code>eqtl_catalogue_paths_imported</code> <code>str</code> <p>Input eQTL Catalogue fine mapping results path.</p> required <code>eqtl_catalogue_study_index_out</code> <code>str</code> <p>Output eQTL Catalogue study index path.</p> required <code>eqtl_catalogue_credible_sets_out</code> <code>str</code> <p>Output eQTL Catalogue credible sets path.</p> required Source code in <code>src/gentropy/eqtl_catalogue.py</code> <pre><code>def __init__(\n    self,\n    session: Session,\n    mqtl_quantification_methods_blacklist: list[str],\n    eqtl_catalogue_paths_imported: str,\n    eqtl_catalogue_study_index_out: str,\n    eqtl_catalogue_credible_sets_out: str,\n) -&gt; None:\n    \"\"\"Run eQTL Catalogue ingestion step.\n\n    Args:\n        session (Session): Session object.\n        mqtl_quantification_methods_blacklist (list[str]): Molecular trait quantification methods that we don't want to ingest. Available options in https://github.com/eQTL-Catalogue/eQTL-Catalogue-resources/blob/master/data_tables/dataset_metadata.tsv\n        eqtl_catalogue_paths_imported (str): Input eQTL Catalogue fine mapping results path.\n        eqtl_catalogue_study_index_out (str): Output eQTL Catalogue study index path.\n        eqtl_catalogue_credible_sets_out (str): Output eQTL Catalogue credible sets path.\n    \"\"\"\n    # Extract\n    studies_metadata = EqtlCatalogueStudyIndex.read_studies_from_source(\n        session, list(mqtl_quantification_methods_blacklist)\n    )\n\n    # Load raw data only for the studies we are interested in ingestion. This makes the proces much lighter.\n    studies_to_ingest = EqtlCatalogueStudyIndex.get_studies_of_interest(\n        studies_metadata\n    )\n    credible_sets_df = EqtlCatalogueFinemapping.read_credible_set_from_source(\n        session,\n        credible_set_path=[\n            f\"{eqtl_catalogue_paths_imported}/{qtd_id}.credible_sets.tsv\"\n            for qtd_id in studies_to_ingest\n        ],\n    )\n    lbf_df = EqtlCatalogueFinemapping.read_lbf_from_source(\n        session,\n        lbf_path=[\n            f\"{eqtl_catalogue_paths_imported}/{qtd_id}.lbf_variable.txt\"\n            for qtd_id in studies_to_ingest\n        ],\n    )\n\n    # Transform\n    processed_susie_df = EqtlCatalogueFinemapping.parse_susie_results(\n        credible_sets_df, lbf_df, studies_metadata\n    )\n    credible_sets = EqtlCatalogueFinemapping.from_susie_results(processed_susie_df)\n    study_index = EqtlCatalogueStudyIndex.from_susie_results(processed_susie_df)\n\n    # Load\n    study_index.df.write.mode(session.write_mode).parquet(\n        eqtl_catalogue_study_index_out\n    )\n    credible_sets.df.write.mode(session.write_mode).parquet(\n        eqtl_catalogue_credible_sets_out\n    )\n</code></pre>"},{"location":"python_api/steps/finngen_studies/","title":"finngen_studies","text":""},{"location":"python_api/steps/finngen_studies/#gentropy.finngen_studies.FinnGenStudiesStep","title":"<code>gentropy.finngen_studies.FinnGenStudiesStep</code>","text":"<p>FinnGen study index generation step.</p> Source code in <code>src/gentropy/finngen_studies.py</code> <pre><code>class FinnGenStudiesStep:\n    \"\"\"FinnGen study index generation step.\"\"\"\n\n    def __init__(self, session: Session, finngen_study_index_out: str) -&gt; None:\n        \"\"\"Run FinnGen study index generation step.\n\n        Args:\n            session (Session): Session object.\n            finngen_study_index_out (str): Output FinnGen study index path.\n        \"\"\"\n        # Fetch study index.\n        FinnGenStudyIndex.from_source(session.spark).df.write.mode(\n            session.write_mode\n        ).parquet(finngen_study_index_out)\n</code></pre>"},{"location":"python_api/steps/finngen_studies/#gentropy.finngen_studies.FinnGenStudiesStep.__init__","title":"<code>__init__(session: Session, finngen_study_index_out: str) -&gt; None</code>","text":"<p>Run FinnGen study index generation step.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Session object.</p> required <code>finngen_study_index_out</code> <code>str</code> <p>Output FinnGen study index path.</p> required Source code in <code>src/gentropy/finngen_studies.py</code> <pre><code>def __init__(self, session: Session, finngen_study_index_out: str) -&gt; None:\n    \"\"\"Run FinnGen study index generation step.\n\n    Args:\n        session (Session): Session object.\n        finngen_study_index_out (str): Output FinnGen study index path.\n    \"\"\"\n    # Fetch study index.\n    FinnGenStudyIndex.from_source(session.spark).df.write.mode(\n        session.write_mode\n    ).parquet(finngen_study_index_out)\n</code></pre>"},{"location":"python_api/steps/finngen_sumstat_preprocess/","title":"finngen_sumstat_preprocess","text":""},{"location":"python_api/steps/finngen_sumstat_preprocess/#gentropy.finngen_sumstat_preprocess.FinnGenSumstatPreprocessStep","title":"<code>gentropy.finngen_sumstat_preprocess.FinnGenSumstatPreprocessStep</code>","text":"<p>FinnGen sumstats preprocessing.</p> Source code in <code>src/gentropy/finngen_sumstat_preprocess.py</code> <pre><code>class FinnGenSumstatPreprocessStep:\n    \"\"\"FinnGen sumstats preprocessing.\"\"\"\n\n    def __init__(\n        self, session: Session, raw_sumstats_path: str, out_sumstats_path: str\n    ) -&gt; None:\n        \"\"\"Run FinnGen summary stats preprocessing step.\n\n        Args:\n            session (Session): Session object.\n            raw_sumstats_path (str): Input raw summary stats path.\n            out_sumstats_path (str): Output summary stats path.\n        \"\"\"\n        # Process summary stats.\n        (\n            FinnGenSummaryStats.from_source(session.spark, raw_file=raw_sumstats_path)\n            .df.write.mode(session.write_mode)\n            .parquet(out_sumstats_path)\n        )\n</code></pre>"},{"location":"python_api/steps/finngen_sumstat_preprocess/#gentropy.finngen_sumstat_preprocess.FinnGenSumstatPreprocessStep.__init__","title":"<code>__init__(session: Session, raw_sumstats_path: str, out_sumstats_path: str) -&gt; None</code>","text":"<p>Run FinnGen summary stats preprocessing step.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Session object.</p> required <code>raw_sumstats_path</code> <code>str</code> <p>Input raw summary stats path.</p> required <code>out_sumstats_path</code> <code>str</code> <p>Output summary stats path.</p> required Source code in <code>src/gentropy/finngen_sumstat_preprocess.py</code> <pre><code>def __init__(\n    self, session: Session, raw_sumstats_path: str, out_sumstats_path: str\n) -&gt; None:\n    \"\"\"Run FinnGen summary stats preprocessing step.\n\n    Args:\n        session (Session): Session object.\n        raw_sumstats_path (str): Input raw summary stats path.\n        out_sumstats_path (str): Output summary stats path.\n    \"\"\"\n    # Process summary stats.\n    (\n        FinnGenSummaryStats.from_source(session.spark, raw_file=raw_sumstats_path)\n        .df.write.mode(session.write_mode)\n        .parquet(out_sumstats_path)\n    )\n</code></pre>"},{"location":"python_api/steps/gene_index/","title":"gene_index","text":""},{"location":"python_api/steps/gene_index/#gentropy.gene_index.GeneIndexStep","title":"<code>gentropy.gene_index.GeneIndexStep</code>","text":"<p>Gene index step.</p> <p>This step generates a gene index dataset from an Open Targets Platform target dataset.</p> Source code in <code>src/gentropy/gene_index.py</code> <pre><code>class GeneIndexStep:\n    \"\"\"Gene index step.\n\n    This step generates a gene index dataset from an Open Targets Platform target dataset.\n    \"\"\"\n\n    def __init__(\n        self,\n        session: Session,\n        target_path: str,\n        gene_index_path: str,\n    ) -&gt; None:\n        \"\"\"Initialize step.\n\n        Args:\n            session (Session): Session object.\n            target_path (str): Input Open Targets Platform target dataset path.\n            gene_index_path (str): Output gene index dataset path.\n        \"\"\"\n        platform_target = session.spark.read.parquet(target_path)\n        # Transform\n        gene_index = OpenTargetsTarget.as_gene_index(platform_target)\n        # Load\n        gene_index.df.write.mode(session.write_mode).parquet(gene_index_path)\n</code></pre>"},{"location":"python_api/steps/gene_index/#gentropy.gene_index.GeneIndexStep.__init__","title":"<code>__init__(session: Session, target_path: str, gene_index_path: str) -&gt; None</code>","text":"<p>Initialize step.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Session object.</p> required <code>target_path</code> <code>str</code> <p>Input Open Targets Platform target dataset path.</p> required <code>gene_index_path</code> <code>str</code> <p>Output gene index dataset path.</p> required Source code in <code>src/gentropy/gene_index.py</code> <pre><code>def __init__(\n    self,\n    session: Session,\n    target_path: str,\n    gene_index_path: str,\n) -&gt; None:\n    \"\"\"Initialize step.\n\n    Args:\n        session (Session): Session object.\n        target_path (str): Input Open Targets Platform target dataset path.\n        gene_index_path (str): Output gene index dataset path.\n    \"\"\"\n    platform_target = session.spark.read.parquet(target_path)\n    # Transform\n    gene_index = OpenTargetsTarget.as_gene_index(platform_target)\n    # Load\n    gene_index.df.write.mode(session.write_mode).parquet(gene_index_path)\n</code></pre>"},{"location":"python_api/steps/gwas_catalog_curation/","title":"gwas_catalog_study_curation","text":""},{"location":"python_api/steps/gwas_catalog_curation/#gentropy.gwas_catalog_study_curation.GWASCatalogStudyCurationStep","title":"<code>gentropy.gwas_catalog_study_curation.GWASCatalogStudyCurationStep</code>","text":"<p>Annotate GWAS Catalog studies with additional curation and create a curation backlog.</p> Source code in <code>src/gentropy/gwas_catalog_study_curation.py</code> <pre><code>class GWASCatalogStudyCurationStep:\n    \"\"\"Annotate GWAS Catalog studies with additional curation and create a curation backlog.\"\"\"\n\n    def __init__(\n        self,\n        session: Session,\n        catalog_study_files: list[str],\n        catalog_ancestry_files: list[str],\n        catalog_sumstats_lut: str,\n        gwas_catalog_study_curation_out: str,\n        gwas_catalog_study_curation_file: str | None,\n    ) -&gt; None:\n        \"\"\"Run step to annotate and create backlog.\n\n        Args:\n            session (Session): Session object.\n            catalog_study_files (list[str]): List of raw GWAS catalog studies file.\n            catalog_ancestry_files (list[str]): List of raw ancestry annotations files from GWAS Catalog.\n            catalog_sumstats_lut (str): GWAS Catalog summary statistics lookup table.\n            gwas_catalog_study_curation_out (str): Path for the updated curation table.\n            gwas_catalog_study_curation_file (str | None): Path to the original curation table. Optinal\n        \"\"\"\n        catalog_studies = session.spark.read.csv(\n            list(catalog_study_files), sep=\"\\t\", header=True\n        )\n        ancestry_lut = session.spark.read.csv(\n            list(catalog_ancestry_files), sep=\"\\t\", header=True\n        )\n        sumstats_lut = session.spark.read.csv(\n            catalog_sumstats_lut, sep=\"\\t\", header=False\n        )\n        gwas_catalog_study_curation = read_curation_table(\n            gwas_catalog_study_curation_file, session\n        )\n\n        # Process GWAS Catalog studies and get list of studies for curation:\n        (\n            StudyIndexGWASCatalogParser.from_source(\n                catalog_studies, ancestry_lut, sumstats_lut\n            )\n            # Adding existing curation:\n            .annotate_from_study_curation(gwas_catalog_study_curation)\n            # Extract new studies for curation:\n            .extract_studies_for_curation(gwas_catalog_study_curation)\n            # Save table:\n            .toPandas()\n            .to_csv(gwas_catalog_study_curation_out, sep=\"\\t\", index=False)\n        )\n</code></pre>"},{"location":"python_api/steps/gwas_catalog_curation/#gentropy.gwas_catalog_study_curation.GWASCatalogStudyCurationStep.__init__","title":"<code>__init__(session: Session, catalog_study_files: list[str], catalog_ancestry_files: list[str], catalog_sumstats_lut: str, gwas_catalog_study_curation_out: str, gwas_catalog_study_curation_file: str | None) -&gt; None</code>","text":"<p>Run step to annotate and create backlog.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Session object.</p> required <code>catalog_study_files</code> <code>list[str]</code> <p>List of raw GWAS catalog studies file.</p> required <code>catalog_ancestry_files</code> <code>list[str]</code> <p>List of raw ancestry annotations files from GWAS Catalog.</p> required <code>catalog_sumstats_lut</code> <code>str</code> <p>GWAS Catalog summary statistics lookup table.</p> required <code>gwas_catalog_study_curation_out</code> <code>str</code> <p>Path for the updated curation table.</p> required <code>gwas_catalog_study_curation_file</code> <code>str | None</code> <p>Path to the original curation table. Optinal</p> required Source code in <code>src/gentropy/gwas_catalog_study_curation.py</code> <pre><code>def __init__(\n    self,\n    session: Session,\n    catalog_study_files: list[str],\n    catalog_ancestry_files: list[str],\n    catalog_sumstats_lut: str,\n    gwas_catalog_study_curation_out: str,\n    gwas_catalog_study_curation_file: str | None,\n) -&gt; None:\n    \"\"\"Run step to annotate and create backlog.\n\n    Args:\n        session (Session): Session object.\n        catalog_study_files (list[str]): List of raw GWAS catalog studies file.\n        catalog_ancestry_files (list[str]): List of raw ancestry annotations files from GWAS Catalog.\n        catalog_sumstats_lut (str): GWAS Catalog summary statistics lookup table.\n        gwas_catalog_study_curation_out (str): Path for the updated curation table.\n        gwas_catalog_study_curation_file (str | None): Path to the original curation table. Optinal\n    \"\"\"\n    catalog_studies = session.spark.read.csv(\n        list(catalog_study_files), sep=\"\\t\", header=True\n    )\n    ancestry_lut = session.spark.read.csv(\n        list(catalog_ancestry_files), sep=\"\\t\", header=True\n    )\n    sumstats_lut = session.spark.read.csv(\n        catalog_sumstats_lut, sep=\"\\t\", header=False\n    )\n    gwas_catalog_study_curation = read_curation_table(\n        gwas_catalog_study_curation_file, session\n    )\n\n    # Process GWAS Catalog studies and get list of studies for curation:\n    (\n        StudyIndexGWASCatalogParser.from_source(\n            catalog_studies, ancestry_lut, sumstats_lut\n        )\n        # Adding existing curation:\n        .annotate_from_study_curation(gwas_catalog_study_curation)\n        # Extract new studies for curation:\n        .extract_studies_for_curation(gwas_catalog_study_curation)\n        # Save table:\n        .toPandas()\n        .to_csv(gwas_catalog_study_curation_out, sep=\"\\t\", index=False)\n    )\n</code></pre>"},{"location":"python_api/steps/gwas_catalog_inclusion/","title":"gwas_catalog_study_inclusion","text":""},{"location":"python_api/steps/gwas_catalog_inclusion/#gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator","title":"<code>gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator</code>","text":"<p>GWAS Catalog study eligibility for ingestion based on curation and the provided criteria.</p> Source code in <code>src/gentropy/gwas_catalog_study_inclusion.py</code> <pre><code>class GWASCatalogStudyInclusionGenerator:\n    \"\"\"GWAS Catalog study eligibility for ingestion based on curation and the provided criteria.\"\"\"\n\n    @staticmethod\n    def flag_eligible_studies(\n        study_index: StudyIndexGWASCatalog, criteria: str\n    ) -&gt; DataFrame:\n        \"\"\"Apply filter on GWAS Catalog studies based on the provided criteria.\n\n        Args:\n            study_index (StudyIndexGWASCatalog): complete study index to be filtered based on the provided filter set\n            criteria (str): name of the filter set to be applied.\n\n        Raises:\n            ValueError: if the provided filter set is not in the accepted values.\n\n        Returns:\n            DataFrame: filtered dataframe containing only eligible studies.\n        \"\"\"\n        filters: dict[str, Column] = {\n            # Filters applied on studies for ingesting curated associations:\n            \"curation\": (study_index.is_gwas() &amp; study_index.has_mapped_trait()),\n            # Filters applied on studies for ingesting summary statistics:\n            \"summary_stats\": (\n                study_index.is_gwas()\n                &amp; study_index.has_mapped_trait()\n                &amp; (~study_index.is_quality_flagged())\n                &amp; study_index.has_summarystats()\n            ),\n        }\n\n        if criteria not in filters:\n            raise ValueError(\n                f'Wrong value as filter set ({criteria}). Accepted: {\",\".join(filters.keys())}'\n            )\n\n        # Applying the relevant filter to the study:\n        return study_index.df.select(\n            \"studyId\",\n            \"studyType\",\n            \"traitFromSource\",\n            \"traitFromSourceMappedIds\",\n            \"qualityControls\",\n            \"hasSumstats\",\n            filters[criteria].alias(\"isEligible\"),\n        )\n\n    @staticmethod\n    def process_harmonised_list(studies: list[str], session: Session) -&gt; DataFrame:\n        \"\"\"Generate spark dataframe from the provided list.\n\n        Args:\n            studies (list[str]): list of path pointing to harmonised summary statistics.\n            session (Session): session\n\n        Returns:\n            DataFrame: column name is consistent with original implementatin\n        \"\"\"\n        return session.spark.createDataFrame([{\"_c0\": path} for path in studies])\n\n    @staticmethod\n    def get_gwas_catalog_study_index(\n        session: Session,\n        variant_annotation_path: str,\n        catalog_study_files: list[str],\n        catalog_ancestry_files: list[str],\n        harmonised_study_file: str,\n        catalog_associations_file: str,\n        gwas_catalog_study_curation_file: str,\n    ) -&gt; StudyIndexGWASCatalog:\n        \"\"\"Return GWAS Catalog study index.\n\n        Args:\n            session (Session): Session object.\n            variant_annotation_path (str): Input variant annotation path.\n            catalog_study_files (list[str]): List of raw GWAS catalog studies file.\n            catalog_ancestry_files (list[str]): List of raw ancestry annotations files from GWAS Catalog.\n            harmonised_study_file (str): GWAS Catalog summary statistics lookup table.\n            catalog_associations_file (str): Raw GWAS catalog associations file.\n            gwas_catalog_study_curation_file (str): file of the curation table. Optional.\n\n        Returns:\n            StudyIndexGWASCatalog: Completely processed and fully annotated study index.\n        \"\"\"\n        # Extract\n        va = VariantAnnotation.from_parquet(session, variant_annotation_path)\n        catalog_studies = session.spark.read.csv(\n            list(catalog_study_files), sep=\"\\t\", header=True\n        )\n        ancestry_lut = session.spark.read.csv(\n            list(catalog_ancestry_files), sep=\"\\t\", header=True\n        )\n        sumstats_lut = session.spark.read.csv(\n            harmonised_study_file, sep=\"\\t\", header=False\n        )\n        catalog_associations = session.spark.read.csv(\n            catalog_associations_file, sep=\"\\t\", header=True\n        ).persist()\n        gwas_catalog_study_curation = read_curation_table(\n            gwas_catalog_study_curation_file, session\n        )\n\n        # Transform\n        study_index, _ = GWASCatalogStudySplitter.split(\n            StudyIndexGWASCatalogParser.from_source(\n                catalog_studies,\n                ancestry_lut,\n                sumstats_lut,\n            ).annotate_from_study_curation(gwas_catalog_study_curation),\n            GWASCatalogCuratedAssociationsParser.from_source(catalog_associations, va),\n        )\n\n        return study_index\n\n    def __init__(\n        self,\n        session: Session,\n        catalog_study_files: list[str],\n        catalog_ancestry_files: list[str],\n        catalog_associations_file: str,\n        gwas_catalog_study_curation_file: str,\n        variant_annotation_path: str,\n        harmonised_study_file: str,\n        criteria: str,\n        inclusion_list_path: str,\n        exclusion_list_path: str,\n    ) -&gt; None:\n        \"\"\"Run step.\n\n        Args:\n            session (Session): Session object.\n            catalog_study_files (list[str]): List of raw GWAS catalog studies file.\n            catalog_ancestry_files (list[str]): List of raw ancestry annotations files from GWAS Catalog.\n            catalog_associations_file (str): Raw GWAS catalog associations file.\n            gwas_catalog_study_curation_file (str): file of the curation table. Optional.\n            variant_annotation_path (str): Input variant annotation path.\n            harmonised_study_file (str): GWAS Catalog summary statistics lookup table.\n            criteria (str): name of the filter set to be applied.\n            inclusion_list_path (str): Output path for the inclusion list.\n            exclusion_list_path (str): Output path for the exclusion list.\n        \"\"\"\n        # Create study index:\n        study_index = self.get_gwas_catalog_study_index(\n            session,\n            variant_annotation_path,\n            catalog_study_files,\n            catalog_ancestry_files,\n            harmonised_study_file,\n            catalog_associations_file,\n            gwas_catalog_study_curation_file,\n        )\n\n        # Get study indices for inclusion:\n        flagged_studies = self.flag_eligible_studies(study_index, criteria)\n\n        # Output inclusion list:\n        eligible = (\n            flagged_studies.filter(f.col(\"isEligible\")).select(\"studyId\").persist()\n        )\n        eligible.write.mode(session.write_mode).parquet(inclusion_list_path)\n\n        # Output exclusion list:\n        excluded = flagged_studies.filter(~f.col(\"isEligible\")).persist()\n        excluded.write.mode(session.write_mode).parquet(exclusion_list_path)\n</code></pre>"},{"location":"python_api/steps/gwas_catalog_inclusion/#gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator.__init__","title":"<code>__init__(session: Session, catalog_study_files: list[str], catalog_ancestry_files: list[str], catalog_associations_file: str, gwas_catalog_study_curation_file: str, variant_annotation_path: str, harmonised_study_file: str, criteria: str, inclusion_list_path: str, exclusion_list_path: str) -&gt; None</code>","text":"<p>Run step.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Session object.</p> required <code>catalog_study_files</code> <code>list[str]</code> <p>List of raw GWAS catalog studies file.</p> required <code>catalog_ancestry_files</code> <code>list[str]</code> <p>List of raw ancestry annotations files from GWAS Catalog.</p> required <code>catalog_associations_file</code> <code>str</code> <p>Raw GWAS catalog associations file.</p> required <code>gwas_catalog_study_curation_file</code> <code>str</code> <p>file of the curation table. Optional.</p> required <code>variant_annotation_path</code> <code>str</code> <p>Input variant annotation path.</p> required <code>harmonised_study_file</code> <code>str</code> <p>GWAS Catalog summary statistics lookup table.</p> required <code>criteria</code> <code>str</code> <p>name of the filter set to be applied.</p> required <code>inclusion_list_path</code> <code>str</code> <p>Output path for the inclusion list.</p> required <code>exclusion_list_path</code> <code>str</code> <p>Output path for the exclusion list.</p> required Source code in <code>src/gentropy/gwas_catalog_study_inclusion.py</code> <pre><code>def __init__(\n    self,\n    session: Session,\n    catalog_study_files: list[str],\n    catalog_ancestry_files: list[str],\n    catalog_associations_file: str,\n    gwas_catalog_study_curation_file: str,\n    variant_annotation_path: str,\n    harmonised_study_file: str,\n    criteria: str,\n    inclusion_list_path: str,\n    exclusion_list_path: str,\n) -&gt; None:\n    \"\"\"Run step.\n\n    Args:\n        session (Session): Session object.\n        catalog_study_files (list[str]): List of raw GWAS catalog studies file.\n        catalog_ancestry_files (list[str]): List of raw ancestry annotations files from GWAS Catalog.\n        catalog_associations_file (str): Raw GWAS catalog associations file.\n        gwas_catalog_study_curation_file (str): file of the curation table. Optional.\n        variant_annotation_path (str): Input variant annotation path.\n        harmonised_study_file (str): GWAS Catalog summary statistics lookup table.\n        criteria (str): name of the filter set to be applied.\n        inclusion_list_path (str): Output path for the inclusion list.\n        exclusion_list_path (str): Output path for the exclusion list.\n    \"\"\"\n    # Create study index:\n    study_index = self.get_gwas_catalog_study_index(\n        session,\n        variant_annotation_path,\n        catalog_study_files,\n        catalog_ancestry_files,\n        harmonised_study_file,\n        catalog_associations_file,\n        gwas_catalog_study_curation_file,\n    )\n\n    # Get study indices for inclusion:\n    flagged_studies = self.flag_eligible_studies(study_index, criteria)\n\n    # Output inclusion list:\n    eligible = (\n        flagged_studies.filter(f.col(\"isEligible\")).select(\"studyId\").persist()\n    )\n    eligible.write.mode(session.write_mode).parquet(inclusion_list_path)\n\n    # Output exclusion list:\n    excluded = flagged_studies.filter(~f.col(\"isEligible\")).persist()\n    excluded.write.mode(session.write_mode).parquet(exclusion_list_path)\n</code></pre>"},{"location":"python_api/steps/gwas_catalog_inclusion/#gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator.flag_eligible_studies","title":"<code>flag_eligible_studies(study_index: StudyIndexGWASCatalog, criteria: str) -&gt; DataFrame</code>  <code>staticmethod</code>","text":"<p>Apply filter on GWAS Catalog studies based on the provided criteria.</p> <p>Parameters:</p> Name Type Description Default <code>study_index</code> <code>StudyIndexGWASCatalog</code> <p>complete study index to be filtered based on the provided filter set</p> required <code>criteria</code> <code>str</code> <p>name of the filter set to be applied.</p> required <p>Raises:</p> Type Description <code>ValueError</code> <p>if the provided filter set is not in the accepted values.</p> <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>filtered dataframe containing only eligible studies.</p> Source code in <code>src/gentropy/gwas_catalog_study_inclusion.py</code> <pre><code>@staticmethod\ndef flag_eligible_studies(\n    study_index: StudyIndexGWASCatalog, criteria: str\n) -&gt; DataFrame:\n    \"\"\"Apply filter on GWAS Catalog studies based on the provided criteria.\n\n    Args:\n        study_index (StudyIndexGWASCatalog): complete study index to be filtered based on the provided filter set\n        criteria (str): name of the filter set to be applied.\n\n    Raises:\n        ValueError: if the provided filter set is not in the accepted values.\n\n    Returns:\n        DataFrame: filtered dataframe containing only eligible studies.\n    \"\"\"\n    filters: dict[str, Column] = {\n        # Filters applied on studies for ingesting curated associations:\n        \"curation\": (study_index.is_gwas() &amp; study_index.has_mapped_trait()),\n        # Filters applied on studies for ingesting summary statistics:\n        \"summary_stats\": (\n            study_index.is_gwas()\n            &amp; study_index.has_mapped_trait()\n            &amp; (~study_index.is_quality_flagged())\n            &amp; study_index.has_summarystats()\n        ),\n    }\n\n    if criteria not in filters:\n        raise ValueError(\n            f'Wrong value as filter set ({criteria}). Accepted: {\",\".join(filters.keys())}'\n        )\n\n    # Applying the relevant filter to the study:\n    return study_index.df.select(\n        \"studyId\",\n        \"studyType\",\n        \"traitFromSource\",\n        \"traitFromSourceMappedIds\",\n        \"qualityControls\",\n        \"hasSumstats\",\n        filters[criteria].alias(\"isEligible\"),\n    )\n</code></pre>"},{"location":"python_api/steps/gwas_catalog_inclusion/#gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator.get_gwas_catalog_study_index","title":"<code>get_gwas_catalog_study_index(session: Session, variant_annotation_path: str, catalog_study_files: list[str], catalog_ancestry_files: list[str], harmonised_study_file: str, catalog_associations_file: str, gwas_catalog_study_curation_file: str) -&gt; StudyIndexGWASCatalog</code>  <code>staticmethod</code>","text":"<p>Return GWAS Catalog study index.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Session object.</p> required <code>variant_annotation_path</code> <code>str</code> <p>Input variant annotation path.</p> required <code>catalog_study_files</code> <code>list[str]</code> <p>List of raw GWAS catalog studies file.</p> required <code>catalog_ancestry_files</code> <code>list[str]</code> <p>List of raw ancestry annotations files from GWAS Catalog.</p> required <code>harmonised_study_file</code> <code>str</code> <p>GWAS Catalog summary statistics lookup table.</p> required <code>catalog_associations_file</code> <code>str</code> <p>Raw GWAS catalog associations file.</p> required <code>gwas_catalog_study_curation_file</code> <code>str</code> <p>file of the curation table. Optional.</p> required <p>Returns:</p> Name Type Description <code>StudyIndexGWASCatalog</code> <code>StudyIndexGWASCatalog</code> <p>Completely processed and fully annotated study index.</p> Source code in <code>src/gentropy/gwas_catalog_study_inclusion.py</code> <pre><code>@staticmethod\ndef get_gwas_catalog_study_index(\n    session: Session,\n    variant_annotation_path: str,\n    catalog_study_files: list[str],\n    catalog_ancestry_files: list[str],\n    harmonised_study_file: str,\n    catalog_associations_file: str,\n    gwas_catalog_study_curation_file: str,\n) -&gt; StudyIndexGWASCatalog:\n    \"\"\"Return GWAS Catalog study index.\n\n    Args:\n        session (Session): Session object.\n        variant_annotation_path (str): Input variant annotation path.\n        catalog_study_files (list[str]): List of raw GWAS catalog studies file.\n        catalog_ancestry_files (list[str]): List of raw ancestry annotations files from GWAS Catalog.\n        harmonised_study_file (str): GWAS Catalog summary statistics lookup table.\n        catalog_associations_file (str): Raw GWAS catalog associations file.\n        gwas_catalog_study_curation_file (str): file of the curation table. Optional.\n\n    Returns:\n        StudyIndexGWASCatalog: Completely processed and fully annotated study index.\n    \"\"\"\n    # Extract\n    va = VariantAnnotation.from_parquet(session, variant_annotation_path)\n    catalog_studies = session.spark.read.csv(\n        list(catalog_study_files), sep=\"\\t\", header=True\n    )\n    ancestry_lut = session.spark.read.csv(\n        list(catalog_ancestry_files), sep=\"\\t\", header=True\n    )\n    sumstats_lut = session.spark.read.csv(\n        harmonised_study_file, sep=\"\\t\", header=False\n    )\n    catalog_associations = session.spark.read.csv(\n        catalog_associations_file, sep=\"\\t\", header=True\n    ).persist()\n    gwas_catalog_study_curation = read_curation_table(\n        gwas_catalog_study_curation_file, session\n    )\n\n    # Transform\n    study_index, _ = GWASCatalogStudySplitter.split(\n        StudyIndexGWASCatalogParser.from_source(\n            catalog_studies,\n            ancestry_lut,\n            sumstats_lut,\n        ).annotate_from_study_curation(gwas_catalog_study_curation),\n        GWASCatalogCuratedAssociationsParser.from_source(catalog_associations, va),\n    )\n\n    return study_index\n</code></pre>"},{"location":"python_api/steps/gwas_catalog_inclusion/#gentropy.gwas_catalog_study_inclusion.GWASCatalogStudyInclusionGenerator.process_harmonised_list","title":"<code>process_harmonised_list(studies: list[str], session: Session) -&gt; DataFrame</code>  <code>staticmethod</code>","text":"<p>Generate spark dataframe from the provided list.</p> <p>Parameters:</p> Name Type Description Default <code>studies</code> <code>list[str]</code> <p>list of path pointing to harmonised summary statistics.</p> required <code>session</code> <code>Session</code> <p>session</p> required <p>Returns:</p> Name Type Description <code>DataFrame</code> <code>DataFrame</code> <p>column name is consistent with original implementatin</p> Source code in <code>src/gentropy/gwas_catalog_study_inclusion.py</code> <pre><code>@staticmethod\ndef process_harmonised_list(studies: list[str], session: Session) -&gt; DataFrame:\n    \"\"\"Generate spark dataframe from the provided list.\n\n    Args:\n        studies (list[str]): list of path pointing to harmonised summary statistics.\n        session (Session): session\n\n    Returns:\n        DataFrame: column name is consistent with original implementatin\n    \"\"\"\n    return session.spark.createDataFrame([{\"_c0\": path} for path in studies])\n</code></pre>"},{"location":"python_api/steps/gwas_catalog_ingestion/","title":"gwas_catalog_ingestion","text":""},{"location":"python_api/steps/gwas_catalog_ingestion/#gentropy.gwas_catalog_ingestion.GWASCatalogIngestionStep","title":"<code>gentropy.gwas_catalog_ingestion.GWASCatalogIngestionStep</code>","text":"<p>GWAS Catalog ingestion step to extract GWASCatalog Study and StudyLocus tables.</p> <p>!!! note This step currently only processes the GWAS Catalog curated list of top hits.</p> Source code in <code>src/gentropy/gwas_catalog_ingestion.py</code> <pre><code>class GWASCatalogIngestionStep:\n    \"\"\"GWAS Catalog ingestion step to extract GWASCatalog Study and StudyLocus tables.\n\n    !!! note This step currently only processes the GWAS Catalog curated list of top hits.\n    \"\"\"\n\n    def __init__(\n        self,\n        session: Session,\n        catalog_study_files: list[str],\n        catalog_ancestry_files: list[str],\n        catalog_sumstats_lut: str,\n        catalog_associations_file: str,\n        variant_annotation_path: str,\n        catalog_studies_out: str,\n        catalog_associations_out: str,\n        gwas_catalog_study_curation_file: str | None = None,\n        inclusion_list_path: str | None = None,\n    ) -&gt; None:\n        \"\"\"Run step.\n\n        Args:\n            session (Session): Session object.\n            catalog_study_files (list[str]): List of raw GWAS catalog studies file.\n            catalog_ancestry_files (list[str]): List of raw ancestry annotations files from GWAS Catalog.\n            catalog_sumstats_lut (str): GWAS Catalog summary statistics lookup table.\n            catalog_associations_file (str): Raw GWAS catalog associations file.\n            variant_annotation_path (str): Input variant annotation path.\n            catalog_studies_out (str): Output GWAS catalog studies path.\n            catalog_associations_out (str): Output GWAS catalog associations path.\n            gwas_catalog_study_curation_file (str | None): file of the curation table. Optional.\n            inclusion_list_path (str | None): optional inclusion list (parquet)\n        \"\"\"\n        # Extract\n        va = VariantAnnotation.from_parquet(session, variant_annotation_path)\n        catalog_studies = session.spark.read.csv(\n            list(catalog_study_files), sep=\"\\t\", header=True\n        )\n        ancestry_lut = session.spark.read.csv(\n            list(catalog_ancestry_files), sep=\"\\t\", header=True\n        )\n        sumstats_lut = session.spark.read.csv(\n            catalog_sumstats_lut, sep=\"\\t\", header=False\n        )\n        catalog_associations = session.spark.read.csv(\n            catalog_associations_file, sep=\"\\t\", header=True\n        ).persist()\n        gwas_catalog_study_curation = read_curation_table(\n            gwas_catalog_study_curation_file, session\n        )\n\n        # Transform\n        study_index, study_locus = GWASCatalogStudySplitter.split(\n            StudyIndexGWASCatalogParser.from_source(\n                catalog_studies, ancestry_lut, sumstats_lut\n            ).annotate_from_study_curation(gwas_catalog_study_curation),\n            GWASCatalogCuratedAssociationsParser.from_source(catalog_associations, va),\n        )\n\n        # if inclusion list is provided apply filter:\n        if inclusion_list_path:\n            inclusion_list = session.spark.read.parquet(\n                inclusion_list_path, sep=\"\\t\", header=True\n            )\n\n            study_index = study_index.apply_inclusion_list(inclusion_list)\n            study_locus = study_locus.apply_inclusion_list(inclusion_list)\n\n        # Load\n        study_index.df.write.mode(session.write_mode).parquet(catalog_studies_out)\n        study_locus.df.write.mode(session.write_mode).parquet(catalog_associations_out)\n</code></pre>"},{"location":"python_api/steps/gwas_catalog_ingestion/#gentropy.gwas_catalog_ingestion.GWASCatalogIngestionStep.__init__","title":"<code>__init__(session: Session, catalog_study_files: list[str], catalog_ancestry_files: list[str], catalog_sumstats_lut: str, catalog_associations_file: str, variant_annotation_path: str, catalog_studies_out: str, catalog_associations_out: str, gwas_catalog_study_curation_file: str | None = None, inclusion_list_path: str | None = None) -&gt; None</code>","text":"<p>Run step.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Session object.</p> required <code>catalog_study_files</code> <code>list[str]</code> <p>List of raw GWAS catalog studies file.</p> required <code>catalog_ancestry_files</code> <code>list[str]</code> <p>List of raw ancestry annotations files from GWAS Catalog.</p> required <code>catalog_sumstats_lut</code> <code>str</code> <p>GWAS Catalog summary statistics lookup table.</p> required <code>catalog_associations_file</code> <code>str</code> <p>Raw GWAS catalog associations file.</p> required <code>variant_annotation_path</code> <code>str</code> <p>Input variant annotation path.</p> required <code>catalog_studies_out</code> <code>str</code> <p>Output GWAS catalog studies path.</p> required <code>catalog_associations_out</code> <code>str</code> <p>Output GWAS catalog associations path.</p> required <code>gwas_catalog_study_curation_file</code> <code>str | None</code> <p>file of the curation table. Optional.</p> <code>None</code> <code>inclusion_list_path</code> <code>str | None</code> <p>optional inclusion list (parquet)</p> <code>None</code> Source code in <code>src/gentropy/gwas_catalog_ingestion.py</code> <pre><code>def __init__(\n    self,\n    session: Session,\n    catalog_study_files: list[str],\n    catalog_ancestry_files: list[str],\n    catalog_sumstats_lut: str,\n    catalog_associations_file: str,\n    variant_annotation_path: str,\n    catalog_studies_out: str,\n    catalog_associations_out: str,\n    gwas_catalog_study_curation_file: str | None = None,\n    inclusion_list_path: str | None = None,\n) -&gt; None:\n    \"\"\"Run step.\n\n    Args:\n        session (Session): Session object.\n        catalog_study_files (list[str]): List of raw GWAS catalog studies file.\n        catalog_ancestry_files (list[str]): List of raw ancestry annotations files from GWAS Catalog.\n        catalog_sumstats_lut (str): GWAS Catalog summary statistics lookup table.\n        catalog_associations_file (str): Raw GWAS catalog associations file.\n        variant_annotation_path (str): Input variant annotation path.\n        catalog_studies_out (str): Output GWAS catalog studies path.\n        catalog_associations_out (str): Output GWAS catalog associations path.\n        gwas_catalog_study_curation_file (str | None): file of the curation table. Optional.\n        inclusion_list_path (str | None): optional inclusion list (parquet)\n    \"\"\"\n    # Extract\n    va = VariantAnnotation.from_parquet(session, variant_annotation_path)\n    catalog_studies = session.spark.read.csv(\n        list(catalog_study_files), sep=\"\\t\", header=True\n    )\n    ancestry_lut = session.spark.read.csv(\n        list(catalog_ancestry_files), sep=\"\\t\", header=True\n    )\n    sumstats_lut = session.spark.read.csv(\n        catalog_sumstats_lut, sep=\"\\t\", header=False\n    )\n    catalog_associations = session.spark.read.csv(\n        catalog_associations_file, sep=\"\\t\", header=True\n    ).persist()\n    gwas_catalog_study_curation = read_curation_table(\n        gwas_catalog_study_curation_file, session\n    )\n\n    # Transform\n    study_index, study_locus = GWASCatalogStudySplitter.split(\n        StudyIndexGWASCatalogParser.from_source(\n            catalog_studies, ancestry_lut, sumstats_lut\n        ).annotate_from_study_curation(gwas_catalog_study_curation),\n        GWASCatalogCuratedAssociationsParser.from_source(catalog_associations, va),\n    )\n\n    # if inclusion list is provided apply filter:\n    if inclusion_list_path:\n        inclusion_list = session.spark.read.parquet(\n            inclusion_list_path, sep=\"\\t\", header=True\n        )\n\n        study_index = study_index.apply_inclusion_list(inclusion_list)\n        study_locus = study_locus.apply_inclusion_list(inclusion_list)\n\n    # Load\n    study_index.df.write.mode(session.write_mode).parquet(catalog_studies_out)\n    study_locus.df.write.mode(session.write_mode).parquet(catalog_associations_out)\n</code></pre>"},{"location":"python_api/steps/gwas_catalog_sumstat_preprocess/","title":"gwas_catalog_sumstat_preprocess","text":""},{"location":"python_api/steps/gwas_catalog_sumstat_preprocess/#gentropy.gwas_catalog_sumstat_preprocess.GWASCatalogSumstatsPreprocessStep","title":"<code>gentropy.gwas_catalog_sumstat_preprocess.GWASCatalogSumstatsPreprocessStep</code>","text":"<p>Step to preprocess GWAS Catalog harmonised summary stats.</p> Source code in <code>src/gentropy/gwas_catalog_sumstat_preprocess.py</code> <pre><code>class GWASCatalogSumstatsPreprocessStep:\n    \"\"\"Step to preprocess GWAS Catalog harmonised summary stats.\"\"\"\n\n    def __init__(\n        self, session: Session, raw_sumstats_path: str, out_sumstats_path: str\n    ) -&gt; None:\n        \"\"\"Run step to preprocess GWAS Catalog harmonised summary stats and produce SummaryStatistics dataset.\n\n        Args:\n            session (Session): Session object.\n            raw_sumstats_path (str): Input GWAS Catalog harmonised summary stats path.\n            out_sumstats_path (str): Output SummaryStatistics dataset path.\n        \"\"\"\n        # Processing dataset:\n        GWASCatalogSummaryStatistics.from_gwas_harmonized_summary_stats(\n            session.spark, raw_sumstats_path\n        ).df.write.mode(session.write_mode).parquet(out_sumstats_path)\n        session.logger.info(\"Processing dataset successfully completed.\")\n</code></pre>"},{"location":"python_api/steps/gwas_catalog_sumstat_preprocess/#gentropy.gwas_catalog_sumstat_preprocess.GWASCatalogSumstatsPreprocessStep.__init__","title":"<code>__init__(session: Session, raw_sumstats_path: str, out_sumstats_path: str) -&gt; None</code>","text":"<p>Run step to preprocess GWAS Catalog harmonised summary stats and produce SummaryStatistics dataset.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Session object.</p> required <code>raw_sumstats_path</code> <code>str</code> <p>Input GWAS Catalog harmonised summary stats path.</p> required <code>out_sumstats_path</code> <code>str</code> <p>Output SummaryStatistics dataset path.</p> required Source code in <code>src/gentropy/gwas_catalog_sumstat_preprocess.py</code> <pre><code>def __init__(\n    self, session: Session, raw_sumstats_path: str, out_sumstats_path: str\n) -&gt; None:\n    \"\"\"Run step to preprocess GWAS Catalog harmonised summary stats and produce SummaryStatistics dataset.\n\n    Args:\n        session (Session): Session object.\n        raw_sumstats_path (str): Input GWAS Catalog harmonised summary stats path.\n        out_sumstats_path (str): Output SummaryStatistics dataset path.\n    \"\"\"\n    # Processing dataset:\n    GWASCatalogSummaryStatistics.from_gwas_harmonized_summary_stats(\n        session.spark, raw_sumstats_path\n    ).df.write.mode(session.write_mode).parquet(out_sumstats_path)\n    session.logger.info(\"Processing dataset successfully completed.\")\n</code></pre>"},{"location":"python_api/steps/l2g/","title":"locus_to_gene","text":""},{"location":"python_api/steps/l2g/#gentropy.l2g.LocusToGeneStep","title":"<code>gentropy.l2g.LocusToGeneStep</code>","text":"<p>Locus to gene step.</p> Source code in <code>src/gentropy/l2g.py</code> <pre><code>class LocusToGeneStep:\n    \"\"\"Locus to gene step.\"\"\"\n\n    def __init__(\n        self,\n        session: Session,\n        run_mode: str,\n        model_path: str,\n        predictions_path: str,\n        credible_set_path: str,\n        variant_gene_path: str,\n        colocalisation_path: str,\n        study_index_path: str,\n        gold_standard_curation_path: str,\n        gene_interactions_path: str,\n        features_list: list[str],\n        hyperparameters: dict[str, Any],\n        wandb_run_name: str | None = None,\n        perform_cross_validation: bool = False,\n    ) -&gt; None:\n        \"\"\"Run step.\n\n        Args:\n            session (Session): Session object.\n            run_mode (str): One of \"train\" or \"predict\".\n            model_path (str): Path to save the model.\n            predictions_path (str): Path to save the predictions.\n            credible_set_path (str): Path to credible set Parquet files.\n            variant_gene_path (str): Path to variant to gene Parquet files.\n            colocalisation_path (str): Path to colocalisation Parquet files.\n            study_index_path (str): Path to study index Parquet files.\n            gold_standard_curation_path (str): Path to gold standard curation JSON files.\n            gene_interactions_path (str): Path to gene interactions Parquet files.\n            features_list (list[str]): List of features to use.\n            hyperparameters (dict[str, Any]): Hyperparameters for the model.\n            wandb_run_name (str | None): Name of the run to be tracked on W&amp;B.\n            perform_cross_validation (bool): Whether to perform cross validation.\n\n        Raises:\n            ValueError: if run_mode is not one of \"train\" or \"predict\".\n        \"\"\"\n        print(\"Sci-kit learn version: \", sklearn.__version__)  # noqa: T201\n        if run_mode not in [\"train\", \"predict\"]:\n            raise ValueError(\n                f\"run_mode must be one of 'train' or 'predict', got {run_mode}\"\n            )\n        # Load common inputs\n        credible_set = StudyLocus.from_parquet(\n            session, credible_set_path, recursiveFileLookup=True\n        )\n        studies = StudyIndex.from_parquet(\n            session, study_index_path, recursiveFileLookup=True\n        )\n        v2g = V2G.from_parquet(session, variant_gene_path)\n        coloc = Colocalisation.from_parquet(\n            session, colocalisation_path, recursiveFileLookup=True\n        )\n\n        if run_mode == \"predict\":\n            if not model_path or not predictions_path:\n                raise ValueError(\n                    \"model_path and predictions_path must be set for predict mode.\"\n                )\n            predictions = L2GPrediction.from_credible_set(\n                model_path, list(features_list), credible_set, studies, v2g, coloc\n            )\n            predictions.df.write.mode(session.write_mode).parquet(predictions_path)\n            session.logger.info(predictions_path)\n        elif (\n            run_mode == \"train\"\n            and gold_standard_curation_path\n            and gene_interactions_path\n        ):\n            # Process gold standard and L2G features\n            gs_curation = session.spark.read.json(gold_standard_curation_path)\n            interactions = session.spark.read.parquet(gene_interactions_path)\n            study_locus_overlap = StudyLocus(\n                # We just extract overlaps of associations in the gold standard. This parsing is a duplication of the one in the gold standard curation,\n                # but we need to do it here to be able to parse gold standards later\n                _df=credible_set.df.join(\n                    f.broadcast(\n                        gs_curation.select(\n                            StudyLocus.assign_study_locus_id(\n                                f.col(\"association_info.otg_id\"),  # studyId\n                                f.concat_ws(  # variantId\n                                    \"_\",\n                                    f.col(\"sentinel_variant.locus_GRCh38.chromosome\"),\n                                    f.col(\"sentinel_variant.locus_GRCh38.position\"),\n                                    f.col(\"sentinel_variant.alleles.reference\"),\n                                    f.col(\"sentinel_variant.alleles.alternative\"),\n                                ),\n                            ).alias(\"studyLocusId\"),\n                        )\n                    ),\n                    \"studyLocusId\",\n                    \"inner\",\n                ),\n                _schema=StudyLocus.get_schema(),\n            ).find_overlaps(studies)\n\n            gold_standards = L2GGoldStandard.from_otg_curation(\n                gold_standard_curation=gs_curation,\n                v2g=v2g,\n                study_locus_overlap=study_locus_overlap,\n                interactions=interactions,\n            )\n\n            fm = L2GFeatureMatrix.generate_features(\n                features_list=features_list,\n                credible_set=credible_set,\n                study_index=studies,\n                variant_gene=v2g,\n                colocalisation=coloc,\n            )\n\n            data = (\n                # Annotate gold standards with features\n                L2GFeatureMatrix(\n                    _df=fm.df.join(\n                        f.broadcast(\n                            gold_standards.df.drop(\"variantId\", \"studyId\", \"sources\")\n                        ),\n                        on=[\"studyLocusId\", \"geneId\"],\n                        how=\"inner\",\n                    ),\n                    _schema=L2GFeatureMatrix.get_schema(),\n                )\n                .fill_na()\n                .select_features(list(features_list))\n            )\n\n            # Instantiate classifier\n            estimator = SparkXGBClassifier(\n                eval_metric=\"logloss\",\n                features_col=\"features\",\n                label_col=\"label\",\n                max_depth=5,\n            )\n            l2g_model = LocusToGeneModel(\n                features_list=list(features_list), estimator=estimator\n            )\n            if perform_cross_validation:\n                # Perform cross validation to extract what are the best hyperparameters\n                cv_folds = hyperparameters.get(\"cross_validation_folds\", 5)\n                LocusToGeneTrainer.cross_validate(\n                    l2g_model=l2g_model,\n                    data=data,\n                    num_folds=cv_folds,\n                )\n            else:\n                # Train model\n                LocusToGeneTrainer.train(\n                    gold_standard_data=data,\n                    l2g_model=l2g_model,\n                    model_path=model_path,\n                    evaluate=True,\n                    wandb_run_name=wandb_run_name,\n                    **hyperparameters,\n                )\n                session.logger.info(model_path)\n</code></pre>"},{"location":"python_api/steps/l2g/#gentropy.l2g.LocusToGeneStep.__init__","title":"<code>__init__(session: Session, run_mode: str, model_path: str, predictions_path: str, credible_set_path: str, variant_gene_path: str, colocalisation_path: str, study_index_path: str, gold_standard_curation_path: str, gene_interactions_path: str, features_list: list[str], hyperparameters: dict[str, Any], wandb_run_name: str | None = None, perform_cross_validation: bool = False) -&gt; None</code>","text":"<p>Run step.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Session object.</p> required <code>run_mode</code> <code>str</code> <p>One of \"train\" or \"predict\".</p> required <code>model_path</code> <code>str</code> <p>Path to save the model.</p> required <code>predictions_path</code> <code>str</code> <p>Path to save the predictions.</p> required <code>credible_set_path</code> <code>str</code> <p>Path to credible set Parquet files.</p> required <code>variant_gene_path</code> <code>str</code> <p>Path to variant to gene Parquet files.</p> required <code>colocalisation_path</code> <code>str</code> <p>Path to colocalisation Parquet files.</p> required <code>study_index_path</code> <code>str</code> <p>Path to study index Parquet files.</p> required <code>gold_standard_curation_path</code> <code>str</code> <p>Path to gold standard curation JSON files.</p> required <code>gene_interactions_path</code> <code>str</code> <p>Path to gene interactions Parquet files.</p> required <code>features_list</code> <code>list[str]</code> <p>List of features to use.</p> required <code>hyperparameters</code> <code>dict[str, Any]</code> <p>Hyperparameters for the model.</p> required <code>wandb_run_name</code> <code>str | None</code> <p>Name of the run to be tracked on W&amp;B.</p> <code>None</code> <code>perform_cross_validation</code> <code>bool</code> <p>Whether to perform cross validation.</p> <code>False</code> <p>Raises:</p> Type Description <code>ValueError</code> <p>if run_mode is not one of \"train\" or \"predict\".</p> Source code in <code>src/gentropy/l2g.py</code> <pre><code>def __init__(\n    self,\n    session: Session,\n    run_mode: str,\n    model_path: str,\n    predictions_path: str,\n    credible_set_path: str,\n    variant_gene_path: str,\n    colocalisation_path: str,\n    study_index_path: str,\n    gold_standard_curation_path: str,\n    gene_interactions_path: str,\n    features_list: list[str],\n    hyperparameters: dict[str, Any],\n    wandb_run_name: str | None = None,\n    perform_cross_validation: bool = False,\n) -&gt; None:\n    \"\"\"Run step.\n\n    Args:\n        session (Session): Session object.\n        run_mode (str): One of \"train\" or \"predict\".\n        model_path (str): Path to save the model.\n        predictions_path (str): Path to save the predictions.\n        credible_set_path (str): Path to credible set Parquet files.\n        variant_gene_path (str): Path to variant to gene Parquet files.\n        colocalisation_path (str): Path to colocalisation Parquet files.\n        study_index_path (str): Path to study index Parquet files.\n        gold_standard_curation_path (str): Path to gold standard curation JSON files.\n        gene_interactions_path (str): Path to gene interactions Parquet files.\n        features_list (list[str]): List of features to use.\n        hyperparameters (dict[str, Any]): Hyperparameters for the model.\n        wandb_run_name (str | None): Name of the run to be tracked on W&amp;B.\n        perform_cross_validation (bool): Whether to perform cross validation.\n\n    Raises:\n        ValueError: if run_mode is not one of \"train\" or \"predict\".\n    \"\"\"\n    print(\"Sci-kit learn version: \", sklearn.__version__)  # noqa: T201\n    if run_mode not in [\"train\", \"predict\"]:\n        raise ValueError(\n            f\"run_mode must be one of 'train' or 'predict', got {run_mode}\"\n        )\n    # Load common inputs\n    credible_set = StudyLocus.from_parquet(\n        session, credible_set_path, recursiveFileLookup=True\n    )\n    studies = StudyIndex.from_parquet(\n        session, study_index_path, recursiveFileLookup=True\n    )\n    v2g = V2G.from_parquet(session, variant_gene_path)\n    coloc = Colocalisation.from_parquet(\n        session, colocalisation_path, recursiveFileLookup=True\n    )\n\n    if run_mode == \"predict\":\n        if not model_path or not predictions_path:\n            raise ValueError(\n                \"model_path and predictions_path must be set for predict mode.\"\n            )\n        predictions = L2GPrediction.from_credible_set(\n            model_path, list(features_list), credible_set, studies, v2g, coloc\n        )\n        predictions.df.write.mode(session.write_mode).parquet(predictions_path)\n        session.logger.info(predictions_path)\n    elif (\n        run_mode == \"train\"\n        and gold_standard_curation_path\n        and gene_interactions_path\n    ):\n        # Process gold standard and L2G features\n        gs_curation = session.spark.read.json(gold_standard_curation_path)\n        interactions = session.spark.read.parquet(gene_interactions_path)\n        study_locus_overlap = StudyLocus(\n            # We just extract overlaps of associations in the gold standard. This parsing is a duplication of the one in the gold standard curation,\n            # but we need to do it here to be able to parse gold standards later\n            _df=credible_set.df.join(\n                f.broadcast(\n                    gs_curation.select(\n                        StudyLocus.assign_study_locus_id(\n                            f.col(\"association_info.otg_id\"),  # studyId\n                            f.concat_ws(  # variantId\n                                \"_\",\n                                f.col(\"sentinel_variant.locus_GRCh38.chromosome\"),\n                                f.col(\"sentinel_variant.locus_GRCh38.position\"),\n                                f.col(\"sentinel_variant.alleles.reference\"),\n                                f.col(\"sentinel_variant.alleles.alternative\"),\n                            ),\n                        ).alias(\"studyLocusId\"),\n                    )\n                ),\n                \"studyLocusId\",\n                \"inner\",\n            ),\n            _schema=StudyLocus.get_schema(),\n        ).find_overlaps(studies)\n\n        gold_standards = L2GGoldStandard.from_otg_curation(\n            gold_standard_curation=gs_curation,\n            v2g=v2g,\n            study_locus_overlap=study_locus_overlap,\n            interactions=interactions,\n        )\n\n        fm = L2GFeatureMatrix.generate_features(\n            features_list=features_list,\n            credible_set=credible_set,\n            study_index=studies,\n            variant_gene=v2g,\n            colocalisation=coloc,\n        )\n\n        data = (\n            # Annotate gold standards with features\n            L2GFeatureMatrix(\n                _df=fm.df.join(\n                    f.broadcast(\n                        gold_standards.df.drop(\"variantId\", \"studyId\", \"sources\")\n                    ),\n                    on=[\"studyLocusId\", \"geneId\"],\n                    how=\"inner\",\n                ),\n                _schema=L2GFeatureMatrix.get_schema(),\n            )\n            .fill_na()\n            .select_features(list(features_list))\n        )\n\n        # Instantiate classifier\n        estimator = SparkXGBClassifier(\n            eval_metric=\"logloss\",\n            features_col=\"features\",\n            label_col=\"label\",\n            max_depth=5,\n        )\n        l2g_model = LocusToGeneModel(\n            features_list=list(features_list), estimator=estimator\n        )\n        if perform_cross_validation:\n            # Perform cross validation to extract what are the best hyperparameters\n            cv_folds = hyperparameters.get(\"cross_validation_folds\", 5)\n            LocusToGeneTrainer.cross_validate(\n                l2g_model=l2g_model,\n                data=data,\n                num_folds=cv_folds,\n            )\n        else:\n            # Train model\n            LocusToGeneTrainer.train(\n                gold_standard_data=data,\n                l2g_model=l2g_model,\n                model_path=model_path,\n                evaluate=True,\n                wandb_run_name=wandb_run_name,\n                **hyperparameters,\n            )\n            session.logger.info(model_path)\n</code></pre>"},{"location":"python_api/steps/ld_clump/","title":"ld_based_clumping","text":""},{"location":"python_api/steps/ld_clump/#gentropy.ld_based_clumping.LDBasedClumpingStep","title":"<code>gentropy.ld_based_clumping.LDBasedClumpingStep</code>","text":"<p>Step to perform LD-based clumping on study locus dataset.</p> <p>As a first step, study locus is enriched with population specific linked-variants. That's why the study index and the ld index is required for this step. Study loci are flaggged in the resulting dataset, which can be explained by a more significant association from the same study.</p> Source code in <code>src/gentropy/ld_based_clumping.py</code> <pre><code>class LDBasedClumpingStep:\n    \"\"\"Step to perform LD-based clumping on study locus dataset.\n\n    As a first step, study locus is enriched with population specific linked-variants.\n    That's why the study index and the ld index is required for this step. Study loci are flaggged\n    in the resulting dataset, which can be explained by a more significant association\n    from the same study.\n    \"\"\"\n\n    def __init__(\n        self,\n        session: Session,\n        study_locus_input_path: str,\n        study_index_path: str,\n        ld_index_path: str,\n        clumped_study_locus_output_path: str,\n    ) -&gt; None:\n        \"\"\"Run LD-based clumping step.\n\n        Args:\n            session (Session): Session object.\n            study_locus_input_path (str): Path to the input study locus.\n            study_index_path (str): Path to the study index.\n            ld_index_path (str): Path to the LD index.\n            clumped_study_locus_output_path (str): path of the resulting, clumped study-locus dataset.\n        \"\"\"\n        study_locus = StudyLocus.from_parquet(session, study_locus_input_path)\n        ld_index = LDIndex.from_parquet(session, ld_index_path)\n        study_index = StudyIndex.from_parquet(session, study_index_path)\n\n        (\n            study_locus\n            # Annotating study locus with LD information:\n            .annotate_ld(study_index, ld_index)\n            .clump()\n            # Save result:\n            .df.write.mode(session.write_mode)\n            .parquet(clumped_study_locus_output_path)\n        )\n</code></pre>"},{"location":"python_api/steps/ld_clump/#gentropy.ld_based_clumping.LDBasedClumpingStep.__init__","title":"<code>__init__(session: Session, study_locus_input_path: str, study_index_path: str, ld_index_path: str, clumped_study_locus_output_path: str) -&gt; None</code>","text":"<p>Run LD-based clumping step.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Session object.</p> required <code>study_locus_input_path</code> <code>str</code> <p>Path to the input study locus.</p> required <code>study_index_path</code> <code>str</code> <p>Path to the study index.</p> required <code>ld_index_path</code> <code>str</code> <p>Path to the LD index.</p> required <code>clumped_study_locus_output_path</code> <code>str</code> <p>path of the resulting, clumped study-locus dataset.</p> required Source code in <code>src/gentropy/ld_based_clumping.py</code> <pre><code>def __init__(\n    self,\n    session: Session,\n    study_locus_input_path: str,\n    study_index_path: str,\n    ld_index_path: str,\n    clumped_study_locus_output_path: str,\n) -&gt; None:\n    \"\"\"Run LD-based clumping step.\n\n    Args:\n        session (Session): Session object.\n        study_locus_input_path (str): Path to the input study locus.\n        study_index_path (str): Path to the study index.\n        ld_index_path (str): Path to the LD index.\n        clumped_study_locus_output_path (str): path of the resulting, clumped study-locus dataset.\n    \"\"\"\n    study_locus = StudyLocus.from_parquet(session, study_locus_input_path)\n    ld_index = LDIndex.from_parquet(session, ld_index_path)\n    study_index = StudyIndex.from_parquet(session, study_index_path)\n\n    (\n        study_locus\n        # Annotating study locus with LD information:\n        .annotate_ld(study_index, ld_index)\n        .clump()\n        # Save result:\n        .df.write.mode(session.write_mode)\n        .parquet(clumped_study_locus_output_path)\n    )\n</code></pre>"},{"location":"python_api/steps/ld_index/","title":"ld_index","text":""},{"location":"python_api/steps/ld_index/#gentropy.ld_index.LDIndexStep","title":"<code>gentropy.ld_index.LDIndexStep</code>","text":"<p>LD index step.</p> <p>This step is resource intensive</p> <p>Suggested params: high memory machine, 5TB of boot disk, no SSDs.</p> Source code in <code>src/gentropy/ld_index.py</code> <pre><code>class LDIndexStep:\n    \"\"\"LD index step.\n\n    !!! warning \"This step is resource intensive\"\n\n        Suggested params: high memory machine, 5TB of boot disk, no SSDs.\n    \"\"\"\n\n    def __init__(self, session: Session, min_r2: float, ld_index_out: str) -&gt; None:\n        \"\"\"Run step.\n\n        Args:\n            session (Session): Session object.\n            min_r2 (float): Minimum r2 to consider when considering variants within a window.\n            ld_index_out (str): Output LD index path.\n        \"\"\"\n        hl.init(sc=session.spark.sparkContext, log=\"/dev/null\")\n        (\n            GnomADLDMatrix()\n            .as_ld_index(min_r2)\n            .df.write.partitionBy(\"chromosome\")\n            .mode(session.write_mode)\n            .parquet(ld_index_out)\n        )\n        session.logger.info(ld_index_out)\n</code></pre>"},{"location":"python_api/steps/ld_index/#gentropy.ld_index.LDIndexStep.__init__","title":"<code>__init__(session: Session, min_r2: float, ld_index_out: str) -&gt; None</code>","text":"<p>Run step.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Session object.</p> required <code>min_r2</code> <code>float</code> <p>Minimum r2 to consider when considering variants within a window.</p> required <code>ld_index_out</code> <code>str</code> <p>Output LD index path.</p> required Source code in <code>src/gentropy/ld_index.py</code> <pre><code>def __init__(self, session: Session, min_r2: float, ld_index_out: str) -&gt; None:\n    \"\"\"Run step.\n\n    Args:\n        session (Session): Session object.\n        min_r2 (float): Minimum r2 to consider when considering variants within a window.\n        ld_index_out (str): Output LD index path.\n    \"\"\"\n    hl.init(sc=session.spark.sparkContext, log=\"/dev/null\")\n    (\n        GnomADLDMatrix()\n        .as_ld_index(min_r2)\n        .df.write.partitionBy(\"chromosome\")\n        .mode(session.write_mode)\n        .parquet(ld_index_out)\n    )\n    session.logger.info(ld_index_out)\n</code></pre>"},{"location":"python_api/steps/pics/","title":"pics","text":""},{"location":"python_api/steps/pics/#gentropy.pics.PICSStep","title":"<code>gentropy.pics.PICSStep</code>","text":"<p>PICS finemapping of LD-annotated StudyLocus.</p> Source code in <code>src/gentropy/pics.py</code> <pre><code>class PICSStep:\n    \"\"\"PICS finemapping of LD-annotated StudyLocus.\"\"\"\n\n    def __init__(\n        self,\n        session: Session,\n        study_locus_ld_annotated_in: str,\n        picsed_study_locus_out: str,\n    ) -&gt; None:\n        \"\"\"Run PICS on LD annotated study-locus.\n\n        Args:\n            session (Session): Session object.\n            study_locus_ld_annotated_in (str): Input LD annotated study-locus path.\n            picsed_study_locus_out (str): Output PICSed study-locus path.\n        \"\"\"\n        # Extract\n        study_locus_ld_annotated = StudyLocus.from_parquet(\n            session, study_locus_ld_annotated_in\n        )\n        # PICS\n        picsed_sl = (\n            PICS.finemap(study_locus_ld_annotated)\n            .annotate_credible_sets()\n            .filter_credible_set(credible_interval=CredibleInterval.IS99)\n        )\n        # Write\n        picsed_sl.df.write.mode(session.write_mode).parquet(picsed_study_locus_out)\n</code></pre>"},{"location":"python_api/steps/pics/#gentropy.pics.PICSStep.__init__","title":"<code>__init__(session: Session, study_locus_ld_annotated_in: str, picsed_study_locus_out: str) -&gt; None</code>","text":"<p>Run PICS on LD annotated study-locus.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Session object.</p> required <code>study_locus_ld_annotated_in</code> <code>str</code> <p>Input LD annotated study-locus path.</p> required <code>picsed_study_locus_out</code> <code>str</code> <p>Output PICSed study-locus path.</p> required Source code in <code>src/gentropy/pics.py</code> <pre><code>def __init__(\n    self,\n    session: Session,\n    study_locus_ld_annotated_in: str,\n    picsed_study_locus_out: str,\n) -&gt; None:\n    \"\"\"Run PICS on LD annotated study-locus.\n\n    Args:\n        session (Session): Session object.\n        study_locus_ld_annotated_in (str): Input LD annotated study-locus path.\n        picsed_study_locus_out (str): Output PICSed study-locus path.\n    \"\"\"\n    # Extract\n    study_locus_ld_annotated = StudyLocus.from_parquet(\n        session, study_locus_ld_annotated_in\n    )\n    # PICS\n    picsed_sl = (\n        PICS.finemap(study_locus_ld_annotated)\n        .annotate_credible_sets()\n        .filter_credible_set(credible_interval=CredibleInterval.IS99)\n    )\n    # Write\n    picsed_sl.df.write.mode(session.write_mode).parquet(picsed_study_locus_out)\n</code></pre>"},{"location":"python_api/steps/variant_annotation_step/","title":"variant_annotation","text":""},{"location":"python_api/steps/variant_annotation_step/#gentropy.variant_annotation.VariantAnnotationStep","title":"<code>gentropy.variant_annotation.VariantAnnotationStep</code>","text":"<p>Variant annotation step.</p> <p>Variant annotation step produces a dataset of the type <code>VariantAnnotation</code> derived from gnomADs <code>gnomad.genomes.vX.X.X.sites.ht</code> Hail's table. This dataset is used to validate variants and as a source of annotation.</p> Source code in <code>src/gentropy/variant_annotation.py</code> <pre><code>class VariantAnnotationStep:\n    \"\"\"Variant annotation step.\n\n    Variant annotation step produces a dataset of the type `VariantAnnotation` derived from gnomADs `gnomad.genomes.vX.X.X.sites.ht` Hail's table. This dataset is used to validate variants and as a source of annotation.\n    \"\"\"\n\n    def __init__(self, session: Session, variant_annotation_path: str) -&gt; None:\n        \"\"\"Run Variant Annotation step.\n\n        Args:\n            session (Session): Session object.\n            variant_annotation_path (str): Variant annotation dataset path.\n        \"\"\"\n        # Initialise hail session.\n        hl.init(sc=session.spark.sparkContext, log=\"/dev/null\")\n        # Run variant annotation.\n        variant_annotation = GnomADVariants().as_variant_annotation()\n        # Write data partitioned by chromosome and position.\n        (\n            variant_annotation.df.write.mode(session.write_mode).parquet(\n                variant_annotation_path\n            )\n        )\n</code></pre>"},{"location":"python_api/steps/variant_annotation_step/#gentropy.variant_annotation.VariantAnnotationStep.__init__","title":"<code>__init__(session: Session, variant_annotation_path: str) -&gt; None</code>","text":"<p>Run Variant Annotation step.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Session object.</p> required <code>variant_annotation_path</code> <code>str</code> <p>Variant annotation dataset path.</p> required Source code in <code>src/gentropy/variant_annotation.py</code> <pre><code>def __init__(self, session: Session, variant_annotation_path: str) -&gt; None:\n    \"\"\"Run Variant Annotation step.\n\n    Args:\n        session (Session): Session object.\n        variant_annotation_path (str): Variant annotation dataset path.\n    \"\"\"\n    # Initialise hail session.\n    hl.init(sc=session.spark.sparkContext, log=\"/dev/null\")\n    # Run variant annotation.\n    variant_annotation = GnomADVariants().as_variant_annotation()\n    # Write data partitioned by chromosome and position.\n    (\n        variant_annotation.df.write.mode(session.write_mode).parquet(\n            variant_annotation_path\n        )\n    )\n</code></pre>"},{"location":"python_api/steps/variant_index_step/","title":"variant_index","text":""},{"location":"python_api/steps/variant_index_step/#gentropy.variant_index.VariantIndexStep","title":"<code>gentropy.variant_index.VariantIndexStep</code>","text":"<p>Run variant index step to only variants in study-locus sets.</p> <p>Using a <code>VariantAnnotation</code> dataset as a reference, this step creates and writes a dataset of the type <code>VariantIndex</code> that includes only variants that have disease-association data with a reduced set of annotations.</p> Source code in <code>src/gentropy/variant_index.py</code> <pre><code>class VariantIndexStep:\n    \"\"\"Run variant index step to only variants in study-locus sets.\n\n    Using a `VariantAnnotation` dataset as a reference, this step creates and writes a dataset of the type `VariantIndex` that includes only variants that have disease-association data with a reduced set of annotations.\n    \"\"\"\n\n    def __init__(\n        self: VariantIndexStep,\n        session: Session,\n        variant_annotation_path: str,\n        credible_set_path: str,\n        variant_index_path: str,\n    ) -&gt; None:\n        \"\"\"Run VariantIndex step.\n\n        Args:\n            session (Session): Session object.\n            variant_annotation_path (str): Variant annotation dataset path.\n            credible_set_path (str): Credible set dataset path.\n            variant_index_path (str): Variant index dataset path.\n        \"\"\"\n        # Extract\n        va = VariantAnnotation.from_parquet(session, variant_annotation_path)\n        credible_set = StudyLocus.from_parquet(\n            session, credible_set_path, recursiveFileLookup=True\n        )\n\n        # Transform\n        vi = VariantIndex.from_variant_annotation(va, credible_set)\n\n        (\n            vi.df.write.partitionBy(\"chromosome\")\n            .mode(session.write_mode)\n            .parquet(variant_index_path)\n        )\n</code></pre>"},{"location":"python_api/steps/variant_index_step/#gentropy.variant_index.VariantIndexStep.__init__","title":"<code>__init__(session: Session, variant_annotation_path: str, credible_set_path: str, variant_index_path: str) -&gt; None</code>","text":"<p>Run VariantIndex step.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Session object.</p> required <code>variant_annotation_path</code> <code>str</code> <p>Variant annotation dataset path.</p> required <code>credible_set_path</code> <code>str</code> <p>Credible set dataset path.</p> required <code>variant_index_path</code> <code>str</code> <p>Variant index dataset path.</p> required Source code in <code>src/gentropy/variant_index.py</code> <pre><code>def __init__(\n    self: VariantIndexStep,\n    session: Session,\n    variant_annotation_path: str,\n    credible_set_path: str,\n    variant_index_path: str,\n) -&gt; None:\n    \"\"\"Run VariantIndex step.\n\n    Args:\n        session (Session): Session object.\n        variant_annotation_path (str): Variant annotation dataset path.\n        credible_set_path (str): Credible set dataset path.\n        variant_index_path (str): Variant index dataset path.\n    \"\"\"\n    # Extract\n    va = VariantAnnotation.from_parquet(session, variant_annotation_path)\n    credible_set = StudyLocus.from_parquet(\n        session, credible_set_path, recursiveFileLookup=True\n    )\n\n    # Transform\n    vi = VariantIndex.from_variant_annotation(va, credible_set)\n\n    (\n        vi.df.write.partitionBy(\"chromosome\")\n        .mode(session.write_mode)\n        .parquet(variant_index_path)\n    )\n</code></pre>"},{"location":"python_api/steps/variant_to_gene_step/","title":"variant_to_gene","text":""},{"location":"python_api/steps/variant_to_gene_step/#gentropy.v2g.V2GStep","title":"<code>gentropy.v2g.V2GStep</code>","text":"<p>Variant-to-gene (V2G) step.</p> <p>This step aims to generate a dataset that contains multiple pieces of evidence supporting the functional association of specific variants with genes. Some of the evidence types include:</p> <ol> <li>Chromatin interaction experiments, e.g. Promoter Capture Hi-C (PCHi-C).</li> <li>In silico functional predictions, e.g. Variant Effect Predictor (VEP) from Ensembl.</li> <li>Distance between the variant and each gene's canonical transcription start site (TSS).</li> </ol> <p>Attributes:</p> Name Type Description <code>session</code> <code>Session</code> <p>Session object.</p> <code>variant_index_path</code> <code>str</code> <p>Input variant index path.</p> <code>variant_annotation_path</code> <code>str</code> <p>Input variant annotation path.</p> <code>gene_index_path</code> <code>str</code> <p>Input gene index path.</p> <code>vep_consequences_path</code> <code>str</code> <p>Input VEP consequences path.</p> <code>liftover_chain_file_path</code> <code>str</code> <p>Path to GRCh37 to GRCh38 chain file.</p> <code>liftover_max_length_difference</code> <code>str</code> <p>Maximum length difference for liftover.</p> <code>max_distance</code> <code>int</code> <p>Maximum distance to consider.</p> <code>approved_biotypes</code> <code>list[str]</code> <p>List of approved biotypes.</p> <code>intervals</code> <code>dict</code> <p>Dictionary of interval sources.</p> <code>v2g_path</code> <code>str</code> <p>Output V2G path.</p> Source code in <code>src/gentropy/v2g.py</code> <pre><code>class V2GStep:\n    \"\"\"Variant-to-gene (V2G) step.\n\n    This step aims to generate a dataset that contains multiple pieces of evidence supporting the functional association of specific variants with genes. Some of the evidence types include:\n\n    1. Chromatin interaction experiments, e.g. Promoter Capture Hi-C (PCHi-C).\n    2. In silico functional predictions, e.g. Variant Effect Predictor (VEP) from Ensembl.\n    3. Distance between the variant and each gene's canonical transcription start site (TSS).\n\n    Attributes:\n        session (Session): Session object.\n        variant_index_path (str): Input variant index path.\n        variant_annotation_path (str): Input variant annotation path.\n        gene_index_path (str): Input gene index path.\n        vep_consequences_path (str): Input VEP consequences path.\n        liftover_chain_file_path (str): Path to GRCh37 to GRCh38 chain file.\n        liftover_max_length_difference: Maximum length difference for liftover.\n        max_distance (int): Maximum distance to consider.\n        approved_biotypes (list[str]): List of approved biotypes.\n        intervals (dict): Dictionary of interval sources.\n        v2g_path (str): Output V2G path.\n    \"\"\"\n\n    def __init__(\n        self,\n        session: Session,\n        variant_index_path: str,\n        variant_annotation_path: str,\n        gene_index_path: str,\n        vep_consequences_path: str,\n        liftover_chain_file_path: str,\n        approved_biotypes: list[str],\n        interval_sources: dict[str, str],\n        v2g_path: str,\n        max_distance: int = 500_000,\n        liftover_max_length_difference: int = 100,\n    ) -&gt; None:\n        \"\"\"Run Variant-to-gene (V2G) step.\n\n        Args:\n            session (Session): Session object.\n            variant_index_path (str): Input variant index path.\n            variant_annotation_path (str): Input variant annotation path.\n            gene_index_path (str): Input gene index path.\n            vep_consequences_path (str): Input VEP consequences path.\n            liftover_chain_file_path (str): Path to GRCh37 to GRCh38 chain file.\n            approved_biotypes (list[str]): List of approved biotypes.\n            interval_sources (dict[str, str]): Dictionary of interval sources.\n            v2g_path (str): Output V2G path.\n            max_distance (int): Maximum distance to consider.\n            liftover_max_length_difference (int): Maximum length difference for liftover.\n        \"\"\"\n        # Read\n        gene_index = GeneIndex.from_parquet(session, gene_index_path)\n        vi = VariantIndex.from_parquet(session, variant_index_path).persist()\n        va = VariantAnnotation.from_parquet(session, variant_annotation_path)\n        vep_consequences = session.spark.read.csv(\n            vep_consequences_path, sep=\"\\t\", header=True\n        ).select(\n            f.element_at(f.split(\"Accession\", r\"/\"), -1).alias(\n                \"variantFunctionalConsequenceId\"\n            ),\n            f.col(\"Term\").alias(\"label\"),\n            f.col(\"v2g_score\").cast(\"double\").alias(\"score\"),\n        )\n\n        # Transform\n        lift = LiftOverSpark(\n            # lift over variants to hg38\n            liftover_chain_file_path,\n            liftover_max_length_difference,\n        )\n        gene_index_filtered = gene_index.filter_by_biotypes(\n            # Filter gene index by approved biotypes to define V2G gene universe\n            list(approved_biotypes)\n        )\n        va_slimmed = va.filter_by_variant_df(\n            # Variant annotation reduced to the variant index to define V2G variant universe\n            vi.df\n        ).persist()\n        intervals = Intervals(\n            _df=reduce(\n                lambda x, y: x.unionByName(y, allowMissingColumns=True),\n                # create interval instances by parsing each source\n                [\n                    Intervals.from_source(\n                        session.spark, source_name, source_path, gene_index, lift\n                    ).df\n                    for source_name, source_path in interval_sources.items()\n                ],\n            ),\n            _schema=Intervals.get_schema(),\n        )\n        v2g_datasets = [\n            va_slimmed.get_distance_to_tss(gene_index_filtered, max_distance),\n            va_slimmed.get_most_severe_vep_v2g(vep_consequences, gene_index_filtered),\n            va_slimmed.get_plof_v2g(gene_index_filtered),\n            intervals.v2g(vi),\n        ]\n        v2g = V2G(\n            _df=reduce(\n                lambda x, y: x.unionByName(y, allowMissingColumns=True),\n                [dataset.df for dataset in v2g_datasets],\n            ).repartition(\"chromosome\"),\n            _schema=V2G.get_schema(),\n        )\n\n        # Load\n        (\n            v2g.df.write.partitionBy(\"chromosome\")\n            .mode(session.write_mode)\n            .parquet(v2g_path)\n        )\n</code></pre>"},{"location":"python_api/steps/variant_to_gene_step/#gentropy.v2g.V2GStep.__init__","title":"<code>__init__(session: Session, variant_index_path: str, variant_annotation_path: str, gene_index_path: str, vep_consequences_path: str, liftover_chain_file_path: str, approved_biotypes: list[str], interval_sources: dict[str, str], v2g_path: str, max_distance: int = 500000, liftover_max_length_difference: int = 100) -&gt; None</code>","text":"<p>Run Variant-to-gene (V2G) step.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Session object.</p> required <code>variant_index_path</code> <code>str</code> <p>Input variant index path.</p> required <code>variant_annotation_path</code> <code>str</code> <p>Input variant annotation path.</p> required <code>gene_index_path</code> <code>str</code> <p>Input gene index path.</p> required <code>vep_consequences_path</code> <code>str</code> <p>Input VEP consequences path.</p> required <code>liftover_chain_file_path</code> <code>str</code> <p>Path to GRCh37 to GRCh38 chain file.</p> required <code>approved_biotypes</code> <code>list[str]</code> <p>List of approved biotypes.</p> required <code>interval_sources</code> <code>dict[str, str]</code> <p>Dictionary of interval sources.</p> required <code>v2g_path</code> <code>str</code> <p>Output V2G path.</p> required <code>max_distance</code> <code>int</code> <p>Maximum distance to consider.</p> <code>500000</code> <code>liftover_max_length_difference</code> <code>int</code> <p>Maximum length difference for liftover.</p> <code>100</code> Source code in <code>src/gentropy/v2g.py</code> <pre><code>def __init__(\n    self,\n    session: Session,\n    variant_index_path: str,\n    variant_annotation_path: str,\n    gene_index_path: str,\n    vep_consequences_path: str,\n    liftover_chain_file_path: str,\n    approved_biotypes: list[str],\n    interval_sources: dict[str, str],\n    v2g_path: str,\n    max_distance: int = 500_000,\n    liftover_max_length_difference: int = 100,\n) -&gt; None:\n    \"\"\"Run Variant-to-gene (V2G) step.\n\n    Args:\n        session (Session): Session object.\n        variant_index_path (str): Input variant index path.\n        variant_annotation_path (str): Input variant annotation path.\n        gene_index_path (str): Input gene index path.\n        vep_consequences_path (str): Input VEP consequences path.\n        liftover_chain_file_path (str): Path to GRCh37 to GRCh38 chain file.\n        approved_biotypes (list[str]): List of approved biotypes.\n        interval_sources (dict[str, str]): Dictionary of interval sources.\n        v2g_path (str): Output V2G path.\n        max_distance (int): Maximum distance to consider.\n        liftover_max_length_difference (int): Maximum length difference for liftover.\n    \"\"\"\n    # Read\n    gene_index = GeneIndex.from_parquet(session, gene_index_path)\n    vi = VariantIndex.from_parquet(session, variant_index_path).persist()\n    va = VariantAnnotation.from_parquet(session, variant_annotation_path)\n    vep_consequences = session.spark.read.csv(\n        vep_consequences_path, sep=\"\\t\", header=True\n    ).select(\n        f.element_at(f.split(\"Accession\", r\"/\"), -1).alias(\n            \"variantFunctionalConsequenceId\"\n        ),\n        f.col(\"Term\").alias(\"label\"),\n        f.col(\"v2g_score\").cast(\"double\").alias(\"score\"),\n    )\n\n    # Transform\n    lift = LiftOverSpark(\n        # lift over variants to hg38\n        liftover_chain_file_path,\n        liftover_max_length_difference,\n    )\n    gene_index_filtered = gene_index.filter_by_biotypes(\n        # Filter gene index by approved biotypes to define V2G gene universe\n        list(approved_biotypes)\n    )\n    va_slimmed = va.filter_by_variant_df(\n        # Variant annotation reduced to the variant index to define V2G variant universe\n        vi.df\n    ).persist()\n    intervals = Intervals(\n        _df=reduce(\n            lambda x, y: x.unionByName(y, allowMissingColumns=True),\n            # create interval instances by parsing each source\n            [\n                Intervals.from_source(\n                    session.spark, source_name, source_path, gene_index, lift\n                ).df\n                for source_name, source_path in interval_sources.items()\n            ],\n        ),\n        _schema=Intervals.get_schema(),\n    )\n    v2g_datasets = [\n        va_slimmed.get_distance_to_tss(gene_index_filtered, max_distance),\n        va_slimmed.get_most_severe_vep_v2g(vep_consequences, gene_index_filtered),\n        va_slimmed.get_plof_v2g(gene_index_filtered),\n        intervals.v2g(vi),\n    ]\n    v2g = V2G(\n        _df=reduce(\n            lambda x, y: x.unionByName(y, allowMissingColumns=True),\n            [dataset.df for dataset in v2g_datasets],\n        ).repartition(\"chromosome\"),\n        _schema=V2G.get_schema(),\n    )\n\n    # Load\n    (\n        v2g.df.write.partitionBy(\"chromosome\")\n        .mode(session.write_mode)\n        .parquet(v2g_path)\n    )\n</code></pre>"},{"location":"python_api/steps/window_based_clumping/","title":"window_based_clumping","text":""},{"location":"python_api/steps/window_based_clumping/#gentropy.window_based_clumping.WindowBasedClumpingStep","title":"<code>gentropy.window_based_clumping.WindowBasedClumpingStep</code>","text":"<p>Apply window based clumping on summary statistics datasets.</p> Source code in <code>src/gentropy/window_based_clumping.py</code> <pre><code>class WindowBasedClumpingStep:\n    \"\"\"Apply window based clumping on summary statistics datasets.\"\"\"\n\n    def __init__(\n        self,\n        session: Session,\n        summary_statistics_input_path: str,\n        study_locus_output_path: str,\n        distance: int = 500_000,\n        collect_locus: bool = False,\n        collect_locus_distance: int = 500_000,\n        inclusion_list_path: str | None = None,\n    ) -&gt; None:\n        \"\"\"Run window-based clumping step.\n\n        Args:\n            session (Session): Session object.\n            summary_statistics_input_path (str): Path to the harmonized summary statistics dataset.\n            study_locus_output_path (str): Output path for the resulting study locus dataset.\n            distance (int): Distance, within which tagging variants are collected around the semi-index. Optional.\n            collect_locus (bool): Whether to collect locus around semi-indices. Optional.\n            collect_locus_distance (int): Distance, within which tagging variants are collected around the semi-index. Optional.\n            inclusion_list_path (str | None): Path to the inclusion list (list of white-listed study identifier). Optional.\n        \"\"\"\n        # If inclusion list path is provided, only these studies will be read:\n        if inclusion_list_path:\n            study_ids_to_ingest = [\n                f'{summary_statistics_input_path}/{row[\"studyId\"]}.parquet'\n                for row in session.spark.read.parquet(inclusion_list_path).collect()\n            ]\n        else:\n            # If no inclusion list is provided, read all summary stats in folder:\n            study_ids_to_ingest = [summary_statistics_input_path]\n\n        ss = SummaryStatistics.from_parquet(\n            session,\n            study_ids_to_ingest,\n            recursiveFileLookup=True,\n        )\n\n        # Clumping:\n        study_locus = ss.window_based_clumping(\n            distance=distance,\n        )\n\n        # Optional locus collection:\n        if collect_locus:\n            # Collecting locus around semi-indices:\n            study_locus = study_locus.annotate_locus_statistics(\n                ss, collect_locus_distance=collect_locus_distance\n            )\n\n        study_locus.df.write.mode(session.write_mode).parquet(study_locus_output_path)\n</code></pre>"},{"location":"python_api/steps/window_based_clumping/#gentropy.window_based_clumping.WindowBasedClumpingStep.__init__","title":"<code>__init__(session: Session, summary_statistics_input_path: str, study_locus_output_path: str, distance: int = 500000, collect_locus: bool = False, collect_locus_distance: int = 500000, inclusion_list_path: str | None = None) -&gt; None</code>","text":"<p>Run window-based clumping step.</p> <p>Parameters:</p> Name Type Description Default <code>session</code> <code>Session</code> <p>Session object.</p> required <code>summary_statistics_input_path</code> <code>str</code> <p>Path to the harmonized summary statistics dataset.</p> required <code>study_locus_output_path</code> <code>str</code> <p>Output path for the resulting study locus dataset.</p> required <code>distance</code> <code>int</code> <p>Distance, within which tagging variants are collected around the semi-index. Optional.</p> <code>500000</code> <code>collect_locus</code> <code>bool</code> <p>Whether to collect locus around semi-indices. Optional.</p> <code>False</code> <code>collect_locus_distance</code> <code>int</code> <p>Distance, within which tagging variants are collected around the semi-index. Optional.</p> <code>500000</code> <code>inclusion_list_path</code> <code>str | None</code> <p>Path to the inclusion list (list of white-listed study identifier). Optional.</p> <code>None</code> Source code in <code>src/gentropy/window_based_clumping.py</code> <pre><code>def __init__(\n    self,\n    session: Session,\n    summary_statistics_input_path: str,\n    study_locus_output_path: str,\n    distance: int = 500_000,\n    collect_locus: bool = False,\n    collect_locus_distance: int = 500_000,\n    inclusion_list_path: str | None = None,\n) -&gt; None:\n    \"\"\"Run window-based clumping step.\n\n    Args:\n        session (Session): Session object.\n        summary_statistics_input_path (str): Path to the harmonized summary statistics dataset.\n        study_locus_output_path (str): Output path for the resulting study locus dataset.\n        distance (int): Distance, within which tagging variants are collected around the semi-index. Optional.\n        collect_locus (bool): Whether to collect locus around semi-indices. Optional.\n        collect_locus_distance (int): Distance, within which tagging variants are collected around the semi-index. Optional.\n        inclusion_list_path (str | None): Path to the inclusion list (list of white-listed study identifier). Optional.\n    \"\"\"\n    # If inclusion list path is provided, only these studies will be read:\n    if inclusion_list_path:\n        study_ids_to_ingest = [\n            f'{summary_statistics_input_path}/{row[\"studyId\"]}.parquet'\n            for row in session.spark.read.parquet(inclusion_list_path).collect()\n        ]\n    else:\n        # If no inclusion list is provided, read all summary stats in folder:\n        study_ids_to_ingest = [summary_statistics_input_path]\n\n    ss = SummaryStatistics.from_parquet(\n        session,\n        study_ids_to_ingest,\n        recursiveFileLookup=True,\n    )\n\n    # Clumping:\n    study_locus = ss.window_based_clumping(\n        distance=distance,\n    )\n\n    # Optional locus collection:\n    if collect_locus:\n        # Collecting locus around semi-indices:\n        study_locus = study_locus.annotate_locus_statistics(\n            ss, collect_locus_distance=collect_locus_distance\n        )\n\n    study_locus.df.write.mode(session.write_mode).parquet(study_locus_output_path)\n</code></pre>"}]}
\ No newline at end of file
diff --git a/sitemap.xml.gz b/sitemap.xml.gz
index c44d53c83..fce3330ba 100644
Binary files a/sitemap.xml.gz and b/sitemap.xml.gz differ
diff --git a/src_snippets/howto/python_api/c_applying_methods.py b/src_snippets/howto/python_api/c_applying_methods.py
index 12eaf61ac..d0bec9edc 100644
--- a/src_snippets/howto/python_api/c_applying_methods.py
+++ b/src_snippets/howto/python_api/c_applying_methods.py
@@ -23,7 +23,7 @@ def apply_class_method_clumping(summary_stats: SummaryStatistics) -> StudyLocus:
     from gentropy.method.window_based_clumping import WindowBasedClumping
 
     clumped_summary_statistics = WindowBasedClumping.clump(
-        summary_stats, window_length=500_000
+        summary_stats, distance=250_000
     )
     # --8<-- [end:apply_class_method_clumping]
     return clumped_summary_statistics