Changeset 5766

Timestamp:

Dec 2, 2019, 10:52:46 AM (3 years ago)

Author:

Nicklas Nordborg

Message:

References #1208: Implement wizard for building database of variant frequencies in SCAN-B samples

Implemented export step for creating a file with patient and VCF file information. This file is intended to be used by the statistics script.

Location:

extensions/net.sf.basedb.reggie/trunk

Files:

• resources/analysis/vcall_build.js (modified) (1 diff)
• resources/analysis/vcall_build.jsp (modified) (1 diff)
• src/net/sf/basedb/reggie/grid/VariantStatisticsJobCreator.java (modified) (4 diffs)
• src/net/sf/basedb/reggie/servlet/VariantCallingServlet.java (modified) (5 diffs)

extensions/net.sf.basedb.reggie/trunk/resources/analysis/vcall_build.js

@@ -189 +189 @@
     url += '&cmd=BuildVariantStatistics';
 
-    Wizard.showLoadingAnimation('Performing registration...');
+    Wizard.showLoadingAnimation('Performing registration...', 'variant-statistics-progress');
     Wizard.asyncJsonRequest(url, vcall.submissionResults, 'POST', JSON.stringify(submitInfo));
   }

extensions/net.sf.basedb.reggie/trunk/resources/analysis/vcall_build.jsp

@@ -121 +121 @@
     
     <div id="wizard-status"></div>
+    <div id="wizard-progress"></div>
   
     <table class="navigation" id="navigation">

extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/grid/VariantStatisticsJobCreator.java

@@ -1 +1 @@
 package net.sf.basedb.reggie.grid;
 
+import java.io.StringWriter;
 import java.util.Arrays;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import net.sf.basedb.core.BioSource;
 import net.sf.basedb.core.DbControl;
+import net.sf.basedb.core.DerivedBioAssay;
+import net.sf.basedb.core.InvalidDataException;
 import net.sf.basedb.core.ItemList;
 import net.sf.basedb.core.ItemNotFoundException;
 import net.sf.basedb.core.ItemParameterType;
+import net.sf.basedb.core.ItemQuery;
 import net.sf.basedb.core.Job;
+import net.sf.basedb.core.ProgressReporter;
 import net.sf.basedb.core.SessionControl;
+import net.sf.basedb.core.Type;
+import net.sf.basedb.core.query.Expressions;
+import net.sf.basedb.core.query.Hql;
+import net.sf.basedb.core.query.Restrictions;
+import net.sf.basedb.core.snapshot.SnapshotManager;
 import net.sf.basedb.opengrid.JobDefinition;
 import net.sf.basedb.opengrid.OpenGridCluster;
@@ -17 +31 @@
 import net.sf.basedb.opengrid.config.ClusterConfig;
 import net.sf.basedb.opengrid.config.JobConfig;
+import net.sf.basedb.opengrid.filetransfer.StringUploadSource;
 import net.sf.basedb.reggie.Reggie;
 import net.sf.basedb.reggie.XmlConfig;
+import net.sf.basedb.reggie.dao.Annotationtype;
+import net.sf.basedb.reggie.dao.Subtype;
 import net.sf.basedb.util.Values;
+import net.sf.basedb.util.export.TableWriter;
 
 /**
@@ -61 +79 @@
     @return The corresponding job in BASE
   */
-  public Job createVariantStatisticsJob(DbControl dc, OpenGridCluster cluster, ItemList list)
+  @SuppressWarnings("unchecked")
+  public Job createVariantStatisticsJob(DbControl dc, OpenGridCluster cluster, ItemList list, ProgressReporter progress)
   {
     SessionControl sc = dc.getSessionControl();
@@ -94 +113 @@
     // TODO -- Create script here
     
-    // TODO -- we need to export a file with paths to raw VCF:s and patient information
-    
-    
     script.progress(99, "Cleaning up temporary folders");
 
+    // Export a list with PATIENT name and path to RAW VCF file
+    StringWriter vcfList = new StringWriter(list.getSize()*80);
+    TableWriter tw = new TableWriter(vcfList);
+    // For each alignment we need to find the patient. We design a query
+    // to load it from the specimen name which we get by cutting the front
+    // of the alignment name
+    ItemQuery<BioSource> findPatBySpecimen = BioSource.getQuery();
+    Subtype.PATIENT.addFilter(dc, findPatBySpecimen);
+    findPatBySpecimen.setIncludes(Reggie.INCLUDE_IN_CURRENT_PROJECT);
+    findPatBySpecimen.join(Hql.innerJoin("childCreationEvents", "cce"));
+    findPatBySpecimen.join(Hql.innerJoin("cce", "event", "evt"));
+    findPatBySpecimen.join(Hql.innerJoin("evt", "bioMaterial", "cse")); // 'cse' should now reference a case
+    findPatBySpecimen.join(Hql.innerJoin("cse", "childCreationEvents", "cce2"));
+    findPatBySpecimen.join(Hql.innerJoin("cce2", "event", "evt2"));
+    findPatBySpecimen.join(Hql.innerJoin("evt2", "bioMaterial", "spm")); // 'spm' should now reference a specimen/nospecimen
+    findPatBySpecimen.restrict(Restrictions.eq(Hql.property("spm", "name"), Expressions.parameter("specimen")));
+    
+    ItemQuery<DerivedBioAssay> query = (ItemQuery<DerivedBioAssay>)list.getMembers();
+    query.setIncludes(Reggie.INCLUDE_IN_CURRENT_PROJECT);
+    List<DerivedBioAssay> alignments = query.list(dc);
+    
+    SnapshotManager manager = new SnapshotManager();
+    int count = 0;
+    int total = alignments.size();
+    Pattern specimenName = Pattern.compile("(\\d+\\.\\d+)\\..*"); // Extract specimen name by taking first part of alignment name
+    for (DerivedBioAssay alignment : alignments)
+    {
+      count++;
+      if (progress != null && count % 100 == 0)
+      {
+        progress.display(count * 90 / total, "Exporting Patient and VCF list (" + count + " of " + total + ")...");
+      }
+      
+      Matcher m = specimenName.matcher(alignment.getName());
+      if (!m.matches())
+      {
+        throw new InvalidDataException("Could not get specimen name for alignment: "+ alignment.getName());
+      }
+      
+      String specimen = m.group(1);
+      findPatBySpecimen.setParameter("specimen", specimen, Type.STRING);
+      List<BioSource> patients = findPatBySpecimen.list(dc);
+      if (patients.size() != 1)
+      {
+        if (patients.size() == 0) throw new ItemNotFoundException("Could not find a patient item for alignment: " +alignment.getName());
+        throw new InvalidDataException("Found "+ patients.size() + " patient items for alignment: " + alignment.getName());
+      }
+      String patient = patients.get(0).getName();
+      String dataFolder = (String)Annotationtype.DATA_FILES_FOLDER.getAnnotationValue(dc, manager, alignment);
+      tw.tablePrintData(patient, dataFolder+"/variants-raw.vcf.gz");
+    }
+    tw.flush();
+    tw.close();
+    
     JobDefinition jobDef = new JobDefinition("VariantStatistics", jobConfig, statJob);
     jobDef.setDebug(debug);
+    jobDef.addFile(new StringUploadSource("vcflist.txt", vcfList.toString()));
     jobDef.setCmd(script.toString());
     

extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/servlet/VariantCallingServlet.java

@@ -29 +29 @@
 import net.sf.basedb.core.RawBioAssay;
 import net.sf.basedb.core.SessionControl;
+import net.sf.basedb.core.SimpleProgressReporter;
 import net.sf.basedb.core.Software;
 import net.sf.basedb.core.Trashcan;
@@ -530 +531 @@
         }
         
+        SimpleProgressReporter progress = new SimpleProgressReporter(null);
+        sc.setSessionSetting("variant-statistics-progress", progress);
+        
         ItemList list = ItemList.getById(dc, listId.intValue());
         
@@ -536 +540 @@
         jobCreator.setPriority(priority == null ? null : priority.intValue());
 
-        Job statJob = jobCreator.createVariantStatisticsJob(dc, cluster, list);
+        Job statJob = jobCreator.createVariantStatisticsJob(dc, cluster, list, progress);
         if (statJob.getStatus() == Job.Status.ERROR)
         {
@@ -545 +549 @@
           jsonMessages.add("Submitted variant statistics job to " + cluster.getConnectionInfo().getName() + " with id " + statJob.getExternalId());
         }
+        progress.display(100, "Done");
         dc.commit();
-
       }
 
@@ -564 +568 @@
       if (dc != null) dc.close();
       json.writeJSONString(resp.getWriter());
+      sc.setSessionSetting("variant-statistics-progress", null);
     }