Changeset 4985

Timestamp:

Sep 28, 2018, 9:50:12 AM (5 years ago)

Author:

Nicklas Nordborg

Message:

References #1054: Even more functionality in the Flagged alignment wizard

The "Start demux and merge" wizard now checks for libraries marked with DoNotUse. It will display information about them and provide an option to either exclude or include them in the demux. The default is to exclude such libraries.

The actual demux process will still demux all data, but it will collect the data to a virtual library "IGNORED". The main reason for doing this is to avoid that the checks that are made efter the demux to generate warnings for high levels of unmapped reads.

Location:

extensions/net.sf.basedb.reggie/trunk

Files:

• resources/analysis/demux_start.js (modified) (9 diffs)
• resources/analysis/demux_start.jsp (modified) (1 diff)
• src/net/sf/basedb/reggie/autoconfirm/SequencingRunAutoConfirmer.java (modified) (1 diff)
• src/net/sf/basedb/reggie/dao/ReggieItem.java (modified) (1 diff)
• src/net/sf/basedb/reggie/grid/DemuxJobCreator.java (modified) (6 diffs)
• src/net/sf/basedb/reggie/plugins/BarcodeFilesForDemuxExporter.java (modified) (3 diffs)
• src/net/sf/basedb/reggie/servlet/DemuxMergeServlet.java (modified) (5 diffs)

extensions/net.sf.basedb.reggie/trunk/resources/analysis/demux_start.js

@@ -128 +128 @@
     if (flowCell.FlowCellType)
     {
-      name += ' [' + Strings.encodeTags(flowCell.FlowCellType) + ']';
-    }
-
+      name += ' [' + flowCell.libraries.length + ' libraries; ' + Strings.encodeTags(flowCell.FlowCellType) + ']';
+    }
+
+    // Count the number of libraries marked with DoNotUse
+    var numDoNotUse = 0;
+    for (var libNo = 0; libNo < flowCell.libraries.length; libNo++)
+    {
+      if (flowCell.libraries[libNo].DO_NOT_USE) numDoNotUse++;
+    }
+    flowCell.numDoNotUse = numDoNotUse;
+    
     // Tooltip is always all pools
     var title = '';
@@ -222 +230 @@
     var numSelected = 0;
     var pools;
+    var numDoNotUse = 0;
     Wizard.setInputStatus('seqRuns');
     for (var seqNo = 0; seqNo < frm.seqRuns.length; seqNo++)
@@ -231 +240 @@
         {
           pools = frm.seqRuns[seqNo].seqRun.flowCell.pools;
+          numDoNotUse = frm.seqRuns[seqNo].seqRun.flowCell.numDoNotUse;
         }
         else
@@ -246 +256 @@
     {
       Wizard.setInputStatus('seqRuns', 'invalid', 'Select at least one sequencing run');
+      return;
+    }
+    
+    if (numDoNotUse > 0)
+    {
+      var msg = numDoNotUse == 1 ? '1 library is marked with DoNotUse.' :
+        numDoNotUse + ' libraries are marked with DoNotUse.'
+      Wizard.setInputStatus('seqRuns', 'warning', msg);
     }
     else
@@ -276 +294 @@
         var seqRun = frm.seqRuns[seqNo].seqRun;
         var fc = seqRun.flowCell;
+        var libs = fc.libraries;
         var readStringId = 'readString.'+seqNo;
         html += '<tr>';
@@ -310 +329 @@
         html += '</tr>';
         
+        // Check for DoNotUse
+        if (fc.numDoNotUse > 0)
+        {
+          var text = fc.numDoNotUse == 1 ? '1 library' : fc.numDoNotUse + ' libraries';
+          html += '<tr>';
+          html += '<td class="subprompt">DoNotUse</td>';
+          html += '<td><label><input type="checkbox" name="excludeDoNotUse.'+seqNo+'" checked>Exclude ' + text + ' marked with DoNotUse</label>';
+          html += ' <img src="../images/info.png" class="link" id="donotuseinfo.'+seqNo+'" data-seq-no="'+seqNo+'" title="More information about the libraries..."></td>';
+          html += '<td class="status warning"></td>';
+          html += '<td class="help">By default, libraries marked with DoNotUse are omitted</td>';
+          html += '</tr>';
+        }
+        
         html += '<tr>';
         html += '<td class="subprompt"></td>';
@@ -327 +359 @@
         Events.addEventHandler('readString.'+seqNo, 'blur', demux.readStringOnBlur);
         Buttons.addClickHandler('check.'+seqNo, demux.checkDataFilesOnClick);
+        Buttons.addClickHandler('donotuseinfo.'+seqNo, demux.showDoNotUseInfo);
       }
     }
@@ -349 +382 @@
   }
 
+  demux.showDoNotUseInfo = function(event)
+  {
+    var frm = document.forms['reggie'];
+    var seqNo = Data.get(event.currentTarget, 'seq-no');
+    var seqRun = frm.seqRuns[seqNo].seqRun;
+    var fc = seqRun.flowCell;
+    var libs = fc.libraries;
+    
+    var msg = '<table>';
+    for (var libNo = 0; libNo < libs.length; libNo++)
+    {
+      var lib = libs[libNo];
+      if (lib.DO_NOT_USE)
+      {
+        msg += '<tr><td><b>'+Strings.encodeTags(lib.name) + '</b></td><td><span class="itemsubtype">[' + lib.DO_NOT_USE + ']</span></td>';
+        msg += '<td>'+ Strings.encodeTags(lib.DO_NOT_USE_COMMENT) + '</td></tr>';
+      }
+    }
+    msg += '</table>';
+    Forms.showNotification(event.currentTarget, msg, 'donotuseinfo', 'pointer-left');
+  }
+  
   demux.clustersLoaded = function(response)
   {
@@ -555 +610 @@
         }
         seqRun.OmitLanes = omitLanes;
+        seqRun.excludeDoNotUse = frm['excludeDoNotUse.'+seqNo].checked;
         sequencingRuns[sequencingRuns.length] = seqRun;
       }

extensions/net.sf.basedb.reggie/trunk/resources/analysis/demux_start.jsp

@@ -27 +27 @@
   scripts="~../reggie-2.js,~demux_start.js" 
   styles="path.css,~../css/reggie-2.css"
-/>
+>
+<style>
+.notify.donotuseinfo
+{
+  width: auto;
+  max-width: 60em;
+  font-weight: normal;
+  background-color: #F8F8E8;
+  border-color: inherit;
+}
+
+.notify.donotuseinfo > div:first-child > div
+{
+  border-color: #F8F8E8;
+}
+
+.notify .notify-message:before
+{
+  content: url('../images/info.png');
+}
+
+.donotuseinfo td
+{
+  vertical-align: top;
+  padding: 2px;
+}
+</style>
+
+</base:head>
 <base:body>
 

extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/autoconfirm/SequencingRunAutoConfirmer.java

@@ -195 +195 @@
       jobCreator.setPriority(priority);
       
-      DemuxDefinition demuxDef = new DemuxDefinition(sr, readString, null); 
+      DemuxDefinition demuxDef = new DemuxDefinition(sr, readString, null, true); 
       Job demuxJob = jobCreator.createDemuxJob(dc, cluster, Arrays.asList(demuxDef), null);     
       dc.commit();

extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/dao/ReggieItem.java

@@ -616 +616 @@
     }
   }
+  
+  /**
+    Sorts a list of items by name.
+    @since 4.20
+  */
+  public static class NameComparator
+    implements Comparator<ReggieItem<?>>
+  {
+    public NameComparator()
+    {}
+    
+    @Override
+    public int compare(ReggieItem<?> item1, ReggieItem<?> item2) 
+    {
+      return item1.getName().compareTo(item2.getName());
+    }
+    
+  }
+  
 }

extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/grid/DemuxJobCreator.java

@@ -423 +423 @@
           List<Extract> libsInPool = libQuery.list(dc);
           List<DerivedBioAssay> mergedInPool = new ArrayList<DerivedBioAssay>(libsInPool.size());
+          List<Extract> doNotUseInPool = new ArrayList<Extract>();
           
           for (Extract lib : libsInPool)
           {
-            // Create MergedSequences item if needed
-            DerivedBioAssay merged = mergedSequences.get(lib);
-            if (merged == null)
+            String doNotUse = (String)Annotationtype.DO_NOT_USE.getAnnotationValue(dc, lib);
+            if (def.excludeDoNotUse && doNotUse != null)
             {
-              Library l = Library.getById(dc, lib.getId());
-              String mergeName = ScriptUtil.checkValidFilename(l.getNextMergedSequencesName(dc));
-              merged = DerivedBioAssay.getNew(dc, false, demuxJob);
-              merged.setItemSubtype(mergedType);
-              merged.setName(mergeName);
-              merged.setSoftware(mergeSoftware);
-              merged.setProtocol(mergeProtocol);
-              merged.setExtract(lib);
-              
-              String dataFilesFolder = ScriptUtil.checkValidPath(MergedSequences.generateDataFilesFolderForProjectArchive(mergeName, debug), true, true);
-              
-              Annotationtype.DATA_FILES_FOLDER.setAnnotationValue(dc, merged, dataFilesFolder);
-              dc.saveItem(merged);
-              mergedSequences.put(lib, merged);
-              
-              //jsonMessages.add("Created " + merged.getName());
+              doNotUseInPool.add(lib);
             }
-            mergedInPool.add(merged);
-            if (libsOnFlowCell.add(lib))
+            else
             {
-              // First time for this lib on the flow cell
-              merged.addParent(demux);
+              // Create MergedSequences item if needed
+              DerivedBioAssay merged = mergedSequences.get(lib);
+              if (merged == null)
+              {
+                Library l = Library.getById(dc, lib.getId());
+                String mergeName = ScriptUtil.checkValidFilename(l.getNextMergedSequencesName(dc));
+                merged = DerivedBioAssay.getNew(dc, false, demuxJob);
+                merged.setItemSubtype(mergedType);
+                merged.setName(mergeName);
+                merged.setSoftware(mergeSoftware);
+                merged.setProtocol(mergeProtocol);
+                merged.setExtract(lib);
+                
+                String dataFilesFolder = ScriptUtil.checkValidPath(MergedSequences.generateDataFilesFolderForProjectArchive(mergeName, debug), true, true);
+                
+                Annotationtype.DATA_FILES_FOLDER.setAnnotationValue(dc, merged, dataFilesFolder);
+                dc.saveItem(merged);
+                mergedSequences.put(lib, merged);
+                
+                //jsonMessages.add("Created " + merged.getName());
+              }
+              mergedInPool.add(merged);
+              if (libsOnFlowCell.add(lib))
+              {
+                // First time for this lib on the flow cell
+                merged.addParent(demux);
+              }
             }
           }
@@ -463 +472 @@
           // Export all required barcode files, 2 files per lane per flow cell
           StringWriter out = new StringWriter();
-          exporter.exportBarcodesFile(dc, demux, laneNo, mergedInPool, out);
+          exporter.exportBarcodesFile(dc, demux, laneNo, mergedInPool, doNotUseInPool, out);
           jobDef.addFile(new StringUploadSource(barcodesFile, out.toString()));
           
           out = new StringWriter();
-          exporter.exportMultiplexFile(dc, demux, laneNo, mergedInPool, out, "fastq");
+          exporter.exportMultiplexFile(dc, demux, laneNo, mergedInPool, doNotUseInPool, out, "fastq");
           jobDef.addFile(new StringUploadSource(multiplexFile, out.toString()));
 
@@ -734 +743 @@
     final String readString;
     final List<Integer> omitLanes;
-    
-    public DemuxDefinition(SequencingRun seqRun, String readString, List<? extends Number> omitLanes)
+    final boolean excludeDoNotUse;
+    
+    public DemuxDefinition(SequencingRun seqRun, String readString, List<? extends Number> omitLanes, boolean excludeDoNotUse)
     {
       this.seqRun = seqRun;
@@ -747 +757 @@
         }
       }
+      this.excludeDoNotUse = excludeDoNotUse;
     }
 
@@ -859 +870 @@
                   sumDemux.addWarning("PF_NNNN_PCT is " + Values.formatNumber(passedFilterPct, 2) + "% on lane " + laneNo + " (" + poolName + ")");
                 }
+              }
+              else if ("IGNORED".equals(libName))
+              {
+                libName = demuxName+".I";
               }
               else if ("UNUSED".equals(libName))
@@ -1068 +1083 @@
           Reads demuxN = sumReads.remove(demuxName+".N");
           Reads demuxUnused = sumReads.remove(demuxName+".U");
+          Reads demuxIgnore = sumReads.remove(demuxName+".I");
           
           DemuxedSequences demux = DemuxedSequences.getByName(dc, demuxName);

extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/plugins/BarcodeFilesForDemuxExporter.java

@@ -53 +53 @@
     when running "picard ExtractIlluminaBarcodes" for the given lane.
   */
-  public void exportBarcodesFile(DbControl dc, DerivedBioAssay demux, int laneNo, List<DerivedBioAssay> mergedInLane, Writer out)
+  public void exportBarcodesFile(DbControl dc, DerivedBioAssay demux, int laneNo, List<DerivedBioAssay> mergedInLane, List<Extract> libIgnore, Writer out)
   {
     TableWriter tw = createTableWriter(out);
@@ -72 +72 @@
       tw.tablePrintData(info);
       barcodes.remove(barcode);
+    }
+    
+    // Libraries that should be ignored
+    // We can't put them under the "UNUSED" tag since the demux will warn if it finds
+    // a large number of unused barcodes!
+    for (Extract lib : libIgnore)
+    {
+      Tag barcode = lib.getTag();
+      if (barcodes.remove(barcode))
+      {
+        info[0] = checkValidBarcodeSequence(dc, barcode);
+        info[1] = checkValidBarcodeName(barcode);
+        info[2] = "IGNORED";
+        tw.tablePrintData(info);
+      }
     }
     
@@ -93 +108 @@
     when running "picard IlluminaBasecallsToFastq" for the given lane.
   */
-  public void exportMultiplexFile(DbControl dc, DerivedBioAssay demux, int laneNo, List<DerivedBioAssay> mergedInLane, Writer out, String outFolder)
+  public void exportMultiplexFile(DbControl dc, DerivedBioAssay demux, int laneNo, List<DerivedBioAssay> mergedInLane, List<Extract> libIgnore, Writer out, String outFolder)
   {
     TableWriter tw = createTableWriter(out);

extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/servlet/DemuxMergeServlet.java

@@ -6 +6 @@
 import java.util.List;
 import java.util.Set;
+import java.util.TreeSet;
 
 import javax.servlet.ServletException;
@@ -52 +53 @@
 import net.sf.basedb.reggie.dao.PooledLibrary;
 import net.sf.basedb.reggie.dao.ReactionPlate;
+import net.sf.basedb.reggie.dao.ReggieItem;
 import net.sf.basedb.reggie.dao.ReggieRole;
 import net.sf.basedb.reggie.dao.Rna;
@@ -344 +346 @@
           String readString = (String)jsonSeqRun.get("ReadString");
           JSONArray jsonOmitLanes = (JSONArray)jsonSeqRun.get("OmitLanes");
+          boolean excludeDoNotUse = Boolean.TRUE.equals(jsonSeqRun.get("excludeDoNotUse"));
           
           SequencingRun seqRun = SequencingRun.getById(dc, seqRunId.intValue());
           Annotationtype.AUTO_PROCESSING.setAnnotationValue(dc, seqRun.getItem(), null);
           
-          demuxDefs.add(new DemuxDefinition(seqRun, readString, jsonOmitLanes));
+          demuxDefs.add(new DemuxDefinition(seqRun, readString, jsonOmitLanes, excludeDoNotUse));
         }
         
@@ -596 +599 @@
     // Load pools on flow cell
     List<PooledLibrary> pools = PooledLibrary.getByFlowCell(dc, fc);
+    Set<Library> libs = new TreeSet<>(new ReggieItem.NameComparator());
+
     JSONArray jsonPools = new JSONArray();
     for (PooledLibrary pool : pools)
     {
+      libs.addAll(Library.getByPool(dc, pool));
       try
       {
@@ -610 +616 @@
     }
     fc.setAnnotation("pools", jsonPools);
+    
+    JSONArray jsonLibs = new JSONArray();
+    for (Library lib : libs)
+    {
+      lib.loadDoNotUseAnnotations(dc, null);
+      jsonLibs.add(lib.asJSONObject());
+    }
+    fc.setAnnotation("libraries", jsonLibs);
+    
     run.setAnnotation("flowCell", fc.asJSONObject());
   }