Changeset 4985


Ignore:
Timestamp:
Sep 28, 2018, 9:50:12 AM (3 years ago)
Author:
Nicklas Nordborg
Message:

References #1054: Even more functionality in the Flagged alignment wizard

The "Start demux and merge" wizard now checks for libraries marked with DoNotUse. It will display information about them and provide an option to either exclude or include them in the demux. The default is to exclude such libraries.

The actual demux process will still demux all data, but it will collect the data to a virtual library "IGNORED". The main reason for doing this is to avoid that the checks that are made efter the demux to generate warnings for high levels of unmapped reads.

Location:
extensions/net.sf.basedb.reggie/trunk
Files:
7 edited

Legend:

Unmodified
Added
Removed
  • extensions/net.sf.basedb.reggie/trunk/resources/analysis/demux_start.js

    r4583 r4985  
    128128    if (flowCell.FlowCellType)
    129129    {
    130       name += ' [' + Strings.encodeTags(flowCell.FlowCellType) + ']';
    131     }
    132 
     130      name += ' [' + flowCell.libraries.length + ' libraries; ' + Strings.encodeTags(flowCell.FlowCellType) + ']';
     131    }
     132
     133    // Count the number of libraries marked with DoNotUse
     134    var numDoNotUse = 0;
     135    for (var libNo = 0; libNo < flowCell.libraries.length; libNo++)
     136    {
     137      if (flowCell.libraries[libNo].DO_NOT_USE) numDoNotUse++;
     138    }
     139    flowCell.numDoNotUse = numDoNotUse;
     140   
    133141    // Tooltip is always all pools
    134142    var title = '';
     
    222230    var numSelected = 0;
    223231    var pools;
     232    var numDoNotUse = 0;
    224233    Wizard.setInputStatus('seqRuns');
    225234    for (var seqNo = 0; seqNo < frm.seqRuns.length; seqNo++)
     
    231240        {
    232241          pools = frm.seqRuns[seqNo].seqRun.flowCell.pools;
     242          numDoNotUse = frm.seqRuns[seqNo].seqRun.flowCell.numDoNotUse;
    233243        }
    234244        else
     
    246256    {
    247257      Wizard.setInputStatus('seqRuns', 'invalid', 'Select at least one sequencing run');
     258      return;
     259    }
     260   
     261    if (numDoNotUse > 0)
     262    {
     263      var msg = numDoNotUse == 1 ? '1 library is marked with DoNotUse.' :
     264        numDoNotUse + ' libraries are marked with DoNotUse.'
     265      Wizard.setInputStatus('seqRuns', 'warning', msg);
    248266    }
    249267    else
     
    276294        var seqRun = frm.seqRuns[seqNo].seqRun;
    277295        var fc = seqRun.flowCell;
     296        var libs = fc.libraries;
    278297        var readStringId = 'readString.'+seqNo;
    279298        html += '<tr>';
     
    310329        html += '</tr>';
    311330       
     331        // Check for DoNotUse
     332        if (fc.numDoNotUse > 0)
     333        {
     334          var text = fc.numDoNotUse == 1 ? '1 library' : fc.numDoNotUse + ' libraries';
     335          html += '<tr>';
     336          html += '<td class="subprompt">DoNotUse</td>';
     337          html += '<td><label><input type="checkbox" name="excludeDoNotUse.'+seqNo+'" checked>Exclude ' + text + ' marked with DoNotUse</label>';
     338          html += ' <img src="../images/info.png" class="link" id="donotuseinfo.'+seqNo+'" data-seq-no="'+seqNo+'" title="More information about the libraries..."></td>';
     339          html += '<td class="status warning"></td>';
     340          html += '<td class="help">By default, libraries marked with DoNotUse are omitted</td>';
     341          html += '</tr>';
     342        }
     343       
    312344        html += '<tr>';
    313345        html += '<td class="subprompt"></td>';
     
    327359        Events.addEventHandler('readString.'+seqNo, 'blur', demux.readStringOnBlur);
    328360        Buttons.addClickHandler('check.'+seqNo, demux.checkDataFilesOnClick);
     361        Buttons.addClickHandler('donotuseinfo.'+seqNo, demux.showDoNotUseInfo);
    329362      }
    330363    }
     
    349382  }
    350383
     384  demux.showDoNotUseInfo = function(event)
     385  {
     386    var frm = document.forms['reggie'];
     387    var seqNo = Data.get(event.currentTarget, 'seq-no');
     388    var seqRun = frm.seqRuns[seqNo].seqRun;
     389    var fc = seqRun.flowCell;
     390    var libs = fc.libraries;
     391   
     392    var msg = '<table>';
     393    for (var libNo = 0; libNo < libs.length; libNo++)
     394    {
     395      var lib = libs[libNo];
     396      if (lib.DO_NOT_USE)
     397      {
     398        msg += '<tr><td><b>'+Strings.encodeTags(lib.name) + '</b></td><td><span class="itemsubtype">[' + lib.DO_NOT_USE + ']</span></td>';
     399        msg += '<td>'+ Strings.encodeTags(lib.DO_NOT_USE_COMMENT) + '</td></tr>';
     400      }
     401    }
     402    msg += '</table>';
     403    Forms.showNotification(event.currentTarget, msg, 'donotuseinfo', 'pointer-left');
     404  }
     405 
    351406  demux.clustersLoaded = function(response)
    352407  {
     
    555610        }
    556611        seqRun.OmitLanes = omitLanes;
     612        seqRun.excludeDoNotUse = frm['excludeDoNotUse.'+seqNo].checked;
    557613        sequencingRuns[sequencingRuns.length] = seqRun;
    558614      }
  • extensions/net.sf.basedb.reggie/trunk/resources/analysis/demux_start.jsp

    r3976 r4985  
    2727  scripts="~../reggie-2.js,~demux_start.js"
    2828  styles="path.css,~../css/reggie-2.css"
    29 />
     29>
     30<style>
     31.notify.donotuseinfo
     32{
     33  width: auto;
     34  max-width: 60em;
     35  font-weight: normal;
     36  background-color: #F8F8E8;
     37  border-color: inherit;
     38}
     39
     40.notify.donotuseinfo > div:first-child > div
     41{
     42  border-color: #F8F8E8;
     43}
     44
     45.notify .notify-message:before
     46{
     47  content: url('../images/info.png');
     48}
     49
     50.donotuseinfo td
     51{
     52  vertical-align: top;
     53  padding: 2px;
     54}
     55</style>
     56
     57</base:head>
    3058<base:body>
    3159
  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/autoconfirm/SequencingRunAutoConfirmer.java

    r4306 r4985  
    195195      jobCreator.setPriority(priority);
    196196     
    197       DemuxDefinition demuxDef = new DemuxDefinition(sr, readString, null);
     197      DemuxDefinition demuxDef = new DemuxDefinition(sr, readString, null, true);
    198198      Job demuxJob = jobCreator.createDemuxJob(dc, cluster, Arrays.asList(demuxDef), null);     
    199199      dc.commit();
  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/dao/ReggieItem.java

    r4983 r4985  
    616616    }
    617617  }
     618 
     619  /**
     620    Sorts a list of items by name.
     621    @since 4.20
     622  */
     623  public static class NameComparator
     624    implements Comparator<ReggieItem<?>>
     625  {
     626    public NameComparator()
     627    {}
     628   
     629    @Override
     630    public int compare(ReggieItem<?> item1, ReggieItem<?> item2)
     631    {
     632      return item1.getName().compareTo(item2.getName());
     633    }
     634   
     635  }
     636 
    618637}
  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/grid/DemuxJobCreator.java

    r4882 r4985  
    423423          List<Extract> libsInPool = libQuery.list(dc);
    424424          List<DerivedBioAssay> mergedInPool = new ArrayList<DerivedBioAssay>(libsInPool.size());
     425          List<Extract> doNotUseInPool = new ArrayList<Extract>();
    425426         
    426427          for (Extract lib : libsInPool)
    427428          {
    428             // Create MergedSequences item if needed
    429             DerivedBioAssay merged = mergedSequences.get(lib);
    430             if (merged == null)
     429            String doNotUse = (String)Annotationtype.DO_NOT_USE.getAnnotationValue(dc, lib);
     430            if (def.excludeDoNotUse && doNotUse != null)
    431431            {
    432               Library l = Library.getById(dc, lib.getId());
    433               String mergeName = ScriptUtil.checkValidFilename(l.getNextMergedSequencesName(dc));
    434               merged = DerivedBioAssay.getNew(dc, false, demuxJob);
    435               merged.setItemSubtype(mergedType);
    436               merged.setName(mergeName);
    437               merged.setSoftware(mergeSoftware);
    438               merged.setProtocol(mergeProtocol);
    439               merged.setExtract(lib);
    440              
    441               String dataFilesFolder = ScriptUtil.checkValidPath(MergedSequences.generateDataFilesFolderForProjectArchive(mergeName, debug), true, true);
    442              
    443               Annotationtype.DATA_FILES_FOLDER.setAnnotationValue(dc, merged, dataFilesFolder);
    444               dc.saveItem(merged);
    445               mergedSequences.put(lib, merged);
    446              
    447               //jsonMessages.add("Created " + merged.getName());
     432              doNotUseInPool.add(lib);
    448433            }
    449             mergedInPool.add(merged);
    450             if (libsOnFlowCell.add(lib))
     434            else
    451435            {
    452               // First time for this lib on the flow cell
    453               merged.addParent(demux);
     436              // Create MergedSequences item if needed
     437              DerivedBioAssay merged = mergedSequences.get(lib);
     438              if (merged == null)
     439              {
     440                Library l = Library.getById(dc, lib.getId());
     441                String mergeName = ScriptUtil.checkValidFilename(l.getNextMergedSequencesName(dc));
     442                merged = DerivedBioAssay.getNew(dc, false, demuxJob);
     443                merged.setItemSubtype(mergedType);
     444                merged.setName(mergeName);
     445                merged.setSoftware(mergeSoftware);
     446                merged.setProtocol(mergeProtocol);
     447                merged.setExtract(lib);
     448               
     449                String dataFilesFolder = ScriptUtil.checkValidPath(MergedSequences.generateDataFilesFolderForProjectArchive(mergeName, debug), true, true);
     450               
     451                Annotationtype.DATA_FILES_FOLDER.setAnnotationValue(dc, merged, dataFilesFolder);
     452                dc.saveItem(merged);
     453                mergedSequences.put(lib, merged);
     454               
     455                //jsonMessages.add("Created " + merged.getName());
     456              }
     457              mergedInPool.add(merged);
     458              if (libsOnFlowCell.add(lib))
     459              {
     460                // First time for this lib on the flow cell
     461                merged.addParent(demux);
     462              }
    454463            }
    455464          }
     
    463472          // Export all required barcode files, 2 files per lane per flow cell
    464473          StringWriter out = new StringWriter();
    465           exporter.exportBarcodesFile(dc, demux, laneNo, mergedInPool, out);
     474          exporter.exportBarcodesFile(dc, demux, laneNo, mergedInPool, doNotUseInPool, out);
    466475          jobDef.addFile(new StringUploadSource(barcodesFile, out.toString()));
    467476         
    468477          out = new StringWriter();
    469           exporter.exportMultiplexFile(dc, demux, laneNo, mergedInPool, out, "fastq");
     478          exporter.exportMultiplexFile(dc, demux, laneNo, mergedInPool, doNotUseInPool, out, "fastq");
    470479          jobDef.addFile(new StringUploadSource(multiplexFile, out.toString()));
    471480
     
    734743    final String readString;
    735744    final List<Integer> omitLanes;
    736    
    737     public DemuxDefinition(SequencingRun seqRun, String readString, List<? extends Number> omitLanes)
     745    final boolean excludeDoNotUse;
     746   
     747    public DemuxDefinition(SequencingRun seqRun, String readString, List<? extends Number> omitLanes, boolean excludeDoNotUse)
    738748    {
    739749      this.seqRun = seqRun;
     
    747757        }
    748758      }
     759      this.excludeDoNotUse = excludeDoNotUse;
    749760    }
    750761
     
    859870                  sumDemux.addWarning("PF_NNNN_PCT is " + Values.formatNumber(passedFilterPct, 2) + "% on lane " + laneNo + " (" + poolName + ")");
    860871                }
     872              }
     873              else if ("IGNORED".equals(libName))
     874              {
     875                libName = demuxName+".I";
    861876              }
    862877              else if ("UNUSED".equals(libName))
     
    10681083          Reads demuxN = sumReads.remove(demuxName+".N");
    10691084          Reads demuxUnused = sumReads.remove(demuxName+".U");
     1085          Reads demuxIgnore = sumReads.remove(demuxName+".I");
    10701086         
    10711087          DemuxedSequences demux = DemuxedSequences.getByName(dc, demuxName);
  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/plugins/BarcodeFilesForDemuxExporter.java

    r4306 r4985  
    5353    when running "picard ExtractIlluminaBarcodes" for the given lane.
    5454  */
    55   public void exportBarcodesFile(DbControl dc, DerivedBioAssay demux, int laneNo, List<DerivedBioAssay> mergedInLane, Writer out)
     55  public void exportBarcodesFile(DbControl dc, DerivedBioAssay demux, int laneNo, List<DerivedBioAssay> mergedInLane, List<Extract> libIgnore, Writer out)
    5656  {
    5757    TableWriter tw = createTableWriter(out);
     
    7272      tw.tablePrintData(info);
    7373      barcodes.remove(barcode);
     74    }
     75   
     76    // Libraries that should be ignored
     77    // We can't put them under the "UNUSED" tag since the demux will warn if it finds
     78    // a large number of unused barcodes!
     79    for (Extract lib : libIgnore)
     80    {
     81      Tag barcode = lib.getTag();
     82      if (barcodes.remove(barcode))
     83      {
     84        info[0] = checkValidBarcodeSequence(dc, barcode);
     85        info[1] = checkValidBarcodeName(barcode);
     86        info[2] = "IGNORED";
     87        tw.tablePrintData(info);
     88      }
    7489    }
    7590   
     
    93108    when running "picard IlluminaBasecallsToFastq" for the given lane.
    94109  */
    95   public void exportMultiplexFile(DbControl dc, DerivedBioAssay demux, int laneNo, List<DerivedBioAssay> mergedInLane, Writer out, String outFolder)
     110  public void exportMultiplexFile(DbControl dc, DerivedBioAssay demux, int laneNo, List<DerivedBioAssay> mergedInLane, List<Extract> libIgnore, Writer out, String outFolder)
    96111  {
    97112    TableWriter tw = createTableWriter(out);
  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/servlet/DemuxMergeServlet.java

    r4597 r4985  
    66import java.util.List;
    77import java.util.Set;
     8import java.util.TreeSet;
    89
    910import javax.servlet.ServletException;
     
    5253import net.sf.basedb.reggie.dao.PooledLibrary;
    5354import net.sf.basedb.reggie.dao.ReactionPlate;
     55import net.sf.basedb.reggie.dao.ReggieItem;
    5456import net.sf.basedb.reggie.dao.ReggieRole;
    5557import net.sf.basedb.reggie.dao.Rna;
     
    344346          String readString = (String)jsonSeqRun.get("ReadString");
    345347          JSONArray jsonOmitLanes = (JSONArray)jsonSeqRun.get("OmitLanes");
     348          boolean excludeDoNotUse = Boolean.TRUE.equals(jsonSeqRun.get("excludeDoNotUse"));
    346349         
    347350          SequencingRun seqRun = SequencingRun.getById(dc, seqRunId.intValue());
    348351          Annotationtype.AUTO_PROCESSING.setAnnotationValue(dc, seqRun.getItem(), null);
    349352         
    350           demuxDefs.add(new DemuxDefinition(seqRun, readString, jsonOmitLanes));
     353          demuxDefs.add(new DemuxDefinition(seqRun, readString, jsonOmitLanes, excludeDoNotUse));
    351354        }
    352355       
     
    596599    // Load pools on flow cell
    597600    List<PooledLibrary> pools = PooledLibrary.getByFlowCell(dc, fc);
     601    Set<Library> libs = new TreeSet<>(new ReggieItem.NameComparator());
     602
    598603    JSONArray jsonPools = new JSONArray();
    599604    for (PooledLibrary pool : pools)
    600605    {
     606      libs.addAll(Library.getByPool(dc, pool));
    601607      try
    602608      {
     
    610616    }
    611617    fc.setAnnotation("pools", jsonPools);
     618   
     619    JSONArray jsonLibs = new JSONArray();
     620    for (Library lib : libs)
     621    {
     622      lib.loadDoNotUseAnnotations(dc, null);
     623      jsonLibs.add(lib.asJSONObject());
     624    }
     625    fc.setAnnotation("libraries", jsonLibs);
     626   
    612627    run.setAnnotation("flowCell", fc.asJSONObject());
    613628  }
Note: See TracChangeset for help on using the changeset viewer.