Changeset 6215

Timestamp:

Apr 16, 2021, 10:16:20 AM (13 months ago)

Author:

Nicklas Nordborg

Message:

References #1295: Registration of specimen handled by external lab

Re-designed the importer to stop at DemuxedSequnces instead of MergedSequences.

There will be one DemuxedSequences for each library instead of one for each flow cell. The DemuxedSequences item will be a named after the library an use 'x' suffix. The suffix is NOT propagated to child MergedSequences which uses 'g' as before. For example:

Lib: 1234.r.lib
Demux: 1234.r.lib.x
Merge: 1234.r.lib.g

The DemuxedSequences will also have information about the original FASTQ files that we recieved from the external lab.

The MergedSequences item is now created by the FASTQ import script. This change makes it easier to redo failed FASTQ import since we can just delete the MergedSequences and try again.

Location:

extensions/net.sf.basedb.reggie/trunk

Files:

• resources/analysis/import_confirm.js (modified) (1 diff)
• resources/analysis/import_confirm.jsp (modified) (3 diffs)
• resources/analysis/import_fastq.js (modified) (10 diffs)
• resources/analysis/import_fastq.jsp (modified) (1 diff)
• resources/index.jsp (modified) (1 diff)
• src/net/sf/basedb/reggie/counter/CounterService.java (modified) (1 diff)
• src/net/sf/basedb/reggie/dao/Annotationtype.java (modified) (1 diff)
• src/net/sf/basedb/reggie/dao/BiomaterialList.java (modified) (1 diff)
• src/net/sf/basedb/reggie/dao/DemuxedSequences.java (modified) (1 diff)
• src/net/sf/basedb/reggie/dao/Library.java (modified) (1 diff)
• src/net/sf/basedb/reggie/dao/MergedSequences.java (modified) (4 diffs)
• src/net/sf/basedb/reggie/grid/ImportFastqJobCreator.java (modified) (12 diffs)
• src/net/sf/basedb/reggie/plugins/FutureSpecimenImporter.java (modified) (7 diffs)
• src/net/sf/basedb/reggie/plugins/cmd/DemuxInfo.java (modified) (2 diffs)
• src/net/sf/basedb/reggie/plugins/cmd/JsonFile.java (modified) (3 diffs)
• src/net/sf/basedb/reggie/plugins/cmd/MergeInfo.java (deleted)
• src/net/sf/basedb/reggie/servlet/FastqServlet.java (modified) (12 diffs)
• src/net/sf/basedb/reggie/servlet/FutureSpecimenServlet.java (modified) (7 diffs)
• src/net/sf/basedb/reggie/servlet/InstallServlet.java (modified) (2 diffs)

extensions/net.sf.basedb.reggie/trunk/resources/analysis/import_confirm.js

@@ -209 +209 @@
     var frm = document.forms['reggie'];
     var submitInfo = {};
+    submitInfo.deleteItemsCreatedByFailedJobs = frm.deleteItemsCreatedByFailedJobs.checked;
     
     var all = [];

extensions/net.sf.basedb.reggie/trunk/resources/analysis/import_confirm.jsp

@@ -53 +53 @@
 }
 
-#mergedSequencesTable tbody td
+#mergedSequencesTable td, #mergedSequencesTable th
 {
-  padding-top: 1px;
-  padding-bottom: 1px;
+  padding: 2px 1px;
 }
 
@@ -64 +63 @@
 }
 
-#mergedSequencesTable .icon-col
+#mergedSequencesTable .warning-col
 {
   width: 20px;
@@ -110 +109 @@
         </td>
       </tr>
+      <tr>
+        <td class="prompt"></td>
+        <td class="input" style="text-align: right; vertical-align: middle;">
+          <label>Delete items selected for re-import
+          <input type="checkbox" name="deleteItemsCreatedByFailedJobs" id="deleteItemsCreatedByFailedJobs" checked>
+          </label>
+        </td>
+        <td class="status" id="deleteItemsCreatedByFailedJobs.status"></td>
+        <td class="help">
+          <span id="deleteItemsCreatedByFailedJobs.message" class="message"></span>
+          Recommended! Items created by jobs selected for re-import are deleted to avoid cluttering in the database.
+        </td>
+      </tr>
       </table>
     </div>

extensions/net.sf.basedb.reggie/trunk/resources/analysis/import_fastq.js

@@ -11 +11 @@
     
     // Step 1
-    Events.addEventHandler('mergedSequences', 'change', fastq.mergedSequencesOnChange);
+    Events.addEventHandler('demuxedSequences', 'change', fastq.demuxedSequencesOnChange);
     Events.addEventHandler('step-1', 'wizard-validate', fastq.validateStep1);
 
@@ -31 +31 @@
     Wizard.showLoadingAnimation('Loading bioassays...');
     var url = '../Fastq.servlet?ID='+App.getSessionId();
-    url += '&cmd=GetMergedSequencesForImport';
+    url += '&cmd=GetDemuxedSequencesForImport';
     Wizard.asyncJsonRequest(url, fastq.initializeStep1);
   }
@@ -38 +38 @@
   fastq.initializeStep1 = function(response)
   {
-    var mergedSequences = response.mergedSequences;
+    var demuxedSequences = response.demuxedSequences;
     var frm = document.forms['reggie']; 
-    if (mergedSequences.length == 0)
-    {
-      Wizard.setFatalError('No merged sequences available for importing.');
+    if (demuxedSequences.length == 0)
+    {
+      Wizard.setFatalError('No demuxed sequences available for importing.');
       return;
     }
 
     var firstPlate = null;
-    for (var msNo=0; msNo < mergedSequences.length; msNo++)
-    {
-      var ms = mergedSequences[msNo];
+    for (var msNo=0; msNo < demuxedSequences.length; msNo++)
+    {
+      var ms = demuxedSequences[msNo];
       var libPlate = ms.lib.bioWell ? ms.lib.bioWell.bioPlate : null;
       if (firstPlate == null && libPlate != null) firstPlate = libPlate.name;
       var option = fastq.createListOption(msNo+1, ms, ms.DO_NOT_USE == null && libPlate != null && libPlate.name==firstPlate)
-      frm.mergedSequences.options[frm.mergedSequences.length] = option;
-    }
-    Events.sendChangeEvent('mergedSequences');
+      frm.demuxedSequences.options[frm.demuxedSequences.length] = option;
+    }
+    Events.sendChangeEvent('demuxedSequences');
     
     Doc.show('step-1');
     Doc.show('gonext');
 
-    frm.mergedSequences.focus();
+    frm.demuxedSequences.focus();
   }
   
@@ -68 +68 @@
   }
   
-  fastq.createListOption = function(index, mergedItem, selected)
-  {
-    var lib = mergedItem.lib;
+  fastq.createListOption = function(index, demuxItem, selected)
+  {
+    var lib = demuxItem.lib;
     var isYellow = lib.specimen && lib.specimen.YellowLabel != null;
     var libPlate = lib.bioWell ? lib.bioWell.bioPlate : null;
@@ -77 +77 @@
     var name = (index) + ': ';
     if (libPlate) name += libPlate.name + ' - ';
-    name += mergedItem.name;
-    
-    if (mergedItem.DO_NOT_USE)
+    name += demuxItem.name;
+    
+    if (demuxItem.DO_NOT_USE)
     {
       name += ' [DoNotUse]';
-      tooltip = 'DoNotUse-'+Strings.encodeTags(mergedItem.DO_NOT_USE+': '+mergedItem.DO_NOT_USE_COMMENT);
-    }
-    else if (mergedItem.AutoProcess == 'ReProcess')
+      tooltip = 'DoNotUse-'+Strings.encodeTags(demuxItem.DO_NOT_USE+': '+demuxItem.DO_NOT_USE_COMMENT);
+    }
+    else if (demuxItem.AutoProcess == 'ReProcess')
     {
       name += ' [R]';
     }
     
-    var option = new Option(name, mergedItem.id, false, selected);
+    var option = new Option(name, demuxItem.id, false, selected);
     if (isYellow) option.className = 'yellow';
     if (tooltip) option.title = tooltip;
-    option.mergedSequences = mergedItem;
+    option.demuxedSequences = demuxItem;
     return option;
   }
   
-  fastq.mergedSequencesOnChange = function()
+  fastq.demuxedSequencesOnChange = function()
   {
     var frm = document.forms['reggie'];
@@ -104 +104 @@
     var invalidSelection = null;
     
-    Wizard.setInputStatus('mergedSequences');
+    Wizard.setInputStatus('demuxedSequences');
     Wizard.hideGoNextConfirmation();
-    for (var msNo = 0; msNo < frm.mergedSequences.length; msNo++)
-    {
-      if (frm.mergedSequences[msNo].selected) 
+    for (var msNo = 0; msNo < frm.demuxedSequences.length; msNo++)
+    {
+      if (frm.demuxedSequences[msNo].selected) 
       {
         numSelected++;
-        var ms = frm.mergedSequences[msNo].mergedSequences;
+        var ms = frm.demuxedSequences[msNo].demuxedSequences;
         if (ms.DO_NOT_USE) numDoNotUse++;
         if (ms.pipeline && ms.pipeline != 'RNAseq')
@@ -121 +121 @@
           invalidSelection = Strings.encodeTags(ms.name+' is missing RawFASTQ information');
         }
-        else if (!ms.rawFastqFolder)
+        else if (!ms.DataFilesFolder)
         {
-          invalidSelection = Strings.encodeTags(ms.name+' demux ('+ms.demuxName+') is missing DataFilesFolder information');
+          invalidSelection = Strings.encodeTags(ms.name+' is missing DataFilesFolder information');
         }
       }
@@ -130 +130 @@
     if (numSelected == 0)
     {
-      Wizard.setInputStatus('mergedSequences', 'invalid', 'Select at least one item');
+      Wizard.setInputStatus('demuxedSequences', 'invalid', 'Select at least one item');
       return;
     }
     if (invalidSelection) 
     {
-      Wizard.setInputStatus('mergedSequences', 'invalid', invalidSelection);
+      Wizard.setInputStatus('demuxedSequences', 'invalid', invalidSelection);
       return;
     }
@@ -142 +142 @@
     if (numDoNotUse > 0)
     {
-      Wizard.setInputStatus('mergedSequences', 'warning', numDoNotUse + ' selected items marked as DoNotUse');
+      Wizard.setInputStatus('demuxedSequences', 'warning', numDoNotUse + ' selected items marked as DoNotUse');
       Wizard.showGoNextConfirmation(true, 'Confirm ' + numDoNotUse + ' items marked as DoNotUse');
     }
     else
     {
-      Wizard.setInputStatus('mergedSequences', 'valid');
+      Wizard.setInputStatus('demuxedSequences', 'valid');
     }
   }
@@ -270 +270 @@
     submitInfo.autoConfirm = frm.autoConfirm.checked;
     
-    var mergedSequences = [];
-    submitInfo.mergedSequences = mergedSequences;
+    var demuxedSequences = [];
+    submitInfo.demuxedSequences = demuxedSequences;
       
-    for (var msNo = 0; msNo < frm.mergedSequences.length; msNo++)
-    {
-      if (frm.mergedSequences[msNo].selected) 
+    for (var msNo = 0; msNo < frm.demuxedSequences.length; msNo++)
+    {
+      if (frm.demuxedSequences[msNo].selected) 
       {
         var ms = {};
-        ms.id = frm.mergedSequences[msNo].mergedSequences.id;
-        mergedSequences[mergedSequences.length] = ms;
+        ms.id = frm.demuxedSequences[msNo].demuxedSequences.id;
+        demuxedSequences[demuxedSequences.length] = ms;
       }
     }

extensions/net.sf.basedb.reggie/trunk/resources/analysis/import_fastq.jsp

@@ -40 +40 @@
   <div class="step auto-hide" id="step-1">
     <div class="step-no">1</div>
-    <div class="step-title">Select merged sequences</div>
+    <div class="step-title">Select demuxed sequences</div>
     <div class="step-content">
     
       <table class="step-form">
       <tr class="align-top">
-        <td class="prompt">Merged sequences</td>
+        <td class="prompt">Demuxed sequences</td>
         <td class="input">
-          <select class="required yellow-label-support" multiple size="30"
-            name="mergedSequences" id="mergedSequences"></select>
+          <select class="required yellow-label-support" multiple size="20"
+            name="demuxedSequences" id="demuxedSequences"></select>
         </td>
-        <td class="status" id="mergedSequences.status"></td>
-        <td class="help"><span id="mergedSequences.message" class="message"></span>
-          Select the sequences that should be imported. The list contains merged sequences 
+        <td class="status" id="demuxedSequences.status"></td>
+        <td class="help"><span id="demuxedSequences.message" class="message"></span>
+          Select the sequences that should be imported. The list contains demuxed sequences 
           from the <b>FASTQ import pipeline</b> item list (max 250).<br><br>
           Items marked with <b>[R]</b> have been marked for re-import due to a failure of

extensions/net.sf.basedb.reggie/trunk/resources/index.jsp

@@ -760 +760 @@
             <li><span class="require-permission" data-role="SecondaryAnalysis" data-link="analysis/import_fastq.jsp?ID=<%=ID%>"
               >Import FASTQ files</span> 
-              <span class="counter" data-counter="merged-sequences-for-fastq-import" 
-                title="Number of merged sequences waiting for FASTQ import">∙</span>
+              <span class="counter" data-counter="demuxed-sequences-for-fastq-import" 
+                title="Number of demuxed sequences waiting for FASTQ import">∙</span>
                 <span id="fastq-import-pipeline-disabled" class="info-warning">auto-processing is disabled</span>
 

extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/counter/CounterService.java

@@ -1171 +1171 @@
     // External merged sequences waiting for FASTQ import
     ItemList fastqImportPipeline = BiomaterialList.FASTQ_IMPORT_PIPELINE.load(dc);
-    json.put("merged-sequences-for-fastq-import", fastqImportPipeline.getSize());
+    json.put("demuxed-sequences-for-fastq-import", fastqImportPipeline.getSize());
     
     // Merged sequences with FASTQ import not confirmed

extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/dao/Annotationtype.java

@@ -1378 +1378 @@
 
   /**
-    The "RawFASTQ" annotation, used for (merged) derived bioassays that
+    The "RawFASTQ" annotation, used for (demuxed) derived bioassays that
     use external FASTQ files for import. This annotation should typically
     have 2 values (one for R1 and one for R2).

extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/dao/BiomaterialList.java

@@ -124 +124 @@
   public static final BiomaterialList FASTQ_IMPORT_PIPELINE = 
     new BiomaterialList("FASTQ Import Pipeline", "net.sf.basedb.reggie.pipeline.fastq-import",
-      Item.DERIVEDBIOASSAY, Subtype.MERGED_SEQUENCES,
-      "This list contains merged sequences items that need FASTQ files to be imported " +
+      Item.DERIVEDBIOASSAY, Subtype.DEMUXED_SEQUENCES,
+      "This list contains demuxed sequences items that need FASTQ files to be imported " +
       "before downstream analysis can take place. Items are normally added and removed from this list " +
       "automatically, but it is possible to also manually add items.");

extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/dao/DemuxedSequences.java

@@ -191 +191 @@
   }
 
+  /**
+    Get the library related to this demux. NOTE! This is only
+    available for externally sequenced where we get raw FASTQ files
+    from the demux.
+    @since 4.32
+  */
+  public Library getLibrary(DbControl dc)
+  {
+    return getItem().getExtract() == null ? null : Library.getById(dc, getItem().getExtract().getId());
+  }
 
 }

extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/dao/Library.java

@@ -318 +318 @@
   }
 
+  /**
+    Find the next name to give a "DemuxedSequences" child item. This assumes that
+    all child items are using the naming convention. foo.x, foo.x2, and
+    so on. NOTE! The first child item have no number!
+    @return The next unused name
+  */
+  public String getNextDemuxedSequencesName(DbControl dc)
+  {
+    Extract lib = getItem();
+    ItemQuery<DerivedBioAssay> query = null;
+    if (lib.isInDatabase())
+    {
+      query = DerivedBioAssay.getQuery();
+      Subtype.DEMUXED_SEQUENCES.addFilter(dc, query);
+      query.setIncludes(Include.ALL);
+      query.restrict(Restrictions.eq(Hql.property("extract"), Hql.entity(lib)));
+    }
+    return getNextChildItemName(dc, query, Subtype.DEMUXED_SEQUENCES.getItemSuffix(), true);
+  }
+
+  
 }

extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/dao/MergedSequences.java

@@ -227 +227 @@
       throw new InvalidDataException(
         "No parent item was found for " + getName() +
-        "This wizard can't be used until that is corrected.");
+        ". This wizard can't be used until that is corrected.");
     }
     
@@ -233 +233 @@
   }
 
-  public DemuxedSequences getSingleDemuxedSequecnces(DbControl dc)
+  public DemuxedSequences getSingleDemuxedSequences(DbControl dc)
   {
     ItemQuery<DerivedBioAssay> query = getItem().getParents();
@@ -244 +244 @@
       throw new InvalidDataException(
         "No demux item was found for " + getName() +
-        "This wizard can't be used until that is corrected.");
+        ". This wizard can't be used until that is corrected.");
     }
     else if (tmp.size() > 1)
@@ -250 +250 @@
       throw new InvalidDataException(
         "More than one demux was found for " + getName() +
-        "This wizard can't be used until that is corrected.");
+        ". This wizard can't be used until that is corrected.");
     }
     return DemuxedSequences.get(tmp.get(0));

extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/grid/ImportFastqJobCreator.java

@@ -22 +22 @@
 import net.sf.basedb.core.Job;
 import net.sf.basedb.core.Path;
-import net.sf.basedb.core.PermissionDeniedException;
 import net.sf.basedb.core.Protocol;
 import net.sf.basedb.core.SessionControl;
@@ -41 +40 @@
 import net.sf.basedb.reggie.dao.Datafiletype;
 import net.sf.basedb.reggie.dao.DemuxedSequences;
+import net.sf.basedb.reggie.dao.DoNotUse;
 import net.sf.basedb.reggie.dao.Fileserver;
 import net.sf.basedb.reggie.dao.Library;
 import net.sf.basedb.reggie.dao.MergedSequences;
 import net.sf.basedb.reggie.dao.Pipeline;
-import net.sf.basedb.reggie.dao.SequencingRun;
 import net.sf.basedb.reggie.dao.Subtype;
 import net.sf.basedb.util.Values;
@@ -119 +118 @@
   
   /**
-    Create a child bioassays for all given merged sequences and schedule
-    jobs on the given cluster for running Hisat alignment.
+    Create a child bioassays for all given demuxed sequences and schedule
+    jobs on the given cluster for importing FASTQ files.
     @return A list with the corresponding jobs in BASE
   */
   @SuppressWarnings("unchecked")
-  public List<JobDefinition> createFastqImportJobs(DbControl dc, OpenGridCluster cluster, List<MergedSequences> mergedSequences)
+  public List<JobDefinition> createFastqImportJobs(DbControl dc, OpenGridCluster cluster, List<DemuxedSequences> demuxedSequences)
   {
     SessionControl sc = dc.getSessionControl();
@@ -160 +159 @@
     // Load common items
     ItemSubtype fastqImportJobType = Subtype.FASTQ_IMPORT_JOB.get(dc);
+    ItemSubtype mergedType = Subtype.MERGED_SEQUENCES.get(dc);
     DataFileType fastqData = Datafiletype.FASTQ.load(dc);
     
@@ -173 +173 @@
     
     // We submit one job for each item to the cluster
-    List<JobDefinition> jobDefs = new ArrayList<JobDefinition>(mergedSequences.size());
-
-    for (MergedSequences ms : mergedSequences)
+    List<JobDefinition> jobDefs = new ArrayList<JobDefinition>(demuxedSequences.size());
+
+    for (DemuxedSequences ds : demuxedSequences)
     {
-      ms = MergedSequences.getById(dc, ms.getId()); // Ensure item is loaded in this transaction
-      DerivedBioAssay merged = ms.getDerivedBioAssay();
-      String mergeName = ScriptUtil.checkValidFilename(merged.getName());
-      
-      List<String> rawFastqNames = (List<String>)Annotationtype.RAW_FASTQ.getAnnotationValues(dc, merged);
+      ds = DemuxedSequences.getById(dc, ds.getId()); // Ensure item is loaded in this transaction
+      DerivedBioAssay demuxed = ds.getDerivedBioAssay();
+      String demuxName = ScriptUtil.checkValidFilename(demuxed.getName());
+      
+      List<String> rawFastqNames = (List<String>)Annotationtype.RAW_FASTQ.getAnnotationValues(dc, demuxed);
       if (rawFastqNames == null || rawFastqNames.size() != 2)
       {
         throw new InvalidDataException("Annotation RawFASTQ on " + 
-            mergeName + " must have two values: " + rawFastqNames);
+            demuxName + " must have two values: " + rawFastqNames);
       }
       
       // Get SequencingRun so that we can get the path to the FASTQ folder.
-      DemuxedSequences demux = ms.getSingleDemuxedSequecnces(dc);
-      String rawFastqFolder = ScriptUtil.checkValidPath((String)Annotationtype.DATA_FILES_FOLDER.getAnnotationValue(dc, demux.getItem()), true, true);
-      
-      if (merged.hasFileSet() && merged.getFileSet().hasMember(fastqData))
-      {
-        throw new PermissionDeniedException("FASTQ files already exists: " + mergeName);
-      }
-      importPipeline.removeItem(merged);
-      Library lib = Library.get(merged.getExtract());
-      boolean isExternal = Reggie.isExternalItem(mergeName);
+      String rawFastqFolder = ScriptUtil.checkValidPath((String)Annotationtype.DATA_FILES_FOLDER.getAnnotationValue(dc, demuxed), true, true);
+      
+      importPipeline.removeItem(demuxed);
+      Library lib = Library.get(demuxed.getExtract());
+      boolean isExternal = Reggie.isExternalItem(demuxName);
       String rootName = isExternal ? lib.getTopExtractOrSample(dc).getName() : null;
       
       // Create job 
+      String mergedName = lib.getNextMergedSequencesName(dc);
       Job importJob = Job.getNew(dc, null, null, null);
       importJob.setItemSubtype(fastqImportJobType);
       importJob.setPluginVersion("reggie-"+Reggie.VERSION);
       importJob.setSendMessage(Values.getBoolean(sc.getUserClientSetting("plugins.sendmessage"), false));
-      importJob.setName("FASTQ Import to " + mergeName);
+      importJob.setName("FASTQ Import to " + mergedName);
       importJob.setParameterValue("pipeline", new StringParameterType(), Pipeline.RNA_SEQ.getId());
       if (debug) importJob.setName(importJob.getName() + " (debug)");
       dc.saveItem(importJob);
-
-      // Update the MergedSequence item with information about this import
+      
+      // Created MERGED derived bioassay set
+      DerivedBioAssay merged = DerivedBioAssay.getNew(dc, demuxed, importJob);
+      merged.setItemSubtype(mergedType);
+      Pipeline.RNA_SEQ.setAnnotation(dc, merged);
+      merged.setName(mergedName);
+      merged.setExtract(lib.getExtract());
       merged.setSoftware(mergeSoftware);
       merged.setProtocol(mergeProtocol);
-      merged.setJob(importJob);
-      
-      String dataFilesFolder = ScriptUtil.checkValidPath(MergedSequences.generateDataFilesFolderForProjectArchive(mergeName, rootName, debug), true, true);
+      DoNotUse.copyDoNotUseAnnotations(dc, lib.getExtract(), merged, false);
+      // Copy READS
+      Annotationtype.READS.copyAnnotationValues(dc, demuxed, merged, false);
+      Annotationtype.PF_READS.copyAnnotationValues(dc, demuxed, merged, false);
+      dc.saveItem(merged);
+      
+      String dataFilesFolder = ScriptUtil.checkValidPath(MergedSequences.generateDataFilesFolderForProjectArchive(mergedName, rootName, debug), true, true);
       Annotationtype.DATA_FILES_FOLDER.setAnnotationValue(dc, merged, dataFilesFolder);
       if (autoConfirm)
@@ -225 +230 @@
       String archiveRoot =  isExternal ? externalRoot : projectRoot;
       String fastqFolder = archiveRoot + dataFilesFolder;
-      String baseFileName = isExternal ? Reggie.removePrefix(mergeName) : mergeName;
+      String baseFileName = isExternal ? Reggie.removePrefix(mergedName) : mergedName;
       String R1_name = baseFileName+"_R1.fastq";
       String R2_name = baseFileName+"_R2.fastq";
@@ -247 +252 @@
   
       // Set file permissions based on consent or external group!
-      String externalGroup = isExternal ? Reggie.getExternalGroup(mergeName) : null;
+      String externalGroup = isExternal ? Reggie.getExternalGroup(mergedName) : null;
       ScriptUtil.setUmaskForItem(dc, lib, externalGroup, script);
 
@@ -294 +299 @@
 
       script.comment("Run Bowtie2");
-      script.progress(40, "Bowtie2: " + mergeName + " (${NumThreads} threads)");
+      script.progress(40, "Bowtie2: " + mergedName + " (${NumThreads} threads)");
       String alignCmd = "./stdwrap.sh " + bowtie_path;
       alignCmd += " -p ${NumThreads}";
@@ -306 +311 @@
       script.cmd(alignCmd);
       script.cmd("./singlecolumnaverager.awk < fastq.aligned/aligned.sam > fastq.aligned/" + fragments_name);
-      script.time("DONE: Bowtie2 " + mergeName);
-      script.newLine();
-      
-      script.progress(60, "Trimmomatic: " + mergeName + " (${NumThreads} threads)");
+      script.time("DONE: Bowtie2 " + mergedName);
+      script.newLine();
+      
+      script.progress(60, "Trimmomatic: " + mergedName + " (${NumThreads} threads)");
       // First Trimmomatic step should filter on Illumina adapters only
       String trimCmd1 = "./stdwrap.sh ./trimmomatic PE";
@@ -340 +345 @@
       trimCmd2 += " >> ${WD}/trimmomatic.out";
       script.cmd(trimCmd1);
-      script.time("DONE: Trimmomatic 1 " + mergeName);
+      script.time("DONE: Trimmomatic 1 " + mergedName);
 
       script.cmd(trimCmd2);
-      script.time("DONE: Trimmomatic 2 " + mergeName);
-      script.newLine();
-
-      script.progress(80, "Archiving FASTQ: " + mergeName);
+      script.time("DONE: Trimmomatic 2 " + mergedName);
+      script.newLine();
+
+      script.progress(80, "Archiving FASTQ: " + mergedName);
       script.cmd("mkdir -p " + fastqFolder);
       script.cmd("rm -f " + fastqFolder + "/*");
@@ -357 +362 @@
       if (externalGroup != null)
       {
-        ScriptUtil.addChgrp(externalGroup, fastqFolder, mergeName, archiveRoot+"/"+Reggie.getPrefix(mergeName), script);
+        ScriptUtil.addChgrp(externalGroup, fastqFolder, mergedName, archiveRoot+"/"+Reggie.getPrefix(mergedName), script);
       }
 
@@ -407 +412 @@
       msg += Values.formatNumber(total.passedFilter/1000000f, 1) + "M passed filter; ";
       msg += Values.formatNumber(total.passedTrimmomatic[1]/1000000f, 1) + "M passed trimmomatic; ";
-/*      if (total.warnings.size() > 0)
-      {
-        msg += total.warnings.size() + " warnings!";
-      }
-*/      return msg;
+      return msg;
     }
     

extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/plugins/FutureSpecimenImporter.java

@@ -25 +25 @@
 import net.sf.basedb.reggie.dao.Library;
 import net.sf.basedb.reggie.dao.Lysate;
-import net.sf.basedb.reggie.dao.MergedSequences;
 import net.sf.basedb.reggie.dao.Pipeline;
 import net.sf.basedb.reggie.dao.PooledLibrary;
@@ -43 +42 @@
 import net.sf.basedb.reggie.plugins.cmd.LibraryInfo;
 import net.sf.basedb.reggie.plugins.cmd.LysateInfo;
-import net.sf.basedb.reggie.plugins.cmd.MergeInfo;
 import net.sf.basedb.reggie.plugins.cmd.PoolInfo;
 import net.sf.basedb.reggie.plugins.cmd.RnaInfo;
@@ -51 +49 @@
 /**
   Import data to FutureSpecimen items and create all downstream items
-  needed (down to MergedSequences) before FASTQ files can be imported.
+  needed (down to DemuxedSequences) before FASTQ files can be imported.
   
   It is also possible to start the import on a Specimen item or an
   RNA item.
-  
-  The implementation is currently a mockup which only create more or
-  less empty items that most likely will cause other wizards to fail.
   
   @author nicklas
@@ -109 +104 @@
     SequencingRunInfo seqRunInfo = jsonFile.getSequencingRun();
     DemuxInfo demuxInfo = jsonFile.getDemuxInfo();
-    MergeInfo mergeInfo = jsonFile.getMergeInfo();
     FastqInfo fastqInfo = jsonFile.getFastqInfo();
 
@@ -140 +134 @@
     
     if (items.sequencingRun == null || !demuxInfo.valid) return null;
-    items.demux = getOrCreateDemux(dc, items.sequencingRun, flowCellInfo, seqRunInfo, demuxInfo);
-    
-    if (items.demux == null || !mergeInfo.valid || !fastqInfo.valid) return null;
-    items.merged = createMergedSequences(dc, items.lib, items.demux, mergeInfo, fastqInfo);
-    
-    return items.merged == null || jsonFile.hasError() ? null : items;
+    items.demux = createDemuxedSequences(dc, items.lib, items.sequencingRun, flowCellInfo, seqRunInfo, demuxInfo, fastqInfo);
+    
+    return items.demux == null || jsonFile.hasError() ? null : items;
   }
   
@@ -505 +496 @@
   
   
-  private DemuxedSequences getOrCreateDemux(DbControl dc, SequencingRun seqRun, FlowCellInfo flowCellInfo, SequencingRunInfo seqRunInfo, DemuxInfo demuxInfo)
-  {
-    String flowCellId = flowCellInfo.flowCellId;
-    if (flowCellId == null) return null;
-
-    DerivedBioAssay demux = demuxInfo.demux;
-    if (demux != null)
-    {
-      addDebugMessage("Found: "+demux.getName()+"[ref="+flowCellId+"]");
-    }
-    else
-    {
-      demux = DerivedBioAssay.getNew(dc, seqRun.getItem(), null);
-      demux.setItemSubtype(Subtype.DEMUXED_SEQUENCES.get(dc));
-      demux.setName(seqRun.getNextDemuxedSequencesName(dc));
-      Pipeline.RNA_SEQ.setAnnotation(dc, demux);
-      demux.setHardware(null);
-      demux.setSoftware(demuxInfo.software);
-      demux.setProtocol(null);
-      Annotationtype.ANALYSIS_RESULT.setAnnotationValue(dc, demux, DemuxedSequences.DEMUX_SUCCESSFUL);
-      Annotationtype.READ_STRING.setAnnotationValue(dc, demux, demuxInfo.readString);   
-      
-      DateFormat df = new SimpleDateFormat("/yyyy/yyMMdd");
-      String dataFilesFolder = df.format(seqRunInfo.startDate)+
-        "_"+seqRunInfo.sequencer.serialNo+
-        "_"+MD5.leftPad(Integer.toString(seqRunInfo.runNumber), '0', 4)+
-        "_"+seqRunInfo.position+flowCellInfo.flowCellId;
-      addDebugMessage("DataFilesFolder: "+dataFilesFolder);
-      Annotationtype.DATA_FILES_FOLDER.setAnnotationValue(dc, demux, dataFilesFolder);
-      
-      dc.saveItem(demux);
-      
-      addDebugMessage("Created: "+ demux.getName());
-    }
+  private DemuxedSequences createDemuxedSequences(DbControl dc, Library lib, SequencingRun seqRun, FlowCellInfo flowCellInfo, SequencingRunInfo seqRunInfo, DemuxInfo demuxInfo, FastqInfo fastqInfo)
+  {
+
+    DerivedBioAssay demux = DerivedBioAssay.getNew(dc, seqRun.getItem(), null);
+    demux.setItemSubtype(Subtype.DEMUXED_SEQUENCES.get(dc));
+    demux.setName(lib.getNextDemuxedSequencesName(dc));
+    Pipeline.RNA_SEQ.setAnnotation(dc, demux);
+    demux.setExtract(lib.getItem());
+    demux.setHardware(null);
+    demux.setSoftware(demuxInfo.software);
+    demux.setProtocol(null);
+    Annotationtype.ANALYSIS_RESULT.setAnnotationValue(dc, demux, DemuxedSequences.DEMUX_SUCCESSFUL);
+    Annotationtype.READ_STRING.setAnnotationValue(dc, demux, demuxInfo.readString);   
+    Annotationtype.READS.setAnnotationValue(dc, demux, demuxInfo.reads);
+    Annotationtype.PF_READS.setAnnotationValue(dc, demux, demuxInfo.pfReads);
+    Annotationtype.RAW_FASTQ.setAnnotationValues(dc, demux, Arrays.asList(fastqInfo.R1.name, fastqInfo.R2.name));
+    
+    DateFormat df = new SimpleDateFormat("/yyyy/yyMMdd");
+    String dataFilesFolder = df.format(seqRunInfo.startDate)+
+      "_"+seqRunInfo.sequencer.serialNo+
+      "_"+MD5.leftPad(Integer.toString(seqRunInfo.runNumber), '0', 4)+
+      "_"+seqRunInfo.position+flowCellInfo.flowCellId;
+    addDebugMessage("DataFilesFolder: "+dataFilesFolder);
+    Annotationtype.DATA_FILES_FOLDER.setAnnotationValue(dc, demux, dataFilesFolder);
+    
+    dc.saveItem(demux);
+    
+    addDebugMessage("Created: "+ demux.getName());
     return DemuxedSequences.get(demux);
-  }
-  
-  private MergedSequences createMergedSequences(DbControl dc, Library lib, DemuxedSequences demux, MergeInfo mergeInfo, FastqInfo fastqInfo)
-  {
-
-    DerivedBioAssay merged = DerivedBioAssay.getNew(dc, demux.getItem(), null);
-    merged.setItemSubtype(Subtype.MERGED_SEQUENCES.get(dc));
-    merged.setName(lib.getNextMergedSequencesName(dc));
-    Pipeline.RNA_SEQ.setAnnotation(dc, merged);
-    merged.setExtract(lib.getItem());
-    Annotationtype.READS.setAnnotationValue(dc, merged, mergeInfo.reads);
-    Annotationtype.PF_READS.setAnnotationValue(dc, merged, mergeInfo.pfReads);
-    Annotationtype.RAW_FASTQ.setAnnotationValues(dc, merged, Arrays.asList(fastqInfo.R1.name, fastqInfo.R2.name));
-    
-    merged.setProtocol(null);
-    merged.setSoftware(null);
-    merged.setHardware(null);
-    
-    dc.saveItem(merged);
-    
-    addDebugMessage("Created: "+ merged.getName());
-    return MergedSequences.get(merged);
   }
 
@@ -579 +542 @@
     public SequencingRun sequencingRun;
     public DemuxedSequences demux;
-    public MergedSequences merged;
   }
   

extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/plugins/cmd/DemuxInfo.java

@@ -4 +4 @@
 
 import net.sf.basedb.core.DbControl;
-import net.sf.basedb.core.DerivedBioAssay;
 import net.sf.basedb.core.ItemQuery;
 import net.sf.basedb.core.Software;
@@ -27 +26 @@
 {
 
-  public DerivedBioAssay demux;
   public Software software;
   public String readString;
-  
-  public boolean valid;
+  public Long pfReads;
+  public Long reads;
 
-  public DemuxInfo(JsonSection section, SequencingRunInfo seqRunInfo, FlowCellInfo flowCellInfo)
+  public boolean valid = false;
+
+  public DemuxInfo(JsonSection demuxSection, JsonSection mergeSection)
   {
-    if (section != null)
+    if (demuxSection != null && mergeSection != null)
     {
-      software = section.getRequiredEntry("Software", SoftwareValidator.INSTANCE);
-      readString = section.getRequiredEntry("ReadString", PatternValidator.READ_STRING);
-      if (seqRunInfo.sequencingRun != null)
-      {
-        demux = findExistingDemux(seqRunInfo.sequencingRun, flowCellInfo.flowCellId, section);
-      }
+      software = demuxSection.getRequiredEntry("Software", SoftwareValidator.INSTANCE);
+      readString = demuxSection.getRequiredEntry("ReadString", PatternValidator.READ_STRING);
+      pfReads = mergeSection.getRequiredEntry("PF_READS", LongValidator.POSITIVE.warnIf(10000000l, null)); // Warn if less than 10M reads
+      reads = pfReads; // 
+      valid = !demuxSection.hasError() && !mergeSection.hasError();
     }
-    valid = section != null && !section.hasError();
-  }
-  
-  
-  private DerivedBioAssay findExistingDemux(DerivedBioAssay seqRun, String flowCellId, JsonSection section)
-  {
-    DerivedBioAssay demux = null;
-    DbControl dc = section.getFile().dc();
-    ItemQuery<DerivedBioAssay> query = seqRun.getChildren();
-    query.setIncludes(Reggie.INCLUDE_IN_CURRENT_PROJECT);
-    Subtype.DEMUXED_SEQUENCES.addFilter(dc, query);
-    List<DerivedBioAssay> list = query.list(dc);
-    if (list.size() > 1)
-    {
-      section.addErrorMessage("Found "+list.size()+" demux items for FlowCellID="+flowCellId);
-    }
-    else if (list.size() == 1)
-    {
-      demux = list.get(0);
-    }
-    return demux;
   }
 

extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/plugins/cmd/JsonFile.java

@@ -95 +95 @@
   private SequencingRunInfo seqRunInfo;
   private DemuxInfo demuxInfo;
-  private MergeInfo mergeInfo;
 
   /**
@@ -260 +259 @@
       flowCellInfo = new FlowCellInfo(getRequiredSection("FlowCell"), poolInfo);
       seqRunInfo = new SequencingRunInfo(getRequiredSection("SequencingRun"), flowCellInfo);
-      demuxInfo = new DemuxInfo(getRequiredSection("DemuxedSequences"), seqRunInfo, flowCellInfo);
-      mergeInfo = new MergeInfo(getRequiredSection("MergedSequences"));
+      demuxInfo = new DemuxInfo(getRequiredSection("DemuxedSequences"), getRequiredSection("MergedSequences"));
       
       if (operator != null) setOperator(operator);
@@ -369 +367 @@
   }
   
-  public MergeInfo getMergeInfo()
-  {
-    return mergeInfo;
-  }
-  
   public FastqInfo getFastqInfo()
   {

extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/servlet/FastqServlet.java

@@ -14 +14 @@
 
 import net.sf.basedb.core.ItemList;
+import net.sf.basedb.core.BasicItem;
 import net.sf.basedb.core.DbControl;
 import net.sf.basedb.core.DerivedBioAssay;
@@ -23 +24 @@
 import net.sf.basedb.core.SessionControl;
 import net.sf.basedb.core.Software;
+import net.sf.basedb.core.Trashcan;
 import net.sf.basedb.core.query.Annotations;
 import net.sf.basedb.core.query.Expressions;
@@ -48 +50 @@
 import net.sf.basedb.reggie.grid.ImportFastqJobCreator;
 import net.sf.basedb.reggie.grid.ScriptUtil;
+import net.sf.basedb.util.Values;
 import net.sf.basedb.util.error.ThrowableUtil;
 
@@ -75 +78 @@
     try
     {
-      if ("GetMergedSequencesForImport".equals(cmd))
+      if ("GetDemuxedSequencesForImport".equals(cmd))
       {
         dc = sc.newDbControl();
         
-        List<MergedSequences> list = null;
+        List<DemuxedSequences> list = null;
         // Load items from the pipeline list
         ItemList fastqImportPipeline = BiomaterialList.FASTQ_IMPORT_PIPELINE.load(dc);
         ItemQuery<DerivedBioAssay> query = fastqImportPipeline.getMembers();
         query.setIncludes(Reggie.INCLUDE_IN_CURRENT_PROJECT);
-        Subtype.MERGED_SEQUENCES.addFilter(dc, query);
+        Subtype.DEMUXED_SEQUENCES.addFilter(dc, query);
         // Join LibPlate for sorting
         query.join(Hql.innerJoin(null, "extract", "lib", true));
@@ -92 +95 @@
         query.order(Orders.asc(Hql.property("name")));
         query.setMaxResults(250);
-        list = MergedSequences.toList(query.list(dc));
+        list = DemuxedSequences.toList(query.list(dc));
 
         SnapshotManager manager = new SnapshotManager();
-        JSONArray jsonMergedSequences = new JSONArray();
-        for (MergedSequences ms : list)
-        {
-          ms.loadAnnotations(dc, "pipeline", Annotationtype.PIPELINE, null);
-          ms.loadAnnotations(dc, "rawFastq", Annotationtype.RAW_FASTQ, null);
-          DemuxedSequences demux = ms.getSingleDemuxedSequecnces(dc);
-          ms.setAnnotation("rawFastqFolder", Annotationtype.DATA_FILES_FOLDER.getAnnotationValue(dc, demux.getItem()));
-          ms.setAnnotation("demuxName", demux.getName());
-          Library lib = ms.getLibrary(dc);
+        JSONArray jsonDemuxedSequences = new JSONArray();
+        for (DemuxedSequences ds : list)
+        {
+          ds.loadAnnotations(dc, "pipeline", Annotationtype.PIPELINE, null);
+          ds.loadAnnotations(dc, "rawFastq", Annotationtype.RAW_FASTQ, null);
+          ds.loadAnnotations(dc, "DataFilesFolder", Annotationtype.DATA_FILES_FOLDER, null);
+          Library lib = ds.getLibrary(dc);
           lib.loadBioPlateLocation();
           Rna r = lib.getRna(dc, true);
           if (r != null) lib.setAnnotation("specimen", r.loadYellowLabelInfo(dc, manager));
-          ms.setAnnotation("lib", lib.asJSONObject());
-          ms.loadAnnotations(dc, manager, "AutoProcess", Annotationtype.AUTO_PROCESSING, null);
-          ms.loadDoNotUseAnnotations(dc, manager);
-          jsonMergedSequences.add(ms.asJSONObject());
-        }
-        json.put("mergedSequences", jsonMergedSequences);       
+          ds.setAnnotation("lib", lib.asJSONObject());
+          ds.loadAnnotations(dc, manager, "AutoProcess", Annotationtype.AUTO_PROCESSING, null);
+          ds.loadDoNotUseAnnotations(dc, manager);
+          jsonDemuxedSequences.add(ds.asJSONObject());
+        }
+        json.put("demuxedSequences", jsonDemuxedSequences);       
       }
       else if ("GetUnconfirmedImportedSequences".equals(cmd))
@@ -220 +221 @@
 
         JSONObject jsonReq = JsonUtil.parseRequest(req);
-        JSONArray jsonMerged = (JSONArray)jsonReq.get("mergedSequences");
+        JSONArray jsonDemuxed = (JSONArray)jsonReq.get("demuxedSequences");
         
         Number mergeSoftwareId = (Number)jsonReq.get("mergeSoftware");
@@ -241 +242 @@
 
         // Load MergedSequences that should be imported
-        List<MergedSequences> mergedSequences = new ArrayList<MergedSequences>();
-        for (int mgNo = 0; mgNo < jsonMerged.size(); mgNo++)
-        {
-          JSONObject jsonMg = (JSONObject)jsonMerged.get(mgNo);
-          Number mergedId = (Number)jsonMg.get("id");
-        
-          MergedSequences ms = MergedSequences.getById(dc, mergedId.intValue());
-          mergedSequences.add(ms);
+        List<DemuxedSequences> demuxedSequences = new ArrayList<DemuxedSequences>();
+        for (int mgNo = 0; mgNo < jsonDemuxed.size(); mgNo++)
+        {
+          JSONObject jsonDx = (JSONObject)jsonDemuxed.get(mgNo);
+          Number demuxedId = (Number)jsonDx.get("id");
+        
+          DemuxedSequences ds = DemuxedSequences.getById(dc, demuxedId.intValue());
+          demuxedSequences.add(ds);
 
           // Reset AUTO_PROCESSING annotation
-          DerivedBioAssay merged = ms.getDerivedBioAssay();
-          Annotationtype.AUTO_PROCESSING.setAnnotationValue(dc, merged, null);
+          DerivedBioAssay demuxed = ds.getDerivedBioAssay();
+          Annotationtype.AUTO_PROCESSING.setAnnotationValue(dc, demuxed, null);
         }
 
@@ -263 +264 @@
         jobCreator.setMergeSoftware(mergeSoftware);
         
-        List<JobDefinition> jobDefs = jobCreator.createFastqImportJobs(dc, cluster, mergedSequences);
+        List<JobDefinition> jobDefs = jobCreator.createFastqImportJobs(dc, cluster, demuxedSequences);
         List<Job> jobs = ScriptUtil.submitJobs(dc, cluster, jobDefs);
         for (Job job : jobs)
@@ -273 +274 @@
           else
           {
-            jsonMessages.add("Submitted FASTQ import job to " + cluster.getConnectionInfo().getName() + " with id " + job.getExternalId());
+            jsonMessages.add("Submitted " + job.getName() + " to " + cluster.getConnectionInfo().getName() + " with id " + job.getExternalId());
           }
         }
@@ -291 +292 @@
         int numLegacyAlign = 0;
         int numConfirmed = 0;
+        int numUnconfirmed = 0;
         int numFlaggedRna = 0;
         int numReImport = 0;
+        int numDeleted = 0;
+        
         ItemList flaggedRna = BiomaterialList.FLAGGED_RNA.load(dc);
         ItemList hisatPipeline = BiomaterialList.HISAT_PIPELINE.load(dc);
         ItemList legacyPipeline = BiomaterialList.LEGACY_PIPELINE.load(dc);
         ItemList fastqImportPipeline = BiomaterialList.FASTQ_IMPORT_PIPELINE.load(dc);
+        
+        boolean deleteItemsCreatedByFailedJobs = Boolean.TRUE.equals(jsonReq.get("deleteItemsCreatedByFailedJobs"));
+        List<BasicItem> toDelete = new ArrayList<BasicItem>();
+        
         for (int mergeNo = 0; mergeNo < jsonMergedSequences.size(); ++mergeNo)
         {
@@ -306 +314 @@
           boolean flag = Boolean.TRUE.equals(jsonMerge.get("flag"));
           boolean align = legacyAlign || hisatAlign;
-          
-          DerivedBioAssay merge = DerivedBioAssay.getById(dc, mergeId.intValue());
-          merge.setDescription((String)jsonMerge.get("comment"));
+          boolean deleted = false;
+
+          MergedSequences mg = MergedSequences.getById(dc, mergeId.intValue());
+          DemuxedSequences dx = mg.getSingleDemuxedSequences(dc);
+          
+          DerivedBioAssay merge = mg.getDerivedBioAssay();
+          DerivedBioAssay demux = dx.getDerivedBioAssay();
+          
+          if (flag && !Reggie.isExternalItem(merge.getName()))
+          {
+            Library lib = Library.get(merge.getExtract());
+            Rna rna = lib.getRna(dc, false);
+            if (rna != null)
+            {
+              Extract r = rna.getItem();
+              Annotationtype.FLAG.setAnnotationValue(dc, r, Rna.FLAG_FASTQ_IMPORT_FAILED);
+              flaggedRna.add(r);
+              numFlaggedRna++;
+            }
+          }
+
+          String comment = Values.getStringOrNull((String)jsonMerge.get("comment"));
+          merge.setDescription(comment);
           
           if (reImport)
           {
-            // TODO -- This will not work in the general case
-            Annotationtype.ANALYSIS_RESULT.setAnnotationValue(dc, merge, null);
-            Annotationtype.AUTO_PROCESSING.setAnnotationValue(dc, merge, "ReProcess");
-            merge.setJob(null); // Must do this to reset for new import
-            fastqImportPipeline.add(merge);
+            // Set ReProcess annotation on demuxed sequences item
+            Annotationtype.AUTO_PROCESSING.setAnnotationValue(dc, demux, "ReProcess");
+            fastqImportPipeline.add(demux);
             numReImport++;
-          }
-          else if (align)
-          {
-            Annotationtype.ANALYSIS_RESULT.setAnnotationValue(dc, merge, MergedSequences.MERGE_SUCCESSFUL);
-            numConfirmed++;
-            if (hisatAlign)
+            
+            if (deleteItemsCreatedByFailedJobs && !align)
             {
-              hisatPipeline.add(merge);
-              numHisatAlign++;
+              // Delete rawbioassay and related items
+              merge.setRemoved(true);
+              toDelete.add(merge);
+              toDelete.addAll(Reggie.removeAttachedFiles(dc, merge));
+              deleted = true;
+              numDeleted++;
+              demux.setDescription(comment);
             }
-  
-            if (legacyAlign)
+          }
+          if (!deleted)
+          {
+            if (align)
             {
-              legacyPipeline.add(merge);
-              numLegacyAlign++;
+              Annotationtype.ANALYSIS_RESULT.setAnnotationValue(dc, merge, MergedSequences.MERGE_SUCCESSFUL);
+              numConfirmed++;
+              if (hisatAlign)
+              {
+                hisatPipeline.add(merge);
+                numHisatAlign++;
+              }
+    
+              if (legacyAlign)
+              {
+                legacyPipeline.add(merge);
+                numLegacyAlign++;
+              }
             }
-          }
-          
-          if (flag)
-          {
-            if (!align) 
+            else if (reImport || flag)
             {
               Annotationtype.ANALYSIS_RESULT.setAnnotationValue(dc, merge, MergedSequences.MERGE_FAILED);
-              Annotationtype.AUTO_PROCESSING.setAnnotationValue(dc, merge, "Disable");
             }
-
-            if (!Reggie.isExternalItem(merge.getName()))
+            else
             {
-              Library lib = Library.get(merge.getExtract());
-              Rna rna = lib.getRna(dc, false);
-              if (rna != null)
-              {
-                Extract r = rna.getItem();
-                Annotationtype.FLAG.setAnnotationValue(dc, r, Rna.FLAG_FASTQ_IMPORT_FAILED);
-                flaggedRna.add(r);
-                numFlaggedRna++;
-              }
+              numUnconfirmed++;
             }
           }
@@ -379 +405 @@
           ActivityDef.FASTQ_IMPORT_CONFIRMED.merge(dc, numConfirmed);
         }
+        if (numReImport > 0)
+        {
+          jsonMessages.add(numReImport + " items flagged for re-importing");
+        }
+        if (numDeleted > 0)
+        {
+          jsonMessages.add(numDeleted + " libraries deleted due to failure");
+          if (toDelete.size() > numDeleted)
+          {
+            jsonMessages.add((toDelete.size() - numDeleted) + " linked items (eg. parent items and files) deleted");
+          }
+        }
+
         if (numFlaggedRna > 0)
         {
           jsonMessages.add("Added " + numFlaggedRna + " RNA items to the '" + flaggedRna.getName() + "' list");
         }
-        
-        if (numReImport > 0)
-        {
-          jsonMessages.add(numReImport + " items flagged for re-importing");
-        }
-        
-        // dc.commit();
+        if (numUnconfirmed > 0)
+        {
+          jsonMessages.add(numUnconfirmed + " libraries remain unconfirmed");
+        }
+        dc.commit();
+        if (toDelete.size() > 0)
+        {
+          try
+          {
+            Trashcan.delete(sc, toDelete, false, null);
+          }
+          catch (RuntimeException ex)
+          {
+            ex.printStackTrace();
+            jsonMessages.add("[Error]Could not delete all items created by failed jobs: " + ex.getMessage());
+          }
+        }
       }
 

extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/servlet/FutureSpecimenServlet.java

@@ -36 +36 @@
 import net.sf.basedb.reggie.dao.Annotationtype;
 import net.sf.basedb.reggie.dao.BiomaterialList;
+import net.sf.basedb.reggie.dao.DemuxedSequences;
 import net.sf.basedb.reggie.dao.Fileserver;
 import net.sf.basedb.reggie.dao.FutureSpecimen;
-import net.sf.basedb.reggie.dao.MergedSequences;
 import net.sf.basedb.reggie.dao.ReggieRole;
 import net.sf.basedb.reggie.dao.SpecimenTube;
@@ -200 +200 @@
 
         OpenGridCluster cluster = null;
-        ImportFastqJobCreator jobCreator = null;
         if (jsonAutoAnalyze != null)
         {
-          boolean debug = Boolean.TRUE.equals(jsonAutoAnalyze.get("debug"));
-          boolean autoConfirm = Boolean.TRUE.equals(jsonAutoAnalyze.get("autoConfirm"));
-          Number priority = (Number)jsonAutoAnalyze.get("priority");
           String clusterId = (String)jsonAutoAnalyze.get("cluster");
-          Number mergeSoftwareId = (Number)jsonAutoAnalyze.get("mergeSoftware");
-          Number mergeProtocolId = (Number)jsonAutoAnalyze.get("mergeProtocol");
-
           cluster = OpenGridService.getInstance().getClusterById(dc, clusterId);
           if (cluster == null)
@@ -215 +208 @@
             throw new ItemNotFoundException("OpenGridCluster[" + clusterId + "]");
           }
-          jobCreator = new ImportFastqJobCreator();
-          jobCreator.setAutoConfirm(autoConfirm);
-          jobCreator.setDebug(debug);
-          if (priority != null) jobCreator.setPriority(priority.intValue());
-          jobCreator.setMergeProtocol(mergeProtocolId != null ? Protocol.getById(dc, mergeProtocolId.intValue()) : null);
-          jobCreator.setMergeSoftware(mergeSoftwareId != null ? Software.getById(dc, mergeSoftwareId.intValue()) : null);
         }
 
@@ -234 +221 @@
           gatewaySession = new RemoteHost(new ConnectionInfo(importGateway)).connect(5);
           archiveSession = new RemoteHost(new ConnectionInfo(importArchive)).connect(5);
-          List<MergedSequences> mergedSequences = new ArrayList<>(); // Collect MergedSequences that has been created by the importer
+          List<DemuxedSequences> demuxedSequences = new ArrayList<>(); // Collect DemuxedSequences that has been created by the importer
           for (int itemNo = 0; itemNo < jsonItems.size(); itemNo++)
           {
@@ -263 +250 @@
             {
               // Add to FASTQ import pipeline
-              BiomaterialList.FASTQ_IMPORT_PIPELINE.get(dc).add(items.merged.getItem());
+              BiomaterialList.FASTQ_IMPORT_PIPELINE.get(dc).add(items.demux.getItem());
               // Save the JSON to Fileserver.IMPORT_ARCHIVE
               String dataFilesFolder = (String)Annotationtype.DATA_FILES_FOLDER.getAnnotationValue(dc, items.demux.getItem());
@@ -269 +256 @@
               dc.commit();
               jsonMessages.add("Imported "+specimen.getName()+" with data from "+jsonFile.getName());
-              mergedSequences.add(items.merged);
+              demuxedSequences.add(items.demux);
               // Remove JSON from Filesever.IMPORT_GATEWAY
               jsonFile.deleteFrom(gatewaySession, gatewayRoot);
@@ -279 +266 @@
             
             jsonMessages.addAll(prefix("[Warning]["+jsonFile.getName()+"] ", jsonFile.getWarningMessages()));
-            //jsonMessages.addAll(prefix("[Debug]["+jsonFile.getName()+"] ", jsonFile.getDebugMessages()));
-          }
-          
-          if (jobCreator != null)
-          {
+            jsonMessages.addAll(prefix("[Debug]["+jsonFile.getName()+"] ", jsonFile.getDebugMessages()));
+          }
+
+          if (jsonAutoAnalyze != null)
+          {
+            boolean debug = Boolean.TRUE.equals(jsonAutoAnalyze.get("debug"));
+            boolean autoConfirm = Boolean.TRUE.equals(jsonAutoAnalyze.get("autoConfirm"));
+            Number priority = (Number)jsonAutoAnalyze.get("priority");
+            String clusterId = (String)jsonAutoAnalyze.get("cluster");
+            Number mergeSoftwareId = (Number)jsonAutoAnalyze.get("mergeSoftware");
+            Number mergeProtocolId = (Number)jsonAutoAnalyze.get("mergeProtocol");
+            
             dc = sc.newDbControl();
-            List<JobDefinition> jobDefs = jobCreator.createFastqImportJobs(dc, cluster, mergedSequences);
-            List<Job> jobs = ScriptUtil.submitJobs(dc, cluster, jobDefs);
-            for (Job job : jobs)
-            {
-              if (job.getStatus() == Job.Status.ERROR)
+            try
+            {
+              ImportFastqJobCreator jobCreator = new ImportFastqJobCreator();
+              jobCreator.setAutoConfirm(autoConfirm);
+              jobCreator.setDebug(debug);
+              if (priority != null) jobCreator.setPriority(priority.intValue());
+              jobCreator.setMergeProtocol(mergeProtocolId != null ? Protocol.getById(dc, mergeProtocolId.intValue()) : null);
+              jobCreator.setMergeSoftware(mergeSoftwareId != null ? Software.getById(dc, mergeSoftwareId.intValue()) : null);
+              
+              List<JobDefinition> jobDefs = jobCreator.createFastqImportJobs(dc, cluster, demuxedSequences);
+              List<Job> jobs = ScriptUtil.submitJobs(dc, cluster, jobDefs);
+              for (Job job : jobs)
               {
-                jsonMessages.add("[Warning]Job submission for '" + job.getName() + "' failed: " + job.getStatusMessage());
+                if (job.getStatus() == Job.Status.ERROR)
+                {
+                  jsonMessages.add("[Warning]Job submission for '" + job.getName() + "' failed: " + job.getStatusMessage());
+                }
+                else
+                {
+                  jsonMessages.add("Submitted "+job.getName()+" to "+cluster.getConnectionInfo().getName()+" with id "+job.getExternalId());
+                }
               }
-              else
-              {
-                jsonMessages.add("Submitted "+job.getName()+" to "+cluster.getConnectionInfo().getName()+" with id "+job.getExternalId());
-              }
-            }
-            dc.commit();
-          }
-          
+              dc.commit();
+            }
+            catch (Exception ex)
+            {
+              jsonMessages.add("[Warning]Job submission for FASTQ import failed: "+ ex.getMessage());
+            }
+          }
         }
         finally

extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/servlet/InstallServlet.java

@@ -1025 +1025 @@
             Annotationtype.READ_STRING, Annotationtype.OMIT_LANES, 
             Annotationtype.ANALYSIS_RESULT, Annotationtype.AUTO_PROCESSING,
-            Annotationtype.DATA_FILES_FOLDER,
+            Annotationtype.DATA_FILES_FOLDER, Annotationtype.RAW_FASTQ,
             Annotationtype.READS, Annotationtype.PF_READS, Annotationtype.PF_NNNN_PCT, 
             Annotationtype.PF_UNUSED_PCT, Annotationtype.DEMUX_WARNINGS, Annotationtype.SKIPPED_TILES,
@@ -1035 +1035 @@
             Annotationtype.READS, Annotationtype.PF_READS, Annotationtype.ADAPTER_READS, Annotationtype.PT_READS,
             Annotationtype.FRAGMENT_SIZE_AVG, Annotationtype.FRAGMENT_SIZE_STDEV,
-            Annotationtype.DATA_FILES_FOLDER, Annotationtype.RAW_FASTQ,
+            Annotationtype.DATA_FILES_FOLDER, 
             Annotationtype.DO_NOT_USE, Annotationtype.DO_NOT_USE_COMMENT,
             Annotationtype.AUTO_PROCESSING,