Changeset 6215


Ignore:
Timestamp:
Apr 16, 2021, 10:16:20 AM (13 months ago)
Author:
Nicklas Nordborg
Message:

References #1295: Registration of specimen handled by external lab

Re-designed the importer to stop at DemuxedSequnces instead of MergedSequences.

There will be one DemuxedSequences for each library instead of one for each flow cell. The DemuxedSequences item will be a named after the library an use 'x' suffix. The suffix is NOT propagated to child MergedSequences which uses 'g' as before. For example:

Lib: 1234.r.lib
Demux: 1234.r.lib.x
Merge: 1234.r.lib.g

The DemuxedSequences will also have information about the original FASTQ files that we recieved from the external lab.

The MergedSequences item is now created by the FASTQ import script. This change makes it easier to redo failed FASTQ import since we can just delete the MergedSequences and try again.

Location:
extensions/net.sf.basedb.reggie/trunk
Files:
1 deleted
18 edited

Legend:

Unmodified
Added
Removed
  • extensions/net.sf.basedb.reggie/trunk/resources/analysis/import_confirm.js

    r6188 r6215  
    209209    var frm = document.forms['reggie'];
    210210    var submitInfo = {};
     211    submitInfo.deleteItemsCreatedByFailedJobs = frm.deleteItemsCreatedByFailedJobs.checked;
    211212   
    212213    var all = [];
  • extensions/net.sf.basedb.reggie/trunk/resources/analysis/import_confirm.jsp

    r6188 r6215  
    5353}
    5454
    55 #mergedSequencesTable tbody td
     55#mergedSequencesTable td, #mergedSequencesTable th
    5656{
    57   padding-top: 1px;
    58   padding-bottom: 1px;
     57  padding: 2px 1px;
    5958}
    6059
     
    6463}
    6564
    66 #mergedSequencesTable .icon-col
     65#mergedSequencesTable .warning-col
    6766{
    6867  width: 20px;
     
    110109        </td>
    111110      </tr>
     111      <tr>
     112        <td class="prompt"></td>
     113        <td class="input" style="text-align: right; vertical-align: middle;">
     114          <label>Delete items selected for re-import
     115          <input type="checkbox" name="deleteItemsCreatedByFailedJobs" id="deleteItemsCreatedByFailedJobs" checked>
     116          </label>
     117        </td>
     118        <td class="status" id="deleteItemsCreatedByFailedJobs.status"></td>
     119        <td class="help">
     120          <span id="deleteItemsCreatedByFailedJobs.message" class="message"></span>
     121          Recommended! Items created by jobs selected for re-import are deleted to avoid cluttering in the database.
     122        </td>
     123      </tr>
    112124      </table>
    113125    </div>
  • extensions/net.sf.basedb.reggie/trunk/resources/analysis/import_fastq.js

    r6214 r6215  
    1111   
    1212    // Step 1
    13     Events.addEventHandler('mergedSequences', 'change', fastq.mergedSequencesOnChange);
     13    Events.addEventHandler('demuxedSequences', 'change', fastq.demuxedSequencesOnChange);
    1414    Events.addEventHandler('step-1', 'wizard-validate', fastq.validateStep1);
    1515
     
    3131    Wizard.showLoadingAnimation('Loading bioassays...');
    3232    var url = '../Fastq.servlet?ID='+App.getSessionId();
    33     url += '&cmd=GetMergedSequencesForImport';
     33    url += '&cmd=GetDemuxedSequencesForImport';
    3434    Wizard.asyncJsonRequest(url, fastq.initializeStep1);
    3535  }
     
    3838  fastq.initializeStep1 = function(response)
    3939  {
    40     var mergedSequences = response.mergedSequences;
     40    var demuxedSequences = response.demuxedSequences;
    4141    var frm = document.forms['reggie'];
    42     if (mergedSequences.length == 0)
    43     {
    44       Wizard.setFatalError('No merged sequences available for importing.');
     42    if (demuxedSequences.length == 0)
     43    {
     44      Wizard.setFatalError('No demuxed sequences available for importing.');
    4545      return;
    4646    }
    4747
    4848    var firstPlate = null;
    49     for (var msNo=0; msNo < mergedSequences.length; msNo++)
    50     {
    51       var ms = mergedSequences[msNo];
     49    for (var msNo=0; msNo < demuxedSequences.length; msNo++)
     50    {
     51      var ms = demuxedSequences[msNo];
    5252      var libPlate = ms.lib.bioWell ? ms.lib.bioWell.bioPlate : null;
    5353      if (firstPlate == null && libPlate != null) firstPlate = libPlate.name;
    5454      var option = fastq.createListOption(msNo+1, ms, ms.DO_NOT_USE == null && libPlate != null && libPlate.name==firstPlate)
    55       frm.mergedSequences.options[frm.mergedSequences.length] = option;
    56     }
    57     Events.sendChangeEvent('mergedSequences');
     55      frm.demuxedSequences.options[frm.demuxedSequences.length] = option;
     56    }
     57    Events.sendChangeEvent('demuxedSequences');
    5858   
    5959    Doc.show('step-1');
    6060    Doc.show('gonext');
    6161
    62     frm.mergedSequences.focus();
     62    frm.demuxedSequences.focus();
    6363  }
    6464 
     
    6868  }
    6969 
    70   fastq.createListOption = function(index, mergedItem, selected)
    71   {
    72     var lib = mergedItem.lib;
     70  fastq.createListOption = function(index, demuxItem, selected)
     71  {
     72    var lib = demuxItem.lib;
    7373    var isYellow = lib.specimen && lib.specimen.YellowLabel != null;
    7474    var libPlate = lib.bioWell ? lib.bioWell.bioPlate : null;
     
    7777    var name = (index) + ': ';
    7878    if (libPlate) name += libPlate.name + ' - ';
    79     name += mergedItem.name;
    80    
    81     if (mergedItem.DO_NOT_USE)
     79    name += demuxItem.name;
     80   
     81    if (demuxItem.DO_NOT_USE)
    8282    {
    8383      name += ' [DoNotUse]';
    84       tooltip = 'DoNotUse-'+Strings.encodeTags(mergedItem.DO_NOT_USE+': '+mergedItem.DO_NOT_USE_COMMENT);
    85     }
    86     else if (mergedItem.AutoProcess == 'ReProcess')
     84      tooltip = 'DoNotUse-'+Strings.encodeTags(demuxItem.DO_NOT_USE+': '+demuxItem.DO_NOT_USE_COMMENT);
     85    }
     86    else if (demuxItem.AutoProcess == 'ReProcess')
    8787    {
    8888      name += ' [R]';
    8989    }
    9090   
    91     var option = new Option(name, mergedItem.id, false, selected);
     91    var option = new Option(name, demuxItem.id, false, selected);
    9292    if (isYellow) option.className = 'yellow';
    9393    if (tooltip) option.title = tooltip;
    94     option.mergedSequences = mergedItem;
     94    option.demuxedSequences = demuxItem;
    9595    return option;
    9696  }
    9797 
    98   fastq.mergedSequencesOnChange = function()
     98  fastq.demuxedSequencesOnChange = function()
    9999  {
    100100    var frm = document.forms['reggie'];
     
    104104    var invalidSelection = null;
    105105   
    106     Wizard.setInputStatus('mergedSequences');
     106    Wizard.setInputStatus('demuxedSequences');
    107107    Wizard.hideGoNextConfirmation();
    108     for (var msNo = 0; msNo < frm.mergedSequences.length; msNo++)
    109     {
    110       if (frm.mergedSequences[msNo].selected)
     108    for (var msNo = 0; msNo < frm.demuxedSequences.length; msNo++)
     109    {
     110      if (frm.demuxedSequences[msNo].selected)
    111111      {
    112112        numSelected++;
    113         var ms = frm.mergedSequences[msNo].mergedSequences;
     113        var ms = frm.demuxedSequences[msNo].demuxedSequences;
    114114        if (ms.DO_NOT_USE) numDoNotUse++;
    115115        if (ms.pipeline && ms.pipeline != 'RNAseq')
     
    121121          invalidSelection = Strings.encodeTags(ms.name+' is missing RawFASTQ information');
    122122        }
    123         else if (!ms.rawFastqFolder)
     123        else if (!ms.DataFilesFolder)
    124124        {
    125           invalidSelection = Strings.encodeTags(ms.name+' demux ('+ms.demuxName+') is missing DataFilesFolder information');
     125          invalidSelection = Strings.encodeTags(ms.name+' is missing DataFilesFolder information');
    126126        }
    127127      }
     
    130130    if (numSelected == 0)
    131131    {
    132       Wizard.setInputStatus('mergedSequences', 'invalid', 'Select at least one item');
     132      Wizard.setInputStatus('demuxedSequences', 'invalid', 'Select at least one item');
    133133      return;
    134134    }
    135135    if (invalidSelection)
    136136    {
    137       Wizard.setInputStatus('mergedSequences', 'invalid', invalidSelection);
     137      Wizard.setInputStatus('demuxedSequences', 'invalid', invalidSelection);
    138138      return;
    139139    }
     
    142142    if (numDoNotUse > 0)
    143143    {
    144       Wizard.setInputStatus('mergedSequences', 'warning', numDoNotUse + ' selected items marked as DoNotUse');
     144      Wizard.setInputStatus('demuxedSequences', 'warning', numDoNotUse + ' selected items marked as DoNotUse');
    145145      Wizard.showGoNextConfirmation(true, 'Confirm ' + numDoNotUse + ' items marked as DoNotUse');
    146146    }
    147147    else
    148148    {
    149       Wizard.setInputStatus('mergedSequences', 'valid');
     149      Wizard.setInputStatus('demuxedSequences', 'valid');
    150150    }
    151151  }
     
    270270    submitInfo.autoConfirm = frm.autoConfirm.checked;
    271271   
    272     var mergedSequences = [];
    273     submitInfo.mergedSequences = mergedSequences;
     272    var demuxedSequences = [];
     273    submitInfo.demuxedSequences = demuxedSequences;
    274274     
    275     for (var msNo = 0; msNo < frm.mergedSequences.length; msNo++)
    276     {
    277       if (frm.mergedSequences[msNo].selected)
     275    for (var msNo = 0; msNo < frm.demuxedSequences.length; msNo++)
     276    {
     277      if (frm.demuxedSequences[msNo].selected)
    278278      {
    279279        var ms = {};
    280         ms.id = frm.mergedSequences[msNo].mergedSequences.id;
    281         mergedSequences[mergedSequences.length] = ms;
     280        ms.id = frm.demuxedSequences[msNo].demuxedSequences.id;
     281        demuxedSequences[demuxedSequences.length] = ms;
    282282      }
    283283    }
  • extensions/net.sf.basedb.reggie/trunk/resources/analysis/import_fastq.jsp

    r6180 r6215  
    4040  <div class="step auto-hide" id="step-1">
    4141    <div class="step-no">1</div>
    42     <div class="step-title">Select merged sequences</div>
     42    <div class="step-title">Select demuxed sequences</div>
    4343    <div class="step-content">
    4444   
    4545      <table class="step-form">
    4646      <tr class="align-top">
    47         <td class="prompt">Merged sequences</td>
     47        <td class="prompt">Demuxed sequences</td>
    4848        <td class="input">
    49           <select class="required yellow-label-support" multiple size="30"
    50             name="mergedSequences" id="mergedSequences"></select>
     49          <select class="required yellow-label-support" multiple size="20"
     50            name="demuxedSequences" id="demuxedSequences"></select>
    5151        </td>
    52         <td class="status" id="mergedSequences.status"></td>
    53         <td class="help"><span id="mergedSequences.message" class="message"></span>
    54           Select the sequences that should be imported. The list contains merged sequences
     52        <td class="status" id="demuxedSequences.status"></td>
     53        <td class="help"><span id="demuxedSequences.message" class="message"></span>
     54          Select the sequences that should be imported. The list contains demuxed sequences
    5555          from the <b>FASTQ import pipeline</b> item list (max 250).<br><br>
    5656          Items marked with <b>[R]</b> have been marked for re-import due to a failure of
  • extensions/net.sf.basedb.reggie/trunk/resources/index.jsp

    r6187 r6215  
    760760            <li><span class="require-permission" data-role="SecondaryAnalysis" data-link="analysis/import_fastq.jsp?ID=<%=ID%>"
    761761              >Import FASTQ files</span>
    762               <span class="counter" data-counter="merged-sequences-for-fastq-import"
    763                 title="Number of merged sequences waiting for FASTQ import">∙</span>
     762              <span class="counter" data-counter="demuxed-sequences-for-fastq-import"
     763                title="Number of demuxed sequences waiting for FASTQ import">∙</span>
    764764                <span id="fastq-import-pipeline-disabled" class="info-warning">auto-processing is disabled</span>
    765765
  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/counter/CounterService.java

    r6209 r6215  
    11711171    // External merged sequences waiting for FASTQ import
    11721172    ItemList fastqImportPipeline = BiomaterialList.FASTQ_IMPORT_PIPELINE.load(dc);
    1173     json.put("merged-sequences-for-fastq-import", fastqImportPipeline.getSize());
     1173    json.put("demuxed-sequences-for-fastq-import", fastqImportPipeline.getSize());
    11741174   
    11751175    // Merged sequences with FASTQ import not confirmed
  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/dao/Annotationtype.java

    r6213 r6215  
    13781378
    13791379  /**
    1380     The "RawFASTQ" annotation, used for (merged) derived bioassays that
     1380    The "RawFASTQ" annotation, used for (demuxed) derived bioassays that
    13811381    use external FASTQ files for import. This annotation should typically
    13821382    have 2 values (one for R1 and one for R2).
  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/dao/BiomaterialList.java

    r6179 r6215  
    124124  public static final BiomaterialList FASTQ_IMPORT_PIPELINE =
    125125    new BiomaterialList("FASTQ Import Pipeline", "net.sf.basedb.reggie.pipeline.fastq-import",
    126       Item.DERIVEDBIOASSAY, Subtype.MERGED_SEQUENCES,
    127       "This list contains merged sequences items that need FASTQ files to be imported " +
     126      Item.DERIVEDBIOASSAY, Subtype.DEMUXED_SEQUENCES,
     127      "This list contains demuxed sequences items that need FASTQ files to be imported " +
    128128      "before downstream analysis can take place. Items are normally added and removed from this list " +
    129129      "automatically, but it is possible to also manually add items.");
  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/dao/DemuxedSequences.java

    r6193 r6215  
    191191  }
    192192
     193  /**
     194    Get the library related to this demux. NOTE! This is only
     195    available for externally sequenced where we get raw FASTQ files
     196    from the demux.
     197    @since 4.32
     198  */
     199  public Library getLibrary(DbControl dc)
     200  {
     201    return getItem().getExtract() == null ? null : Library.getById(dc, getItem().getExtract().getId());
     202  }
    193203
    194204}
  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/dao/Library.java

    r6193 r6215  
    318318  }
    319319
     320  /**
     321    Find the next name to give a "DemuxedSequences" child item. This assumes that
     322    all child items are using the naming convention. foo.x, foo.x2, and
     323    so on. NOTE! The first child item have no number!
     324    @return The next unused name
     325  */
     326  public String getNextDemuxedSequencesName(DbControl dc)
     327  {
     328    Extract lib = getItem();
     329    ItemQuery<DerivedBioAssay> query = null;
     330    if (lib.isInDatabase())
     331    {
     332      query = DerivedBioAssay.getQuery();
     333      Subtype.DEMUXED_SEQUENCES.addFilter(dc, query);
     334      query.setIncludes(Include.ALL);
     335      query.restrict(Restrictions.eq(Hql.property("extract"), Hql.entity(lib)));
     336    }
     337    return getNextChildItemName(dc, query, Subtype.DEMUXED_SEQUENCES.getItemSuffix(), true);
     338  }
     339
     340 
    320341}
  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/dao/MergedSequences.java

    r6214 r6215  
    227227      throw new InvalidDataException(
    228228        "No parent item was found for " + getName() +
    229         "This wizard can't be used until that is corrected.");
     229        ". This wizard can't be used until that is corrected.");
    230230    }
    231231   
     
    233233  }
    234234
    235   public DemuxedSequences getSingleDemuxedSequecnces(DbControl dc)
     235  public DemuxedSequences getSingleDemuxedSequences(DbControl dc)
    236236  {
    237237    ItemQuery<DerivedBioAssay> query = getItem().getParents();
     
    244244      throw new InvalidDataException(
    245245        "No demux item was found for " + getName() +
    246         "This wizard can't be used until that is corrected.");
     246        ". This wizard can't be used until that is corrected.");
    247247    }
    248248    else if (tmp.size() > 1)
     
    250250      throw new InvalidDataException(
    251251        "More than one demux was found for " + getName() +
    252         "This wizard can't be used until that is corrected.");
     252        ". This wizard can't be used until that is corrected.");
    253253    }
    254254    return DemuxedSequences.get(tmp.get(0));
  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/grid/ImportFastqJobCreator.java

    r6214 r6215  
    2222import net.sf.basedb.core.Job;
    2323import net.sf.basedb.core.Path;
    24 import net.sf.basedb.core.PermissionDeniedException;
    2524import net.sf.basedb.core.Protocol;
    2625import net.sf.basedb.core.SessionControl;
     
    4140import net.sf.basedb.reggie.dao.Datafiletype;
    4241import net.sf.basedb.reggie.dao.DemuxedSequences;
     42import net.sf.basedb.reggie.dao.DoNotUse;
    4343import net.sf.basedb.reggie.dao.Fileserver;
    4444import net.sf.basedb.reggie.dao.Library;
    4545import net.sf.basedb.reggie.dao.MergedSequences;
    4646import net.sf.basedb.reggie.dao.Pipeline;
    47 import net.sf.basedb.reggie.dao.SequencingRun;
    4847import net.sf.basedb.reggie.dao.Subtype;
    4948import net.sf.basedb.util.Values;
     
    119118 
    120119  /**
    121     Create a child bioassays for all given merged sequences and schedule
    122     jobs on the given cluster for running Hisat alignment.
     120    Create a child bioassays for all given demuxed sequences and schedule
     121    jobs on the given cluster for importing FASTQ files.
    123122    @return A list with the corresponding jobs in BASE
    124123  */
    125124  @SuppressWarnings("unchecked")
    126   public List<JobDefinition> createFastqImportJobs(DbControl dc, OpenGridCluster cluster, List<MergedSequences> mergedSequences)
     125  public List<JobDefinition> createFastqImportJobs(DbControl dc, OpenGridCluster cluster, List<DemuxedSequences> demuxedSequences)
    127126  {
    128127    SessionControl sc = dc.getSessionControl();
     
    160159    // Load common items
    161160    ItemSubtype fastqImportJobType = Subtype.FASTQ_IMPORT_JOB.get(dc);
     161    ItemSubtype mergedType = Subtype.MERGED_SEQUENCES.get(dc);
    162162    DataFileType fastqData = Datafiletype.FASTQ.load(dc);
    163163   
     
    173173   
    174174    // We submit one job for each item to the cluster
    175     List<JobDefinition> jobDefs = new ArrayList<JobDefinition>(mergedSequences.size());
    176 
    177     for (MergedSequences ms : mergedSequences)
     175    List<JobDefinition> jobDefs = new ArrayList<JobDefinition>(demuxedSequences.size());
     176
     177    for (DemuxedSequences ds : demuxedSequences)
    178178    {
    179       ms = MergedSequences.getById(dc, ms.getId()); // Ensure item is loaded in this transaction
    180       DerivedBioAssay merged = ms.getDerivedBioAssay();
    181       String mergeName = ScriptUtil.checkValidFilename(merged.getName());
    182      
    183       List<String> rawFastqNames = (List<String>)Annotationtype.RAW_FASTQ.getAnnotationValues(dc, merged);
     179      ds = DemuxedSequences.getById(dc, ds.getId()); // Ensure item is loaded in this transaction
     180      DerivedBioAssay demuxed = ds.getDerivedBioAssay();
     181      String demuxName = ScriptUtil.checkValidFilename(demuxed.getName());
     182     
     183      List<String> rawFastqNames = (List<String>)Annotationtype.RAW_FASTQ.getAnnotationValues(dc, demuxed);
    184184      if (rawFastqNames == null || rawFastqNames.size() != 2)
    185185      {
    186186        throw new InvalidDataException("Annotation RawFASTQ on " +
    187             mergeName + " must have two values: " + rawFastqNames);
     187            demuxName + " must have two values: " + rawFastqNames);
    188188      }
    189189     
    190190      // Get SequencingRun so that we can get the path to the FASTQ folder.
    191       DemuxedSequences demux = ms.getSingleDemuxedSequecnces(dc);
    192       String rawFastqFolder = ScriptUtil.checkValidPath((String)Annotationtype.DATA_FILES_FOLDER.getAnnotationValue(dc, demux.getItem()), true, true);
    193      
    194       if (merged.hasFileSet() && merged.getFileSet().hasMember(fastqData))
    195       {
    196         throw new PermissionDeniedException("FASTQ files already exists: " + mergeName);
    197       }
    198       importPipeline.removeItem(merged);
    199       Library lib = Library.get(merged.getExtract());
    200       boolean isExternal = Reggie.isExternalItem(mergeName);
     191      String rawFastqFolder = ScriptUtil.checkValidPath((String)Annotationtype.DATA_FILES_FOLDER.getAnnotationValue(dc, demuxed), true, true);
     192     
     193      importPipeline.removeItem(demuxed);
     194      Library lib = Library.get(demuxed.getExtract());
     195      boolean isExternal = Reggie.isExternalItem(demuxName);
    201196      String rootName = isExternal ? lib.getTopExtractOrSample(dc).getName() : null;
    202197     
    203198      // Create job
     199      String mergedName = lib.getNextMergedSequencesName(dc);
    204200      Job importJob = Job.getNew(dc, null, null, null);
    205201      importJob.setItemSubtype(fastqImportJobType);
    206202      importJob.setPluginVersion("reggie-"+Reggie.VERSION);
    207203      importJob.setSendMessage(Values.getBoolean(sc.getUserClientSetting("plugins.sendmessage"), false));
    208       importJob.setName("FASTQ Import to " + mergeName);
     204      importJob.setName("FASTQ Import to " + mergedName);
    209205      importJob.setParameterValue("pipeline", new StringParameterType(), Pipeline.RNA_SEQ.getId());
    210206      if (debug) importJob.setName(importJob.getName() + " (debug)");
    211207      dc.saveItem(importJob);
    212 
    213       // Update the MergedSequence item with information about this import
     208     
     209      // Created MERGED derived bioassay set
     210      DerivedBioAssay merged = DerivedBioAssay.getNew(dc, demuxed, importJob);
     211      merged.setItemSubtype(mergedType);
     212      Pipeline.RNA_SEQ.setAnnotation(dc, merged);
     213      merged.setName(mergedName);
     214      merged.setExtract(lib.getExtract());
    214215      merged.setSoftware(mergeSoftware);
    215216      merged.setProtocol(mergeProtocol);
    216       merged.setJob(importJob);
    217      
    218       String dataFilesFolder = ScriptUtil.checkValidPath(MergedSequences.generateDataFilesFolderForProjectArchive(mergeName, rootName, debug), true, true);
     217      DoNotUse.copyDoNotUseAnnotations(dc, lib.getExtract(), merged, false);
     218      // Copy READS
     219      Annotationtype.READS.copyAnnotationValues(dc, demuxed, merged, false);
     220      Annotationtype.PF_READS.copyAnnotationValues(dc, demuxed, merged, false);
     221      dc.saveItem(merged);
     222     
     223      String dataFilesFolder = ScriptUtil.checkValidPath(MergedSequences.generateDataFilesFolderForProjectArchive(mergedName, rootName, debug), true, true);
    219224      Annotationtype.DATA_FILES_FOLDER.setAnnotationValue(dc, merged, dataFilesFolder);
    220225      if (autoConfirm)
     
    225230      String archiveRoot =  isExternal ? externalRoot : projectRoot;
    226231      String fastqFolder = archiveRoot + dataFilesFolder;
    227       String baseFileName = isExternal ? Reggie.removePrefix(mergeName) : mergeName;
     232      String baseFileName = isExternal ? Reggie.removePrefix(mergedName) : mergedName;
    228233      String R1_name = baseFileName+"_R1.fastq";
    229234      String R2_name = baseFileName+"_R2.fastq";
     
    247252 
    248253      // Set file permissions based on consent or external group!
    249       String externalGroup = isExternal ? Reggie.getExternalGroup(mergeName) : null;
     254      String externalGroup = isExternal ? Reggie.getExternalGroup(mergedName) : null;
    250255      ScriptUtil.setUmaskForItem(dc, lib, externalGroup, script);
    251256
     
    294299
    295300      script.comment("Run Bowtie2");
    296       script.progress(40, "Bowtie2: " + mergeName + " (${NumThreads} threads)");
     301      script.progress(40, "Bowtie2: " + mergedName + " (${NumThreads} threads)");
    297302      String alignCmd = "./stdwrap.sh " + bowtie_path;
    298303      alignCmd += " -p ${NumThreads}";
     
    306311      script.cmd(alignCmd);
    307312      script.cmd("./singlecolumnaverager.awk < fastq.aligned/aligned.sam > fastq.aligned/" + fragments_name);
    308       script.time("DONE: Bowtie2 " + mergeName);
    309       script.newLine();
    310      
    311       script.progress(60, "Trimmomatic: " + mergeName + " (${NumThreads} threads)");
     313      script.time("DONE: Bowtie2 " + mergedName);
     314      script.newLine();
     315     
     316      script.progress(60, "Trimmomatic: " + mergedName + " (${NumThreads} threads)");
    312317      // First Trimmomatic step should filter on Illumina adapters only
    313318      String trimCmd1 = "./stdwrap.sh ./trimmomatic PE";
     
    340345      trimCmd2 += " >> ${WD}/trimmomatic.out";
    341346      script.cmd(trimCmd1);
    342       script.time("DONE: Trimmomatic 1 " + mergeName);
     347      script.time("DONE: Trimmomatic 1 " + mergedName);
    343348
    344349      script.cmd(trimCmd2);
    345       script.time("DONE: Trimmomatic 2 " + mergeName);
    346       script.newLine();
    347 
    348       script.progress(80, "Archiving FASTQ: " + mergeName);
     350      script.time("DONE: Trimmomatic 2 " + mergedName);
     351      script.newLine();
     352
     353      script.progress(80, "Archiving FASTQ: " + mergedName);
    349354      script.cmd("mkdir -p " + fastqFolder);
    350355      script.cmd("rm -f " + fastqFolder + "/*");
     
    357362      if (externalGroup != null)
    358363      {
    359         ScriptUtil.addChgrp(externalGroup, fastqFolder, mergeName, archiveRoot+"/"+Reggie.getPrefix(mergeName), script);
     364        ScriptUtil.addChgrp(externalGroup, fastqFolder, mergedName, archiveRoot+"/"+Reggie.getPrefix(mergedName), script);
    360365      }
    361366
     
    407412      msg += Values.formatNumber(total.passedFilter/1000000f, 1) + "M passed filter; ";
    408413      msg += Values.formatNumber(total.passedTrimmomatic[1]/1000000f, 1) + "M passed trimmomatic; ";
    409 /*      if (total.warnings.size() > 0)
    410       {
    411         msg += total.warnings.size() + " warnings!";
    412       }
    413 */      return msg;
     414      return msg;
    414415    }
    415416   
  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/plugins/FutureSpecimenImporter.java

    r6214 r6215  
    2525import net.sf.basedb.reggie.dao.Library;
    2626import net.sf.basedb.reggie.dao.Lysate;
    27 import net.sf.basedb.reggie.dao.MergedSequences;
    2827import net.sf.basedb.reggie.dao.Pipeline;
    2928import net.sf.basedb.reggie.dao.PooledLibrary;
     
    4342import net.sf.basedb.reggie.plugins.cmd.LibraryInfo;
    4443import net.sf.basedb.reggie.plugins.cmd.LysateInfo;
    45 import net.sf.basedb.reggie.plugins.cmd.MergeInfo;
    4644import net.sf.basedb.reggie.plugins.cmd.PoolInfo;
    4745import net.sf.basedb.reggie.plugins.cmd.RnaInfo;
     
    5149/**
    5250  Import data to FutureSpecimen items and create all downstream items
    53   needed (down to MergedSequences) before FASTQ files can be imported.
     51  needed (down to DemuxedSequences) before FASTQ files can be imported.
    5452 
    5553  It is also possible to start the import on a Specimen item or an
    5654  RNA item.
    57  
    58   The implementation is currently a mockup which only create more or
    59   less empty items that most likely will cause other wizards to fail.
    6055 
    6156  @author nicklas
     
    109104    SequencingRunInfo seqRunInfo = jsonFile.getSequencingRun();
    110105    DemuxInfo demuxInfo = jsonFile.getDemuxInfo();
    111     MergeInfo mergeInfo = jsonFile.getMergeInfo();
    112106    FastqInfo fastqInfo = jsonFile.getFastqInfo();
    113107
     
    140134   
    141135    if (items.sequencingRun == null || !demuxInfo.valid) return null;
    142     items.demux = getOrCreateDemux(dc, items.sequencingRun, flowCellInfo, seqRunInfo, demuxInfo);
    143    
    144     if (items.demux == null || !mergeInfo.valid || !fastqInfo.valid) return null;
    145     items.merged = createMergedSequences(dc, items.lib, items.demux, mergeInfo, fastqInfo);
    146    
    147     return items.merged == null || jsonFile.hasError() ? null : items;
     136    items.demux = createDemuxedSequences(dc, items.lib, items.sequencingRun, flowCellInfo, seqRunInfo, demuxInfo, fastqInfo);
     137   
     138    return items.demux == null || jsonFile.hasError() ? null : items;
    148139  }
    149140 
     
    505496 
    506497 
    507   private DemuxedSequences getOrCreateDemux(DbControl dc, SequencingRun seqRun, FlowCellInfo flowCellInfo, SequencingRunInfo seqRunInfo, DemuxInfo demuxInfo)
    508   {
    509     String flowCellId = flowCellInfo.flowCellId;
    510     if (flowCellId == null) return null;
    511 
    512     DerivedBioAssay demux = demuxInfo.demux;
    513     if (demux != null)
    514     {
    515       addDebugMessage("Found: "+demux.getName()+"[ref="+flowCellId+"]");
    516     }
    517     else
    518     {
    519       demux = DerivedBioAssay.getNew(dc, seqRun.getItem(), null);
    520       demux.setItemSubtype(Subtype.DEMUXED_SEQUENCES.get(dc));
    521       demux.setName(seqRun.getNextDemuxedSequencesName(dc));
    522       Pipeline.RNA_SEQ.setAnnotation(dc, demux);
    523       demux.setHardware(null);
    524       demux.setSoftware(demuxInfo.software);
    525       demux.setProtocol(null);
    526       Annotationtype.ANALYSIS_RESULT.setAnnotationValue(dc, demux, DemuxedSequences.DEMUX_SUCCESSFUL);
    527       Annotationtype.READ_STRING.setAnnotationValue(dc, demux, demuxInfo.readString);   
    528      
    529       DateFormat df = new SimpleDateFormat("/yyyy/yyMMdd");
    530       String dataFilesFolder = df.format(seqRunInfo.startDate)+
    531         "_"+seqRunInfo.sequencer.serialNo+
    532         "_"+MD5.leftPad(Integer.toString(seqRunInfo.runNumber), '0', 4)+
    533         "_"+seqRunInfo.position+flowCellInfo.flowCellId;
    534       addDebugMessage("DataFilesFolder: "+dataFilesFolder);
    535       Annotationtype.DATA_FILES_FOLDER.setAnnotationValue(dc, demux, dataFilesFolder);
    536      
    537       dc.saveItem(demux);
    538      
    539       addDebugMessage("Created: "+ demux.getName());
    540     }
     498  private DemuxedSequences createDemuxedSequences(DbControl dc, Library lib, SequencingRun seqRun, FlowCellInfo flowCellInfo, SequencingRunInfo seqRunInfo, DemuxInfo demuxInfo, FastqInfo fastqInfo)
     499  {
     500
     501    DerivedBioAssay demux = DerivedBioAssay.getNew(dc, seqRun.getItem(), null);
     502    demux.setItemSubtype(Subtype.DEMUXED_SEQUENCES.get(dc));
     503    demux.setName(lib.getNextDemuxedSequencesName(dc));
     504    Pipeline.RNA_SEQ.setAnnotation(dc, demux);
     505    demux.setExtract(lib.getItem());
     506    demux.setHardware(null);
     507    demux.setSoftware(demuxInfo.software);
     508    demux.setProtocol(null);
     509    Annotationtype.ANALYSIS_RESULT.setAnnotationValue(dc, demux, DemuxedSequences.DEMUX_SUCCESSFUL);
     510    Annotationtype.READ_STRING.setAnnotationValue(dc, demux, demuxInfo.readString);   
     511    Annotationtype.READS.setAnnotationValue(dc, demux, demuxInfo.reads);
     512    Annotationtype.PF_READS.setAnnotationValue(dc, demux, demuxInfo.pfReads);
     513    Annotationtype.RAW_FASTQ.setAnnotationValues(dc, demux, Arrays.asList(fastqInfo.R1.name, fastqInfo.R2.name));
     514   
     515    DateFormat df = new SimpleDateFormat("/yyyy/yyMMdd");
     516    String dataFilesFolder = df.format(seqRunInfo.startDate)+
     517      "_"+seqRunInfo.sequencer.serialNo+
     518      "_"+MD5.leftPad(Integer.toString(seqRunInfo.runNumber), '0', 4)+
     519      "_"+seqRunInfo.position+flowCellInfo.flowCellId;
     520    addDebugMessage("DataFilesFolder: "+dataFilesFolder);
     521    Annotationtype.DATA_FILES_FOLDER.setAnnotationValue(dc, demux, dataFilesFolder);
     522   
     523    dc.saveItem(demux);
     524   
     525    addDebugMessage("Created: "+ demux.getName());
    541526    return DemuxedSequences.get(demux);
    542   }
    543  
    544   private MergedSequences createMergedSequences(DbControl dc, Library lib, DemuxedSequences demux, MergeInfo mergeInfo, FastqInfo fastqInfo)
    545   {
    546 
    547     DerivedBioAssay merged = DerivedBioAssay.getNew(dc, demux.getItem(), null);
    548     merged.setItemSubtype(Subtype.MERGED_SEQUENCES.get(dc));
    549     merged.setName(lib.getNextMergedSequencesName(dc));
    550     Pipeline.RNA_SEQ.setAnnotation(dc, merged);
    551     merged.setExtract(lib.getItem());
    552     Annotationtype.READS.setAnnotationValue(dc, merged, mergeInfo.reads);
    553     Annotationtype.PF_READS.setAnnotationValue(dc, merged, mergeInfo.pfReads);
    554     Annotationtype.RAW_FASTQ.setAnnotationValues(dc, merged, Arrays.asList(fastqInfo.R1.name, fastqInfo.R2.name));
    555    
    556     merged.setProtocol(null);
    557     merged.setSoftware(null);
    558     merged.setHardware(null);
    559    
    560     dc.saveItem(merged);
    561    
    562     addDebugMessage("Created: "+ merged.getName());
    563     return MergedSequences.get(merged);
    564527  }
    565528
     
    579542    public SequencingRun sequencingRun;
    580543    public DemuxedSequences demux;
    581     public MergedSequences merged;
    582544  }
    583545 
  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/plugins/cmd/DemuxInfo.java

    r6205 r6215  
    44
    55import net.sf.basedb.core.DbControl;
    6 import net.sf.basedb.core.DerivedBioAssay;
    76import net.sf.basedb.core.ItemQuery;
    87import net.sf.basedb.core.Software;
     
    2726{
    2827
    29   public DerivedBioAssay demux;
    3028  public Software software;
    3129  public String readString;
    32  
    33   public boolean valid;
     30  public Long pfReads;
     31  public Long reads;
    3432
    35   public DemuxInfo(JsonSection section, SequencingRunInfo seqRunInfo, FlowCellInfo flowCellInfo)
     33  public boolean valid = false;
     34
     35  public DemuxInfo(JsonSection demuxSection, JsonSection mergeSection)
    3636  {
    37     if (section != null)
     37    if (demuxSection != null && mergeSection != null)
    3838    {
    39       software = section.getRequiredEntry("Software", SoftwareValidator.INSTANCE);
    40       readString = section.getRequiredEntry("ReadString", PatternValidator.READ_STRING);
    41       if (seqRunInfo.sequencingRun != null)
    42       {
    43         demux = findExistingDemux(seqRunInfo.sequencingRun, flowCellInfo.flowCellId, section);
    44       }
     39      software = demuxSection.getRequiredEntry("Software", SoftwareValidator.INSTANCE);
     40      readString = demuxSection.getRequiredEntry("ReadString", PatternValidator.READ_STRING);
     41      pfReads = mergeSection.getRequiredEntry("PF_READS", LongValidator.POSITIVE.warnIf(10000000l, null)); // Warn if less than 10M reads
     42      reads = pfReads; //
     43      valid = !demuxSection.hasError() && !mergeSection.hasError();
    4544    }
    46     valid = section != null && !section.hasError();
    47   }
    48  
    49  
    50   private DerivedBioAssay findExistingDemux(DerivedBioAssay seqRun, String flowCellId, JsonSection section)
    51   {
    52     DerivedBioAssay demux = null;
    53     DbControl dc = section.getFile().dc();
    54     ItemQuery<DerivedBioAssay> query = seqRun.getChildren();
    55     query.setIncludes(Reggie.INCLUDE_IN_CURRENT_PROJECT);
    56     Subtype.DEMUXED_SEQUENCES.addFilter(dc, query);
    57     List<DerivedBioAssay> list = query.list(dc);
    58     if (list.size() > 1)
    59     {
    60       section.addErrorMessage("Found "+list.size()+" demux items for FlowCellID="+flowCellId);
    61     }
    62     else if (list.size() == 1)
    63     {
    64       demux = list.get(0);
    65     }
    66     return demux;
    6745  }
    6846
  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/plugins/cmd/JsonFile.java

    r6212 r6215  
    9595  private SequencingRunInfo seqRunInfo;
    9696  private DemuxInfo demuxInfo;
    97   private MergeInfo mergeInfo;
    9897
    9998  /**
     
    260259      flowCellInfo = new FlowCellInfo(getRequiredSection("FlowCell"), poolInfo);
    261260      seqRunInfo = new SequencingRunInfo(getRequiredSection("SequencingRun"), flowCellInfo);
    262       demuxInfo = new DemuxInfo(getRequiredSection("DemuxedSequences"), seqRunInfo, flowCellInfo);
    263       mergeInfo = new MergeInfo(getRequiredSection("MergedSequences"));
     261      demuxInfo = new DemuxInfo(getRequiredSection("DemuxedSequences"), getRequiredSection("MergedSequences"));
    264262     
    265263      if (operator != null) setOperator(operator);
     
    369367  }
    370368 
    371   public MergeInfo getMergeInfo()
    372   {
    373     return mergeInfo;
    374   }
    375  
    376369  public FastqInfo getFastqInfo()
    377370  {
  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/servlet/FastqServlet.java

    r6214 r6215  
    1414
    1515import net.sf.basedb.core.ItemList;
     16import net.sf.basedb.core.BasicItem;
    1617import net.sf.basedb.core.DbControl;
    1718import net.sf.basedb.core.DerivedBioAssay;
     
    2324import net.sf.basedb.core.SessionControl;
    2425import net.sf.basedb.core.Software;
     26import net.sf.basedb.core.Trashcan;
    2527import net.sf.basedb.core.query.Annotations;
    2628import net.sf.basedb.core.query.Expressions;
     
    4850import net.sf.basedb.reggie.grid.ImportFastqJobCreator;
    4951import net.sf.basedb.reggie.grid.ScriptUtil;
     52import net.sf.basedb.util.Values;
    5053import net.sf.basedb.util.error.ThrowableUtil;
    5154
     
    7578    try
    7679    {
    77       if ("GetMergedSequencesForImport".equals(cmd))
     80      if ("GetDemuxedSequencesForImport".equals(cmd))
    7881      {
    7982        dc = sc.newDbControl();
    8083       
    81         List<MergedSequences> list = null;
     84        List<DemuxedSequences> list = null;
    8285        // Load items from the pipeline list
    8386        ItemList fastqImportPipeline = BiomaterialList.FASTQ_IMPORT_PIPELINE.load(dc);
    8487        ItemQuery<DerivedBioAssay> query = fastqImportPipeline.getMembers();
    8588        query.setIncludes(Reggie.INCLUDE_IN_CURRENT_PROJECT);
    86         Subtype.MERGED_SEQUENCES.addFilter(dc, query);
     89        Subtype.DEMUXED_SEQUENCES.addFilter(dc, query);
    8790        // Join LibPlate for sorting
    8891        query.join(Hql.innerJoin(null, "extract", "lib", true));
     
    9295        query.order(Orders.asc(Hql.property("name")));
    9396        query.setMaxResults(250);
    94         list = MergedSequences.toList(query.list(dc));
     97        list = DemuxedSequences.toList(query.list(dc));
    9598
    9699        SnapshotManager manager = new SnapshotManager();
    97         JSONArray jsonMergedSequences = new JSONArray();
    98         for (MergedSequences ms : list)
    99         {
    100           ms.loadAnnotations(dc, "pipeline", Annotationtype.PIPELINE, null);
    101           ms.loadAnnotations(dc, "rawFastq", Annotationtype.RAW_FASTQ, null);
    102           DemuxedSequences demux = ms.getSingleDemuxedSequecnces(dc);
    103           ms.setAnnotation("rawFastqFolder", Annotationtype.DATA_FILES_FOLDER.getAnnotationValue(dc, demux.getItem()));
    104           ms.setAnnotation("demuxName", demux.getName());
    105           Library lib = ms.getLibrary(dc);
     100        JSONArray jsonDemuxedSequences = new JSONArray();
     101        for (DemuxedSequences ds : list)
     102        {
     103          ds.loadAnnotations(dc, "pipeline", Annotationtype.PIPELINE, null);
     104          ds.loadAnnotations(dc, "rawFastq", Annotationtype.RAW_FASTQ, null);
     105          ds.loadAnnotations(dc, "DataFilesFolder", Annotationtype.DATA_FILES_FOLDER, null);
     106          Library lib = ds.getLibrary(dc);
    106107          lib.loadBioPlateLocation();
    107108          Rna r = lib.getRna(dc, true);
    108109          if (r != null) lib.setAnnotation("specimen", r.loadYellowLabelInfo(dc, manager));
    109           ms.setAnnotation("lib", lib.asJSONObject());
    110           ms.loadAnnotations(dc, manager, "AutoProcess", Annotationtype.AUTO_PROCESSING, null);
    111           ms.loadDoNotUseAnnotations(dc, manager);
    112           jsonMergedSequences.add(ms.asJSONObject());
    113         }
    114         json.put("mergedSequences", jsonMergedSequences);       
     110          ds.setAnnotation("lib", lib.asJSONObject());
     111          ds.loadAnnotations(dc, manager, "AutoProcess", Annotationtype.AUTO_PROCESSING, null);
     112          ds.loadDoNotUseAnnotations(dc, manager);
     113          jsonDemuxedSequences.add(ds.asJSONObject());
     114        }
     115        json.put("demuxedSequences", jsonDemuxedSequences);       
    115116      }
    116117      else if ("GetUnconfirmedImportedSequences".equals(cmd))
     
    220221
    221222        JSONObject jsonReq = JsonUtil.parseRequest(req);
    222         JSONArray jsonMerged = (JSONArray)jsonReq.get("mergedSequences");
     223        JSONArray jsonDemuxed = (JSONArray)jsonReq.get("demuxedSequences");
    223224       
    224225        Number mergeSoftwareId = (Number)jsonReq.get("mergeSoftware");
     
    241242
    242243        // Load MergedSequences that should be imported
    243         List<MergedSequences> mergedSequences = new ArrayList<MergedSequences>();
    244         for (int mgNo = 0; mgNo < jsonMerged.size(); mgNo++)
    245         {
    246           JSONObject jsonMg = (JSONObject)jsonMerged.get(mgNo);
    247           Number mergedId = (Number)jsonMg.get("id");
    248        
    249           MergedSequences ms = MergedSequences.getById(dc, mergedId.intValue());
    250           mergedSequences.add(ms);
     244        List<DemuxedSequences> demuxedSequences = new ArrayList<DemuxedSequences>();
     245        for (int mgNo = 0; mgNo < jsonDemuxed.size(); mgNo++)
     246        {
     247          JSONObject jsonDx = (JSONObject)jsonDemuxed.get(mgNo);
     248          Number demuxedId = (Number)jsonDx.get("id");
     249       
     250          DemuxedSequences ds = DemuxedSequences.getById(dc, demuxedId.intValue());
     251          demuxedSequences.add(ds);
    251252
    252253          // Reset AUTO_PROCESSING annotation
    253           DerivedBioAssay merged = ms.getDerivedBioAssay();
    254           Annotationtype.AUTO_PROCESSING.setAnnotationValue(dc, merged, null);
     254          DerivedBioAssay demuxed = ds.getDerivedBioAssay();
     255          Annotationtype.AUTO_PROCESSING.setAnnotationValue(dc, demuxed, null);
    255256        }
    256257
     
    263264        jobCreator.setMergeSoftware(mergeSoftware);
    264265       
    265         List<JobDefinition> jobDefs = jobCreator.createFastqImportJobs(dc, cluster, mergedSequences);
     266        List<JobDefinition> jobDefs = jobCreator.createFastqImportJobs(dc, cluster, demuxedSequences);
    266267        List<Job> jobs = ScriptUtil.submitJobs(dc, cluster, jobDefs);
    267268        for (Job job : jobs)
     
    273274          else
    274275          {
    275             jsonMessages.add("Submitted FASTQ import job to " + cluster.getConnectionInfo().getName() + " with id " + job.getExternalId());
     276            jsonMessages.add("Submitted " + job.getName() + " to " + cluster.getConnectionInfo().getName() + " with id " + job.getExternalId());
    276277          }
    277278        }
     
    291292        int numLegacyAlign = 0;
    292293        int numConfirmed = 0;
     294        int numUnconfirmed = 0;
    293295        int numFlaggedRna = 0;
    294296        int numReImport = 0;
     297        int numDeleted = 0;
     298       
    295299        ItemList flaggedRna = BiomaterialList.FLAGGED_RNA.load(dc);
    296300        ItemList hisatPipeline = BiomaterialList.HISAT_PIPELINE.load(dc);
    297301        ItemList legacyPipeline = BiomaterialList.LEGACY_PIPELINE.load(dc);
    298302        ItemList fastqImportPipeline = BiomaterialList.FASTQ_IMPORT_PIPELINE.load(dc);
     303       
     304        boolean deleteItemsCreatedByFailedJobs = Boolean.TRUE.equals(jsonReq.get("deleteItemsCreatedByFailedJobs"));
     305        List<BasicItem> toDelete = new ArrayList<BasicItem>();
     306       
    299307        for (int mergeNo = 0; mergeNo < jsonMergedSequences.size(); ++mergeNo)
    300308        {
     
    306314          boolean flag = Boolean.TRUE.equals(jsonMerge.get("flag"));
    307315          boolean align = legacyAlign || hisatAlign;
    308          
    309           DerivedBioAssay merge = DerivedBioAssay.getById(dc, mergeId.intValue());
    310           merge.setDescription((String)jsonMerge.get("comment"));
     316          boolean deleted = false;
     317
     318          MergedSequences mg = MergedSequences.getById(dc, mergeId.intValue());
     319          DemuxedSequences dx = mg.getSingleDemuxedSequences(dc);
     320         
     321          DerivedBioAssay merge = mg.getDerivedBioAssay();
     322          DerivedBioAssay demux = dx.getDerivedBioAssay();
     323         
     324          if (flag && !Reggie.isExternalItem(merge.getName()))
     325          {
     326            Library lib = Library.get(merge.getExtract());
     327            Rna rna = lib.getRna(dc, false);
     328            if (rna != null)
     329            {
     330              Extract r = rna.getItem();
     331              Annotationtype.FLAG.setAnnotationValue(dc, r, Rna.FLAG_FASTQ_IMPORT_FAILED);
     332              flaggedRna.add(r);
     333              numFlaggedRna++;
     334            }
     335          }
     336
     337          String comment = Values.getStringOrNull((String)jsonMerge.get("comment"));
     338          merge.setDescription(comment);
    311339         
    312340          if (reImport)
    313341          {
    314             // TODO -- This will not work in the general case
    315             Annotationtype.ANALYSIS_RESULT.setAnnotationValue(dc, merge, null);
    316             Annotationtype.AUTO_PROCESSING.setAnnotationValue(dc, merge, "ReProcess");
    317             merge.setJob(null); // Must do this to reset for new import
    318             fastqImportPipeline.add(merge);
     342            // Set ReProcess annotation on demuxed sequences item
     343            Annotationtype.AUTO_PROCESSING.setAnnotationValue(dc, demux, "ReProcess");
     344            fastqImportPipeline.add(demux);
    319345            numReImport++;
    320           }
    321           else if (align)
    322           {
    323             Annotationtype.ANALYSIS_RESULT.setAnnotationValue(dc, merge, MergedSequences.MERGE_SUCCESSFUL);
    324             numConfirmed++;
    325             if (hisatAlign)
     346           
     347            if (deleteItemsCreatedByFailedJobs && !align)
    326348            {
    327               hisatPipeline.add(merge);
    328               numHisatAlign++;
     349              // Delete rawbioassay and related items
     350              merge.setRemoved(true);
     351              toDelete.add(merge);
     352              toDelete.addAll(Reggie.removeAttachedFiles(dc, merge));
     353              deleted = true;
     354              numDeleted++;
     355              demux.setDescription(comment);
    329356            }
    330  
    331             if (legacyAlign)
     357          }
     358          if (!deleted)
     359          {
     360            if (align)
    332361            {
    333               legacyPipeline.add(merge);
    334               numLegacyAlign++;
     362              Annotationtype.ANALYSIS_RESULT.setAnnotationValue(dc, merge, MergedSequences.MERGE_SUCCESSFUL);
     363              numConfirmed++;
     364              if (hisatAlign)
     365              {
     366                hisatPipeline.add(merge);
     367                numHisatAlign++;
     368              }
     369   
     370              if (legacyAlign)
     371              {
     372                legacyPipeline.add(merge);
     373                numLegacyAlign++;
     374              }
    335375            }
    336           }
    337          
    338           if (flag)
    339           {
    340             if (!align)
     376            else if (reImport || flag)
    341377            {
    342378              Annotationtype.ANALYSIS_RESULT.setAnnotationValue(dc, merge, MergedSequences.MERGE_FAILED);
    343               Annotationtype.AUTO_PROCESSING.setAnnotationValue(dc, merge, "Disable");
    344379            }
    345 
    346             if (!Reggie.isExternalItem(merge.getName()))
     380            else
    347381            {
    348               Library lib = Library.get(merge.getExtract());
    349               Rna rna = lib.getRna(dc, false);
    350               if (rna != null)
    351               {
    352                 Extract r = rna.getItem();
    353                 Annotationtype.FLAG.setAnnotationValue(dc, r, Rna.FLAG_FASTQ_IMPORT_FAILED);
    354                 flaggedRna.add(r);
    355                 numFlaggedRna++;
    356               }
     382              numUnconfirmed++;
    357383            }
    358384          }
     
    379405          ActivityDef.FASTQ_IMPORT_CONFIRMED.merge(dc, numConfirmed);
    380406        }
     407        if (numReImport > 0)
     408        {
     409          jsonMessages.add(numReImport + " items flagged for re-importing");
     410        }
     411        if (numDeleted > 0)
     412        {
     413          jsonMessages.add(numDeleted + " libraries deleted due to failure");
     414          if (toDelete.size() > numDeleted)
     415          {
     416            jsonMessages.add((toDelete.size() - numDeleted) + " linked items (eg. parent items and files) deleted");
     417          }
     418        }
     419
    381420        if (numFlaggedRna > 0)
    382421        {
    383422          jsonMessages.add("Added " + numFlaggedRna + " RNA items to the '" + flaggedRna.getName() + "' list");
    384423        }
    385        
    386         if (numReImport > 0)
    387         {
    388           jsonMessages.add(numReImport + " items flagged for re-importing");
    389         }
    390        
    391         // dc.commit();
     424        if (numUnconfirmed > 0)
     425        {
     426          jsonMessages.add(numUnconfirmed + " libraries remain unconfirmed");
     427        }
     428        dc.commit();
     429        if (toDelete.size() > 0)
     430        {
     431          try
     432          {
     433            Trashcan.delete(sc, toDelete, false, null);
     434          }
     435          catch (RuntimeException ex)
     436          {
     437            ex.printStackTrace();
     438            jsonMessages.add("[Error]Could not delete all items created by failed jobs: " + ex.getMessage());
     439          }
     440        }
    392441      }
    393442
  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/servlet/FutureSpecimenServlet.java

    r6214 r6215  
    3636import net.sf.basedb.reggie.dao.Annotationtype;
    3737import net.sf.basedb.reggie.dao.BiomaterialList;
     38import net.sf.basedb.reggie.dao.DemuxedSequences;
    3839import net.sf.basedb.reggie.dao.Fileserver;
    3940import net.sf.basedb.reggie.dao.FutureSpecimen;
    40 import net.sf.basedb.reggie.dao.MergedSequences;
    4141import net.sf.basedb.reggie.dao.ReggieRole;
    4242import net.sf.basedb.reggie.dao.SpecimenTube;
     
    200200
    201201        OpenGridCluster cluster = null;
    202         ImportFastqJobCreator jobCreator = null;
    203202        if (jsonAutoAnalyze != null)
    204203        {
    205           boolean debug = Boolean.TRUE.equals(jsonAutoAnalyze.get("debug"));
    206           boolean autoConfirm = Boolean.TRUE.equals(jsonAutoAnalyze.get("autoConfirm"));
    207           Number priority = (Number)jsonAutoAnalyze.get("priority");
    208204          String clusterId = (String)jsonAutoAnalyze.get("cluster");
    209           Number mergeSoftwareId = (Number)jsonAutoAnalyze.get("mergeSoftware");
    210           Number mergeProtocolId = (Number)jsonAutoAnalyze.get("mergeProtocol");
    211 
    212205          cluster = OpenGridService.getInstance().getClusterById(dc, clusterId);
    213206          if (cluster == null)
     
    215208            throw new ItemNotFoundException("OpenGridCluster[" + clusterId + "]");
    216209          }
    217           jobCreator = new ImportFastqJobCreator();
    218           jobCreator.setAutoConfirm(autoConfirm);
    219           jobCreator.setDebug(debug);
    220           if (priority != null) jobCreator.setPriority(priority.intValue());
    221           jobCreator.setMergeProtocol(mergeProtocolId != null ? Protocol.getById(dc, mergeProtocolId.intValue()) : null);
    222           jobCreator.setMergeSoftware(mergeSoftwareId != null ? Software.getById(dc, mergeSoftwareId.intValue()) : null);
    223210        }
    224211
     
    234221          gatewaySession = new RemoteHost(new ConnectionInfo(importGateway)).connect(5);
    235222          archiveSession = new RemoteHost(new ConnectionInfo(importArchive)).connect(5);
    236           List<MergedSequences> mergedSequences = new ArrayList<>(); // Collect MergedSequences that has been created by the importer
     223          List<DemuxedSequences> demuxedSequences = new ArrayList<>(); // Collect DemuxedSequences that has been created by the importer
    237224          for (int itemNo = 0; itemNo < jsonItems.size(); itemNo++)
    238225          {
     
    263250            {
    264251              // Add to FASTQ import pipeline
    265               BiomaterialList.FASTQ_IMPORT_PIPELINE.get(dc).add(items.merged.getItem());
     252              BiomaterialList.FASTQ_IMPORT_PIPELINE.get(dc).add(items.demux.getItem());
    266253              // Save the JSON to Fileserver.IMPORT_ARCHIVE
    267254              String dataFilesFolder = (String)Annotationtype.DATA_FILES_FOLDER.getAnnotationValue(dc, items.demux.getItem());
     
    269256              dc.commit();
    270257              jsonMessages.add("Imported "+specimen.getName()+" with data from "+jsonFile.getName());
    271               mergedSequences.add(items.merged);
     258              demuxedSequences.add(items.demux);
    272259              // Remove JSON from Filesever.IMPORT_GATEWAY
    273260              jsonFile.deleteFrom(gatewaySession, gatewayRoot);
     
    279266           
    280267            jsonMessages.addAll(prefix("[Warning]["+jsonFile.getName()+"] ", jsonFile.getWarningMessages()));
    281             //jsonMessages.addAll(prefix("[Debug]["+jsonFile.getName()+"] ", jsonFile.getDebugMessages()));
    282           }
    283          
    284           if (jobCreator != null)
    285           {
     268            jsonMessages.addAll(prefix("[Debug]["+jsonFile.getName()+"] ", jsonFile.getDebugMessages()));
     269          }
     270
     271          if (jsonAutoAnalyze != null)
     272          {
     273            boolean debug = Boolean.TRUE.equals(jsonAutoAnalyze.get("debug"));
     274            boolean autoConfirm = Boolean.TRUE.equals(jsonAutoAnalyze.get("autoConfirm"));
     275            Number priority = (Number)jsonAutoAnalyze.get("priority");
     276            String clusterId = (String)jsonAutoAnalyze.get("cluster");
     277            Number mergeSoftwareId = (Number)jsonAutoAnalyze.get("mergeSoftware");
     278            Number mergeProtocolId = (Number)jsonAutoAnalyze.get("mergeProtocol");
     279           
    286280            dc = sc.newDbControl();
    287             List<JobDefinition> jobDefs = jobCreator.createFastqImportJobs(dc, cluster, mergedSequences);
    288             List<Job> jobs = ScriptUtil.submitJobs(dc, cluster, jobDefs);
    289             for (Job job : jobs)
    290             {
    291               if (job.getStatus() == Job.Status.ERROR)
     281            try
     282            {
     283              ImportFastqJobCreator jobCreator = new ImportFastqJobCreator();
     284              jobCreator.setAutoConfirm(autoConfirm);
     285              jobCreator.setDebug(debug);
     286              if (priority != null) jobCreator.setPriority(priority.intValue());
     287              jobCreator.setMergeProtocol(mergeProtocolId != null ? Protocol.getById(dc, mergeProtocolId.intValue()) : null);
     288              jobCreator.setMergeSoftware(mergeSoftwareId != null ? Software.getById(dc, mergeSoftwareId.intValue()) : null);
     289             
     290              List<JobDefinition> jobDefs = jobCreator.createFastqImportJobs(dc, cluster, demuxedSequences);
     291              List<Job> jobs = ScriptUtil.submitJobs(dc, cluster, jobDefs);
     292              for (Job job : jobs)
    292293              {
    293                 jsonMessages.add("[Warning]Job submission for '" + job.getName() + "' failed: " + job.getStatusMessage());
     294                if (job.getStatus() == Job.Status.ERROR)
     295                {
     296                  jsonMessages.add("[Warning]Job submission for '" + job.getName() + "' failed: " + job.getStatusMessage());
     297                }
     298                else
     299                {
     300                  jsonMessages.add("Submitted "+job.getName()+" to "+cluster.getConnectionInfo().getName()+" with id "+job.getExternalId());
     301                }
    294302              }
    295               else
    296               {
    297                 jsonMessages.add("Submitted "+job.getName()+" to "+cluster.getConnectionInfo().getName()+" with id "+job.getExternalId());
    298               }
    299             }
    300             dc.commit();
    301           }
    302          
     303              dc.commit();
     304            }
     305            catch (Exception ex)
     306            {
     307              jsonMessages.add("[Warning]Job submission for FASTQ import failed: "+ ex.getMessage());
     308            }
     309          }
    303310        }
    304311        finally
  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/servlet/InstallServlet.java

    r6214 r6215  
    10251025            Annotationtype.READ_STRING, Annotationtype.OMIT_LANES,
    10261026            Annotationtype.ANALYSIS_RESULT, Annotationtype.AUTO_PROCESSING,
    1027             Annotationtype.DATA_FILES_FOLDER,
     1027            Annotationtype.DATA_FILES_FOLDER, Annotationtype.RAW_FASTQ,
    10281028            Annotationtype.READS, Annotationtype.PF_READS, Annotationtype.PF_NNNN_PCT,
    10291029            Annotationtype.PF_UNUSED_PCT, Annotationtype.DEMUX_WARNINGS, Annotationtype.SKIPPED_TILES,
     
    10351035            Annotationtype.READS, Annotationtype.PF_READS, Annotationtype.ADAPTER_READS, Annotationtype.PT_READS,
    10361036            Annotationtype.FRAGMENT_SIZE_AVG, Annotationtype.FRAGMENT_SIZE_STDEV,
    1037             Annotationtype.DATA_FILES_FOLDER, Annotationtype.RAW_FASTQ,
     1037            Annotationtype.DATA_FILES_FOLDER,
    10381038            Annotationtype.DO_NOT_USE, Annotationtype.DO_NOT_USE_COMMENT,
    10391039            Annotationtype.AUTO_PROCESSING,
Note: See TracChangeset for help on using the changeset viewer.