Changeset 6210


Ignore:
Timestamp:
Apr 14, 2021, 8:35:52 AM (13 months ago)
Author:
Nicklas Nordborg
Message:

References #1295: Registration of specimen handled by external lab

After a successful import the JSON file is now moved from the ImportGateway to the ImportArchive and the resulting merged bioassay is added to the FASTQ import pipeline. Auto-confirm support remains to be implemented.

Location:
extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie
Files:
4 edited

Legend:

Unmodified
Added
Removed
  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/plugins/FutureSpecimenImporter.java

    r6209 r6210  
    9090  /**
    9191    Import to a Specimen or FutureSpecimen item.
     92    If the import is successful the imported items are returned,
     93    otherwise null.
    9294  */
    93   public void doImport(DbControl dc, Sample sample, JsonFile jsonFile)
     95  public ImportedItems doImport(DbControl dc, Sample sample, JsonFile jsonFile)
    9496  {
    9597    this.currentFile = jsonFile;
    9698   
    9799    JSONObject json = jsonFile.getJSON();
    98    
    99100    JsonSection jsonSpecimen = jsonFile.getRequiredSection("Specimen");
    100101   
     
    111112    FastqInfo fastqInfo = jsonFile.getFastqInfo();
    112113
    113     if (jsonSpecimen == null) return;
    114    
     114    if (jsonSpecimen == null) return null;
     115
     116    ImportedItems items = new ImportedItems();
     117
    115118    SpecimenTube specimen = importToSpecimen(dc, sample, jsonSpecimen);
    116119   
    117     if (specimen == null || !lysateInfo.valid) return;
    118     Lysate lysate = createLysate(dc, specimen, lysateInfo);
    119    
    120     if (lysate == null || !rnaInfo.valid || !dnaInfo.valid || !ftInfo.valid) return;
    121     Rna rna = createRna(dc, lysate, rnaInfo);
    122     RnaQc rnaQc = rna != null ? createRnaQc(dc, rna, rnaInfo) : null;
    123     Dna dna = createDna(dc, lysate, dnaInfo);
    124     FlowThrough ft = createFlowThrough(dc, lysate, ftInfo);
    125    
    126     if (rna == null || !libInfo.valid) return;
    127     Library lib = createLibrary(dc, rna, libInfo);
    128    
    129     if (lib == null || !poolInfo.valid) return;
    130     PooledLibrary pool = getOrCreatePool(dc, lib, poolInfo);
    131    
    132     if (pool == null || !flowCellInfo.valid || !seqRunInfo.valid) return;
    133     FlowCell flowCell = getOrCreateFlowCell(dc, pool, flowCellInfo, seqRunInfo);
    134    
    135     if (flowCell == null) return;
    136     SequencingRun seqRun = getOrCreateSequencingRun(dc, flowCell, flowCellInfo, seqRunInfo);
    137    
    138     if (seqRun == null || !demuxInfo.valid) return;
    139     DemuxedSequences demux = getOrCreateDemux(dc, seqRun, flowCellInfo, demuxInfo);
    140    
    141     if (demux == null || !mergeInfo.valid || !fastqInfo.valid) return;
    142     MergedSequences merged = createMergedSequences(dc, lib, demux, mergeInfo, fastqInfo);
    143    
     120    if (specimen == null || !lysateInfo.valid) return null;
     121    items.lysate = createLysate(dc, specimen, lysateInfo);
     122   
     123    if (items.lysate == null || !rnaInfo.valid || !dnaInfo.valid || !ftInfo.valid) return null;
     124    items.rna = createRna(dc, items.lysate, rnaInfo);
     125    if (items.rna != null) items.rnaQc = createRnaQc(dc, items.rna, rnaInfo);
     126    items.dna = createDna(dc, items.lysate, dnaInfo);
     127    items.flowThrough = createFlowThrough(dc, items.lysate, ftInfo);
     128   
     129    if (items.rna == null || !libInfo.valid) return null;
     130    items.lib = createLibrary(dc, items.rna, libInfo);
     131   
     132    if (items.lib == null || !poolInfo.valid) return null;
     133    items.pool = getOrCreatePool(dc, items.lib, poolInfo);
     134   
     135    if (items.pool == null || !flowCellInfo.valid || !seqRunInfo.valid) return null;
     136    items.flowCell = getOrCreateFlowCell(dc, items.pool, flowCellInfo, seqRunInfo);
     137   
     138    if (items.flowCell == null) return null;
     139    items.sequencingRun = getOrCreateSequencingRun(dc, items.flowCell, flowCellInfo, seqRunInfo);
     140   
     141    if (items.sequencingRun == null || !demuxInfo.valid) return null;
     142    items.demux = getOrCreateDemux(dc, items.sequencingRun, flowCellInfo, demuxInfo);
     143   
     144    if (items.demux == null || !mergeInfo.valid || !fastqInfo.valid) return null;
     145    items.merged = createMergedSequences(dc, items.lib, items.demux, mergeInfo, fastqInfo);
     146   
     147    return items.merged == null || jsonFile.hasError() ? null : items;
    144148  }
    145149 
     
    558562  }
    559563
     564  /**
     565    Hold all imported items.
     566  */
     567  public static class ImportedItems
     568  {
     569    public Lysate lysate;
     570    public FlowThrough flowThrough;
     571    public Dna dna;
     572    public Rna rna;
     573    public RnaQc rnaQc;
     574    public Library lib;
     575    public PooledLibrary pool;
     576    public FlowCell flowCell;
     577    public SequencingRun sequencingRun;
     578    public DemuxedSequences demux;
     579    public MergedSequences merged;
     580  }
     581 
    560582}
  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/plugins/cmd/FastqInfo.java

    r6201 r6210  
    3838  }
    3939 
    40   public void loadFileInfo(RemoteSession session, String rootDir)
     40  public void loadFileInfo(RemoteSession session, String directory)
    4141  {
    42     loadMetadata(session, rootDir, R1);
    43     loadMetadata(session, rootDir, R2);
     42    loadMetadata(session, directory, R1);
     43    loadMetadata(session, directory, R2);
    4444  }
    4545 
    4646 
    47   private void loadMetadata(RemoteSession session, String rootDir, FastqFile f)
     47  private void loadMetadata(RemoteSession session, String directory, FastqFile f)
    4848  {
    4949    InputStream tmp = null;
     
    5151    {
    5252      FileMetaData info = new FileMetaData();
    53       tmp = session.readFile(rootDir+f.name, info);
     53      tmp = session.readFile(directory+f.name, info);
    5454      f.size = info.getSize();
    5555      f.lastModified = info.getLastModifiedTime();
  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/plugins/cmd/JsonFile.java

    r6209 r6210  
    205205    Download and parse the JSON file from the remote server.
    206206  */
    207   public void downloadAndParse(DbControl dc, RemoteSession session, String rootDir)
    208   {
    209     if (!rootDir.endsWith("/")) rootDir += "/";
    210     String path = rootDir + name;
     207  public void downloadAndParse(DbControl dc, RemoteSession session, String directory)
     208  {
     209    if (!directory.endsWith("/")) directory += "/";
     210    String path = directory + name;
    211211
    212212    ByteArrayDownloadTarget download = new ByteArrayDownloadTarget(path);
     
    231231     
    232232      fastqInfo = new FastqInfo(getRequiredSection("fastq"));
    233       if (fastqInfo.valid) fastqInfo.loadFileInfo(session, rootDir);
     233      if (fastqInfo.valid) fastqInfo.loadFileInfo(session, directory);
    234234
    235235      lysateInfo = new LysateInfo(getRequiredSection("Lysate"));
     
    252252  }
    253253 
    254  
    255   public void copyTo(RemoteSession session, String rootDir)
    256   {
    257     if (!rootDir.endsWith("/")) rootDir += "/";
    258     String path = rootDir + name;
     254  /**
     255    Delete the JSON file from the remote server.
     256  */
     257  public void deleteFrom(RemoteSession session, String directory)
     258  {
     259    if (!directory.endsWith("/")) directory += "/";
     260    String path = directory + name;
    259261
    260262    UploadSource upload = new StringUploadSource(name, rawData);
    261263    try
    262264    {
    263       session.executeCmd("mkdir -p " + rootDir, 5);
     265      session.executeCmd("rm -f " + path, 5).throwExceptionIfNonZeroExitStatus();
     266    }
     267    catch (Exception ex)
     268    {
     269      addErrorMessage("Could not delete '"+ path + "': " + ex.getMessage());
     270      return;
     271    }
     272  }
     273 
     274  /**
     275    Save the JSON file to the remote server in the given directory.
     276  */
     277  public void saveTo(RemoteSession session, String directory)
     278  {
     279    if (!directory.endsWith("/")) directory += "/";
     280    String path = directory + name;
     281
     282    UploadSource upload = new StringUploadSource(name, rawData);
     283    try
     284    {
     285      session.executeCmd("mkdir -p " + directory, 5).throwExceptionIfNonZeroExitStatus();
    264286      session.uploadFile(upload, path, FilePermission.USER_RW);
    265287    }
    266288    catch (Exception ex)
    267289    {
    268       addErrorMessage("Could not upload '"+ path + "': " + ex.getMessage());
     290      addErrorMessage("Could not save to '"+ path + "': " + ex.getMessage());
    269291      return;
    270292    }
  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/servlet/FutureSpecimenServlet.java

    r6209 r6210  
    3030import net.sf.basedb.reggie.Reggie;
    3131import net.sf.basedb.reggie.counter.CounterService;
     32import net.sf.basedb.reggie.dao.Annotationtype;
     33import net.sf.basedb.reggie.dao.BiomaterialList;
    3234import net.sf.basedb.reggie.dao.Fileserver;
    3335import net.sf.basedb.reggie.dao.FutureSpecimen;
     
    3739import net.sf.basedb.reggie.grid.ScriptUtil;
    3840import net.sf.basedb.reggie.plugins.FutureSpecimenImporter;
     41import net.sf.basedb.reggie.plugins.FutureSpecimenImporter.ImportedItems;
    3942import net.sf.basedb.reggie.plugins.cmd.JsonFile;
    4043import net.sf.basedb.util.error.ThrowableUtil;
     
    212215          for (int itemNo = 0; itemNo < jsonItems.size(); itemNo++)
    213216          {
     217            // We use one transaction per JSON file
     218            if (dc.isClosed()) dc = sc.newDbControl();
     219           
    214220            JSONObject jsonItem = (JSONObject)jsonItems.get(itemNo);
    215221            Number itemId = (Number)jsonItem.get("id");
     
    218224           
    219225            Sample specimen = Sample.getById(dc, itemId.intValue());
    220            
     226            ImportedItems items = null;
    221227            if (!jsonFile.hasError())
    222228            {
    223229              FutureSpecimenImporter importer = new FutureSpecimenImporter();
    224               importer.doImport(dc, specimen, jsonFile);
    225              
    226               if (!jsonFile.hasError())
    227               {
    228                 // Add to FASTQ import pipeline
    229                 // BiomaterialList.FASTQ_IMPORT_PIPELINE.get(dc).add(merged);
    230                 // Copy JSON to Fileserver.IMPORT_ARCHIVE
    231                 // jsonFile.copyTo(archiveSession, archiveRoot);
    232                 // Remove JSON from Filesever.IMPORT_GATEWAY
    233               }
     230              items = importer.doImport(dc, specimen, jsonFile);
    234231            }
    235232           
     
    242239            else
    243240            {
    244               dc.commit();
    245               jsonMessages.add("Imported " + specimen.getName());
     241             
     242              // Add to FASTQ import pipeline
     243              BiomaterialList.FASTQ_IMPORT_PIPELINE.get(dc).add(items.merged.getItem());
     244              // Save the JSON to Fileserver.IMPORT_ARCHIVE
     245              String dataFilesFolder = (String)Annotationtype.DATA_FILES_FOLDER.getAnnotationValue(dc, items.sequencingRun.getItem());
     246              jsonFile.saveTo(archiveSession, archiveRoot + dataFilesFolder);
     247              //dc.commit();
     248              jsonMessages.add("Imported "+specimen.getName()+" with data from "+jsonFile.getName());
     249              // Remove JSON from Filesever.IMPORT_GATEWAY
     250              //jsonFile.deleteFrom(gatewaySession, gatewayRoot);
     251              if (jsonFile.hasError())
     252              {
     253                jsonMessages.addAll(prefix("[Warning]["+jsonFile.getName()+"] ", jsonFile.getErrorMessages()));
     254              }
     255              if (autoConfirm)
     256              {
     257                jsonMessages.add(items.merged.getName()+" added to FASTQ import pipeline");
     258                // TODO -- implement this
     259              }
     260             
    246261            }
    247262           
    248263            jsonMessages.addAll(prefix("[Warning]["+jsonFile.getName()+"] ", jsonFile.getWarningMessages()));
    249             jsonMessages.addAll(prefix("[Debug]["+jsonFile.getName()+"] ", jsonFile.getDebugMessages()));
    250 
    251             dc = sc.newDbControl();
     264            if (debug)
     265            {
     266              jsonMessages.addAll(prefix("[Debug]["+jsonFile.getName()+"] ", jsonFile.getDebugMessages()));
     267            }
    252268          }
    253 
    254269        }
    255270        finally
Note: See TracChangeset for help on using the changeset viewer.