Changeset 1131


Ignore:
Timestamp:
Jun 16, 2009, 1:38:21 PM (12 years ago)
Author:
Nicklas Nordborg
Message:

References #218: Proof-of-concept: run a selected GenePattern? module from BASE

Changed file transfer code and error handling so that stderr.txt is actually loaded into BASE.
This concludes the major parts of the plug-in. It should be possible to run any GP module/pipeline. User-friendlyness is a remaining issue. Everything sent to GP must be in a file and there is very little logic for finding/creating input files. Results are also only stored as file and can't be imported as new spot data.

File:
1 edited

Legend:

Unmodified
Added
Removed
  • extensions/net.sf.basedb.genepattern/trunk/src/net/sf/basedb/genepattern/plugin/GenePatternAnalysisPlugin.java

    r1128 r1131  
    99import org.genepattern.webservice.WebServiceException;
    1010
    11 import net.sf.basedb.clients.web.extensions.ExtensionsControl;
    1211import net.sf.basedb.clients.web.formatter.FormatterFactory;
    1312import net.sf.basedb.core.AnnotationType;
    1413import net.sf.basedb.core.AnyToAny;
    15 import net.sf.basedb.core.Application;
    1614import net.sf.basedb.core.BaseException;
    1715import net.sf.basedb.core.BioAssaySet;
     
    4442import net.sf.basedb.core.plugin.Request;
    4543import net.sf.basedb.core.plugin.Response;
    46 import net.sf.basedb.core.signal.SignalException;
    4744import net.sf.basedb.core.signal.SignalHandler;
    4845import net.sf.basedb.core.signal.SignalTarget;
     
    5249import net.sf.basedb.genepattern.file.BaseFileProxy;
    5350import net.sf.basedb.genepattern.file.FileTransferGateway;
     51import net.sf.basedb.genepattern.servlet.Download;
    5452import net.sf.basedb.genepattern.wrapper.GPClient;
    5553import net.sf.basedb.genepattern.wrapper.JobResult;
    5654import net.sf.basedb.genepattern.wrapper.ParameterInfo;
    57 
     55import net.sf.basedb.util.ChainedProgressReporter;
     56
     57/**
     58  Plug-in that interfaces with GenePattern analysis modules. This plug-in must be
     59  configured before it can be used. The configuration sequence starts by selecting
     60  the GenePattern module or pipeline that is going to be used by this plug-in.
     61  In the second step default values for the module parameters can be specfied
     62  and also some information about the input file types that will make it easier
     63  for the plug-in to automatically find a suitable file.
     64  <p>
     65  When using this plug-in the user selects the input files and parameter values
     66  for the module/pipeline. The execution phase will first copy all files to
     67  a temporary location that can be accessed by the GenePattern server (via the
     68  BASE webserver and the {@link Download} servlet) without access permission
     69  controls.
     70  <p>
     71  NOTE! The temporary location is open to anyone and in theory
     72  files can be downloaded without permission checks. However, the file path
     73  is randomized and is hard to guess and the files are removed as soon as
     74  the job on the GenePattern server has finished.
     75  <p>
     76  When the files are in place a web service request is sent to the GenePattern
     77  server which is allowed to do it's work. When the analysis is complete all
     78  results files are downloaded and added to the BASE file system and attached to
     79  a child bioassay set.
     80 
     81  @author nicklas
     82  @since 1.0
     83*/
    5884public class GenePatternAnalysisPlugin
    5985  extends AbstractAnalysisPlugin
     
    111137    try
    112138    {
    113       // Export files and prepare parameters to GenePattern
    114       if (progress != null) progress.display(0, "Copying files to GenePattern server");
    115       checkInterrupted();
     139      // Initialize things that we need
    116140      String gpModule = (String)configuration.getValue("gpModule");
    117141      GPServer gpServer = GPServer.get(sc, (String)job.getValue("gpServer"));
    118      
    119       gw = new FileTransferGateway(gpServer.getBaseUrl() + "/" + GenePattern.getServletUrl("Download"));
     142      if (progress != null) progress.display(0, "Initializing " + gpModule + "...");
    120143      GPClient gp = new GPClient(gpServer);
    121144      ParameterInfo[] info = gp.getParameters(gpModule, null);     
    122145      Parameter[] parameters = new Parameter[info.length];
    123       int index = 0;
    124       for (ParameterInfo pi : info)
    125       {
    126         checkInterrupted();
     146     
     147      // Create parameters
     148      gw = new FileTransferGateway(gpServer.getBaseUrl() + "/" + GenePattern.getServletUrl("Download"));
     149      for (int index = 0; index < info.length; ++index)
     150      {
     151        ParameterInfo pi = info[index];
    127152        String name = pi.getName();
    128153        Parameter param = null;
     
    132157          if (file != null)
    133158          {
    134             if (progress != null) progress.display(5, "Copying file: " + file.getName());
    135159            param = gw.addFile(name, new BaseFileProxy(file));
    136160          }
     
    145169          param = new Parameter(name, value == null ? "" : value.toString());
    146170        }
    147         if (param != null)
    148         {
    149           parameters[index] = param;
    150           ++index;
    151         }
    152       }
     171        parameters[index] = param;
     172      }
     173     
     174      // Transfer files
     175      ChainedProgressReporter chainedProgress = null;
     176      if (progress != null)
     177      {
     178        chainedProgress = new ChainedProgressReporter(progress);
     179        chainedProgress.setRange(5, 35);
     180      }
     181      int fileTransfered = gw.prepareUpload(chainedProgress);
    153182 
    154183      // Execute the job on the GenePattern server
    155184      if (progress != null)
    156185      {
    157         progress.display(25, "Running analysis on GenePattern server: " + gpServer.getUrl());
     186        progress.display(35, "Running " + gpModule + " on GenePattern server: " + gpServer.getUrl());
    158187      }
    159188      checkInterrupted();
    160       int jobId = gp.runAnalysisNoWait(gpModule, parameters);
    161       JobResult result = null;
    162       try
    163       {
    164         result = gp.waitFor(jobId, 5);
    165       }
    166       catch (InterruptedException ex)
    167       {
    168         throw new SignalException("Aborted by user");
    169       }
    170       checkInterrupted();
    171      
     189      JobResult result = gp.runAnalysis(gpModule, parameters);
     190     
     191      // Was the job successful or not?
    172192      boolean failed = result.hasStandardError();
    173       if (progress != null)
    174       {
    175         if (failed)
    176         {
    177           progress.display(75, "Analysis failed. Downloading error information...");
    178         }
    179         else
    180         {
    181           progress.display(75, "Analysis complete. Downloading result...");
    182         }
    183       }
    184      
    185       // Get parameters
    186       dc = sc.newDbControl();
    187       BioAssaySet source = getSourceBioAssaySet(dc);
    188       Job currentJob = getCurrentJob(dc);
    189       String transformationName = (String)job.getValue(TRANSFORMATION_NAME);
    190       String childName = (String)job.getValue(CHILD_NAME);
    191       String childDescription = (String)job.getValue(CHILD_DESCRIPTION);
    192       boolean copyAnnotations = Boolean.TRUE.equals(job.getValue(COPY_ANNOTATIONS));
    193       boolean keepSpotData = Boolean.TRUE.equals(job.getValue("keepSpotData"));
    194      
    195       // Create child items
    196       Transformation t = source.newTransformation(currentJob);
    197       BioAssaySet child = t.newProduct(null, keepSpotData ? null : "new", keepSpotData);
    198       t.setName(transformationName);
    199       child.setName(childName);
    200       child.setDescription(childDescription);
    201       dc.saveItem(t);
    202       dc.saveItem(child);
    203      
    204       // Copy annotations
    205       if (copyAnnotations)
    206       {
    207         PluginConfiguration cfg = getCurrentConfiguration(dc);
    208         if (cfg != null && cfg.isAnnotated()) child.getAnnotationSet().copyFrom(cfg, false);
    209       }
    210 
    211       String resultDirectory = (String)job.getValue("resultDirectory");
    212       Directory dir = Directory.getNew(dc, new Path(resultDirectory, Path.Type.DIRECTORY));
    213 
    214       String[] files = result.getOutputFileNames();
    215       int i = 0;
    216       int total = files.length;
    217       for (String f : files)
    218       {
    219         checkInterrupted();
    220         File baseFile = File.getFile(dc, dir, f, true);
    221         if (!baseFile.isInDatabase()) dc.saveItem(baseFile);
    222         if (progress != null)
    223         {
    224           progress.display(80 + (20 * i) / total, "Downloading " + f + "...");
    225         }
    226         gw.downloadResultFile(result, f, baseFile.getUploadStream(false));
    227         AnyToAny ata = AnyToAny.getNew(dc, child, baseFile, f, false);
    228         dc.saveItem(ata);
    229         i++;
    230       }
    231       dc.commit();
    232      
    233193      if (failed)
    234194      {
    235         response.setError("Failed", null);
     195        if (progress != null) progress.display(75, "Analysis failed. Downloading error information...");
     196       
     197        String error = gw.downloadAsString(result, "stderr.txt", "UTF-8");
     198        response.setError(error, null);
    236199      }
    237200      else
    238201      {
    239         response.setDone("okidoki");
     202        if (progress != null) progress.display(75, "Analysis complete. Downloading result files...");
     203     
     204        // Prepare to create transformation, child bioassay set, etc.
     205        dc = sc.newDbControl();
     206        BioAssaySet source = getSourceBioAssaySet(dc);
     207        Job currentJob = getCurrentJob(dc);
     208        String transformationName = (String)job.getValue(TRANSFORMATION_NAME);
     209        String childName = (String)job.getValue(CHILD_NAME);
     210        String childDescription = (String)job.getValue(CHILD_DESCRIPTION);
     211        boolean copyAnnotations = Boolean.TRUE.equals(job.getValue(COPY_ANNOTATIONS));
     212        boolean keepSpotData = Boolean.TRUE.equals(job.getValue("keepSpotData"));
     213     
     214        // Create child items
     215        Transformation t = source.newTransformation(currentJob);
     216        BioAssaySet child = t.newProduct(null, keepSpotData ? null : "new", keepSpotData);
     217        t.setName(transformationName);
     218        child.setName(childName);
     219        child.setDescription(childDescription);
     220        dc.saveItem(t);
     221        dc.saveItem(child);
     222     
     223        // Copy annotations
     224        if (copyAnnotations)
     225        {
     226          PluginConfiguration cfg = getCurrentConfiguration(dc);
     227          if (cfg != null && cfg.isAnnotated()) child.getAnnotationSet().copyFrom(cfg, false);
     228        }
     229
     230        // Download files
     231        String resultDirectory = (String)job.getValue("resultDirectory");
     232        Directory dir = Directory.getNew(dc, new Path(resultDirectory, Path.Type.DIRECTORY));
     233        String[] files = result.getOutputFileNames();
     234        int i = 0;
     235        int total = files.length;
     236        for (String f : files)
     237        {
     238          checkInterrupted();
     239          File baseFile = File.getFile(dc, dir, f, true);
     240          if (!baseFile.isInDatabase()) dc.saveItem(baseFile);
     241          if (progress != null)
     242          {
     243            progress.display(80 + (20 * i) / total, "Downloading " + f + "...");
     244          }
     245          gw.downloadResultFile(result, f, baseFile.getUploadStream(false));
     246          AnyToAny ata = AnyToAny.getNew(dc, child, baseFile, f, false);
     247          dc.saveItem(ata);
     248          i++;
     249        }
     250        dc.commit();
     251     
     252        response.setDone(gpModule + " completed successfully.");
    240253      }
    241254    }
Note: See TracChangeset for help on using the changeset viewer.