Changeset 1131 for extensions/net.sf.basedb.genepattern

Timestamp:

Jun 16, 2009, 1:38:21 PM (14 years ago)

Author:

Nicklas Nordborg

Message:

References #218: Proof-of-concept: run a selected GenePattern? module from BASE

Changed file transfer code and error handling so that stderr.txt is actually loaded into BASE.
This concludes the major parts of the plug-in. It should be possible to run any GP module/pipeline. User-friendlyness is a remaining issue. Everything sent to GP must be in a file and there is very little logic for finding/creating input files. Results are also only stored as file and can't be imported as new spot data.

File:

• extensions/net.sf.basedb.genepattern/trunk/src/net/sf/basedb/genepattern/plugin/GenePatternAnalysisPlugin.java (modified) (6 diffs)

extensions/net.sf.basedb.genepattern/trunk/src/net/sf/basedb/genepattern/plugin/GenePatternAnalysisPlugin.java

@@ -9 +9 @@
 import org.genepattern.webservice.WebServiceException;
 
-import net.sf.basedb.clients.web.extensions.ExtensionsControl;
 import net.sf.basedb.clients.web.formatter.FormatterFactory;
 import net.sf.basedb.core.AnnotationType;
 import net.sf.basedb.core.AnyToAny;
-import net.sf.basedb.core.Application;
 import net.sf.basedb.core.BaseException;
 import net.sf.basedb.core.BioAssaySet;
@@ -44 +42 @@
 import net.sf.basedb.core.plugin.Request;
 import net.sf.basedb.core.plugin.Response;
-import net.sf.basedb.core.signal.SignalException;
 import net.sf.basedb.core.signal.SignalHandler;
 import net.sf.basedb.core.signal.SignalTarget;
@@ -52 +49 @@
 import net.sf.basedb.genepattern.file.BaseFileProxy;
 import net.sf.basedb.genepattern.file.FileTransferGateway;
+import net.sf.basedb.genepattern.servlet.Download;
 import net.sf.basedb.genepattern.wrapper.GPClient;
 import net.sf.basedb.genepattern.wrapper.JobResult;
 import net.sf.basedb.genepattern.wrapper.ParameterInfo;
-
+import net.sf.basedb.util.ChainedProgressReporter;
+
+/**
+  Plug-in that interfaces with GenePattern analysis modules. This plug-in must be
+  configured before it can be used. The configuration sequence starts by selecting
+  the GenePattern module or pipeline that is going to be used by this plug-in.
+  In the second step default values for the module parameters can be specfied
+  and also some information about the input file types that will make it easier
+  for the plug-in to automatically find a suitable file.
+  <p>
+  When using this plug-in the user selects the input files and parameter values
+  for the module/pipeline. The execution phase will first copy all files to 
+  a temporary location that can be accessed by the GenePattern server (via the
+  BASE webserver and the {@link Download} servlet) without access permission
+  controls. 
+  <p>
+  NOTE! The temporary location is open to anyone and in theory
+  files can be downloaded without permission checks. However, the file path
+  is randomized and is hard to guess and the files are removed as soon as
+  the job on the GenePattern server has finished.
+  <p>
+  When the files are in place a web service request is sent to the GenePattern
+  server which is allowed to do it's work. When the analysis is complete all
+  results files are downloaded and added to the BASE file system and attached to
+  a child bioassay set.
+  
+  @author nicklas
+  @since 1.0
+*/
 public class GenePatternAnalysisPlugin 
   extends AbstractAnalysisPlugin 
@@ -111 +137 @@
     try
     {
-      // Export files and prepare parameters to GenePattern
-      if (progress != null) progress.display(0, "Copying files to GenePattern server");
-      checkInterrupted();
+      // Initialize things that we need
       String gpModule = (String)configuration.getValue("gpModule"); 
       GPServer gpServer = GPServer.get(sc, (String)job.getValue("gpServer"));
-      
-      gw = new FileTransferGateway(gpServer.getBaseUrl() + "/" + GenePattern.getServletUrl("Download"));
+      if (progress != null) progress.display(0, "Initializing " + gpModule + "...");
       GPClient gp = new GPClient(gpServer);
       ParameterInfo[] info = gp.getParameters(gpModule, null);      
       Parameter[] parameters = new Parameter[info.length];
-      int index = 0;
-      for (ParameterInfo pi : info)
-      {
-        checkInterrupted();
+      
+      // Create parameters 
+      gw = new FileTransferGateway(gpServer.getBaseUrl() + "/" + GenePattern.getServletUrl("Download"));
+      for (int index = 0; index < info.length; ++index)
+      {
+        ParameterInfo pi = info[index];
         String name = pi.getName();
         Parameter param = null;
@@ -132 +157 @@
           if (file != null)
           {
-            if (progress != null) progress.display(5, "Copying file: " + file.getName());
             param = gw.addFile(name, new BaseFileProxy(file));
           }
@@ -145 +169 @@
           param = new Parameter(name, value == null ? "" : value.toString());
         }
-        if (param != null)
-        {
-          parameters[index] = param;
-          ++index;
-        }
-      }
+        parameters[index] = param;
+      }
+      
+      // Transfer files
+      ChainedProgressReporter chainedProgress = null;
+      if (progress != null)
+      {
+        chainedProgress = new ChainedProgressReporter(progress);
+        chainedProgress.setRange(5, 35);
+      }
+      int fileTransfered = gw.prepareUpload(chainedProgress);
   
       // Execute the job on the GenePattern server
       if (progress != null) 
       {
-        progress.display(25, "Running analysis on GenePattern server: " + gpServer.getUrl());
+        progress.display(35, "Running " + gpModule + " on GenePattern server: " + gpServer.getUrl());
       }
       checkInterrupted();
-      int jobId = gp.runAnalysisNoWait(gpModule, parameters);
-      JobResult result = null;
-      try
-      {
-        result = gp.waitFor(jobId, 5);
-      }
-      catch (InterruptedException ex)
-      {
-        throw new SignalException("Aborted by user");
-      }
-      checkInterrupted();
-      
+      JobResult result = gp.runAnalysis(gpModule, parameters);
+      
+      // Was the job successful or not?
       boolean failed = result.hasStandardError();
-      if (progress != null) 
-      {
-        if (failed)
-        {
-          progress.display(75, "Analysis failed. Downloading error information...");
-        }
-        else
-        {
-          progress.display(75, "Analysis complete. Downloading result...");
-        }
-      }
-      
-      // Get parameters
-      dc = sc.newDbControl();
-      BioAssaySet source = getSourceBioAssaySet(dc);
-      Job currentJob = getCurrentJob(dc);
-      String transformationName = (String)job.getValue(TRANSFORMATION_NAME);
-      String childName = (String)job.getValue(CHILD_NAME);
-      String childDescription = (String)job.getValue(CHILD_DESCRIPTION);
-      boolean copyAnnotations = Boolean.TRUE.equals(job.getValue(COPY_ANNOTATIONS));
-      boolean keepSpotData = Boolean.TRUE.equals(job.getValue("keepSpotData"));
-      
-      // Create child items
-      Transformation t = source.newTransformation(currentJob);
-      BioAssaySet child = t.newProduct(null, keepSpotData ? null : "new", keepSpotData);
-      t.setName(transformationName);
-      child.setName(childName);
-      child.setDescription(childDescription);
-      dc.saveItem(t);
-      dc.saveItem(child);
-      
-      // Copy annotations
-      if (copyAnnotations)
-      {
-        PluginConfiguration cfg = getCurrentConfiguration(dc);
-        if (cfg != null && cfg.isAnnotated()) child.getAnnotationSet().copyFrom(cfg, false);
-      }
-
-      String resultDirectory = (String)job.getValue("resultDirectory");
-      Directory dir = Directory.getNew(dc, new Path(resultDirectory, Path.Type.DIRECTORY));
-
-      String[] files = result.getOutputFileNames();
-      int i = 0;
-      int total = files.length;
-      for (String f : files)
-      {
-        checkInterrupted();
-        File baseFile = File.getFile(dc, dir, f, true);
-        if (!baseFile.isInDatabase()) dc.saveItem(baseFile);
-        if (progress != null) 
-        {
-          progress.display(80 + (20 * i) / total, "Downloading " + f + "...");
-        }
-        gw.downloadResultFile(result, f, baseFile.getUploadStream(false));
-        AnyToAny ata = AnyToAny.getNew(dc, child, baseFile, f, false);
-        dc.saveItem(ata);
-        i++;
-      }
-      dc.commit();
-      
       if (failed)
       {
-        response.setError("Failed", null);
+        if (progress != null) progress.display(75, "Analysis failed. Downloading error information...");
+        
+        String error = gw.downloadAsString(result, "stderr.txt", "UTF-8");
+        response.setError(error, null);
       }
       else
       {
-        response.setDone("okidoki");
+        if (progress != null) progress.display(75, "Analysis complete. Downloading result files...");
+      
+        // Prepare to create transformation, child bioassay set, etc.
+        dc = sc.newDbControl();
+        BioAssaySet source = getSourceBioAssaySet(dc);
+        Job currentJob = getCurrentJob(dc);
+        String transformationName = (String)job.getValue(TRANSFORMATION_NAME);
+        String childName = (String)job.getValue(CHILD_NAME);
+        String childDescription = (String)job.getValue(CHILD_DESCRIPTION);
+        boolean copyAnnotations = Boolean.TRUE.equals(job.getValue(COPY_ANNOTATIONS));
+        boolean keepSpotData = Boolean.TRUE.equals(job.getValue("keepSpotData"));
+      
+        // Create child items
+        Transformation t = source.newTransformation(currentJob);
+        BioAssaySet child = t.newProduct(null, keepSpotData ? null : "new", keepSpotData);
+        t.setName(transformationName);
+        child.setName(childName);
+        child.setDescription(childDescription);
+        dc.saveItem(t);
+        dc.saveItem(child);
+      
+        // Copy annotations
+        if (copyAnnotations)
+        {
+          PluginConfiguration cfg = getCurrentConfiguration(dc);
+          if (cfg != null && cfg.isAnnotated()) child.getAnnotationSet().copyFrom(cfg, false);
+        }
+
+        // Download files
+        String resultDirectory = (String)job.getValue("resultDirectory");
+        Directory dir = Directory.getNew(dc, new Path(resultDirectory, Path.Type.DIRECTORY));
+        String[] files = result.getOutputFileNames();
+        int i = 0;
+        int total = files.length;
+        for (String f : files)
+        {
+          checkInterrupted();
+          File baseFile = File.getFile(dc, dir, f, true);
+          if (!baseFile.isInDatabase()) dc.saveItem(baseFile);
+          if (progress != null) 
+          {
+            progress.display(80 + (20 * i) / total, "Downloading " + f + "...");
+          }
+          gw.downloadResultFile(result, f, baseFile.getUploadStream(false));
+          AnyToAny ata = AnyToAny.getNew(dc, child, baseFile, f, false);
+          dc.saveItem(ata);
+          i++;
+        }
+        dc.commit();
+      
+        response.setDone(gpModule + " completed successfully.");
       }
     }