Changeset 231 for trunk/uk/ac/scri/batchimporter/src/sbrn

Timestamp:

Feb 14, 2007, 12:45:29 PM (16 years ago)

Author:

mbayer

Message:

fixed a number of issues:

• implemented support for Affy files
• have removed need for additional jar by including time stamp code in this class directly
• exception messages now show at end of import in popup
• removed hardcoding of id of RawDataImporter? def

File:

• trunk/uk/ac/scri/batchimporter/src/sbrn/base/BatchDataImport.java (modified) (19 diffs)

trunk/uk/ac/scri/batchimporter/src/sbrn/base/BatchDataImport.java

@@ -17 +17 @@
 
 See also http://www.gnu.org/licenses/gpl.txt. 
-*/
+ */
 package sbrn.base;
 
+import net.sf.basedb.core.Affymetrix;
+import net.sf.basedb.core.ArrayDesign;
 import net.sf.basedb.core.BaseException;
 import net.sf.basedb.core.DbControl;
@@ -50 +52 @@
 import net.sf.basedb.util.zip.FileUnpacker;
 
-import sbrn.commons.TimeStampUtils;
-
 import java.io.InputStream;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
+import java.text.SimpleDateFormat;
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.List;
+import java.util.Locale;
 
 
@@ -64 +67 @@
   and that it is of the same data type as the data that are being imported. Currently only supports the creation of 
   RawBioAssay objects as part of the import but could be extended to create other objects as needed. 
-  
+
   @author Micha Bayer - Plant Bioinformatics Group, Scottish Crop Research Institute
   email: sbrn@scri.ac.uk, web http://www.scri.ac.uk
@@ -87 +90 @@
   //the Experiment that this import will be attached to
   private Experiment experiment = null;
-  
+
   //the progress reporter for the overall  batch import
   private ProgressReporter progress = null;
-  
+
   //the percentage complete of the currently executing file import (subjob)
   private double currentJobPercentComplete = 0;
-  
+
   //the currently executing file import job
   private Job currentJob = null;
-  
+
+  //the cdf file for the array desing for Affy data
+  private File cdfFile = null;
 
 //  ================================c'tor=================================
@@ -115 +120 @@
    (minus the extension). 
    */
-  public boolean unpackZipFile()
+  public boolean unpackZipFile() throws Exception
   {
     User user = null;
@@ -135 +140 @@
       //the zip file should then be unzipped in here
       //give it the same name as that of the zip file plus a timestamp
-      String timeStamp = TimeStampUtils.getTime("ddMMyy_hhmmss");
+      String timeStamp = getTime("ddMMyy_HHmmss");
       Directory dataDir = Directory.getByPath(dc, new Path(homeDir.getPath().toString()+"/raw data",Path.Type.DIRECTORY));            
       Directory subDir = Directory.getNew(dc,dataDir);  
@@ -173 +178 @@
     catch (Exception e)
     {
-      // TODO Auto-generated catch block
       e.printStackTrace();
+      throw new Exception("zip file unpacking failed: "+ e.getMessage());
     }
     return unpackSuccess;
@@ -191 +196 @@
     boolean importSuccess = false;
     boolean unpackSuccess = unpackZipFile();
-    if(!unpackSuccess)
-      throw new Exception("zip file unpacking failed");
 
     System.out.println("BatchDataImport importData");
     System.out.println("dir = " + dir.getName());
+
+    //get the raw data type
+    RawDataType rawDataType = experiment.getRawDataType(); 
+    System.out.println("rawDataType = " + rawDataType.getName());
 
     ItemQuery<File> fileQuery = null;
@@ -208 +215 @@
       fileQuery = dir.getFiles();
       fileList = fileQuery.list(dc);  
-      File [] files = new File[fileList.size()];
+
+      File [] dataFiles = null; 
       if(fileQuery!=null)
       {
-        for (File f : fileList)
-        {     
-          System.out.println("file = " + f.getName());
+        //for Affy data the zip file uploaded should contain the cdf file for the chip and all the cel files
+        //need to separate these out
+        if (rawDataType.getName().equalsIgnoreCase("affymetrix"))
+        {
+
+          //iterate over all the files and check their extensions
+          int dataFileCount =0; 
+          //initiate a shorter array for the data files since this should not include the cdf file
+          dataFiles = new File[fileList.size()-1];  
+          System.out.println("instantiating dataFiles array of length " + dataFiles.length);
+          
+          System.out.println("affy data -- iterating over file names");
+          for (int i = 0; i < fileList.size(); i++)
+          {
+            //get the current file and check its extension
+            File file = fileList.get(i);
+            String fileName = file.getName();
+            String extension = fileName.substring(fileName.lastIndexOf(".")+1);
+            System.out.println("file name = " +fileName + ", extension = " + extension);
+
+            //if it's the cdf file, point the global var for this at it
+            if(extension.equalsIgnoreCase("cdf"))
+            {
+              System.out.println("cdf file found");
+              cdfFile = file;
+              System.out.println("cdf file = " + cdfFile);              
+            }
+            //if it's a cel file, add it to the dataFiles array
+            if(extension.equalsIgnoreCase("cel"))
+            {
+                System.out.println("setting dataFiles["+dataFileCount+"] to file " + file.getName());
+                dataFiles[dataFileCount] = file;
+                dataFileCount++;
+            }
+          }
+        }
+        else//non-Affy data types
+        {
+          //in this case all the files in the zip file should be data files
+          dataFiles = new File[fileList.size()];  
+          System.out.println("instantiating dataFiles array of length " + dataFiles.length);
+
+          //put the files from the list into the array for homogeneity's sake
+          fileList.toArray(dataFiles);
+        }
+
+        //list all the data files
+        System.out.println("data files: "); 
+        for (int i = 0; i< dataFiles.length; i++)
+        {               
+          System.out.println(dataFiles[i].getName());
         }
       }
@@ -222 +278 @@
       System.out.println("exampleFile = " + exampleFile.getName());
 
-      //get the appropriate plugin config
-      PluginConfigDetector detector = new PluginConfigDetector(sc);
-      PluginConfiguration importConfig = detector.detectPluginConfig(exampleFile);      
-      System.out.println("PluginConfiguration importConfig = " + importConfig);
-
-      //now need to figure out the raw data type
-      //this can be done by looking at the plugin config which should have it stored in it
-      //it's needed so we can configure the RawBioAssay and Experiment objects appropriately
-      List rdtVal = importConfig.getParameterValues("rawDataType");
-      String rawDataTypeStr = (String)rdtVal.get(0);
-      System.out.println("rawDataType = " + rawDataTypeStr);  
-      RawDataType rawDataType = RawDataTypes.getRawDataType(rawDataTypeStr); 
-
-      // for each file
+      //only do this for non-Affy data types
+      PluginConfiguration importConfig = null;
+      if (!rawDataType.getName().equalsIgnoreCase("affymetrix"))
+      {
+        //get the appropriate plugin config
+        PluginConfigDetector detector = new PluginConfigDetector(sc);
+        importConfig = detector.detectPluginConfig(exampleFile);      
+        System.out.println("PluginConfiguration importConfig = " + importConfig);
+      }
+
+      // for each data file
       //create a new RawBioAssay
-      List<RawBioAssay> rbaList = new ArrayList<RawBioAssay>();
-      for (int i = 0; i < fileList.size(); i++)
+      RawBioAssay [] rawBioAssays = new RawBioAssay [dataFiles.length];
+      System.out.println("dataFiles.length = " + dataFiles.length);
+      for (int i = 0; i < dataFiles.length; i++)
       {
         if(dc.isClosed())
           dc = sc.newDbControl();
-
-        File file = fileList.get(i);
-        files[i]=file;
-        System.out.println("current file is " + file.getName());
-        RawBioAssay rba = createRawBioAssay(dc, "raw bioassay "+ file.getName(),rawDataType);
-        rbaList.add(rba);
+        System.out.println("current data file is " + dataFiles[i].getName());
+        RawBioAssay rba = createRawBioAssay(dc, "raw bioassay "+ dataFiles[i].getName(),rawDataType);
+        rawBioAssays[i] = rba;
       } 
 
@@ -256 +307 @@
       if(dc.isClosed())
         dc = sc.newDbControl(); 
+
+      //monitor the job progress
+      ProgressMonitorThread pm = new ProgressMonitorThread(progress,rawBioAssays.length,this);
+      pm.start();
+
+      //import the data for each RawBioAssay 
+      //only do this for non-Affy data types
+      if (!rawDataType.getName().equalsIgnoreCase("affymetrix"))
+      {
+        for (int i = 0; i< rawBioAssays.length; i++)
+        {
+          SimpleProgressReporter pr = new SimpleProgressReporter(null);
+          pm.setPr(pr);
+          importRawData(importConfig, dataFiles[i], rawBioAssays[i], pr);
+          pm.setCount(i);
+        }
+      } 
+      else//Affy data 
+      {
+        System.out.println("Affymetrix experiment -- attaching cel files to raw bioassays");
+
+        //make a new ArrayDesign for this affy chip
+        ArrayDesign arrayDesign = ArrayDesign.getNew(dc, true);
+        arrayDesign.setName(cdfFile.getName());
+        dc.saveItem(arrayDesign);
+        dc.commit();
+        //ArrayDesign arrayDesign = ArrayDesign.getById(dc, 7);
+        System.out.println("using arrayDesign object " + arrayDesign.getName());
+        
+        //make sure we have a working dbcontrol object
+        if(dc.isClosed())
+          dc = sc.newDbControl();         
+
+        //check the cdf file exists -- this should have been included in the zip file
+        //else fail the whole import
+        if(cdfFile == null)
+        {
+          throw new Exception("no cdf file found for Affy data -- check that the zip file included this");
+        }
+
+        //now set the cdf file on the array design
+        System.out.println("setting cdf file on the array design object");
+        dc.reattachItem(arrayDesign);
+        dc.reattachItem(cdfFile);
+        Affymetrix.setCdfFile(arrayDesign, cdfFile);
+        
+        //make sure we have a working dbcontrol object
+        if(dc.isClosed())
+          dc = sc.newDbControl(); 
+
+        System.out.println("adding cel files to raw bioassays");
+        for (int i = 0; i< rawBioAssays.length; i++)
+        { 
+          RawBioAssay rba = RawBioAssay.getById(dc, rawBioAssays[i].getId());
+          
+          System.out.println("iteration " + i);
+          System.out.println("rba = " + rba.getName());
+          System.out.println("file = " + dataFiles[i].getName());
+          
+          //must set the array design
+          rba.setArrayDesign(arrayDesign);
+
+          //need to attach the cel file to the bioassay object 
+          Affymetrix.setCelFile(rba, dataFiles[i]); 
+        }
+      }
       
-      int count = 0;  
-      double numFiles = rbaList.size();
-
-      ProgressMonitorThread pm = new ProgressMonitorThread(progress,numFiles,this);
-      pm.start();
-      
-      //import the data for each RawBioAssay 
-      for(RawBioAssay rba : rbaList)
-      {
-        SimpleProgressReporter pr = new SimpleProgressReporter(null);
-        pm.setPr(pr);
-        importRawData(importConfig,files[count], rba,pr);     
-        count++;
-        pm.setCount(count);
-      }
+      //commit the changes to the database
+      dc.commit();
 
       //now attach all the new RawBioAssay objects to the current Experiment item
       //this is the Experiment the user will be running the import from
-      configureExperiment(dc, rawDataType,rbaList); 
-      
+      configureExperiment(dc, rawDataType,rawBioAssays); 
+
       //make sure we have a working dbcontrol object
       if(dc.isClosed())
@@ -284 +388 @@
       //if item 0 in the list of RawBioAssay objects can be retrieved, we assume it all worked
       ItemQuery<RawBioAssay> query = RawBioAssay.getQuery();
-      query.restrict(Restrictions.eq(Hql.property("name"), Expressions.string(rbaList.get(0).getName())));
+      query.restrict(Restrictions.eq(Hql.property("name"), Expressions.string(rawBioAssays[0].getName())));
       RawBioAssay testRba = query.list(dc).get(0);
       if(testRba!=null)
@@ -295 +399 @@
     {
       e.printStackTrace();
+      throw new Exception(e);
     }
     finally
@@ -327 +432 @@
       job.setParameterValue("missingReporterError", new StringParameterType(), "skip");
       job.setParameterValue("featureMismatchError", new StringParameterType(), "skip");
-      
+
       currentJob = job;
-      
+
       dc.saveItem(job);
       dc.commit();
@@ -369 +474 @@
     System.out.println("Executing job: " + job.getName() );
     job = Job.getById(dc, job.getId());
-    
+
     PluginExecutionRequest request = job.execute(pr, localhost);
     dc.commit();
@@ -375 +480 @@
     //execute the job request
     PluginResponse response = request.invoke();
-    
+
     //check the job ended sucessfully
     if (response.getStatus() == Response.Status.ERROR)
@@ -393 +498 @@
    Adds the raw bioassays to our experiment.
    */
-  private void configureExperiment(DbControl dc, RawDataType rawDataType, List<RawBioAssay> rawBioAssays) throws Exception 
+  private void configureExperiment(DbControl dc, RawDataType rawDataType, RawBioAssay [] rawBioAssays) throws Exception 
   {
     try
@@ -420 +525 @@
       int expID = experiment.getId();
       Experiment exp = Experiment.getById(dc, expID);
-      
+
       for (RawBioAssay rba : rawBioAssays)
       {
@@ -459 +564 @@
 
 //  ------------------------------------------------------------------------------------------------------------------------------------------- 
-  
+
+  private String getTime(String pattern)
+  {
+    SimpleDateFormat formatter;
+    Locale currentLocale = Locale.getDefault();
+    formatter = new SimpleDateFormat(pattern, currentLocale);
+    String timeStamp = formatter.format(new Date());  
+    return timeStamp;
+  }
+
+//  ------------------------------------------------------------------------------------------------------------------------------------------- 
+
   public Job getCurrentJob()
   {