Changeset 239 for trunk/uk/ac/scri/batchimporter/src

Timestamp:

Feb 20, 2007, 5:21:08 PM (17 years ago)

Author:

mbayer

Message:

several bug fixes:

• change handling of dirs and files so that file is unzipped in its current dir
• unclosed DbControl? during finalize at BatchDataImport?.java:109 -- now fixed
• User Quota for files needs to be greater than the file size to be imported -- code now checks
• misreporting of raw data type on check fixed

File:

• trunk/uk/ac/scri/batchimporter/src/sbrn/base/BatchDataImport.java (modified) (23 diffs)

trunk/uk/ac/scri/batchimporter/src/sbrn/base/BatchDataImport.java

@@ -1 +1 @@
 /*
-Copyright (C) Authors contributing to this file. 
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See also http://www.gnu.org/licenses/gpl.txt. 
+ Copyright (C) Authors contributing to this file. 
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+ See also http://www.gnu.org/licenses/gpl.txt. 
  */
 package sbrn.base;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Enumeration;
+import java.util.List;
+import java.util.Locale;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
+import java.util.zip.ZipInputStream;
 
 import net.sf.basedb.core.Affymetrix;
@@ -32 +45 @@
 import net.sf.basedb.core.ItemResultList;
 import net.sf.basedb.core.Job;
-import net.sf.basedb.core.Path;
+import net.sf.basedb.core.Location;
 import net.sf.basedb.core.Permission;
 import net.sf.basedb.core.PluginConfiguration;
@@ -39 +52 @@
 import net.sf.basedb.core.PluginResponse;
 import net.sf.basedb.core.ProgressReporter;
+import net.sf.basedb.core.Quota;
+import net.sf.basedb.core.QuotaType;
 import net.sf.basedb.core.RawBioAssay;
 import net.sf.basedb.core.RawDataType;
-import net.sf.basedb.core.RawDataTypes;
 import net.sf.basedb.core.SessionControl;
 import net.sf.basedb.core.SimpleProgressReporter;
 import net.sf.basedb.core.StringParameterType;
+import net.sf.basedb.core.SystemItems;
 import net.sf.basedb.core.User;
 import net.sf.basedb.core.plugin.Response;
@@ -52 +67 @@
 import net.sf.basedb.util.zip.FileUnpacker;
 
-import java.io.InputStream;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-import java.util.Locale;
-
-
 /**
-  Imports raw data from a batch of flat files into BASE2. The data files must be packaged as a zip file and conform
-  to the same data format. It is assumed that this code is executed after an Experiment object has been created already
-  and that it is of the same data type as the data that are being imported. Currently only supports the creation of 
-  RawBioAssay objects as part of the import but could be extended to create other objects as needed. 
-
-  @author Micha Bayer - Plant Bioinformatics Group, Scottish Crop Research Institute
-  email: sbrn@scri.ac.uk, web http://www.scri.ac.uk
+ * Imports raw data from a batch of flat files into BASE2. The data files must
+ * be packaged as a zip file and conform to the same data format. It is assumed
+ * that this code is executed after an Experiment object has been created
+ * already and that it is of the same data type as the data that are being
+ * imported. Currently only supports the creation of RawBioAssay objects as part
+ * of the import but could be extended to create other objects as needed.
+ * 
+ * @author Micha Bayer - Plant Bioinformatics Group, Scottish Crop Research
+ *         Institute email: sbrn@scri.ac.uk, web http://www.scri.ac.uk
  */
 public class BatchDataImport
 {
-//  ======================================vars============================= 
-
-  //BASE client infrastructure
-  private DbControl dc  = null;
+  // ======================================vars=============================
+
+  // BASE client infrastructure
+  private DbControl dc = null;
+
   private SessionControl sc = null;
 
-  //the directory the files for import are located in
+  // the directory the files for import are located in
   private Directory dir = null;
 
-  //the name of the zip file minus the extension
+  // the name of the zip file minus the extension
   private String fileName = null;
 
-  //the zip file we want to import
+  // the zip file we want to import
   private File zipFile = null;
 
-  //the Experiment that this import will be attached to
+  // the Experiment that this import will be attached to
   private Experiment experiment = null;
 
-  //the progress reporter for the overall  batch import
+  // the progress reporter for the overall batch import
   private ProgressReporter progress = null;
 
-  //the percentage complete of the currently executing file import (subjob)
+  // the percentage complete of the currently executing file import
+  // (subjob)
   private double currentJobPercentComplete = 0;
 
-  //the currently executing file import job
+  // the currently executing file import job
   private Job currentJob = null;
 
-  //the cdf file for the array desing for Affy data
+  // the cdf file for the array desing for Affy data
   private File cdfFile = null;
 
-//  ================================c'tor=================================
-
-  public BatchDataImport(SessionControl sc,File zipFile,Experiment experiment,ProgressReporter progress)
+  // ================================c'tor=================================
+
+  public BatchDataImport(SessionControl sc, File zipFile, Experiment experiment, ProgressReporter progress)
   {
     dc = sc.newDbControl();
@@ -114 +123 @@
   }
 
-//  ================================methods==================================
+  // ================================methods==================================
 
   /**
-   Unpacks the zip file in a new directory in the user's raw data directory. The new dir is named after the name of the zip file
-   (minus the extension). 
+   * Unpacks the zip file in a new subdirectory of the directory where the zip file is located.
+   * The new dir is named after the name of the zip file (minus the extension).
    */
   public boolean unpackZipFile() throws Exception
   {
-    User user = null;
-    Directory homeDir = null;   
+
     int numUnpackedFiles = 0;
-    boolean unpackSuccess = false; 
+    boolean unpackSuccess = false;
 
     try
     {
-      // get User id and home dir
+      // figure out where the zip file is located
+      Directory zipFileDir = zipFile.getDirectory();
+      System.out.println("zipFile is located in dir " + zipFileDir.toString());
+
+      // make a new subdir in that dir
+      // the zip file should then be unzipped in here
+      // give it the same name as that of the zip file plus a
+      // timestamp
+      String timeStamp = getTime("ddMMyy_HHmmss");
+      Directory subDir = Directory.getNew(dc, zipFileDir);
+      fileName = zipFile.getName();
+      String fileNameNoExt = fileName.substring(0, fileName.indexOf("."));
+      System.out.println("new subDir = " + fileNameNoExt + timeStamp);
+      subDir.setName(fileNameNoExt + timeStamp);
+
+      // save the new subDir
+      dc.saveItem(subDir);
+      dir = subDir;
+      
+      //check the size of the zip file content
+      //if unzipping this file we must not execeed our disk quota
+      InputStream inStream = zipFile.getDownloadStream(0);      
+      long zipFileSize = getZipfileEntriesSize(inStream); 
+      
+      //get the user's current disk usage and quota
       int userID = sc.getLoggedInUserId();
-      user = User.getById(dc, userID);
-      homeDir = user.getHomeDirectory();
-      homeDir.checkPermission(Permission.WRITE);
-      homeDir.checkPermission(Permission.CREATE);     
-      System.out.println("user = " + user.getName());
-
-      //make a new subdir in the user's data dir
-      //the zip file should then be unzipped in here
-      //give it the same name as that of the zip file plus a timestamp
-      String timeStamp = getTime("ddMMyy_HHmmss");
-      Directory dataDir = Directory.getByPath(dc, new Path(homeDir.getPath().toString()+"/raw data",Path.Type.DIRECTORY));            
-      Directory subDir = Directory.getNew(dc,dataDir);  
-      fileName = zipFile.getName();
-      String fileNameNoExt = fileName.substring(0, fileName.indexOf("."));      
-      System.out.println("new subDir = " + fileNameNoExt+timeStamp);
-      subDir.setName(fileNameNoExt+timeStamp);
-
-      //save the new subDir
-      dc.saveItem(subDir);  
-      dir = subDir;
-
-      //now unzip the zip file into the new subdir
+      User user = User.getById(dc, userID);
+      Quota quota = user.getQuota();
+      QuotaType totalQuotaType = QuotaType.getById(dc, SystemItems.getId(QuotaType.TOTAL));
+      //unlimited storage (e.g. for user root) will return -1 here so need to check for this further down
+      long quotaValue = quota.getQuotaValue(totalQuotaType, Location.PRIMARY);      
+      long currentDiskUsage = user.getDiskUsage(totalQuotaType, Location.PRIMARY);
+      
+      //check whether remaining disk space and zip file size are compatible
+      long remainingSpace = quotaValue - currentDiskUsage;
+      System.out.println("quotaValue for user " + user.toString() + " = " + quotaValue);
+      System.out.println("currentDiskUsage = " + currentDiskUsage);
+      System.out.println("remainingSpace = " + remainingSpace);
+      System.out.println("zipFileSize = " + zipFileSize);
+      if(remainingSpace < zipFileSize && quotaValue!=-1L)
+      {
+        throw new Exception("insufficient disk space remaining for unzipping of file -- please contact your BASE administrator or remove some files");
+      }
+
+      // now unzip the zip file into the new subdir
       if (zipFile != null)
-      {             
+      {
         PluginDefinition plugin = PluginDefinition.getByClassName(dc, "net.sf.basedb.plugins.ZipFileUnpacker");
-        FileUnpacker unpacker = plugin.newInstance(net.sf.basedb.plugins.ZipFileUnpacker.class,null,sc,null,null);        
+        FileUnpacker unpacker = plugin.newInstance(net.sf.basedb.plugins.ZipFileUnpacker.class, null, sc, null, null);
         InputStream uploadStream = zipFile.getDownloadStream(0);
         if (unpacker != null)
@@ -162 +191 @@
           System.out.println("unpacking");
           numUnpackedFiles = unpacker.unpack(dc, subDir, uploadStream, false, null);
-        }       
+        }
         uploadStream.close();
       }
 
-      //commit the changes
+      // commit the changes
       dc.commit();
 
-      //can't check for all the files so we have to assume that if any files were unpacked ok, they all worked
-      if(numUnpackedFiles > 0)
+      // can't check for all the files so we have to assume
+      // that if any files were unpacked ok, they all worked
+      if (numUnpackedFiles > 0)
       {
         unpackSuccess = true;
@@ -179 +209 @@
     {
       e.printStackTrace();
-      throw new Exception("zip file unpacking failed: "+ e.getMessage());
+      throw new Exception("zip file unpacking failed: " + e.getMessage());
+    }
+    finally
+    {
+      if (dc != null)
+        dc.close();
     }
     return unpackSuccess;
   }
 
-//  -----------------------------------------------------------------------------------------------------------------------------------------------
-
+  // -----------------------------------------------------------------------------------------------------------------------------------------------
 
   /**
-   Imports the data from a batch of text files into new RawBioAssay objects (one per file), then creates a new
-   experiment with all of these RBAs attached. 
-   @return boolean success -- true if everything worked as defined 
+   * Imports the data from a batch of text files into new RawBioAssay
+   * objects (one per file), then creates a new experiment with all of
+   * these RBAs attached.
+   * 
+   * @return boolean success -- true if everything worked as defined
    */
   public boolean importData() throws Exception
@@ -200 +236 @@
     System.out.println("dir = " + dir.getName());
 
-    //get the raw data type from the experiment
-    RawDataType rawDataType = experiment.getRawDataType(); 
+    // get the raw data type from the experiment
+    RawDataType rawDataType = experiment.getRawDataType();
     System.out.println("rawDataType = " + rawDataType.getName());
 
     ItemQuery<File> fileQuery = null;
-    ItemResultList<File> fileList = null; 
-
-    if(dc == null || !dc.isConnected())
-      dc=sc.newDbControl();
+    ItemResultList<File> fileList = null;
+
+    if (dc == null || !dc.isConnected())
+      dc = sc.newDbControl();
 
     try
-    {     
-      //list all files in dir -- this should be what we unzipped from the zip file
+    {
+      // list all files in dir -- this should be what we
+      // unzipped from the zip file
       fileQuery = dir.getFiles();
-      fileList = fileQuery.list(dc);  
-
-      File [] dataFiles = null; 
-      if(fileQuery!=null)
-      {
-        //for Affy data the zip file uploaded should contain the cdf file for the chip and all the cel files
-        //need to separate these out
+      fileList = fileQuery.list(dc);
+
+      File[] dataFiles = null;
+      if (fileQuery != null)
+      {
+        // for Affy data the zip file uploaded should
+        // contain the cdf file for the chip and all the
+        // cel files
+        // need to separate these out
         if (rawDataType.getName().equalsIgnoreCase("affymetrix"))
         {
 
-          //iterate over all the files and check their extensions
-          int dataFileCount =0; 
-          //initiate a shorter array for the data files since this should not include the cdf file
-          dataFiles = new File[fileList.size()-1];  
+          // iterate over all the files and check
+          // their extensions
+          int dataFileCount = 0;
+          // initiate a shorter array for the data
+          // files since this should not include
+          // the cdf file
+          dataFiles = new File[fileList.size() - 1];
           System.out.println("instantiating dataFiles array of length " + dataFiles.length);
-          
+
           System.out.println("affy data -- iterating over file names");
           for (int i = 0; i < fileList.size(); i++)
           {
-            //get the current file and check its extension
+            // get the current file and
+            // check its extension
             File file = fileList.get(i);
             String fileName = file.getName();
-            String extension = fileName.substring(fileName.lastIndexOf(".")+1);
-            System.out.println("file name = " +fileName + ", extension = " + extension);
-
-            //if it's the cdf file, point the global var for this at it
-            if(extension.equalsIgnoreCase("cdf"))
+            String extension = fileName.substring(fileName.lastIndexOf(".") + 1);
+            System.out.println("file name = " + fileName + ", extension = " + extension);
+
+            // if it's the cdf file, point
+            // the global var for this at it
+            if (extension.equalsIgnoreCase("cdf"))
             {
               System.out.println("cdf file found");
               cdfFile = file;
-              System.out.println("cdf file = " + cdfFile);              
+              System.out.println("cdf file = " + cdfFile);
             }
-            //if it's a cel file, add it to the dataFiles array
-            else if(extension.equalsIgnoreCase("cel"))
+            // if it's a cel file, add it to
+            // the dataFiles array
+            else if (extension.equalsIgnoreCase("cel"))
             {
-                System.out.println("setting dataFiles["+dataFileCount+"] to file " + file.getName());
-                dataFiles[dataFileCount] = file;
-                dataFileCount++;
+              System.out.println("setting dataFiles[" + dataFileCount + "] to file " + file.getName());
+              dataFiles[dataFileCount] = file;
+              dataFileCount++;
             }
-            //if it is neither a cel file nor a cdf file then the user must have defined the Experiment as
-            //being Affy but the data they are importing is not Affy
-            //in that case we need to throw an Exception and abort the whole import
+            // if it is neither a cel file
+            // nor a cdf file then the user
+            // must have defined the
+            // Experiment as
+            // being Affy but the data they
+            // are importing is not Affy
+            // in that case we need to throw
+            // an Exception and abort the
+            // whole import
             else
             {
@@ -262 +313 @@
           }
         }
-        else//non-Affy data types
+        else
+        // non-Affy data types
         {
-          //in this case all the files in the zip file should be data files
-          dataFiles = new File[fileList.size()];  
+          // in this case all the files in the zip
+          // file should be data files
+          dataFiles = new File[fileList.size()];
           System.out.println("instantiating dataFiles array of length " + dataFiles.length);
 
-          //put the files from the list into the array for homogeneity's sake
+          // put the files from the list into the
+          // array for homogeneity's sake
           fileList.toArray(dataFiles);
         }
 
-        //list all the data files
-        System.out.println("data files: "); 
-        for (int i = 0; i< dataFiles.length; i++)
-        {               
+        // list all the data files
+        System.out.println("data files: ");
+        for (int i = 0; i < dataFiles.length; i++)
+        {
           System.out.println(dataFiles[i].getName());
         }
       }
 
-      //earmark this one file for autodetecting the file format
-      //all files must have same format
+      // earmark this one file for autodetecting the file
+      // format
+      // all files must have same format
       File exampleFile = fileList.get(0);
       System.out.println("exampleFile = " + exampleFile.getName());
-      
-      //check we don't have a discrepancy between the raw data type and the data files here
-      //if users have specified a non-affy data type and the data is affy we need to abort the import with an appropriate error
-      //get the current file and check its extension
+
+      // check we don't have a discrepancy between the raw
+      // data type and the data files here
+      // if users have specified a non-affy data type and the
+      // data is affy we need to abort the import with an
+      // appropriate error
+      // get the current file and check its extension
       String exampleFileName = exampleFile.getName();
-      String exampleFileExtension = exampleFileName.substring(exampleFileName.lastIndexOf(".")+1);
-      if(exampleFileExtension.equalsIgnoreCase("cdf") || exampleFileExtension.equalsIgnoreCase("cel"))
+      String exampleFileExtension = exampleFileName.substring(exampleFileName.lastIndexOf(".") + 1);
+      if ((exampleFileExtension.equalsIgnoreCase("cdf") || exampleFileExtension.equalsIgnoreCase("cel")) && !rawDataType.getName().equalsIgnoreCase("affymetrix"))
       {
         throw new Exception("Affymetrix data found but Experiment data type was specified as non-Affy");
       }
 
-      //for non-Affy data types, find a suitable plugin configuration
+      // for non-Affy data types, find a suitable plugin
+      // configuration
       PluginConfiguration importConfig = null;
       if (!rawDataType.getName().equalsIgnoreCase("affymetrix"))
       {
-        //get the appropriate plugin config
+        // get the appropriate plugin config
         PluginConfigDetector detector = new PluginConfigDetector(sc);
-        importConfig = detector.detectPluginConfig(exampleFile);      
+        importConfig = detector.detectPluginConfig(exampleFile);
         System.out.println("PluginConfiguration importConfig = " + importConfig);
-        
-        //now need to figure out the raw data type to make sure it matches that of the Experiment we are importing into
-        //this can be done by looking at the plugin config which should have it stored in it
+
+        // now need to figure out the raw data type to
+        // make sure it matches that of the Experiment
+        // we are importing into
+        // this can be done by looking at the plugin
+        // config which should have it stored in it
         List rdtVal = importConfig.getParameterValues("rawDataType");
-        String rawDataTypeStr = (String)rdtVal.get(0);
-        System.out.println("rawDataType detected from files = " + rawDataTypeStr);  
-        //compare this to the raw data type of the experiment
-        if(!rawDataType.getName().equalsIgnoreCase(rawDataTypeStr))
+        String rawDataTypeStr = (String) rdtVal.get(0);
+        System.out.println("rawDataType detected from files = " + rawDataTypeStr);
+        // compare this to the raw data type of the
+        // experiment
+        if (!rawDataType.getName().equalsIgnoreCase(rawDataTypeStr))
         {
           throw new Exception("raw data types of experiment and data files do not match");
         }
-        
+
       }
 
       // for each data file
-      //create a new RawBioAssay
-      RawBioAssay [] rawBioAssays = new RawBioAssay [dataFiles.length];
+      // create a new RawBioAssay
+      RawBioAssay[] rawBioAssays = new RawBioAssay[dataFiles.length];
       System.out.println("dataFiles.length = " + dataFiles.length);
       for (int i = 0; i < dataFiles.length; i++)
       {
-        if(dc.isClosed())
+        if (dc.isClosed())
           dc = sc.newDbControl();
         System.out.println("current data file is " + dataFiles[i].getName());
-        RawBioAssay rba = createRawBioAssay(dc, "raw bioassay "+ dataFiles[i].getName(),rawDataType);
+        RawBioAssay rba = createRawBioAssay(dc, "raw bioassay " + dataFiles[i].getName(), rawDataType);
         rawBioAssays[i] = rba;
-      } 
-
-      //write all this to the database
+      }
+
+      // write all this to the database
       dc.commit();
 
-      //make sure we have a working dbcontrol object
-      if(dc.isClosed())
-        dc = sc.newDbControl(); 
-
-      //monitor the job progress
-      ProgressMonitorThread pm = new ProgressMonitorThread(progress,rawBioAssays.length,this);
+      // make sure we have a working dbcontrol object
+      if (dc.isClosed())
+        dc = sc.newDbControl();
+
+      // monitor the job progress
+      ProgressMonitorThread pm = new ProgressMonitorThread(progress, rawBioAssays.length, this);
       pm.start();
 
-      //import the data for each RawBioAssay 
-      //only do this for non-Affy data types
+      // import the data for each RawBioAssay
+      // only do this for non-Affy data types
       if (!rawDataType.getName().equalsIgnoreCase("affymetrix"))
       {
-        for (int i = 0; i< rawBioAssays.length; i++)
+        for (int i = 0; i < rawBioAssays.length; i++)
         {
           SimpleProgressReporter pr = new SimpleProgressReporter(null);
@@ -352 +415 @@
           pm.setCount(i);
         }
-      } 
-      else//Affy data 
+      }
+      else
+      // Affy data
       {
         System.out.println("Affymetrix experiment -- attaching cel files to raw bioassays");
 
-        //make a new ArrayDesign for this affy chip
+        // make a new ArrayDesign for this affy chip
         ArrayDesign arrayDesign = ArrayDesign.getNew(dc, true);
         arrayDesign.setName(cdfFile.getName());
         dc.saveItem(arrayDesign);
         dc.commit();
-        //ArrayDesign arrayDesign = ArrayDesign.getById(dc, 7);
+        // ArrayDesign arrayDesign =
+        // ArrayDesign.getById(dc, 7);
         System.out.println("using arrayDesign object " + arrayDesign.getName());
-        
-        //make sure we have a working dbcontrol object
-        if(dc.isClosed())
-          dc = sc.newDbControl();         
-
-        //check the cdf file exists -- this should have been included in the zip file
-        //else fail the whole import
-        if(cdfFile == null)
+
+        // make sure we have a working dbcontrol object
+        if (dc.isClosed())
+          dc = sc.newDbControl();
+
+        // check the cdf file exists -- this should have
+        // been included in the zip file
+        // else fail the whole import
+        if (cdfFile == null)
         {
           throw new Exception("no cdf file found for Affy data -- check that the zip file included this");
         }
 
-        //now set the cdf file on the array design
+        // now set the cdf file on the array design
         System.out.println("setting cdf file on the array design object");
         dc.reattachItem(arrayDesign);
         dc.reattachItem(cdfFile);
         Affymetrix.setCdfFile(arrayDesign, cdfFile);
-        
-        //make sure we have a working dbcontrol object
-        if(dc.isClosed())
-          dc = sc.newDbControl(); 
+
+        // make sure we have a working dbcontrol object
+        if (dc.isClosed())
+          dc = sc.newDbControl();
 
         System.out.println("adding cel files to raw bioassays");
-        for (int i = 0; i< rawBioAssays.length; i++)
-        { 
+        for (int i = 0; i < rawBioAssays.length; i++)
+        {
           RawBioAssay rba = RawBioAssay.getById(dc, rawBioAssays[i].getId());
-          
+
           System.out.println("iteration " + i);
           System.out.println("rba = " + rba.getName());
           System.out.println("file = " + dataFiles[i].getName());
-          
-          //must set the array design
+
+          // must set the array design
           rba.setArrayDesign(arrayDesign);
 
-          //need to attach the cel file to the bioassay object 
-          Affymetrix.setCelFile(rba, dataFiles[i]); 
+          // need to attach the cel file to the
+          // bioassay object
+          Affymetrix.setCelFile(rba, dataFiles[i]);
         }
       }
-      
-      //make sure we have a working dbcontrol object
-      if(dc.isClosed())
-        dc = sc.newDbControl();     
-      //commit the changes to the database
+
+      // make sure we have a working dbcontrol object
+      if (dc.isClosed())
+        dc = sc.newDbControl();
+      // commit the changes to the database
       dc.commit();
 
-      //now attach all the new RawBioAssay objects to the current Experiment item
-      //this is the Experiment the user will be running the import from
-      configureExperiment(dc, rawDataType,rawBioAssays); 
-
-      //make sure we have a working dbcontrol object
-      if(dc.isClosed())
-        dc = sc.newDbControl(); 
-
-      //check whether the import worked
-      //if item 0 in the list of RawBioAssay objects can be retrieved, we assume it all worked
+      // now attach all the new RawBioAssay objects to the
+      // current Experiment item
+      // this is the Experiment the user will be running the
+      // import from
+      configureExperiment(dc, rawDataType, rawBioAssays);
+
+      // make sure we have a working dbcontrol object
+      if (dc.isClosed())
+        dc = sc.newDbControl();
+
+      // check whether the import worked
+      // if item 0 in the list of RawBioAssay objects can be
+      // retrieved, we assume it all worked
       ItemQuery<RawBioAssay> query = RawBioAssay.getQuery();
       query.restrict(Restrictions.eq(Hql.property("name"), Expressions.string(rawBioAssays[0].getName())));
       RawBioAssay testRba = query.list(dc).get(0);
-      if(testRba!=null)
+      if (testRba != null)
       {
         importSuccess = true;
@@ -435 +505 @@
     finally
     {
-      if(dc!=null)
+      if (dc != null)
         dc.close();
     }
@@ -445 +515 @@
 
   /**
-   Import features.
+   * Import features.
    */
-  private void importRawData(PluginConfiguration config, File file, RawBioAssay rba,SimpleProgressReporter pr) throws Exception
-  {
-    System.out.println("importing raw data from file: " + file.getName() );
+  private void importRawData(PluginConfiguration config, File file, RawBioAssay rba, SimpleProgressReporter pr) throws Exception
+  {
+    System.out.println("importing raw data from file: " + file.getName());
     Job job = null;
     try
     {
-      //make sure we have a working dbcontrol object
-      if(dc.isClosed())
+      // make sure we have a working dbcontrol object
+      if (dc.isClosed())
         dc = sc.newDbControl();
 
@@ -469 +539 @@
       dc.saveItem(job);
       dc.commit();
-      executeJob(job,pr);
-    }
-    catch(Exception x)
+      executeJob(job, pr);
+    }
+    catch (Exception x)
     {
       x.printStackTrace();
@@ -477 +547 @@
     finally
     {
-      if(!dc.isClosed())
+      if (!dc.isClosed())
         dc.close();
     }
@@ -485 +555 @@
 
   /**
-   Execute a job.
-  @param job
+   * Execute a job.
+   * 
+   * @param job
    */
-  private void executeJob(Job job,SimpleProgressReporter pr)
-  {
-    //make sure we have a working dbcontrol object
-    if(dc.isClosed())
+  private void executeJob(Job job, SimpleProgressReporter pr)
+  {
+    // make sure we have a working dbcontrol object
+    if (dc.isClosed())
       dc = sc.newDbControl();
 
@@ -497 +568 @@
     try
     {
-      localhost =InetAddress.getLocalHost().toString();
+      localhost = InetAddress.getLocalHost().toString();
     }
     catch (UnknownHostException e)
@@ -504 +575 @@
     }
 
-    System.out.println("Executing job: " + job.getName() );
+    System.out.println("Executing job: " + job.getName());
     job = Job.getById(dc, job.getId());
 
@@ -510 +581 @@
     dc.commit();
 
-    //execute the job request
+    // execute the job request
     PluginResponse response = request.invoke();
 
-    //check the job ended sucessfully
+    // check the job ended sucessfully
     if (response.getStatus() == Response.Status.ERROR)
     {
       throw new BaseException(response.getMessage(), response.getErrorList().get(0));
     }
-    System.out.println("Job ended successfully: " + response.getMessage() );
-
-    if(!dc.isClosed())
+    System.out.println("Job ended successfully: " + response.getMessage());
+
+    if (!dc.isClosed())
       dc.close();
   }
 
-
   // -------------------------------------------------------------------------------------------------------------------------------------------
 
   /**
-   Adds the raw bioassays to our experiment.
+   * Adds the raw bioassays to our experiment.
    */
-  private void configureExperiment(DbControl dc, RawDataType rawDataType, RawBioAssay [] rawBioAssays) throws Exception 
+  private void configureExperiment(DbControl dc, RawDataType rawDataType, RawBioAssay[] rawBioAssays) throws Exception
   {
     try
     {
-      //make sure we have a working dbcontrol object
-      if(dc.isClosed())
+      // make sure we have a working dbcontrol object
+      if (dc.isClosed())
         dc = sc.newDbControl();
 
       System.out.println("Configuring experiment " + experiment);
 
-      //make sure the raw data type is set to the correct value
-      //rather than relying on the one specified by the user, we want to use the one that we have 
-      //actually detected by looking at the data to be imported (see above)
-      //check what the user has set is correct -- if not, we need to fail the whole run
+      // make sure the raw data type is set to the correct
+      // value
+      // rather than relying on the one specified by the user,
+      // we want to use the one that we have
+      // actually detected by looking at the data to be
+      // imported (see above)
+      // check what the user has set is correct -- if not, we
+      // need to fail the whole run
       RawDataType actualRdt = experiment.getRawDataType();
       if (!actualRdt.getName().equals(rawDataType.getName()))
       {
-        throw new BaseException("Import failed -- the raw data type of the selected experiment and that of the "+
-                "data to be imported don't match. Please change your data type to " +rawDataType.getName() +
-        " and try again.");
-      }
-
-      //now attach the raw bioassays
-      //tried to do this directly on the Experiment object passed into this class -- that fails
-      //the following is a workaround -- just make a fresh Experiment object from the ID of the one passed in
+        throw new BaseException("Import failed -- the raw data type of the selected experiment and that of the " + "data to be imported don't match. Please change your data type to " + rawDataType.getName() + " and try again.");
+      }
+
+      // now attach the raw bioassays
+      // tried to do this directly on the Experiment object
+      // passed into this class -- that fails
+      // the following is a workaround -- just make a fresh
+      // Experiment object from the ID of the one passed in
       int expID = experiment.getId();
       Experiment exp = Experiment.getById(dc, expID);
@@ -565 +639 @@
       }
 
-      //housekeeping
+      // housekeeping
       dc.commit();
-      if(!dc.isClosed())
+      if (!dc.isClosed())
         dc.close();
     }
@@ -577 +651 @@
   }
 
-
   // -------------------------------------------------------------------------------------------------------------------------------------------
 
   /**
-   Create a raw bioassay.
+   * Create a raw bioassay.
    */
-  private RawBioAssay createRawBioAssay(DbControl dc, String name,RawDataType rawDataType)
-  {
-    if(dc.isClosed())
+  private RawBioAssay createRawBioAssay(DbControl dc, String name, RawDataType rawDataType)
+  {
+    if (dc.isClosed())
       dc = sc.newDbControl();
 
-    System.out.println("Creating raw bioassay: " + name );
+    System.out.println("Creating raw bioassay: " + name);
     RawBioAssay rba = RawBioAssay.getNew(dc, rawDataType);
     rba.setName(name);
@@ -595 +668 @@
   }
 
-//  ------------------------------------------------------------------------------------------------------------------------------------------- 
+  // -------------------------------------------------------------------------------------------------------------------------------------------
 
   private String getTime(String pattern)
@@ -602 +675 @@
     Locale currentLocale = Locale.getDefault();
     formatter = new SimpleDateFormat(pattern, currentLocale);
-    String timeStamp = formatter.format(new Date());  
+    String timeStamp = formatter.format(new Date());
     return timeStamp;
   }
 
-//  ------------------------------------------------------------------------------------------------------------------------------------------- 
+  // -------------------------------------------------------------------------------------------------------------------------------------------
 
   public Job getCurrentJob()
@@ -613 +686 @@
   }
 
-
-
-//  -------------------------------------------------------------------------------------------------------------------------------------------
+  // -------------------------------------------------------------------------------------------------------------------------------------------
+  
+  /**
+   * Checks the total size of a given zip file (uncompressed)
+   * @param InputStream fileInputStream -- an input stream from the zip file
+   * @return long - the total size of the zip file uncompressed, in bytes
+   * 
+   */
+  public static long getZipfileEntriesSize(InputStream fileInputStream)
+  {
+    long totalSize =0;
+    try
+    {
+      ZipInputStream zipStream = new ZipInputStream(fileInputStream);     
+      ZipEntry entry = zipStream.getNextEntry();
+      while(entry!=null)
+      {
+        long size = entry.getSize();
+        totalSize += size;
+        entry = zipStream.getNextEntry();
+      }
+    }
+    catch (IOException ioe)
+    {
+      ioe.printStackTrace();
+    }
+    return totalSize;
+  }
+  
+//   -------------------------------------------------------------------------------------------------------------------------------------------
 
 }// end class