Changeset 239


Ignore:
Timestamp:
Feb 20, 2007, 5:21:08 PM (15 years ago)
Author:
mbayer
Message:

several bug fixes:

  • change handling of dirs and files so that file is unzipped in its current dir
  • unclosed DbControl? during finalize at BatchDataImport?.java:109 -- now fixed
  • User Quota for files needs to be greater than the file size to be imported -- code now checks
  • misreporting of raw data type on check fixed
File:
1 edited

Legend:

Unmodified
Added
Removed
  • trunk/uk/ac/scri/batchimporter/src/sbrn/base/BatchDataImport.java

    r234 r239  
    11/*
    2 Copyright (C) Authors contributing to this file.
    3 
    4 This program is free software; you can redistribute it and/or modify
    5 it under the terms of the GNU General Public License as published by
    6 the Free Software Foundation; either version 2 of the License, or
    7 (at your option) any later version.
    8 
    9 This program is distributed in the hope that it will be useful,
    10 but WITHOUT ANY WARRANTY; without even the implied warranty of
    11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    12 GNU General Public License for more details.
    13 
    14 You should have received a copy of the GNU General Public License along
    15 with this program; if not, write to the Free Software Foundation, Inc.,
    16 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
    17 
    18 See also http://www.gnu.org/licenses/gpl.txt.
     2 Copyright (C) Authors contributing to this file.
     3
     4 This program is free software; you can redistribute it and/or modify
     5 it under the terms of the GNU General Public License as published by
     6 the Free Software Foundation; either version 2 of the License, or
     7 (at your option) any later version.
     8
     9 This program is distributed in the hope that it will be useful,
     10 but WITHOUT ANY WARRANTY; without even the implied warranty of
     11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
     12 GNU General Public License for more details.
     13
     14 You should have received a copy of the GNU General Public License along
     15 with this program; if not, write to the Free Software Foundation, Inc.,
     16 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
     17
     18 See also http://www.gnu.org/licenses/gpl.txt.
    1919 */
    2020package sbrn.base;
     21
     22import java.io.IOException;
     23import java.io.InputStream;
     24import java.net.InetAddress;
     25import java.net.UnknownHostException;
     26import java.text.SimpleDateFormat;
     27import java.util.Date;
     28import java.util.Enumeration;
     29import java.util.List;
     30import java.util.Locale;
     31import java.util.zip.ZipEntry;
     32import java.util.zip.ZipFile;
     33import java.util.zip.ZipInputStream;
    2134
    2235import net.sf.basedb.core.Affymetrix;
     
    3245import net.sf.basedb.core.ItemResultList;
    3346import net.sf.basedb.core.Job;
    34 import net.sf.basedb.core.Path;
     47import net.sf.basedb.core.Location;
    3548import net.sf.basedb.core.Permission;
    3649import net.sf.basedb.core.PluginConfiguration;
     
    3952import net.sf.basedb.core.PluginResponse;
    4053import net.sf.basedb.core.ProgressReporter;
     54import net.sf.basedb.core.Quota;
     55import net.sf.basedb.core.QuotaType;
    4156import net.sf.basedb.core.RawBioAssay;
    4257import net.sf.basedb.core.RawDataType;
    43 import net.sf.basedb.core.RawDataTypes;
    4458import net.sf.basedb.core.SessionControl;
    4559import net.sf.basedb.core.SimpleProgressReporter;
    4660import net.sf.basedb.core.StringParameterType;
     61import net.sf.basedb.core.SystemItems;
    4762import net.sf.basedb.core.User;
    4863import net.sf.basedb.core.plugin.Response;
     
    5267import net.sf.basedb.util.zip.FileUnpacker;
    5368
    54 import java.io.InputStream;
    55 import java.net.InetAddress;
    56 import java.net.UnknownHostException;
    57 import java.text.SimpleDateFormat;
    58 import java.util.ArrayList;
    59 import java.util.Date;
    60 import java.util.List;
    61 import java.util.Locale;
    62 
    63 
    6469/**
    65   Imports raw data from a batch of flat files into BASE2. The data files must be packaged as a zip file and conform
    66   to the same data format. It is assumed that this code is executed after an Experiment object has been created already
    67   and that it is of the same data type as the data that are being imported. Currently only supports the creation of
    68   RawBioAssay objects as part of the import but could be extended to create other objects as needed.
    69 
    70   @author Micha Bayer - Plant Bioinformatics Group, Scottish Crop Research Institute
    71   email: sbrn@scri.ac.uk, web http://www.scri.ac.uk
     70 * Imports raw data from a batch of flat files into BASE2. The data files must
     71 * be packaged as a zip file and conform to the same data format. It is assumed
     72 * that this code is executed after an Experiment object has been created
     73 * already and that it is of the same data type as the data that are being
     74 * imported. Currently only supports the creation of RawBioAssay objects as part
     75 * of the import but could be extended to create other objects as needed.
     76 *
     77 * @author Micha Bayer - Plant Bioinformatics Group, Scottish Crop Research
     78 *         Institute email: sbrn@scri.ac.uk, web http://www.scri.ac.uk
    7279 */
    7380public class BatchDataImport
    7481{
    75 //  ======================================vars=============================
    76 
    77   //BASE client infrastructure
    78   private DbControl dc  = null;
     82  // ======================================vars=============================
     83
     84  // BASE client infrastructure
     85  private DbControl dc = null;
     86
    7987  private SessionControl sc = null;
    8088
    81   //the directory the files for import are located in
     89  // the directory the files for import are located in
    8290  private Directory dir = null;
    8391
    84   //the name of the zip file minus the extension
     92  // the name of the zip file minus the extension
    8593  private String fileName = null;
    8694
    87   //the zip file we want to import
     95  // the zip file we want to import
    8896  private File zipFile = null;
    8997
    90   //the Experiment that this import will be attached to
     98  // the Experiment that this import will be attached to
    9199  private Experiment experiment = null;
    92100
    93   //the progress reporter for the overall batch import
     101  // the progress reporter for the overall batch import
    94102  private ProgressReporter progress = null;
    95103
    96   //the percentage complete of the currently executing file import (subjob)
     104  // the percentage complete of the currently executing file import
     105  // (subjob)
    97106  private double currentJobPercentComplete = 0;
    98107
    99   //the currently executing file import job
     108  // the currently executing file import job
    100109  private Job currentJob = null;
    101110
    102   //the cdf file for the array desing for Affy data
     111  // the cdf file for the array desing for Affy data
    103112  private File cdfFile = null;
    104113
    105 //  ================================c'tor=================================
    106 
    107   public BatchDataImport(SessionControl sc,File zipFile,Experiment experiment,ProgressReporter progress)
     114  // ================================c'tor=================================
     115
     116  public BatchDataImport(SessionControl sc, File zipFile, Experiment experiment, ProgressReporter progress)
    108117  {
    109118    dc = sc.newDbControl();
     
    114123  }
    115124
    116 //  ================================methods==================================
     125  // ================================methods==================================
    117126
    118127  /**
    119    Unpacks the zip file in a new directory in the user's raw data directory. The new dir is named after the name of the zip file
    120    (minus the extension).
     128   * Unpacks the zip file in a new subdirectory of the directory where the zip file is located.
     129   * The new dir is named after the name of the zip file (minus the extension).
    121130   */
    122131  public boolean unpackZipFile() throws Exception
    123132  {
    124     User user = null;
    125     Directory homeDir = null;   
     133
    126134    int numUnpackedFiles = 0;
    127     boolean unpackSuccess = false; 
     135    boolean unpackSuccess = false;
    128136
    129137    try
    130138    {
    131       // get User id and home dir
     139      // figure out where the zip file is located
     140      Directory zipFileDir = zipFile.getDirectory();
     141      System.out.println("zipFile is located in dir " + zipFileDir.toString());
     142
     143      // make a new subdir in that dir
     144      // the zip file should then be unzipped in here
     145      // give it the same name as that of the zip file plus a
     146      // timestamp
     147      String timeStamp = getTime("ddMMyy_HHmmss");
     148      Directory subDir = Directory.getNew(dc, zipFileDir);
     149      fileName = zipFile.getName();
     150      String fileNameNoExt = fileName.substring(0, fileName.indexOf("."));
     151      System.out.println("new subDir = " + fileNameNoExt + timeStamp);
     152      subDir.setName(fileNameNoExt + timeStamp);
     153
     154      // save the new subDir
     155      dc.saveItem(subDir);
     156      dir = subDir;
     157     
     158      //check the size of the zip file content
     159      //if unzipping this file we must not execeed our disk quota
     160      InputStream inStream = zipFile.getDownloadStream(0);     
     161      long zipFileSize = getZipfileEntriesSize(inStream);
     162     
     163      //get the user's current disk usage and quota
    132164      int userID = sc.getLoggedInUserId();
    133       user = User.getById(dc, userID);
    134       homeDir = user.getHomeDirectory();
    135       homeDir.checkPermission(Permission.WRITE);
    136       homeDir.checkPermission(Permission.CREATE);     
    137       System.out.println("user = " + user.getName());
    138 
    139       //make a new subdir in the user's data dir
    140       //the zip file should then be unzipped in here
    141       //give it the same name as that of the zip file plus a timestamp
    142       String timeStamp = getTime("ddMMyy_HHmmss");
    143       Directory dataDir = Directory.getByPath(dc, new Path(homeDir.getPath().toString()+"/raw data",Path.Type.DIRECTORY));           
    144       Directory subDir = Directory.getNew(dc,dataDir); 
    145       fileName = zipFile.getName();
    146       String fileNameNoExt = fileName.substring(0, fileName.indexOf("."));     
    147       System.out.println("new subDir = " + fileNameNoExt+timeStamp);
    148       subDir.setName(fileNameNoExt+timeStamp);
    149 
    150       //save the new subDir
    151       dc.saveItem(subDir); 
    152       dir = subDir;
    153 
    154       //now unzip the zip file into the new subdir
     165      User user = User.getById(dc, userID);
     166      Quota quota = user.getQuota();
     167      QuotaType totalQuotaType = QuotaType.getById(dc, SystemItems.getId(QuotaType.TOTAL));
     168      //unlimited storage (e.g. for user root) will return -1 here so need to check for this further down
     169      long quotaValue = quota.getQuotaValue(totalQuotaType, Location.PRIMARY);     
     170      long currentDiskUsage = user.getDiskUsage(totalQuotaType, Location.PRIMARY);
     171     
     172      //check whether remaining disk space and zip file size are compatible
     173      long remainingSpace = quotaValue - currentDiskUsage;
     174      System.out.println("quotaValue for user " + user.toString() + " = " + quotaValue);
     175      System.out.println("currentDiskUsage = " + currentDiskUsage);
     176      System.out.println("remainingSpace = " + remainingSpace);
     177      System.out.println("zipFileSize = " + zipFileSize);
     178      if(remainingSpace < zipFileSize && quotaValue!=-1L)
     179      {
     180        throw new Exception("insufficient disk space remaining for unzipping of file -- please contact your BASE administrator or remove some files");
     181      }
     182
     183      // now unzip the zip file into the new subdir
    155184      if (zipFile != null)
    156       {             
     185      {
    157186        PluginDefinition plugin = PluginDefinition.getByClassName(dc, "net.sf.basedb.plugins.ZipFileUnpacker");
    158         FileUnpacker unpacker = plugin.newInstance(net.sf.basedb.plugins.ZipFileUnpacker.class,null,sc,null,null);       
     187        FileUnpacker unpacker = plugin.newInstance(net.sf.basedb.plugins.ZipFileUnpacker.class, null, sc, null, null);
    159188        InputStream uploadStream = zipFile.getDownloadStream(0);
    160189        if (unpacker != null)
     
    162191          System.out.println("unpacking");
    163192          numUnpackedFiles = unpacker.unpack(dc, subDir, uploadStream, false, null);
    164         }       
     193        }
    165194        uploadStream.close();
    166195      }
    167196
    168       //commit the changes
     197      // commit the changes
    169198      dc.commit();
    170199
    171       //can't check for all the files so we have to assume that if any files were unpacked ok, they all worked
    172       if(numUnpackedFiles > 0)
     200      // can't check for all the files so we have to assume
     201      // that if any files were unpacked ok, they all worked
     202      if (numUnpackedFiles > 0)
    173203      {
    174204        unpackSuccess = true;
     
    179209    {
    180210      e.printStackTrace();
    181       throw new Exception("zip file unpacking failed: "+ e.getMessage());
     211      throw new Exception("zip file unpacking failed: " + e.getMessage());
     212    }
     213    finally
     214    {
     215      if (dc != null)
     216        dc.close();
    182217    }
    183218    return unpackSuccess;
    184219  }
    185220
    186 //  -----------------------------------------------------------------------------------------------------------------------------------------------
    187 
     221  // -----------------------------------------------------------------------------------------------------------------------------------------------
    188222
    189223  /**
    190    Imports the data from a batch of text files into new RawBioAssay objects (one per file), then creates a new
    191    experiment with all of these RBAs attached.
    192    @return boolean success -- true if everything worked as defined
     224   * Imports the data from a batch of text files into new RawBioAssay
     225   * objects (one per file), then creates a new experiment with all of
     226   * these RBAs attached.
     227   *
     228   * @return boolean success -- true if everything worked as defined
    193229   */
    194230  public boolean importData() throws Exception
     
    200236    System.out.println("dir = " + dir.getName());
    201237
    202     //get the raw data type from the experiment
    203     RawDataType rawDataType = experiment.getRawDataType(); 
     238    // get the raw data type from the experiment
     239    RawDataType rawDataType = experiment.getRawDataType();
    204240    System.out.println("rawDataType = " + rawDataType.getName());
    205241
    206242    ItemQuery<File> fileQuery = null;
    207     ItemResultList<File> fileList = null; 
    208 
    209     if(dc == null || !dc.isConnected())
    210       dc=sc.newDbControl();
     243    ItemResultList<File> fileList = null;
     244
     245    if (dc == null || !dc.isConnected())
     246      dc = sc.newDbControl();
    211247
    212248    try
    213     {     
    214       //list all files in dir -- this should be what we unzipped from the zip file
     249    {
     250      // list all files in dir -- this should be what we
     251      // unzipped from the zip file
    215252      fileQuery = dir.getFiles();
    216       fileList = fileQuery.list(dc); 
    217 
    218       File [] dataFiles = null;
    219       if(fileQuery!=null)
    220       {
    221         //for Affy data the zip file uploaded should contain the cdf file for the chip and all the cel files
    222         //need to separate these out
     253      fileList = fileQuery.list(dc);
     254
     255      File[] dataFiles = null;
     256      if (fileQuery != null)
     257      {
     258        // for Affy data the zip file uploaded should
     259        // contain the cdf file for the chip and all the
     260        // cel files
     261        // need to separate these out
    223262        if (rawDataType.getName().equalsIgnoreCase("affymetrix"))
    224263        {
    225264
    226           //iterate over all the files and check their extensions
    227           int dataFileCount =0;
    228           //initiate a shorter array for the data files since this should not include the cdf file
    229           dataFiles = new File[fileList.size()-1]; 
     265          // iterate over all the files and check
     266          // their extensions
     267          int dataFileCount = 0;
     268          // initiate a shorter array for the data
     269          // files since this should not include
     270          // the cdf file
     271          dataFiles = new File[fileList.size() - 1];
    230272          System.out.println("instantiating dataFiles array of length " + dataFiles.length);
    231          
     273
    232274          System.out.println("affy data -- iterating over file names");
    233275          for (int i = 0; i < fileList.size(); i++)
    234276          {
    235             //get the current file and check its extension
     277            // get the current file and
     278            // check its extension
    236279            File file = fileList.get(i);
    237280            String fileName = file.getName();
    238             String extension = fileName.substring(fileName.lastIndexOf(".")+1);
    239             System.out.println("file name = " +fileName + ", extension = " + extension);
    240 
    241             //if it's the cdf file, point the global var for this at it
    242             if(extension.equalsIgnoreCase("cdf"))
     281            String extension = fileName.substring(fileName.lastIndexOf(".") + 1);
     282            System.out.println("file name = " + fileName + ", extension = " + extension);
     283
     284            // if it's the cdf file, point
     285            // the global var for this at it
     286            if (extension.equalsIgnoreCase("cdf"))
    243287            {
    244288              System.out.println("cdf file found");
    245289              cdfFile = file;
    246               System.out.println("cdf file = " + cdfFile);             
     290              System.out.println("cdf file = " + cdfFile);
    247291            }
    248             //if it's a cel file, add it to the dataFiles array
    249             else if(extension.equalsIgnoreCase("cel"))
     292            // if it's a cel file, add it to
     293            // the dataFiles array
     294            else if (extension.equalsIgnoreCase("cel"))
    250295            {
    251                 System.out.println("setting dataFiles["+dataFileCount+"] to file " + file.getName());
    252                 dataFiles[dataFileCount] = file;
    253                 dataFileCount++;
     296              System.out.println("setting dataFiles[" + dataFileCount + "] to file " + file.getName());
     297              dataFiles[dataFileCount] = file;
     298              dataFileCount++;
    254299            }
    255             //if it is neither a cel file nor a cdf file then the user must have defined the Experiment as
    256             //being Affy but the data they are importing is not Affy
    257             //in that case we need to throw an Exception and abort the whole import
     300            // if it is neither a cel file
     301            // nor a cdf file then the user
     302            // must have defined the
     303            // Experiment as
     304            // being Affy but the data they
     305            // are importing is not Affy
     306            // in that case we need to throw
     307            // an Exception and abort the
     308            // whole import
    258309            else
    259310            {
     
    262313          }
    263314        }
    264         else//non-Affy data types
     315        else
     316        // non-Affy data types
    265317        {
    266           //in this case all the files in the zip file should be data files
    267           dataFiles = new File[fileList.size()]; 
     318          // in this case all the files in the zip
     319          // file should be data files
     320          dataFiles = new File[fileList.size()];
    268321          System.out.println("instantiating dataFiles array of length " + dataFiles.length);
    269322
    270           //put the files from the list into the array for homogeneity's sake
     323          // put the files from the list into the
     324          // array for homogeneity's sake
    271325          fileList.toArray(dataFiles);
    272326        }
    273327
    274         //list all the data files
    275         System.out.println("data files: "); 
    276         for (int i = 0; i< dataFiles.length; i++)
    277         {               
     328        // list all the data files
     329        System.out.println("data files: ");
     330        for (int i = 0; i < dataFiles.length; i++)
     331        {
    278332          System.out.println(dataFiles[i].getName());
    279333        }
    280334      }
    281335
    282       //earmark this one file for autodetecting the file format
    283       //all files must have same format
     336      // earmark this one file for autodetecting the file
     337      // format
     338      // all files must have same format
    284339      File exampleFile = fileList.get(0);
    285340      System.out.println("exampleFile = " + exampleFile.getName());
    286      
    287       //check we don't have a discrepancy between the raw data type and the data files here
    288       //if users have specified a non-affy data type and the data is affy we need to abort the import with an appropriate error
    289       //get the current file and check its extension
     341
     342      // check we don't have a discrepancy between the raw
     343      // data type and the data files here
     344      // if users have specified a non-affy data type and the
     345      // data is affy we need to abort the import with an
     346      // appropriate error
     347      // get the current file and check its extension
    290348      String exampleFileName = exampleFile.getName();
    291       String exampleFileExtension = exampleFileName.substring(exampleFileName.lastIndexOf(".")+1);
    292       if(exampleFileExtension.equalsIgnoreCase("cdf") || exampleFileExtension.equalsIgnoreCase("cel"))
     349      String exampleFileExtension = exampleFileName.substring(exampleFileName.lastIndexOf(".") + 1);
     350      if ((exampleFileExtension.equalsIgnoreCase("cdf") || exampleFileExtension.equalsIgnoreCase("cel")) && !rawDataType.getName().equalsIgnoreCase("affymetrix"))
    293351      {
    294352        throw new Exception("Affymetrix data found but Experiment data type was specified as non-Affy");
    295353      }
    296354
    297       //for non-Affy data types, find a suitable plugin configuration
     355      // for non-Affy data types, find a suitable plugin
     356      // configuration
    298357      PluginConfiguration importConfig = null;
    299358      if (!rawDataType.getName().equalsIgnoreCase("affymetrix"))
    300359      {
    301         //get the appropriate plugin config
     360        // get the appropriate plugin config
    302361        PluginConfigDetector detector = new PluginConfigDetector(sc);
    303         importConfig = detector.detectPluginConfig(exampleFile);     
     362        importConfig = detector.detectPluginConfig(exampleFile);
    304363        System.out.println("PluginConfiguration importConfig = " + importConfig);
    305        
    306         //now need to figure out the raw data type to make sure it matches that of the Experiment we are importing into
    307         //this can be done by looking at the plugin config which should have it stored in it
     364
     365        // now need to figure out the raw data type to
     366        // make sure it matches that of the Experiment
     367        // we are importing into
     368        // this can be done by looking at the plugin
     369        // config which should have it stored in it
    308370        List rdtVal = importConfig.getParameterValues("rawDataType");
    309         String rawDataTypeStr = (String)rdtVal.get(0);
    310         System.out.println("rawDataType detected from files = " + rawDataTypeStr); 
    311         //compare this to the raw data type of the experiment
    312         if(!rawDataType.getName().equalsIgnoreCase(rawDataTypeStr))
     371        String rawDataTypeStr = (String) rdtVal.get(0);
     372        System.out.println("rawDataType detected from files = " + rawDataTypeStr);
     373        // compare this to the raw data type of the
     374        // experiment
     375        if (!rawDataType.getName().equalsIgnoreCase(rawDataTypeStr))
    313376        {
    314377          throw new Exception("raw data types of experiment and data files do not match");
    315378        }
    316        
     379
    317380      }
    318381
    319382      // for each data file
    320       //create a new RawBioAssay
    321       RawBioAssay [] rawBioAssays = new RawBioAssay [dataFiles.length];
     383      // create a new RawBioAssay
     384      RawBioAssay[] rawBioAssays = new RawBioAssay[dataFiles.length];
    322385      System.out.println("dataFiles.length = " + dataFiles.length);
    323386      for (int i = 0; i < dataFiles.length; i++)
    324387      {
    325         if(dc.isClosed())
     388        if (dc.isClosed())
    326389          dc = sc.newDbControl();
    327390        System.out.println("current data file is " + dataFiles[i].getName());
    328         RawBioAssay rba = createRawBioAssay(dc, "raw bioassay "+ dataFiles[i].getName(),rawDataType);
     391        RawBioAssay rba = createRawBioAssay(dc, "raw bioassay " + dataFiles[i].getName(), rawDataType);
    329392        rawBioAssays[i] = rba;
    330       } 
    331 
    332       //write all this to the database
     393      }
     394
     395      // write all this to the database
    333396      dc.commit();
    334397
    335       //make sure we have a working dbcontrol object
    336       if(dc.isClosed())
    337         dc = sc.newDbControl(); 
    338 
    339       //monitor the job progress
    340       ProgressMonitorThread pm = new ProgressMonitorThread(progress,rawBioAssays.length,this);
     398      // make sure we have a working dbcontrol object
     399      if (dc.isClosed())
     400        dc = sc.newDbControl();
     401
     402      // monitor the job progress
     403      ProgressMonitorThread pm = new ProgressMonitorThread(progress, rawBioAssays.length, this);
    341404      pm.start();
    342405
    343       //import the data for each RawBioAssay
    344       //only do this for non-Affy data types
     406      // import the data for each RawBioAssay
     407      // only do this for non-Affy data types
    345408      if (!rawDataType.getName().equalsIgnoreCase("affymetrix"))
    346409      {
    347         for (int i = 0; i< rawBioAssays.length; i++)
     410        for (int i = 0; i < rawBioAssays.length; i++)
    348411        {
    349412          SimpleProgressReporter pr = new SimpleProgressReporter(null);
     
    352415          pm.setCount(i);
    353416        }
    354       }
    355       else//Affy data
     417      }
     418      else
     419      // Affy data
    356420      {
    357421        System.out.println("Affymetrix experiment -- attaching cel files to raw bioassays");
    358422
    359         //make a new ArrayDesign for this affy chip
     423        // make a new ArrayDesign for this affy chip
    360424        ArrayDesign arrayDesign = ArrayDesign.getNew(dc, true);
    361425        arrayDesign.setName(cdfFile.getName());
    362426        dc.saveItem(arrayDesign);
    363427        dc.commit();
    364         //ArrayDesign arrayDesign = ArrayDesign.getById(dc, 7);
     428        // ArrayDesign arrayDesign =
     429        // ArrayDesign.getById(dc, 7);
    365430        System.out.println("using arrayDesign object " + arrayDesign.getName());
    366        
    367         //make sure we have a working dbcontrol object
    368         if(dc.isClosed())
    369           dc = sc.newDbControl();         
    370 
    371         //check the cdf file exists -- this should have been included in the zip file
    372         //else fail the whole import
    373         if(cdfFile == null)
     431
     432        // make sure we have a working dbcontrol object
     433        if (dc.isClosed())
     434          dc = sc.newDbControl();
     435
     436        // check the cdf file exists -- this should have
     437        // been included in the zip file
     438        // else fail the whole import
     439        if (cdfFile == null)
    374440        {
    375441          throw new Exception("no cdf file found for Affy data -- check that the zip file included this");
    376442        }
    377443
    378         //now set the cdf file on the array design
     444        // now set the cdf file on the array design
    379445        System.out.println("setting cdf file on the array design object");
    380446        dc.reattachItem(arrayDesign);
    381447        dc.reattachItem(cdfFile);
    382448        Affymetrix.setCdfFile(arrayDesign, cdfFile);
    383        
    384         //make sure we have a working dbcontrol object
    385         if(dc.isClosed())
    386           dc = sc.newDbControl(); 
     449
     450        // make sure we have a working dbcontrol object
     451        if (dc.isClosed())
     452          dc = sc.newDbControl();
    387453
    388454        System.out.println("adding cel files to raw bioassays");
    389         for (int i = 0; i< rawBioAssays.length; i++)
    390         { 
     455        for (int i = 0; i < rawBioAssays.length; i++)
     456        {
    391457          RawBioAssay rba = RawBioAssay.getById(dc, rawBioAssays[i].getId());
    392          
     458
    393459          System.out.println("iteration " + i);
    394460          System.out.println("rba = " + rba.getName());
    395461          System.out.println("file = " + dataFiles[i].getName());
    396          
    397           //must set the array design
     462
     463          // must set the array design
    398464          rba.setArrayDesign(arrayDesign);
    399465
    400           //need to attach the cel file to the bioassay object
    401           Affymetrix.setCelFile(rba, dataFiles[i]);
     466          // need to attach the cel file to the
     467          // bioassay object
     468          Affymetrix.setCelFile(rba, dataFiles[i]);
    402469        }
    403470      }
    404      
    405       //make sure we have a working dbcontrol object
    406       if(dc.isClosed())
    407         dc = sc.newDbControl();     
    408       //commit the changes to the database
     471
     472      // make sure we have a working dbcontrol object
     473      if (dc.isClosed())
     474        dc = sc.newDbControl();
     475      // commit the changes to the database
    409476      dc.commit();
    410477
    411       //now attach all the new RawBioAssay objects to the current Experiment item
    412       //this is the Experiment the user will be running the import from
    413       configureExperiment(dc, rawDataType,rawBioAssays);
    414 
    415       //make sure we have a working dbcontrol object
    416       if(dc.isClosed())
    417         dc = sc.newDbControl();
    418 
    419       //check whether the import worked
    420       //if item 0 in the list of RawBioAssay objects can be retrieved, we assume it all worked
     478      // now attach all the new RawBioAssay objects to the
     479      // current Experiment item
     480      // this is the Experiment the user will be running the
     481      // import from
     482      configureExperiment(dc, rawDataType, rawBioAssays);
     483
     484      // make sure we have a working dbcontrol object
     485      if (dc.isClosed())
     486        dc = sc.newDbControl();
     487
     488      // check whether the import worked
     489      // if item 0 in the list of RawBioAssay objects can be
     490      // retrieved, we assume it all worked
    421491      ItemQuery<RawBioAssay> query = RawBioAssay.getQuery();
    422492      query.restrict(Restrictions.eq(Hql.property("name"), Expressions.string(rawBioAssays[0].getName())));
    423493      RawBioAssay testRba = query.list(dc).get(0);
    424       if(testRba!=null)
     494      if (testRba != null)
    425495      {
    426496        importSuccess = true;
     
    435505    finally
    436506    {
    437       if(dc!=null)
     507      if (dc != null)
    438508        dc.close();
    439509    }
     
    445515
    446516  /**
    447    Import features.
     517   * Import features.
    448518   */
    449   private void importRawData(PluginConfiguration config, File file, RawBioAssay rba,SimpleProgressReporter pr) throws Exception
    450   {
    451     System.out.println("importing raw data from file: " + file.getName() );
     519  private void importRawData(PluginConfiguration config, File file, RawBioAssay rba, SimpleProgressReporter pr) throws Exception
     520  {
     521    System.out.println("importing raw data from file: " + file.getName());
    452522    Job job = null;
    453523    try
    454524    {
    455       //make sure we have a working dbcontrol object
    456       if(dc.isClosed())
     525      // make sure we have a working dbcontrol object
     526      if (dc.isClosed())
    457527        dc = sc.newDbControl();
    458528
     
    469539      dc.saveItem(job);
    470540      dc.commit();
    471       executeJob(job,pr);
    472     }
    473     catch(Exception x)
     541      executeJob(job, pr);
     542    }
     543    catch (Exception x)
    474544    {
    475545      x.printStackTrace();
     
    477547    finally
    478548    {
    479       if(!dc.isClosed())
     549      if (!dc.isClosed())
    480550        dc.close();
    481551    }
     
    485555
    486556  /**
    487    Execute a job.
    488   @param job
     557   * Execute a job.
     558   *
     559   * @param job
    489560   */
    490   private void executeJob(Job job,SimpleProgressReporter pr)
    491   {
    492     //make sure we have a working dbcontrol object
    493     if(dc.isClosed())
     561  private void executeJob(Job job, SimpleProgressReporter pr)
     562  {
     563    // make sure we have a working dbcontrol object
     564    if (dc.isClosed())
    494565      dc = sc.newDbControl();
    495566
     
    497568    try
    498569    {
    499       localhost =InetAddress.getLocalHost().toString();
     570      localhost = InetAddress.getLocalHost().toString();
    500571    }
    501572    catch (UnknownHostException e)
     
    504575    }
    505576
    506     System.out.println("Executing job: " + job.getName() );
     577    System.out.println("Executing job: " + job.getName());
    507578    job = Job.getById(dc, job.getId());
    508579
     
    510581    dc.commit();
    511582
    512     //execute the job request
     583    // execute the job request
    513584    PluginResponse response = request.invoke();
    514585
    515     //check the job ended sucessfully
     586    // check the job ended sucessfully
    516587    if (response.getStatus() == Response.Status.ERROR)
    517588    {
    518589      throw new BaseException(response.getMessage(), response.getErrorList().get(0));
    519590    }
    520     System.out.println("Job ended successfully: " + response.getMessage() );
    521 
    522     if(!dc.isClosed())
     591    System.out.println("Job ended successfully: " + response.getMessage());
     592
     593    if (!dc.isClosed())
    523594      dc.close();
    524595  }
    525596
    526 
    527597  // -------------------------------------------------------------------------------------------------------------------------------------------
    528598
    529599  /**
    530    Adds the raw bioassays to our experiment.
     600   * Adds the raw bioassays to our experiment.
    531601   */
    532   private void configureExperiment(DbControl dc, RawDataType rawDataType, RawBioAssay [] rawBioAssays) throws Exception
     602  private void configureExperiment(DbControl dc, RawDataType rawDataType, RawBioAssay[] rawBioAssays) throws Exception
    533603  {
    534604    try
    535605    {
    536       //make sure we have a working dbcontrol object
    537       if(dc.isClosed())
     606      // make sure we have a working dbcontrol object
     607      if (dc.isClosed())
    538608        dc = sc.newDbControl();
    539609
    540610      System.out.println("Configuring experiment " + experiment);
    541611
    542       //make sure the raw data type is set to the correct value
    543       //rather than relying on the one specified by the user, we want to use the one that we have
    544       //actually detected by looking at the data to be imported (see above)
    545       //check what the user has set is correct -- if not, we need to fail the whole run
     612      // make sure the raw data type is set to the correct
     613      // value
     614      // rather than relying on the one specified by the user,
     615      // we want to use the one that we have
     616      // actually detected by looking at the data to be
     617      // imported (see above)
     618      // check what the user has set is correct -- if not, we
     619      // need to fail the whole run
    546620      RawDataType actualRdt = experiment.getRawDataType();
    547621      if (!actualRdt.getName().equals(rawDataType.getName()))
    548622      {
    549         throw new BaseException("Import failed -- the raw data type of the selected experiment and that of the "+
    550                 "data to be imported don't match. Please change your data type to " +rawDataType.getName() +
    551         " and try again.");
    552       }
    553 
    554       //now attach the raw bioassays
    555       //tried to do this directly on the Experiment object passed into this class -- that fails
    556       //the following is a workaround -- just make a fresh Experiment object from the ID of the one passed in
     623        throw new BaseException("Import failed -- the raw data type of the selected experiment and that of the " + "data to be imported don't match. Please change your data type to " + rawDataType.getName() + " and try again.");
     624      }
     625
     626      // now attach the raw bioassays
     627      // tried to do this directly on the Experiment object
     628      // passed into this class -- that fails
     629      // the following is a workaround -- just make a fresh
     630      // Experiment object from the ID of the one passed in
    557631      int expID = experiment.getId();
    558632      Experiment exp = Experiment.getById(dc, expID);
     
    565639      }
    566640
    567       //housekeeping
     641      // housekeeping
    568642      dc.commit();
    569       if(!dc.isClosed())
     643      if (!dc.isClosed())
    570644        dc.close();
    571645    }
     
    577651  }
    578652
    579 
    580653  // -------------------------------------------------------------------------------------------------------------------------------------------
    581654
    582655  /**
    583    Create a raw bioassay.
     656   * Create a raw bioassay.
    584657   */
    585   private RawBioAssay createRawBioAssay(DbControl dc, String name,RawDataType rawDataType)
    586   {
    587     if(dc.isClosed())
     658  private RawBioAssay createRawBioAssay(DbControl dc, String name, RawDataType rawDataType)
     659  {
     660    if (dc.isClosed())
    588661      dc = sc.newDbControl();
    589662
    590     System.out.println("Creating raw bioassay: " + name );
     663    System.out.println("Creating raw bioassay: " + name);
    591664    RawBioAssay rba = RawBioAssay.getNew(dc, rawDataType);
    592665    rba.setName(name);
     
    595668  }
    596669
    597 //  -------------------------------------------------------------------------------------------------------------------------------------------
     670  // -------------------------------------------------------------------------------------------------------------------------------------------
    598671
    599672  private String getTime(String pattern)
     
    602675    Locale currentLocale = Locale.getDefault();
    603676    formatter = new SimpleDateFormat(pattern, currentLocale);
    604     String timeStamp = formatter.format(new Date()); 
     677    String timeStamp = formatter.format(new Date());
    605678    return timeStamp;
    606679  }
    607680
    608 //  -------------------------------------------------------------------------------------------------------------------------------------------
     681  // -------------------------------------------------------------------------------------------------------------------------------------------
    609682
    610683  public Job getCurrentJob()
     
    613686  }
    614687
    615 
    616 
    617 //  -------------------------------------------------------------------------------------------------------------------------------------------
     688  // -------------------------------------------------------------------------------------------------------------------------------------------
     689 
     690  /**
     691   * Checks the total size of a given zip file (uncompressed)
     692   * @param InputStream fileInputStream -- an input stream from the zip file
     693   * @return long - the total size of the zip file uncompressed, in bytes
     694   *
     695   */
     696  public static long getZipfileEntriesSize(InputStream fileInputStream)
     697  {
     698    long totalSize =0;
     699    try
     700    {
     701      ZipInputStream zipStream = new ZipInputStream(fileInputStream);     
     702      ZipEntry entry = zipStream.getNextEntry();
     703      while(entry!=null)
     704      {
     705        long size = entry.getSize();
     706        totalSize += size;
     707        entry = zipStream.getNextEntry();
     708      }
     709    }
     710    catch (IOException ioe)
     711    {
     712      ioe.printStackTrace();
     713    }
     714    return totalSize;
     715  }
     716 
     717//   -------------------------------------------------------------------------------------------------------------------------------------------
    618718
    619719}// end class
Note: See TracChangeset for help on using the changeset viewer.