Changeset 374

Timestamp:

Aug 8, 2007, 12:22:46 PM (15 years ago)

Author:

dominic

Message:

 

Location:

trunk/uk/ac/ebi/Tab2MageImporter/src/uk/ac/ebi/nugo/plugins

Files:

• AnnotationTypeCvImporter.java (modified) (2 diffs)
• Tab2MageImporter.java (modified) (19 diffs)
• tab2mageimport/io/AnnotationTypeFileWriter.java (modified) (4 diffs)
• tab2mageimport/io/AnnotationTypesWriter.java (modified) (5 diffs)
• tab2mageimport/io/Base2Tab2MageFileReader.java (modified) (2 diffs)
• tab2mageimport/io/RawDataImport.java (modified) (5 diffs)
• tab2mageimport/io/Tab2MageDataImportFactory.java (modified) (7 diffs)
• tab2mageimport/io/Tab2MageExperimentWriter.java (modified) (7 diffs)
• tab2mageimport/io/Tab2MageHybWriter.java (modified) (31 diffs)
• tab2mageimport/io/Tab2MageProtocolWriter.java (modified) (5 diffs)
• tab2mageimport/io/Tab2MageWriterFactory.java (modified) (14 diffs)
• tab2mageimport/mail/Tab2MageEMessaging.java (modified) (5 diffs)
• tab2mageimport/validation/AgilentValidation.java (modified) (1 diff)
• tab2mageimport/validation/GenePixFileValidation.java (modified) (2 diffs)
• tab2mageimport/validation/RawDataValidation.java (modified) (1 diff)
• tab2mageimport/validation/Tab2MageImportValidatorUtils.java (modified) (10 diffs)
• tab2mageimport/validation/Tab2MageValidationFactory.java (modified) (22 diffs)
• util/PluginConfigDetector.java (modified) (5 diffs)
• util/Tab2MageHelper.java (modified) (7 diffs)
• util/ZipUnpacker.java (modified) (7 diffs)

trunk/uk/ac/ebi/Tab2MageImporter/src/uk/ac/ebi/nugo/plugins/AnnotationTypeCvImporter.java

@@ -34 +34 @@
 import net.sf.basedb.util.parser.FlatFileParser;
 import net.sf.basedb.util.parser.Mapper;
-import uk.ac.ebi.nugo.plugins.util.Values;
+import net.sf.basedb.util.Values;
 
 
@@ -419 +419 @@
   private ItemResultList<AnnotationTypeCategory> categories;
   private ItemQuery<AnnotationTypeCategory> categoryQuery;
-  private boolean updateExisting, enumeration,found;
+  private boolean updateExisting, enumeration;
   private String separator;
   private int numInserted;

trunk/uk/ac/ebi/Tab2MageImporter/src/uk/ac/ebi/nugo/plugins/Tab2MageImporter.java

@@ -8 +8 @@
 import java.io.IOException;
 import java.io.InputStream;
-import java.net.InetAddress;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Calendar;
 import java.util.Collections;
-import java.util.Date;
 import java.util.List;
 import java.util.Set;
@@ -19 +16 @@
 import java.util.zip.ZipInputStream;
 
-import uk.ac.ebi.nugo.plugins.tab2mageimport.io.AnnotationTypesWriter;
-import uk.ac.ebi.nugo.plugins.tab2mageimport.io.Tab2MageDataImportFactory;
-import uk.ac.ebi.nugo.plugins.tab2mageimport.io.Tab2MageExperimentWriter;
-import uk.ac.ebi.nugo.plugins.tab2mageimport.io.Tab2MageWriterFactory;
-import uk.ac.ebi.nugo.plugins.tab2mageimport.validation.Tab2MageValidationFactory;
-import uk.ac.ebi.nugo.plugins.tidy.RollBack;
-import uk.ac.ebi.nugo.plugins.util.NutribasePluginException;
-import uk.ac.ebi.nugo.plugins.util.Tab2MageHelper;
-import uk.ac.ebi.nugo.plugins.util.ZipUnpacker;
+import org.apache.log4j.Logger;
 
 import net.sf.basedb.core.BaseException;
-import net.sf.basedb.core.BooleanParameterType;
-import net.sf.basedb.core.DateParameterType;
 import net.sf.basedb.core.DbControl;
 import net.sf.basedb.core.Directory;
@@ -39 +26 @@
 import net.sf.basedb.core.Item;
 import net.sf.basedb.core.ItemParameterType;
-import net.sf.basedb.core.Path;
-import net.sf.basedb.core.PathParameterType;
 import net.sf.basedb.core.PluginParameter;
 import net.sf.basedb.core.ProgressReporter;
-import net.sf.basedb.core.RawBioAssay;
 import net.sf.basedb.core.RawDataType;
 import net.sf.basedb.core.RequestInformation;
-import net.sf.basedb.core.SessionControl;
 import net.sf.basedb.core.StringParameterType;
 import net.sf.basedb.core.User;
@@ -58 +41 @@
 import net.sf.basedb.core.plugin.Request;
 import net.sf.basedb.core.plugin.Response;
+import uk.ac.ebi.nugo.plugins.tab2mageimport.io.AnnotationTypesWriter;
+import uk.ac.ebi.nugo.plugins.tab2mageimport.io.Tab2MageDataImportFactory;
+import uk.ac.ebi.nugo.plugins.tab2mageimport.io.Tab2MageExperimentWriter;
+import uk.ac.ebi.nugo.plugins.tab2mageimport.io.Tab2MageWriterFactory;
 import uk.ac.ebi.nugo.plugins.tab2mageimport.mail.Tab2MageEMessaging;
-import uk.ac.ebi.nugo.plugins.tab2mageimport.ExperimentPlatformsLibrary;
+import uk.ac.ebi.nugo.plugins.tab2mageimport.validation.Tab2MageValidationFactory;
+import uk.ac.ebi.nugo.plugins.util.Tab2MageHelper;
+import uk.ac.ebi.nugo.plugins.util.ZipUnpacker;
 
 /**
@@ -83 +72 @@
   //  the gui email server parameter
   private PluginParameter<String> emailHostParameter;
+  // logger
+  private static Logger log = Logger.getLogger(Tab2MageImporter.class);
   
   private static final About about = 
@@ -179 +170 @@
     {
       response.setError(ex.getMessage(), Arrays.asList(ex));
-      ex.printStackTrace();
     }
     
@@ -202 +192 @@
   {
     String message = null;
-    if (context.equals(experimentItemGuiContext))
-    {
-      if (item == null)
-      {
-        message = "The object is null";
-      }
-      else if (!(item instanceof Experiment))
-      {
-        message = "The object is not an Experiment: " + item;
-      }
+    if (item == null)
+    {
+      message = "The object is null";
+      
+    }
+    else if (!(item instanceof Experiment))
+    {
+      message = "The object is not an Experiment: " + item;
+    }
+    else
+    {
+      Experiment exp = (Experiment)item;
+      RawDataType rawDataType = exp.getRawDataType(); 
+      if (!Tab2MageHelper.supportedPlatforms(rawDataType.getId().toLowerCase()))
+      {
+        message="This importer only supports Affymetrix, Agilent and Genepix Platforms. Your experiment's platform is :" + rawDataType.getId().toLowerCase();
+      } 
     }
     return message;
@@ -250 +247 @@
     {
       //who is using this plugin
-      //[ DC MESSAGE : dc 1 is created ]
+      //[ DC MESSAGE : Manager dc  is created ]
       dc= sc.newDbControl();
       User user= User.getById(dc, sc.getLoggedInUserId());
@@ -267 +264 @@
       RawDataType rdt= experiment.getRawDataType(); // experiment platform
             
-      boolean supported = Tab2MageHelper.supportedPlatforms(rdt.getId().toLowerCase());
-      if (!supported)
-      {
-        response.setError("IMPORT FAILS: This importer only supports Affymetrix, Agilent and Genepix Platforms" ,Arrays.asList(new Throwable()));
-        return;
-      }
       /*
         create the roll back object here for testing
@@ -279 +270 @@
       
       /*
-        unzip the files into user directory
-       */
-      Tab2MageHelper.write("The zipfile name is "+ file.getName());
-      ZipUnpacker importUnpacker = new ZipUnpacker(file, user, dc);
-      importUnpacker.unpack();
-      
-      // if unzip is sucessfull. (to check this return boolean value, we need to reimplement the util.ZipUnpacker, will affect the affyArrayDesignBatchImporter)
-      
-      //[ DC MESSAGE : dc 1 is closed ]
-      
-      // check that provided files conform to platform / experiment's platform, exit if not
-      
-      //    [ DC MESSAGE : dc 2 is created ]
-      
+        FILE UNZIP MODULE: unzip the files into user directory
+       */
+      ZipUnpacker importUnpacker = new ZipUnpacker(file, user, sc);
+      importUnpacker.unpack(); // may change the the util.ZipUnpacker to return a boolean will affect affyArrayDesignBatchImporter too
+      Directory dir= importUnpacker.getUnzippedFilesDirectory(); // this may return the id of the unzipped file directory instead
+      if (dir==null)
+      {
+        response.setError("IMPORT FAILS: The directory containing the unzipped files could not be found. You may NOT have sufficient Disk Space" ,Arrays.asList(new Throwable()));
+        return;
+      }
+      
+      // -- end of file unzip module
+      
+      /*
+        VALIDATION MODULE
+       */
+        //[ DC MESSAGE : if the manager dc is no longer available creates a new one.  It should be] 
       if (dc == null || !dc.isConnected()) dc = sc.newDbControl();
-      Directory dir= importUnpacker.getUnzippedFilesDirectory(); // this may return the id of the unzipped file direcotyr instead
-      if (dir==null)
-      {
-        response.setError("IMPORT FAILS: The directory containing the unzipped files could not be found. You may NOT have sufficient Disk Space" ,Arrays.asList(new Throwable()));
-        return;
-      }
-      
-      Tab2MageHelper.write("The directory containing the unzippped files is: "+ dir.getPath());
-          
+      
+        // validate files supplied to platform
       validPlatform=Tab2MageHelper.validatePlatform(dir, dc, rdt);
       if (!validPlatform)
@@ -310 +297 @@
             }
       
-      // validate image file
+        // validate image file
       List<File> imageFiles=Tab2MageHelper.getImageFiles(dir, dc) ;
-      if (imageFiles.isEmpty() || imageFiles!=null) Tab2MageHelper.write("No image file supplied"); // remove or log this later message
-      
       if (!imageFiles.isEmpty()|| imageFiles==null) 
       {
@@ -324 +309 @@
       }
       
-      // validate the tab2mage file (do initialisation here too)
-      Tab2MageValidationFactory validator = new Tab2MageValidationFactory(tab2MageFileName, dc, dir, rdt.getId());
+        // validate the tab2mage file (do initialisation here too)
+      Tab2MageValidationFactory validator = new Tab2MageValidationFactory(tab2MageFileName, sc, dir, rdt.getId());
       validTab2mageFile=validator.getValidated();
       if (!validTab2mageFile)
@@ -332 +317 @@
         return;
       }
-      
-      //  [ DC MESSAGE : dc 2 is closed ]
-      
-      //  Do import() of the tab2mage file.
-      
-      // Call and initiate the roll back routines
-      
-      //create annotation types
+      // -- end of validation module
+      
+      /*
+        TAB2MAGE FILE IMPORT MODULE.
+       */ 
+          //  create annotation types
       AnnotationTypesWriter atWriter= new AnnotationTypesWriter (sc, validator.getAtFile(),progress);
       boolean writtenAT =atWriter.write(); // note if this action failed the transaction is completely rolled back
@@ -348 +331 @@
       }
       /*
-        A new db control object necessary here to help with roll back
-       */
-        //write protocols and hybs
+        A new db control object may be necessary here to help with roll back
+       */
+        // write protocols and hybs
       Tab2MageWriterFactory hibernateItemsWriter = new Tab2MageWriterFactory
                           (sc,Tab2MageValidationFactory.getCatchedProtocol(),
-                            Tab2MageValidationFactory.getCatchedHybs(),rdt );
-      
-      //  [ DC MESSAGE : dc 3 is created ]
-      if (dc == null || !dc.isConnected()|| dc.isClosed()) dc = sc.newDbControl();
-      
-      // get raw data file before closing dc is closed
-      List<File> rawDataFiles= Tab2MageHelper.getRawDataFiles(dir, dc, rdt.getId(), tab2MageFileName);
-      
-      //NOTE: THIS IS ANOTHER DB TRANSACTION
-      // if the protocol type is not found, write it into the database.
-      // alternative is to fail at validation instead of a warning, if the protocol type 
-      // provided in tab2mage file is not in Base2 database. May may have to rework this.
-      hibernateItemsWriter.writeProtocolTypes(dc); // dc is closed here
-      
-      // dc 3 is closed
+                            Tab2MageValidationFactory.getCatchedHybs(),rdt,
+                            Tab2MageValidationFactory.getSampleExtractMap(),
+                            Tab2MageValidationFactory.getBioSourceSampleMap(),
+                            Tab2MageValidationFactory.getExtractLabeledExtractMap());
+      
+        // if the protocol type is not found, write it into the database.
+        // alternative is to fail at validation instead of a warning, if the protocol type 
+        // provided in tab2mage file is not in Base2 database. May may have to rework this.
+      
+      hibernateItemsWriter.writeProtocolTypes(sc); 
       
       boolean writtenHibItems = hibernateItemsWriter.write();
@@ -375 +353 @@
         return;
       }
-      for(String  fact : hibernateItemsWriter.getExpFactors())
-      {
-        Tab2MageHelper.write("\nThe factor values obtained : "+ fact);
-        //TODO need to clear the memory holding expfactor
-      }
-      /*
-        Do Data Import
-        A new db control object necessary here to help with roll back
-       */
-      
+      
+      // -- end of Tab2Mage file import module 
+      
+      /*
+        RAW DATA IMPORT MODULE
+        
+       */
+        //[ DC MESSAGE : if the manager dc is no longer available creates a new one.  It should be]
+      if (dc == null || !dc.isConnected()|| dc.isClosed()) dc = sc.newDbControl();
+      
+        // get raw data files
+      List<File> rawDataFiles= Tab2MageHelper.getRawDataFiles(dir, dc, rdt.getId(), tab2MageFileName);
+        
+        // do data import
       Tab2MageDataImportFactory dataImport = new Tab2MageDataImportFactory(sc,rdt.getId(),rawDataFiles,
                         hibernateItemsWriter.getRbas(), progress);
@@ -393 +375 @@
         return;
       }
-      
-      /*
-        Configure and load the experiment
-        A new dc is created here for another roll back point.
+      // -- end of raw data import module
+      
+      /*
+        EXPERIMENT CONFIGURATION AND LOADING MODULE
+        
        */     
+        //write experiment, attaching raw bio-assays
       Tab2MageExperimentWriter expWriter= new Tab2MageExperimentWriter(hibernateItemsWriter.getRbas(),sc,
                                       experiment,hibernateItemsWriter.getExpFactors(),
                                       validator.getTExperiment());
       boolean writtenExp= expWriter.LoadExperiment();
-      //  send an email to the user on the success of the experiment (if email server name and email address are provided) 
-      if (hostName!="" && hostName.length()>=5)
-      {
-        if (user.getEmail()!="" && user.getEmail()!=null )
+      // -- end of experiment configuration and loading module
+      
+      /*
+        MESSAGING MODULE
+       */
+        //  send an email to the user on the success of the experiment (if email server name and email address are provided) 
+      if (hostName!= null && hostName.length()>=5)
+      {
+        if (!user.getEmail().equals("") && user.getEmail()!=null )
         {
           Tab2MageEMessaging message= new Tab2MageEMessaging(writtenExp,"", user.getEmail(),tab2MageFileName, user.getName(), experiment.getName(),hostName );
@@ -411 +400 @@
         }
       }
-      
-      //  release object from memory and do clean up.
-      if (!dir.isRemoved()) dir.setRemoved(true);
+      // -- end of messaging module
+      
       response.setDone("Import plugin ended successfully");
     }
@@ -419 +407 @@
     {
       response.setError(tr.getMessage(), Arrays.asList(tr));
-        tr.printStackTrace();
-    }
+      }
     finally
     {
@@ -448 +435 @@
       catch (Exception e)
       {
-        Tab2MageHelper.write("File is not a zip file and cannot be imported by this plugin.");
-        e.printStackTrace();
+        log.error("File is not a zip file and cannot be imported by this plugin.");
         return false;
       }

trunk/uk/ac/ebi/Tab2MageImporter/src/uk/ac/ebi/nugo/plugins/tab2mageimport/io/AnnotationTypeFileWriter.java

@@ -47 +47 @@
     catch(Throwable t )
     {
-      t.printStackTrace();
+      throw new NutribasePluginException(t.getMessage());
     }
   }
@@ -78 +78 @@
     catch (IOException ioe)
     {
-      ioe.printStackTrace();
       throw new NutribasePluginException(ioe.getMessage());
     }
@@ -95 +94 @@
         if (at==null)
         {
-          System.out.println("\nAT is null\n");
           return;
         }
@@ -115 +113 @@
     catch (IOException ioe)
     {
-      ioe.printStackTrace();
       throw new NutribasePluginException(ioe.getMessage());
     }

trunk/uk/ac/ebi/Tab2MageImporter/src/uk/ac/ebi/nugo/plugins/tab2mageimport/io/AnnotationTypesWriter.java

@@ -69 +69 @@
   public AnnotationTypesWriter(SessionControl sc, File expFile, ProgressReporter progress)
   {
-    System.out.println("Annotation Type Writer Object Created");
     this.sc= sc;
     this.exampleFile= expFile;
@@ -88 +87 @@
       return written=false;
     }
-    System.out.println("PluginConfiguration importConfig = " + importConfig);
+    log.info("PluginConfiguration importConfig = " + importConfig);
     Job job = null;
     
     SimpleProgressReporter spr = new    SimpleProgressReporter(progress);
     if (dc ==null)        dc = sc.newDbControl();
-    System.out.println("+++Begin to execute job+++");
+    log.info("+++Begin to execute job+++");
     importConfig = PluginConfiguration.getById(dc, importConfig.getId());
     job = Job.getNew(dc, importConfig.getPluginDefinition(), importConfig);
@@ -111 +110 @@
         {
           written= false;
-          e.printStackTrace();
+          log.error(e.getMessage());
         }
     
-        System.out.println("Executing job: " + job.getName());
+        log.info("Executing job: " + job.getName());
         
         PluginExecutionRequest request = job.execute(spr, localhost);
@@ -128 +127 @@
           throw new BaseException(response.getMessage(), response.getErrorList().get(0));
         }
-        System.out.println("Job ended successfully: " + response.getMessage());
+        log.info("Job ended successfully: " + response.getMessage());
         return written;
     }
@@ -134 +133 @@
       {
       dc.close();
-      e.printStackTrace();
+      log.error(e.getMessage());
       return false;
-      //throw new Exception(e);
       }
       finally

trunk/uk/ac/ebi/Tab2MageImporter/src/uk/ac/ebi/nugo/plugins/tab2mageimport/io/Base2Tab2MageFileReader.java

@@ -332 +332 @@
   }
   
-  private void createAnnotationTypeFile() throws TAB2MAGEException
-  {
-    String filename= "anntype.txt";
-    try
-    {
-      // create a file named: anntype.txt into the subdirectory (temp) created above
-      atFile =  File.getFile(dc, subDir, filename, true);
-      atFile.setMimeType(file.getMimeType());
-      if (file.getFileType()!=null)
-      {
-        atFile.setFileType(file.getFileType());
-      }
-      // if the new file exist in the database,  
-      if (atFile.isInDatabase())
-      {
-        throw new ItemAlreadyExistsException("Another file with the same name already exists");
-        
-      }
-      else // save the file to the database.
-      {
-        dc.saveItem(atFile);
-      }
-    }
-    catch (Throwable e) 
-    {
-      log.error("Error: " + e.getMessage());
-      throw new TAB2MAGEException(e.getMessage());
-    }
-  }
-  
   private void createExperimentFile() throws TAB2MAGEException
   {
@@ -697 +667 @@
               {
                 protocol.set( key, rowValues.get(j) );
-                //System.out.println("This protocol row value thing is : "+rowValues.get(j).toString());
               }
               catch(IndexOutOfBoundsException e)
               {
                 protocol.set( key, "" );
-                //System.out.println("This protocol row value thing is null and changed to empty string");
               }
               continue cont;

trunk/uk/ac/ebi/Tab2MageImporter/src/uk/ac/ebi/nugo/plugins/tab2mageimport/io/RawDataImport.java

@@ -26 +26 @@
 
 import org.apache.log4j.Logger;
-
-import uk.ac.ebi.nugo.plugins.util.Tab2MageHelper;
 /**
   This class writes raw data for multi - channel experiments.
@@ -89 +87 @@
     public boolean importRawData() throws Exception
     {
-      System.out.println("++++ Importing raw data from file: " + file.getName());
-      //DbControl dc =null;
-    boolean written = true;
+      log.info("++++ Importing raw data from file: " + file.getName());
+      boolean written = true;
     try
     {
       Job job = null;
       SimpleProgressReporter spr = new    SimpleProgressReporter(progress);
-      Tab2MageHelper.write("+++Begin to execute job+++");
+      log.info("+++Begin to execute job+++");
       importConfig = PluginConfiguration.getById(dc, importConfig.getId());
       job = Job.getNew(dc, importConfig.getPluginDefinition(), importConfig);
@@ -120 +117 @@
       {
         written= false;
-        e.printStackTrace();
+        log.error(e.getMessage());
       }
       
-      Tab2MageHelper.write("Executing job: " + job.getName());
+      log.info("Executing job: " + job.getName());
       
       PluginExecutionRequest request = job.execute(spr, localhost);
@@ -137 +134 @@
         throw new BaseException(response.getMessage(), response.getErrorList().get(0));
       }
-      System.out.println("Job ended successfully: " + response.getMessage());
+      log.info("Job ended successfully: " + response.getMessage());
       return written;
     }
@@ -144 +141 @@
       dc.close();
       log.error("The raw data plugin could not load raw data");
-      e.printStackTrace();
       return false;
       }

trunk/uk/ac/ebi/Tab2MageImporter/src/uk/ac/ebi/nugo/plugins/tab2mageimport/io/Tab2MageDataImportFactory.java

@@ -109 +109 @@
       this.file =file;
     }
-    //System.out.println("++Raw Data File already set is : "+ file.getName());
+    log.info("++Raw Data File already set is : "+ file.getName());
   }
   
@@ -119 +119 @@
   public boolean importData()
   {
-    Tab2MageHelper.write("\n++++Begin Data Import\n");
+    log.info("\n++++Begin Data Import\n");
     DbControl dc =null;
     boolean success= true;
@@ -127 +127 @@
       //  :) extra validation is good for all
       checkFiles(); 
-      //System.out.println("file check is successful");
       if (platform.equalsIgnoreCase("affymetrix"))
       {
         if (dc ==null)
           dc = sc.newDbControl();
-        //System.out.println("adding cel files to raw bioassays");
+        log.info("adding cel files to raw bioassays");
             for (RawBioAssay rbas  :rawBioAssays )
             {
@@ -140 +139 @@
               if (file!=null)
               {
-               Tab2MageHelper.write("The file is: " + file.getName() );
                // need to attach the cel file to the bioassay object
                Affymetrix.setCelFile(rba, file);
@@ -149 +147 @@
       else if (!platform.equalsIgnoreCase("affymetrix"))// 2. import 2-channel -- include checkpoints and rollback points
       {
-        Tab2MageHelper.write(".......beginning ["+platform + "] data loading");
+        log.info(".......beginning ["+platform + "] data loading");
         if (dc ==null)
           dc = sc.newDbControl();
@@ -167 +165 @@
             dc = sc.newDbControl();
           RawBioAssay rba = RawBioAssay.getById(dc,rbas.getId());
-          //System.out.println("rba = " + rba.getName());
+          
           // get the file using file name. the file name is the raw bioassay name
           File file = getFile(rba.getName(), dc);
           if (file!=null)
           {
-            Tab2MageHelper.write("The name of the import configuration being used: "+ importConfig.getName());
+            log.info("The name of the import configuration being used: "+ importConfig.getName());
             RawDataImport rawDataImport = new RawDataImport(dc, importConfig, progress, file, rba);
             boolean writtenRawData = rawDataImport.importRawData(); // Problem: closes the db control
@@ -184 +182 @@
     {
       success= false;
-      t.printStackTrace();
       log.error("DataImport: writing raw data failed");
       dc.close();

trunk/uk/ac/ebi/Tab2MageImporter/src/uk/ac/ebi/nugo/plugins/tab2mageimport/io/Tab2MageExperimentWriter.java

@@ -19 +19 @@
 import org.embl.cg.bloader.models.Tab2MAGEExperiment;
 
-import uk.ac.ebi.nugo.plugins.util.Tab2MageHelper;
+import uk.ac.ebi.nugo.plugins.util.ValueFormatter;
 /**
   This class writes an experiment into the BASE 2 / Nutribase Database 
@@ -86 +86 @@
     {
       //write experiment
-      Tab2MageHelper.write("Loading experiment " + experiment);
+      log.info("Loading experiment " + experiment);
       int expID = experiment.getId();
       Experiment exp = Experiment.getById(dc, expID);
@@ -93 +93 @@
       {
         rba = RawBioAssay.getById(dc, rba.getId());
-        Tab2MageHelper.write("adding raw bioassay to experiment : " + rba);
+        log.info("adding raw bioassay to experiment : " + rba);
         exp.addRawBioAssay(rba);
         }
@@ -101 +101 @@
         AnnotationType at= getAnnotationTypeByName(factor);
         ats.add(at);
-        //Tab2MageHelper.write("The expr factor to add is : "+ at.toString());
       }
       if (!ats.isEmpty())
@@ -122 +121 @@
       exp.setPublication(publication);
       exp.setPublicationDate(Values.getDate(tab2mageExperiment.getPublication_date()));
-      //Tab2MageHelper.write("The pub med id is : "+ Values.getEmptyStringIfNull(tab2mageExperiment.getPubmed_id()));
-      exp.setPubMedId((String)Values.getEmptyStringIfNull(tab2mageExperiment.getPubmed_id()));
+      exp.setPubMedId((String)ValueFormatter.getEmptyStringIfNull(tab2mageExperiment.getPubmed_id()));
       exp.setTitle(tab2mageExperiment.getPublication_title());
       dc.commit();
@@ -129 +127 @@
     catch (Exception e)
     {
-      e.printStackTrace();
       done= false;
       dc.close(); // rollback if any problem arises
@@ -144 +141 @@
   }
   
-  //remove getAnnotationType and use the common one in the Tab2MageHelper class.
   
   /**

trunk/uk/ac/ebi/Tab2MageImporter/src/uk/ac/ebi/nugo/plugins/tab2mageimport/io/Tab2MageHybWriter.java

@@ -36 +36 @@
 import net.sf.basedb.core.Scan;
 import net.sf.basedb.core.Type;
-
+import net.sf.basedb.core.BioMaterial;
 
 import org.apache.log4j.Logger;
@@ -109 +109 @@
   {
     boolean done = true;
-    Tab2MageHelper.write ("\n++++ Writing Hybridizations ++++\n");
+    log.info ("\n++++ Writing Hybridizations ++++\n");
     // declare protocols and parameter temporary catche
     Map<String, String> protocolCatche = new HashMap<String, String>();
@@ -129 +129 @@
         for (Tab2MAGEHybridization hyb : hybs)
         {   
-          Tab2MageHelper.write("\n------Handling Hybridization :" + hyb.getName()+"------\n");
+          log.info("\n------Handling Hybridization :" + hyb.getName()+"------\n");
           List<LabeledExtract> labeledExtracts = new ArrayList<LabeledExtract>();
           i++;
@@ -163 +163 @@
                 for (Map.Entry<String, String> fact : factorMap.entrySet())
                 {
-                  //System.out.println(" Factor Value is " + fact.getKey()+ ":" +fact.getValue());
                   expFactors.add(fact.getKey());
                 }
@@ -191 +190 @@
                 return done=false;
               } 
-              //System.out.println("The array Design is : " + arrayDesign.getName());
               dc.reattachItem(arrayDesign);
               
@@ -205 +203 @@
               
               //write Scan
-                Scan scan = createScan(hyb.getScanner(), newHyb, parameterCatche, protocolCatche);
+                Scan scan = createScan( newHyb, parameterCatche, protocolCatche);
               
               //write Raw Bio Assay
                 RawBioAssay rba = createRawBioAssay(hyb.getFileRaw(), rdt, scan, arrayDesign, parameterCatche, protocolCatche, factorValueCatche);
                 rbaCatche.add(rba);
-                //System.out.println("Raw Bio Assay"+ rba.getName()+ "is added to catch");
+                
         }// end of each Hyb 
       }
       catch(Throwable tr)
       {
-        tr.printStackTrace();
+        log.warn(tr.getMessage());
         return done=false;
       }
@@ -242 +240 @@
   {
     BioSource bio;
-    Tab2MageHelper.write("--Creating biosource: " + name + "\n");
+    log.info("--Creating biosource: " + name + "\n");
     if (!bioSources.isEmpty() &&  bioSources.containsKey(name))
     {
       bio= (BioSource)bioSources.get(name);
-      System.out.println("--The biosource obtained is :" + bio.getName());
     }
     else
@@ -260 +257 @@
         {
           log.warn("Annotation for BioMaterial Characteristics cannot be set. Check the allowed values for the annotation type");
-          //roll back done in the write method
         }
         dc.saveItem(bio);
         bioSources.put(bio.getName(),bio);
-        System.out.println("--The biosource created is :" + bio.getName()); 
+        log.info("--The biosource created is :" + bio.getName()); 
     }
     return bio;
@@ -281 +277 @@
   {
     Sample sample;
-    Tab2MageHelper.write("--Creating sample: " + name + "\n");
+    log.info("--Creating sample: " + name + "\n");
     if (!samples.isEmpty() &&  samples.containsKey(name))
     {
       sample= (Sample)samples.get(name);
-      Tab2MageHelper.write("--The sample obtained is :" + sample.getName());
+      log.info("--The sample obtained is :" + sample.getName());
     }
     else
@@ -307 +303 @@
         {
           log.warn("Annotation for Sampling Protocol cannot be set. Check the allowed values for the annotation type");
-          //roll back done in the write method
         }
         
@@ -313 +308 @@
         dc.saveItem(sample);
         samples.put(sample.getName(),sample);
-        Tab2MageHelper.write("--The sample created is :" + sample.getName()+"\n");
+        log.info("--The sample created is :" + sample.getName()+"\n");
     }
     return sample;
@@ -328 +323 @@
   private Extract createExtract(String name,Sample sample, Map<String, BasicItem> extracts, Map<String, String> param, Map<String, String> protMap )
   {
-    Tab2MageHelper.write("--Creating extract: " + name + "\n");
+    log.info("--Creating extract: " + name + "\n");
     Extract extract;
     if (!extracts.isEmpty() &&  extracts.containsKey(name))
     {
       extract= (Extract)extracts.get(name);
-      Tab2MageHelper.write("--The extract obtained is :" + extract.getName());
+      log.info("--The extract obtained is :" + extract.getName());
     }
     else
@@ -352 +347 @@
         {
           log.warn("Annotation for Extraction Protocol cannot be set.Check the allowed values for the annotation type");
-          //roll back done in the write method
         }
         // add the extract annotations for MAG- TAB sample characteristics
         dc.saveItem(extract);
         extracts.put(extract.getName(),extract);
-        Tab2MageHelper.write("--The extract created is :" + extract.getName()+"\n");
+        log.info("--The extract created is :" + extract.getName()+"\n");
     }
     return extract;
@@ -374 +368 @@
   private LabeledExtract createLabeledExtract(String name, String dye, Label lb, Extract extract, Map<String, BasicItem> lExtracts, Map<String, String> param, Map<String, String> protMap)
   {
-    Tab2MageHelper.write("--Creating labeled extract: " + name + "\n");
+    log.info("--Creating labeled extract: " + name + "\n");
     LabeledExtract lextract;
     if (!lExtracts.isEmpty() &&  lExtracts.containsKey(name))
     {
       lextract= (LabeledExtract)lExtracts.get(name);
-      Tab2MageHelper.write("--The extract obtained is :" + extract.getName());
+      log.info("--The extract obtained is :" + extract.getName());
     }
     else
@@ -397 +391 @@
       {
         log.warn("Annotation for Labeling Protocol cannot be set. Check the allowed values for the annotation type");
-        //roll back done in the write method
       }
       //     add the labeled extract annotations for MAG- TAB sample characteristics
       dc.saveItem(lextract);
       lExtracts.put(lextract.getName(),lextract);
-      Tab2MageHelper.write("--The labeled extract created is :" + lextract.getName()+"\n");
+      log.info("--The labeled extract created is :" + lextract.getName()+"\n");
     }
     return lextract;
@@ -418 +411 @@
       Map<String, String> parCatche, Map<String,String> proCatche )
   {
-    Tab2MageHelper.write("--Creating hybridization: " + name + "\n");
-    //     used dc.commitif labelled extract is saved.
+    log.info("--Creating hybridization: " + name + "\n");
     Hybridization newHyb = Hybridization.getNew(dc);
     newHyb.setName(name);
@@ -430 +422 @@
     }
     String accessionNo = getProtocolAccesion(proCatche,"hybridization"); //
-    // temp comment this out and test......
-    //BioMaterialEvent newHybEvent=newHyb.getCreationEvent();
     bioMaterialEvent.setProtocol(getProtocol(accessionNo));
     // add the  annotation for protocol parameters
@@ -441 +431 @@
     {
       log.warn("Annotation for Hybridization Protocol cannot be set. Check that");
-      //roll back done in the write method
     }
     dc.saveItem(newHyb);
-    Tab2MageHelper.write("--The Hybridization created is :" + newHyb.getName()+"\n");
+    log.info("--The Hybridization created is :" + newHyb.getName()+"\n");
     return newHyb;
   }
@@ -455 +444 @@
     @return
    */
-  private Scan createScan(String name, Hybridization hyb, Map<String, String> parCatche, Map<String, String> proCatche)
-  {
-    //Scan scan= hyb.newScan();
-    Tab2MageHelper.write("--Creating scan: " + name + "\n");
+  private Scan createScan(Hybridization hyb, Map<String, String> parCatche, Map<String, String> proCatche)
+  {
+    log.info("--Creating scan: " +hyb.getName()+ "\n");
     Scan scan = Scan.getNew(dc, hyb);
-    if (name!="" && name!= null)
-      scan.setName(name);
-    else
-      scan.setName("scan"+hyb.getName());  //to make scan unique per Hyb add hyb name to the scan name
+    scan.setName("scan"+hyb.getName());  //to make scan unique per Hyb add hyb name to the scan name
     String accessionNo = getProtocolAccesion(proCatche,"scanning"); //
     scan.setProtocol(getProtocol(accessionNo));
@@ -474 +459 @@
     {
       log.warn("Annotation for Scanning Protocol cannot be set. Check that");
-      //roll back done in the write method
     }
     dc.saveItem(scan);
-    Tab2MageHelper.write("--The scan created is :" + scan.getName()+"\n");
+    log.info("--The scan created is :" + scan.getName()+"\n");
     return scan;
   }
@@ -494 +478 @@
       Map<String,String> parCatche, Map<String,String> protCatche, Map<String, String>factCatche)
   {
-    Tab2MageHelper.write("--Creating raw bioassay: " + name +"\n");
+    log.info("--Creating raw bioassay: " + name +"\n");
       RawBioAssay rba = RawBioAssay.getNew(dc, rawDataType);
       rba.setName(name);
@@ -510 +494 @@
     {
       log.warn("Annotation for Feature Extraction Protocol cannot be set. Check the allowed values for the annotation type");
-      //roll back done in the write method
     }
     //  add the raw bioassay  annotations for implementing factor value annotation.
     //  This is a work around, avoiding the inherited annotation problem: i.e items to inherit factor value
-    //  from are not available in tab2mage file neither si there an association.
+    //  from are not available in tab2mage file neither is there an association.
     
-    //Tab2MageHelper.write(" \n+++++++ Checking out the Content of the Factor Vaue Map +++++\n");
-    //for (Map.Entry<String, String> fact : factorValueCatche.entrySet())
-    //{
-    //  Tab2MageHelper.write(" Factor Val is " + fact.getKey()+ ":" +fact.getValue());
-    //}
     try
     {
@@ -531 +509 @@
     }
     dc.saveItem(rba);
-    Tab2MageHelper.write("--The raw bio assay created is :" + rba.getName()+"\n");   
+    log.info("--The raw bio assay created is :" + rba.getName()+"\n");   
     return rba;
   }
@@ -549 +527 @@
       as.setName(arrayDesign.getName()+ "-Slide#" +i);
       dc.saveItem(as);
-      //Tab2MageHelper.write("The new array  slide is :" + as.getName());
+      log.info("The new array  slide is :" + as.getName());
     }
     return as;
@@ -580 +558 @@
   private String getProtocolAccesion(Map<String, String> protMap, String type)
   {
-    //Tab2MageHelper.write(" \n+++++++ Checking out the Hyb Protocols +++++\n");
     String accession="";
     for (Map.Entry<String, String> prot: protMap.entrySet())
     {
-      //Tab2MageHelper.write(" Hyb Protocol is " + prot.getKey()+ ":" +prot.getValue());
       if (prot.getKey().equals(type.toUpperCase()))
       {
@@ -599 +575 @@
   private Protocol getProtocol(String protocolAccesion)
   {
-    //Tab2MageHelper.write(" \n+++++++ Getting Protocols +++++\n");
     Protocol protocol=null;
     for (Map.Entry<String, Protocol> prot: protocolAccessionMap.entrySet())
     {
-      //Tab2MageHelper.write(" Protocol Accession Map is: " + prot.getKey()+ ":" +prot.getValue());
       if (prot.getKey().equalsIgnoreCase(protocolAccesion))
       {
@@ -637 +611 @@
       for (Map.Entry<AnnotationType, String> annt : annotationTypes.entrySet())
       {
-        //Tab2MageHelper.write("Annotation type - Value mapping is " + annt.getKey().toString()+ ":" +annt.getValue());
-          
-        // remove annotations if the annotatable item has annotations already
-        // this is because, we consider the annotatable item as a new item  that
-        // should not reference an already existing annotation  of an item with 
-        //the same name, even if the item has been deleted.
-        //if (annotatable.isAnnotated()) annotatable.removeAnnotations(); 
         if (annotatable instanceof  RawBioAssay)
         {
-          //Tab2MageHelper.write("YES, IT IS AN INSTANCE OF RAWBIOASSAY");
           AnnotationType t = annt.getKey();
           if (t.isEnabledForItem(Item.RAWBIOASSAY)) // if the annotation type is enable for raw bioassay item, set annotations
@@ -652 +618 @@
             AnnotationSet as = annotatable.getAnnotationSet();
             String values= annt.getValue();
-            Tab2MageHelper.write("Inserting: "+ annt.getValue()+" annotation for: "+ annotatable.toString());
+            log.info("Inserting: "+ annt.getValue()+" annotation for: "+ annotatable.toString());
             
             if (t.getValueType().equals(Type.FLOAT))
             {
-              Tab2MageHelper.write("YES, the annotation value type is float");
               Float v = new Float(Float.parseFloat(values));
               as.getAnnotation(annt.getKey()).setValue(v);
@@ -671 +636 @@
         {
           AnnotationSet as = annotatable.getAnnotationSet();
-          //Tab2MageHelper.write("Inserting: "+ annt.getValue()+" annotation for: "+ annotatable.toString());
+          //log.info("Inserting: "+ annt.getValue()+" annotation for: "+ annotatable.toString());
           as.getAnnotation(annt.getKey()).setValue(annt.getValue());
         }
@@ -689 +654 @@
     for (Map.Entry<String, String> an : annotationMap.entrySet())
     {
-      //Tab2MageHelper.write(" Annotation type [BioMat,Param] Value is " + an.getKey()+ ":" +an.getValue());
       AnnotationType at =Tab2MageHelper.getAnnotationTypeByName(an.getKey(),dc);
       if (at!=null)
@@ -705 +669 @@
           {
             AnnotationSet as = annotatable.getAnnotationSet();
-            //Tab2MageHelper.write("Inserting: "+ anv.getValue()+" annotation for: "+ annotatable.toString());
+            log.info("Inserting: "+ anv.getValue()+" annotation for: "+ annotatable.toString());
             as.getAnnotation(anv.getKey()).setValue(anv.getValue());
           }

trunk/uk/ac/ebi/Tab2MageImporter/src/uk/ac/ebi/nugo/plugins/tab2mageimport/io/Tab2MageProtocolWriter.java

@@ -65 +65 @@
       {
         boolean exist= checkIfProtocolExist(protocol, dc);
-        Tab2MageHelper.write(exist? "Protocol Exist " : "Protocol DoNot Exist");
+        log.info(exist? "Protocol Exist " : "Protocol DoNot Exist");
         if (!exist) // if protocol is not found in the protocol table  in BASE 2
         {         
@@ -84 +84 @@
           for (String res : result)
           {
-            Tab2MageHelper.write("The protocol parameter to convert to annotation type is : "+ res);
+            log.info("The protocol parameter to convert to annotation type is : "+ res);
             AnnotationType annotationType= getAnnotationTypeFromProtocolParameter(res.trim(), dc);
             try
@@ -91 +91 @@
               {
                 newProtocol.addParameter(annotationType);
-                System.out.printf("The annotation type: %s is added to protocol\n", annotationType.getName());
               }
             }
@@ -104 +103 @@
           
           dc.saveItem(newProtocol);
-          Tab2MageHelper.write("The protocol name to write is : " + newProtocol.getName());
+          log.info("The protocol name to write is : " + newProtocol.getName());
         }
       }
@@ -134 +133 @@
     if (names.contains(prot.getName().toLowerCase())) 
     {
-      System.out.printf("The protocol: %s is found in the database\n", prot.getName().toString());
       found = true;
     } 

trunk/uk/ac/ebi/Tab2MageImporter/src/uk/ac/ebi/nugo/plugins/tab2mageimport/io/Tab2MageWriterFactory.java

@@ -21 +21 @@
 import net.sf.basedb.core.ArraySlide;
 import net.sf.basedb.core.BasicItem;
+import net.sf.basedb.core.BioMaterial;
 import net.sf.basedb.core.BioMaterialEvent;
 import net.sf.basedb.core.BioSource;
@@ -33 +34 @@
 import net.sf.basedb.core.Label;
 import net.sf.basedb.core.LabeledExtract;
+import net.sf.basedb.core.MeasuredBioMaterial;
 import net.sf.basedb.core.Protocol;
 import net.sf.basedb.core.ProtocolType;
@@ -47 +49 @@
 import org.embl.cg.bloader.models.Tab2MAGEProtocol;
 
+import uk.ac.ebi.nugo.plugins.tab2mageimport.validation.Tab2MageValidationFactory;
 import uk.ac.ebi.nugo.plugins.util.NutribasePluginException;
 import uk.ac.ebi.nugo.plugins.util.Tab2MageHelper;
@@ -96 +99 @@
   private Set<String> expFactors= new HashSet<String>();
   /**
-    List of protocol object for experiment.
-   */
-  //private List<Protocol> usedProtocols =  new ArrayList<Protocol>();
+    Map of pooled sample -to- extract.
+   */
+    private Map<String, List<String>> sampleExtMap =  new HashMap<String, List<String>>();
+    
+  /**
+    Map of pooled biosource -to- sample.
+   */
+    private Map<String, List<String>> bioSourceSamMap =  new HashMap<String, List<String>>();
+    
+  /**
+    Map of pooled extract -to- labeled extract.
+   */
+    private Map<String, List<String>> extractLEMap =  new HashMap<String, List<String>>();
   /**
     creates a new tab2mage writer factory
@@ -106 +119 @@
       @param rdt
    */
-  public Tab2MageWriterFactory(SessionControl sc, Set<Tab2MAGEProtocol> protocols, ArrayList<Tab2MAGEHybridization> hybs, RawDataType rdt)
+  public Tab2MageWriterFactory(SessionControl sc, Set<Tab2MAGEProtocol> protocols, 
+                ArrayList<Tab2MAGEHybridization> hybs, RawDataType rdt,
+                Map<String, List<String>> sampleExtract,
+                Map<String, List<String>> bioSourceSample,
+                Map<String, List<String>> extractLabExtract)
   {     
     this.hybs= hybs;
     this.rdt= rdt;
     this.protocols = protocols;
-    dc= sc.newDbControl(); // create a single dbcontrol for this writer factory
+    dc= sc.newDbControl(); // create a single db control for this writer factory
+    this.sc= sc;
+    if (sampleExtract!=null && !sampleExtract.isEmpty() ) this.sampleExtMap.putAll(sampleExtract);
+    if (bioSourceSample!=null && !bioSourceSample.isEmpty()) this.bioSourceSamMap.putAll(bioSourceSample);
+    if (extractLabExtract!=null && !extractLabExtract.isEmpty()) this.extractLEMap.putAll(extractLabExtract);
   }
   
@@ -135 +156 @@
         return success= false;
       }
+      // store rawbioassay and experimental factors for later use by the experiment writer
       this.rbas=hybWriter.getRbaCatche();
       this.expFactors=hybWriter.getExpFactors();
@@ -142 +164 @@
     {
       success= false;
-      t.printStackTrace();
       log.error("WriterFactory: writing Protocol and Hybridization failed");
       dc.close();
@@ -166 +187 @@
   }
   
+  // this should only be executed if the preceding #write() works fine
+
+  @SuppressWarnings("unchecked")
+  public boolean writePooledMetaData()
+  {
+    boolean done= true;
+    //Tab2MageHelper.write(",,, handling pooled data........");
+    try 
+    {
+      // create a new db control since the last once would have been closed anyway
+      if (dc == null || !dc.isConnected()|| dc.isClosed()) dc = sc.newDbControl();
+    
+      Tab2MageHelper.write("Entering the try loop");
+      for (Map.Entry<String , List<String>> entry : sampleExtMap.entrySet())
+      {
+        Tab2MageHelper.write("\n++ The sample extract from the SampleExractMap is : "+entry.getKey()+":"+ entry.getValue());
+        List<String> parents= entry.getValue();
+        List<BasicItem> children = new ArrayList<BasicItem>();
+        for (String parent : parents )
+        {
+          Tab2MageHelper.write("++The current parent being checked is "+ parent);
+          Sample p = Tab2MageHelper.findSample(dc, parent); // get the parent from the database
+          ItemQuery<Extract> childrenQuery  = p.getExtracts(); // get all the extracts ( children of parents -Sample)
+          childrenQuery.include(Include.MINE, Include.NOT_REMOVED); // include those children created by the loggedin user only 
+          if (childrenQuery.list(dc).size()!=0)
+            children.add(childrenQuery.list(dc).get(0));  // pick the first child or extract in this case
+        }
+        String childofParent = entry.getKey(); // in this example this is the extract
+        Tab2MageHelper.write("...The key from map is : " + childofParent );
+        Extract e= Tab2MageHelper.findExtract(dc, childofParent);
+        Tab2MageHelper.write("...the already created object fetched is : " + e.getName() );
+        dc.reattachItem(e);
+        e.setPooled(true);
+        for (BasicItem<?> child : children)
+        {
+          e.getCreationEvent().addSource((MeasuredBioMaterial) child, Float.parseFloat("0.10"));
+        }
+        
+      }
+      dc.commit();
+    }
+    catch(Throwable tr)
+    {
+      done= false;
+      log.error("WriterFactory: setting pooled items failed");
+      dc.close();
+      throw new NutribasePluginException(tr.getMessage());
+    }
+    finally
+    {
+      if (dc!=null)
+        dc.close();
+    }
+    done=false;
+    return done;
+  }
+  
   /**
     writes the  protocol type not found in the database which must be within the allowed range in tab2mage specification
@@ -171 +249 @@
     @param localDc
    */
-  public void writeProtocolTypes(DbControl localDc)
-  {
+  public void writeProtocolTypes(SessionControl sc)
+  {
+    DbControl localDc=sc.newDbControl();
     try
     {
@@ -178 +257 @@
       {
         boolean foundType =Tab2MageHelper.checkIfProtocolTypesExist(protocol,localDc);
-        //System.out.println(foundType? "ProtocolType Exist " : "ProtocolType DoNot Exist");
         if (!foundType)
         {
@@ -192 +270 @@
     {
       log.error("ProtocolTypes cannot be created");
-      tr.printStackTrace();
       localDc.close(); // if there is an error. the creation of annotation types is rolled back.
       throw new NutribasePluginException(tr.getMessage());  
@@ -201 +278 @@
         localDc.close();
     }
-  }
-    
-  
-  /**
-    creates a new array batch if one does not exist already  
-      @param arrayDesign
-    @return array batch
-   */
-  private ArrayBatch createArrayBatch (ArrayDesign arrayDesign, int i )
-  {
-    String accession = arrayDesign.getName()+"-Batch#" + i; // accession is thename of the slide
-    ArrayBatch arrayBatch = Tab2MageHelper.findArrayBatch(dc, accession);
-    if (arrayBatch== null)
-    {
-      arrayBatch = ArrayBatch.getNew(dc, arrayDesign);
-      arrayBatch.setName(accession);
-      dc.saveItem(arrayBatch);
-    }
-    return arrayBatch;
-  }
-  /**
-    creates a raw bioassay
-      @param name
-      @param rawDataType
-    @return
-   */
-  private RawBioAssay createRawBioAssay( String name, RawDataType rawDataType)
-  {
-    //System.out.println("Creating raw bioassay: " + name);
-      RawBioAssay rba = RawBioAssay.getNew(dc, rawDataType);
-      rba.setName(name);
-      dc.saveItem(rba);
-      return rba;
-    }
-  /**
-    gets the protocol accession number  given
-    @param protMap, the Hybline protocol map
-    @param type, the protocol type
-    @return the accession number
-   */ 
-
-  private String getProtocolAccesion(Map<String, String> protMap, String type)
-  {
-    //System.out.println(" \n+++++++ Checking out the Hyb Protocols +++++\n");
-    String accession="";
-    for (Map.Entry<String, String> prot: protMap.entrySet())
-    {
-      //System.out.println(" Hyb Protocol is " + prot.getKey()+ ":" +prot.getValue());
-      if (prot.getKey().equals(type.toUpperCase()))
-      {
-        accession =prot.getValue();
-        break;
-      }
-    }
-    return accession;
-  }
-  /**
-    get protocol given the protocolname and type
-   * @return
-   */
-  private Protocol getProtocol(String protocolAccesion)
-  {
-    //System.out.println(" \n+++++++ Getting Protocols +++++\n");
-    Protocol protocol=null;
-    for (Map.Entry<String, Protocol> prot: protocolAccessionMap.entrySet())
-    {
-      //System.out.println(" Protocol Accession Map is: " + prot.getKey()+ ":" +prot.getValue());
-      if (prot.getKey().equalsIgnoreCase(protocolAccesion))
-      {
-        protocol =prot.getValue();
-        break;
-      }
-    }
-    return protocol;
   }
   
@@ -286 +289 @@
   }
   
-  private void setAnnotations(Annotatable annotatable, Map<String,String>annotationMap)
-  {
-    Map<AnnotationType, String> annotationTypes = new HashMap<AnnotationType, String>();
-    for (Map.Entry<String, String> an : annotationMap.entrySet())
-    {
-      AnnotationType at =Tab2MageHelper.getAnnotationTypeByName(an.getKey(),dc);
-      if (at!=null)
-      {
-        annotationTypes.put(at, an.getValue());
-      }
-    } 
-    if (!annotationTypes.isEmpty())
-    {
-      for (Map.Entry<AnnotationType, String> annt : annotationTypes.entrySet())
-      {
-        //System.out.println("Annotation type - Value mapping is " + annt.getKey().toString()+ ":" +annt.getValue());
-          
-        // remove annotations if the annotatable item has annotations already
-        // this is because, we consider the annotatable item as a new item  that
-        // should not reference an already existing annotation  of an item with 
-        //the same name, even if the item has been deleted.
-        //if (annotatable.isAnnotated()) annotatable.removeAnnotations(); 
-        if (annotatable instanceof  RawBioAssay)
-        {
-          //System.out.println("YES, IT IS AN INSTANCE OF RAWBIOASSAY");
-          AnnotationType t = annt.getKey();
-          if (t.isEnabledForItem(Item.RAWBIOASSAY)) // if the annotation type is enable for raw bioassay item, set annotations
-          {
-            AnnotationSet as = annotatable.getAnnotationSet();
-            String values= annt.getValue();
-            System.out.println("Inserting: "+ annt.getValue()+" annotation for: "+ annotatable.toString());
-            
-            if (t.getValueType().equals(Type.FLOAT))
-            {
-              System.out.println("YES, the annotation value type is float");
-              Float v = new Float(Float.parseFloat(values));
-              as.getAnnotation(annt.getKey()).setValue(v);
-              
-            }
-            else //if( t.getValueType().equals(Type.STRING)) 
-            {
-              as.getAnnotation(annt.getKey()).setValue(annt.getValue());
-            }
-          }
-          // otherwise do nothing
-        }
-        else
-        {
-          AnnotationSet as = annotatable.getAnnotationSet();
-          //System.out.println("Inserting: "+ annt.getValue()+" annotation for: "+ annotatable.toString());
-          as.getAnnotation(annt.getKey()).setValue(annt.getValue());
-        }
-        
-      } 
-    }
-  }
-  
-  /**
-    sets annotations for protocol parameters
-    @param annotatable, the item that uses the protocol
-    @param annotationMap, map of [annotation types - value] pair
-   */
-  private void setParameterAnnotations(Annotatable annotatable, Map<String,String>annotationMap)
-  {
-    Map<AnnotationType, String> ats = new HashMap<AnnotationType, String>();
-    for (Map.Entry<String, String> an : annotationMap.entrySet())
-    {
-      //System.out.println(" Annotation type [BioMat,Param] Value is " + an.getKey()+ ":" +an.getValue());
-      AnnotationType at =Tab2MageHelper.getAnnotationTypeByName(an.getKey(),dc);
-      if (at!=null)
-      {
-        ats.put(at, an.getValue());
-      }
-    }
-    if (!ats.isEmpty())
-    {
-      for (Map.Entry<AnnotationType, String> anv : ats.entrySet())
-      {
-        if (annotatable.getProtocol()!=null)
-        {
-          if (annotatable.getProtocol().isParameter(anv.getKey()))
-          {
-            AnnotationSet as = annotatable.getAnnotationSet();
-            //System.out.println("Inserting: "+ anv.getValue()+" annotation for: "+ annotatable.toString());
-            as.getAnnotation(anv.getKey()).setValue(anv.getValue());
-          }
-        }
-      }
-    }
-  }
-
   /**
     get the list of raw bio asssay
@@ -385 +297 @@
     return rbas;
   }
-
-
 }

trunk/uk/ac/ebi/Tab2MageImporter/src/uk/ac/ebi/nugo/plugins/tab2mageimport/mail/Tab2MageEMessaging.java

@@ -20 +20 @@
 
 import org.apache.log4j.Logger;
+
+import uk.ac.ebi.nugo.plugins.util.NutribasePluginException;
 /**
   This class is the e messaging for the Tab2Mage import and export. This is 
@@ -33 +35 @@
   private String mailFrom = "";
   private String mailTo = "";
-  //private String host = "smtp.ebi.ac.uk";
   private String host="";
   private String body = "";
@@ -44 +45 @@
     this.mailFrom = from;
     this.mailTo= to;
-    if (fileName!="")tab2MageFileName = fileName;
-    if (userName!="")loggedinUserName=userName;
+    if (!fileName.equals("")) tab2MageFileName = fileName;
+    if (!userName.equals("")) loggedinUserName=userName;
     this.host= hostname;
     if (status) // if successful
@@ -107 +108 @@
       message.setSentDate(new Date());
       
-      System.out.println("sending message.......");
+      log.info("sending message.......");
       //sending the tab2mage message
       Transport.send(message);
@@ -114 +115 @@
     {
       log.error("Email message cannot be sent, your email address maybe invalid. Or " +me.getMessage()+ ". Contact your smtp sever adminstrator");
-      me.printStackTrace();
-      //throw new NutribasePluginException("Email message cannot be sent, your email address maybe invalid. Or " +me.getMessage()+ ". Contact your smtp sever adminstrator");
+      throw new NutribasePluginException("Email message cannot be sent, your email address maybe invalid. Or " +me.getMessage()+ ". Contact your smtp sever adminstrator");
     }
     catch (UnsupportedEncodingException e) 
     {
       log.error("You have some unsupported characters in your base/nutribase username or email address.");
-      e.printStackTrace();
-      //throw new NutribasePluginException(e.getMessage());
+      throw new NutribasePluginException(e.getMessage());
     }
   }

trunk/uk/ac/ebi/Tab2MageImporter/src/uk/ac/ebi/nugo/plugins/tab2mageimport/validation/AgilentValidation.java

@@ -60 +60 @@
       for (File agFile : agilentFiles )
       {
-        Tab2MageHelper.write("Validating agilent file : "+ agFile.getName());
+        log.info("Validating agilent file : "+ agFile.getName());
         AgilentRDParser agParser= new AgilentRDParser(agFile, dc); 
         String [] headers= agParser.getHeaders();
         if (headers==null) continue;
-        //Tab2MageHelper.write("\n..........The list of agilent headers\n");
         List<String> headerList = Tab2MageHelper.arrayContentToLowercase(headers);
-        //int i=0;
-        //for (String header :headerList )
-        //{
-        //  i++;
-        //  Tab2MageHelper.write("Header " + i + ": "+ header);
-        //}
-        //Tab2MageHelper.write("End of agilent headers\n");
         List<AgilentDataColumns> mandatoryColumns= Arrays.asList(AgilentDataColumns.values());
         for (AgilentDataColumns man :mandatoryColumns )

trunk/uk/ac/ebi/Tab2MageImporter/src/uk/ac/ebi/nugo/plugins/tab2mageimport/validation/GenePixFileValidation.java

@@ -71 +71 @@
         Base2GenePixRDParser gParser= new Base2GenePixRDParser(gprFile, dc); 
         String [] headers= gParser.getHeaders();
-        //Tab2MageHelper.write("\n..........The list of Genepix headers\n");
         List<String> headerList = Tab2MageHelper.arrayContentToLowercase(headers);
-        //int i=0;
-        //for (String header :headerList )
-        //{
-        //  i++;
-        //  Tab2MageHelper.write("Header " + i + ": "+ header);
-        //}
-      //  Tab2MageHelper.write("End of Genepix headers\n");
         List<GenePixDataColumns> mandatoryColumns= Arrays.asList(GenePixDataColumns.values());
         for (GenePixDataColumns man :mandatoryColumns )
@@ -95 +87 @@
       success = false;
       log.error("The gpr files cannot be validated"+tr.getMessage());
-      tr.printStackTrace();
     }
     return success;

trunk/uk/ac/ebi/Tab2MageImporter/src/uk/ac/ebi/nugo/plugins/tab2mageimport/validation/RawDataValidation.java

@@ -7 +7 @@
 
 /**
-  This inteface is for the validation of raw data files. 
+  This interface is for the validation of raw data files. 
   Note: seperate implementation to base1/ baseloader
     @author Dominic Oyeniran

trunk/uk/ac/ebi/Tab2MageImporter/src/uk/ac/ebi/nugo/plugins/tab2mageimport/validation/Tab2MageImportValidatorUtils.java

@@ -29 +29 @@
 import uk.ac.ebi.nugo.plugins.tab2mageimport.AnnotationTypeFileMapping;
 import uk.ac.ebi.nugo.plugins.util.NutribasePluginException;
+import uk.ac.ebi.nugo.plugins.util.Tab2MageConstants;
 import uk.ac.ebi.nugo.plugins.util.Tab2MageHelper;
 
@@ -80 +81 @@
   List<String> annotationTypes = new LinkedList<String>();
   
-
+  //annotate this guy later.
+  private static Map< String, List<String>> BioSourceSamplePoolMap= new HashMap< String, List<String>>();
+  private static Map< String, List<String>> SampleExractPoolMap= new HashMap< String, List<String>>();
+  private static Map<String, List<String>> ExractLabeledExtractPoolMap= new HashMap< String, List<String>>();
+  
   /**
       creates the hyb validator util
@@ -126 +131 @@
         createAnnotationTypeValueMap(parameterAnnotationType);
         createAnnotationTypeValueMap(factorValueAnnotationType);  
+      
       }
     }
@@ -138 +144 @@
     for(Map.Entry<String, String> e : map.entrySet())
     {
-      //Tab2MageHelper.write(e.getKey() + ": " + e.getValue());
       if (Values.getFloat(e.getValue())== 0)
       {
@@ -174 +179 @@
         if (parameterAnnotationType.containsKey(e.getKey()))
         {
-          //Tab2MageHelper.write("Found Parameter key : [" + e.getKey()+ "] and value is :" + parameterAnnotationType.get(e.getKey()));
           parameter="true";
         }
-        //Tab2MageHelper.write(".....checking biomat map");
+        
         if (bioMaterialAnnotationType.containsKey(e.getKey())) 
         {
-          //Tab2MageHelper.write("Found BioMat key: [" + e.getKey()+ "] and value is :" + bioMaterialAnnotationType.get(e.getKey()));
           items.add("biosource");
         }
-        //Tab2MageHelper.write(".....checking factor value map"); 
         if (factorValueAnnotationType.containsKey(e.getKey())) 
         {
-          //Tab2MageHelper.write("Found Factor Value key: [" + e.getKey()+ "] and value is :" + factorValueAnnotationType.get(e.getKey()));
           items.add("rawbioassay");
         }
-              
         AnnotationTypeFileMapping at = new AnnotationTypeFileMapping(name, valuetype, parameter,items);
         annotationTypeFileMappings.add(at);
@@ -197 +197 @@
     {
       log.error("Annotation File cannot be created");
-      t.printStackTrace();
-      
+            
     }
     return annotationTypeFileMappings;
@@ -233 +232 @@
     try
     {
-      //Tab2MageHelper.write("creating the subdirectory ....");
       if (!Directory.exists(dc, dir, directoryName)) 
       {
@@ -240 +238 @@
         dc.saveItem(subDir);
       }
-      Tab2MageHelper.write("creating the file.....");       
+      log.info("creating the file.....");       
       // create a file : filename into the subdirectory (temp) created above
       file =  File.getFile(dc, subDir, filename, true);
@@ -263 +261 @@
     {
       log.error("Error: " + tex.getMessage());
-      tex.printStackTrace();
       throw new NutribasePluginException("Temporary File cannot be created");
     }
@@ -278 +275 @@
     return file;
   }
+/**
+    check if the hybridization headers contain Protocol[pool]
+      @return TRUE if yes and FALSE otherwise.
+   */
+  public boolean checkForPoolingProtocol()
+  {
+    boolean pooled = header.contains(Tab2MageConstants.protocol_pool.getName())? true : false;
+    return pooled;
+  }
+  // if pooled do handlepool()
+  public void handlePool()
+  {
+    //call checkForPoolingProtocol and if pooled?
+    Map<String, String> BioSourceSampleMap= new HashMap<String,String>();
+    Map<String, String> SampleExtractMap= new HashMap<String,String>();
+    Map<String, String> ExtractLabeledExtractMap= new HashMap<String,String>();
+    
+    // clear maps of parent child relationships
+    if (!BioSourceSampleMap.isEmpty()) BioSourceSampleMap.clear();
+    if (!SampleExtractMap.isEmpty()) SampleExtractMap.clear();
+    if (!ExtractLabeledExtractMap.isEmpty()) ExtractLabeledExtractMap.clear();
+    try
+    {
+      for (Tab2MAGEHybridization hyb: hybs)
+      {
+        for (Tab2MAGEHybridizationLine line : hyb.getLines())
+        {
+          // populate maps of parent child relationships
+          BioSourceSampleMap.put(line.getBioSource(), line.getSample());
+          SampleExtractMap.put(line.getSample(), line.getExtract());
+          ExtractLabeledExtractMap.put(line.getExtract(), line.getLabeledExtract());
+        }
+      }
+      
+      SampleExractPoolMap.putAll(getParentChildMap(SampleExtractMap));
+      BioSourceSamplePoolMap.putAll(getParentChildMap(BioSourceSampleMap));
+      ExractLabeledExtractPoolMap.putAll(getParentChildMap(ExtractLabeledExtractMap));
+    }
+    catch(Throwable tr)
+    {
+      log.error("cannot processing pooling" + tr.getMessage());
+    }
+  }
+  
+  private Map<String, List<String>> getParentChildMap(Map<String, String> childParentMap)
+  {
+    Map<String, List<String>> map = new HashMap<String, List<String>>();
+    for (Map.Entry<String, String> childParentEntry : childParentMap.entrySet())
+    {
+      String child= childParentEntry.getValue();
+      List<String> parents= getParents(child, childParentMap);
+      if (parents.size()>=2) // means that the child has more than one parents
+      {
+        map.put(child, parents);
+      }
+    }
+    return map;
+  }
+  
+  private List<String> getParents(String child, Map<String, String> childParentMap)
+  {
+    List<String> parents= new ArrayList<String>();
+    //if (!child.equals("")|| child.length()>=1 )
+    if (!child.equals("") && child.length()>=1 ) //ensure that child is not an empty string
+    {
+      for (Map.Entry<String, String> entry : childParentMap.entrySet())
+      {
+        if (child.equalsIgnoreCase(entry.getValue()))
+        {
+          parents.add(entry.getKey());
+        }
+      }
+    }
+    return parents;
+  }
+
+  public Map<String, List<String>> getBioSourceSamplePoolMap() 
+  {
+    return BioSourceSamplePoolMap;
+  }
+
+  public Map<String, List<String>> getExractLabeledExtractPoolMap() 
+  {
+    return ExractLabeledExtractPoolMap;
+  }
+
+  public Map<String, List<String>> getSampleExractPoolMap() 
+  {
+    return SampleExractPoolMap;
+  }
 }

trunk/uk/ac/ebi/Tab2MageImporter/src/uk/ac/ebi/nugo/plugins/tab2mageimport/validation/Tab2MageValidationFactory.java

@@ -10 +10 @@
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
@@ -52 +53 @@
    private static Set<Tab2MAGEProtocol> catchedProtocol=null;
    private static ArrayList<Tab2MAGEHybridization> catchedHybs=null;
+   
+   private static Map<String , List<String>> sampleExtractMap= null;
+   private static Map<String, List<String>> bioSourceSampleMap =null;
+   private static Map<String, List<String>>  extractLabeledExtractMap = null;
+   
    /**
       Filename with extension e.g tab2mage.txt
@@ -60 +66 @@
     */
    private Directory dir;
+   /**
+      the db control created from session
+    */
    private DbControl dc;
+   /**
+      session control passed as parameter on creating this class
+    */
    private SessionControl sc;
    /**
@@ -81 +93 @@
    
   private Tab2MAGEExperiment tExperiment;
+/**
+      variable that indicates if experiment is pooled or not.
+    */
+   private boolean pooled;
+
    /**
       
@@ -91 +108 @@
       @param expPlatform
     */
-  public Tab2MageValidationFactory(String fileName, DbControl dControl, Directory fileDirectory, String expPlatform)
+  public Tab2MageValidationFactory(String fileName, SessionControl sControl, Directory fileDirectory, String expPlatform)
   {
     filename= fileName;
-    dc = dControl;
-    sc =  dc.getSessionControl();
+    dc = sControl.newDbControl();
+    sc = sControl;
     dir=fileDirectory;
     platform= expPlatform;
-    //this call the Tab2mage file validation routine
-    Tab2MageHelper.write("About to begin tab2mage file validation");
+    //this calls the Tab2mage file validation routine
     validateTab2MageFile();
   }
@@ -152 +168 @@
       if (isFileValid)
       {
-        //Tab2MageHelper.write("Is the tab2magefile format valid? "  + isFileValid);
         //  parse and read the tab2magefile
         reader = new Base2Tab2MageFileReader(tab2MageFile, dc);
@@ -158 +173 @@
         
         //validate experiment 
-        Tab2MageHelper.write("Begin Experiment validation");
+        log.info("Begin Experiment validation");
         tExperiment= reader.getExperiment(); 
         Tab2MageExperimentValidation experimentVal= new Tab2MageExperimentValidation(reader);
         boolean expValidated= experimentVal.validate(); // return true or false
         
-        //Tab2MageHelper.write(expValidated ? "yes the exp is validated" : "No exp is not validated");
         if (!expValidated)
         {
@@ -169 +183 @@
           throw new NutribasePluginException("SORRY: The experiment section of the tab2mage file: " +filename+ " is invalid. Must be rectified to proceed");
         }
-        Tab2MageHelper.write("Finished  experiment validation");    
+        log.info("Finished  experiment validation");    
         
         //validate protocols
         if (dc == null || !dc.isConnected() || dc.isClosed()) //dc = dir.getDbControl();
           dc= sc.newDbControl();
-        Tab2MageHelper.write("Begin Protocol validation");
+        log.info("Begin Protocol validation");
         Set<Tab2MAGEProtocol> protocols = reader.getProtocols(); // we get all the  protocols
         Tab2MageProtocolValidation protocolVal= new Tab2MageProtocolValidation(protocols, dc);
@@ -183 +197 @@
           throw new NutribasePluginException("SORRY: The protocol section of the tab2mage file: " +filename+ " is invalid. Must be rectified to proceed. See the log file for more details");
         }
-        // Display the protocols only for debugging     
-        //Tab2MageHelper.write("\n+++++++++++++++++++++++ALL THE PROTOCOLS++++++++++++++++++++++++++++++++++++++\n");
-        //for (Tab2MAGEProtocol p : protocols)
-        //{
-        //  Tab2MageHelper.write(p.getAccession()+ "\t" +p.getText()+ "\t" +p.getName() + "\t" +p.getType()+ "\t" +p.getParameters());
-        //}
-        //Display all the parameters, only for debugging
-        
-        //Tab2MageHelper.write("\n++++++++++++++++++++ The parameters ++++++++++++++\n");
-        //List<String> params= protocolVal.getProtocolParameters();
-        //for (String p : params)
-        //{
-        //  Tab2MageHelper.write(p);
-        //}
-              
-        Tab2MageHelper.write("Finished  protocol validation");
+                      
+        log.info("Finished  protocol validation");
         
         /*
@@ -204 +204 @@
         */
         if (dc == null || !dc.isConnected()) dc = sc.newDbControl();
-        Tab2MageHelper.write("Begin Hybridization validation");
+        log.info("Begin Hybridization validation");
         ArrayList<Tab2MAGEHybridization> hybs = reader.getHybridizations();
         
-        //Tab2MageHelper.write( "Finished getting hybs from the reader object, now +++++ listing hyb names+++++ ");
-        //for (Tab2MAGEHybridization hyb : hybs)
-        //{
-        //  System.out.printf("\nThe hyridization name for this hyb is %s\n",hyb.getName());
-        //}
           // validate headers
         boolean validateHeaders= Tab2MageHybridizationValidation.validateHeader(new ArrayList<String>(Arrays.asList(reader.getHeaders())));
@@ -220 +215 @@
         }
           //validate rows /lines
-        //Tab2MageHelper.write("Begin validate hyb rows after headers"); //NOTE: the db control is closed here
+          
         Tab2MageHybridizationValidation validateHyb = new Tab2MageHybridizationValidation(dir, platform, hybs, reader, dc);
         if (!validateHyb.validate())
@@ -227 +222 @@
           throw new ValidationException("There is an error in the Tab2Mage Hybridization section!");
         }
-        Tab2MageHelper.write("Finished  Hybridization validation");
+        log.info("Finished  Hybridization validation");
         //NOTE: the db control is closed here
         
@@ -233 +228 @@
           
         if (dc == null || !dc.isConnected() || dc.isClosed()) dc = sc.newDbControl();
-        //Tab2MageHelper.write("\n+++++++++++ List AT - value type map ++++++++++++++\n ");
+        
         Tab2MageImportValidatorUtils utils = new Tab2MageImportValidatorUtils(hybs, new ArrayList<String>(Arrays.asList(reader.getHeaders())));
         utils.initialiseMaps();
@@ -239 +234 @@
         
         tab2MageFile= File.getByPath(dc, tab2MageFile.getPath(), false);  
-        //Tab2MageHelper.write("create file and directory");
+        
         String filename= "anntype.txt";
-        Tab2MageHelper.write("The ta2mage file mime and file type is: " + tab2MageFile.getMimeType()+ " -- "+tab2MageFile.getFileType());
-        
         utils.createFile(sc, dir, "tempAT",  tab2MageFile.getMimeType(),tab2MageFile.getFileType() , filename);
-        //Tab2MageHelper.write("Finished createFiles()");
         atFile = utils.getFile();
         
@@ -253 +245 @@
         writer.writeFile(); 
         dc.commit();
-        Tab2MageHelper.write("Writing file :"+atFile.getName()+ "completed!");        
+        
+        log.info("Writing temporary annotation type file :"+atFile.getName()+ "completed!");        
         
         /*
@@ -259 +252 @@
            
          */
-        //System.out.println("The experiment platform is :"+ platform);
+        
+        // [ create a new DC  here because the last one has been closed]
+        
         if (dc == null || !dc.isConnected() || dc.isClosed()) dc = sc.newDbControl();
         dir= Directory.getByPath(dc, dir.getPath());
         if (platform.equalsIgnoreCase("genepix"))  
         {
-          Tab2MageHelper.write("Begin GenePix File validation");    
+          log.info("Begin GenePix File validation");    
           List<File> gprFiles= Tab2MageHelper.getRawDataFiles(dir, dc, platform, filename);
           GenePixFileValidation gprValidation= new GenePixFileValidation(gprFiles, dc);
@@ -273 +268 @@
           }
           GenePixValidation.validateRawFiles(hybs);
-          Tab2MageHelper.write("Finished GenePix File validation");
+          log.info("Finished GenePix File validation");
         }
         /*
@@ -283 +278 @@
           // we need to parse the headers
           // obtain and check if the headers is within the allowed range
-          Tab2MageHelper.write("Begin Agilent File validation");    
+          log.info("Begin Agilent File validation");    
           List<File> agFiles= Tab2MageHelper.getRawDataFiles(dir, dc, platform, filename);
           AgilentValidation agValidation= new AgilentValidation(agFiles,dc);
@@ -291 +286 @@
             throw new ValidationException("An agilent file is missing once or more mandatory columns!");
           }
-          Tab2MageHelper.write("Finished Agilent File validation");
+          log.info("Finished Agilent File validation");
         }
           
@@ -320 +315 @@
   }
   /**
-    returns true if validation is succesful and false otherwise.
+    returns true if validation is successful and false otherwise.
       @return validated;
    */
@@ -363 +358 @@
     get the protocols in the tab2mage File
     @return protocols.
- */
+   */
   public static Set<Tab2MAGEProtocol> getCatchedProtocol() 
   {
@@ -381 +376 @@
     return tExperiment;
   }
+  
+  /**
+    check if experiment is pooled , indicated by the protocol header.
+      @return pooled TRUE if pooled indicated by Protocol[pool] and false otherwise.
+   */
+  public boolean isPooled() 
+  {
+    return pooled;
+  }
+  
+  /**
+    get Sample -to- List of Extract map.
+      @return  Map<String, List<String>
+   */
+  public static Map<String, List<String>> getSampleExtractMap() 
+  {
+    return sampleExtractMap;
+  }
+  /**
+      get BioSource -to- List of Sample map.
+      @return map
+   */
+  public static Map<String, List<String>> getBioSourceSampleMap() 
+  {
+    return bioSourceSampleMap;
+  }
+  /**
+     get Extract -to- List of Labeled Extract map.
+    @return map
+   */
+  public static Map<String, List<String>> getExtractLabeledExtractMap() 
+  {
+    return extractLabeledExtractMap;
+  }
 }

trunk/uk/ac/ebi/Tab2MageImporter/src/uk/ac/ebi/nugo/plugins/util/PluginConfigDetector.java

@@ -84 +84 @@
     public PluginConfiguration detectPluginConfig(File exampleFile) throws Exception
     {   
-      System.out.println("auto-detecting PluginConfiguration");
+      log.info("auto-detecting PluginConfiguration");
   
       DbControl dc = sc.newDbControl();
@@ -126 +126 @@
       {
         log.error("No suitable plugin configuration found for this plugin. Please ensure you have a valid " + pluginType!=null? pluginType.toString(): "" + " configuration in Base2");
-        x.printStackTrace();
+        
         throw new Exception("No suitable plugin configuration found for this plugin. Please ensure you have a valid " + pluginType!=null? pluginType.toString(): "" + " configuration in Base2");
       }
@@ -173 +173 @@
               if (importer.isImportable(in))
                 configs.add(config);
-              System.out.println("The name of the plugin configuration is: "+config.getName());
+                  log.info("The name of the plugin configuration found is: "+config.getName());
             }
             else if(pluginType== DetectablePluginTypes.ANNOTATIONTYPES)
@@ -190 +190 @@
               configs.add(config);
          }
-                 
-          //close the stream
+           //close the stream
           in.close();
         }
@@ -197 +196 @@
       catch (Exception x)
       {
-        x.printStackTrace();
+         log.error(x.getMessage());
       }
       return configs;

trunk/uk/ac/ebi/Tab2MageImporter/src/uk/ac/ebi/nugo/plugins/util/Tab2MageHelper.java

@@ -255 +255 @@
        //    try to locate the files in subdirectories too. /home/dominic/test/test/ instead
        if (fileQuery.count(dc)==0)
-         //dir.getSubDirectories();
+       {
+         System.out.println( "The direcrtory may already exist");//dir.getSubDirectories();
          return validPlatform;
+       }
        ItemResultList<File> fileList = fileQuery.list(dc);
        System.out.println("The number of files in Dir : " + dir.getPath()+ " is : " +fileList.size());
@@ -271 +273 @@
         if (!ext.equalsIgnoreCase(ExperimentPlatformsLibrary.TXT.getFileExtension()))
         { 
-          System.out.println("++++ Now checking platform +++++++++++");
+          //System.out.println("++++ Now checking platform +++++++++++");
           ExperimentPlatformsLibrary platform = ExperimentPlatformsLibrary.fromExtension(ext);    
           if (platform!=null) // platform (raw data files provided) is supported
@@ -277 +279 @@
             if (platform.getPlatformName().equalsIgnoreCase(rdt.getId())) //does the files match experiment platform?
             {
-              System.out.println("Platform ["+ platform.getPlatformName() +"] is supported" );
+              //System.out.println("Platform ["+ platform.getPlatformName() +"] is supported" );
               validPlatform= true;
             }
           } else if (platform == null) //raw data files -platforms are not supported.
           {
-            System.out.println("Platform not supported. Check the zip file content");
+            //System.out.println("Platform not supported. Check the zip file content");
             validPlatform= false;
           }
@@ -292 +294 @@
      catch (Throwable t)
      {
-       t.printStackTrace();
+      // t.printStackTrace();
        return false;
      }
@@ -364 +366 @@
      catch (Throwable t)
      {
-       t.printStackTrace();
+       //t.printStackTrace();
        throw new NutribasePluginException(t.getMessage());
      }
@@ -401 +403 @@
      catch (Throwable t)
      {
-      t.printStackTrace();
+      //t.printStackTrace();
       throw new NutribasePluginException(t.getMessage());
      }
@@ -586 +588 @@
     return s;
   }
-  
+  /**
+   * 
+   * @param dc
+   * @param sampleName
+   */
+  
+  public static Sample findSample(DbControl dc, String sampleName)
+  {
+    Sample s= null;
+    ItemQuery<Sample> query = Sample.getQuery();
+    //query.include(Include.MINE, Include.IN_PROJECT, Include.NOT_REMOVED);
+    query.include(Include.MINE, Include.NOT_REMOVED);
+    query.restrict(Restrictions.eq(Hql.property("name"), Expressions.string(sampleName)));
+    if (query.list(dc).size()!=0)
+      return query.list(dc).get(0);
+    else
+      return s; 
+    
+    
+  }
+  public static Extract findExtract(DbControl dc, String extractName)
+  {
+    Extract e= null;
+    ItemQuery<Extract> query = Extract.getQuery();
+    //query.include(Include.MINE, Include.IN_PROJECT, Include.NOT_REMOVED);
+    query.include(Include.MINE, Include.NOT_REMOVED);
+    query.restrict(Restrictions.eq(Hql.property("name"), Expressions.string(extractName)));
+    if (query.list(dc).size()!=0)
+      return query.list(dc).get(0);
+    else
+      return e; 
+    
+  }
   
   

trunk/uk/ac/ebi/Tab2MageImporter/src/uk/ac/ebi/nugo/plugins/util/ZipUnpacker.java

@@ -9 +9 @@
 import java.io.IOException;
 import java.io.InputStream;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.Locale;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
-
-import org.apache.log4j.Logger;
-
-import uk.ac.ebi.nugo.plugins.tab2mageimport.validation.Tab2MageValidationFactory;
 
 import net.sf.basedb.core.DbControl;
@@ -23 +16 @@
 import net.sf.basedb.core.File;
 import net.sf.basedb.core.Location;
-import net.sf.basedb.core.Path;
-import net.sf.basedb.core.SessionControl;
-import net.sf.basedb.core.Path.Type;
+import net.sf.basedb.core.PluginDefinition;
 import net.sf.basedb.core.Quota;
 import net.sf.basedb.core.QuotaType;
+import net.sf.basedb.core.SessionControl;
 import net.sf.basedb.core.SystemItems;
 import net.sf.basedb.core.User;
-import net.sf.basedb.core.PluginDefinition;
 import net.sf.basedb.util.zip.FileUnpacker;
+
+import org.apache.log4j.Logger;
 
 
@@ -46 +39 @@
 public class ZipUnpacker 
 {
-  protected static Logger log = Logger.getLogger(Tab2MageValidationFactory.class);
+  protected static Logger log = Logger.getLogger(ZipUnpacker.class);
   private User user;
   private File file; 
@@ -55 +48 @@
   private SessionControl sc;
   
-  public ZipUnpacker(File zipFile, User user, DbControl dc)
+  public ZipUnpacker(File zipFile, User user, SessionControl sc)
   {
     this.user = user;
-    this.dc=dc;
+    this.dc=sc.newDbControl();
     this.file= zipFile;
-    this.sc= dc.getSessionControl();
+    //this.sc= dc.getSessionControl();
   }
   /**
     Unpacks a zip file, checking that there is enough space to accommodate 
     the zip file once it is unpacked. 
-   * @throws NutribasePluginException
+      @throws NutribasePluginException
    */
   public void unpack() throws NutribasePluginException
   {
-    System.out.println("Begin to unpack the files");
+    log.info("Begin to unpack the files");
     try
     {
@@ -94 +87 @@
     catch (Throwable exc)
     {
+      dc.close();
+      log.info("File cannot be unpacked", exc);
       new NutribasePluginException(exc.getMessage());
     }
     finally
     {
-      
+      if (dc!=null)
+        dc.close();
     }
   }
@@ -160 +156 @@
       catch (IOException ioe)
       {
-        ioe.printStackTrace();
+        //ioe.printStackTrace();
+        log.error(ioe.getMessage());
       }
       return totalSize;
     }
-  
+  /**
+    create a sub-directory to contain the unzip files
+    @param zipFile, the zip file to unpack
+    @param user, currently logged-in user
+    @param dc, the database control
+   */
   public void createSubDirectory(File zipFile, User user , DbControl dc)
   {
@@ -170 +172 @@
       
         Directory zipFileDir = zipFile.getDirectory();
-        System.out.println("zipFile is located in dir " + zipFileDir.toString());
+        log.info("zipFile is located in dir " + zipFileDir.toString());
         Directory subDir = Directory.getNew(dc, zipFileDir);
         String fileName = zipFile.getName();