Changeset 4594

Timestamp:

Oct 21, 2008, 9:20:40 AM (15 years ago)

Author:

Nicklas Nordborg

Message:

Fixes #1145: Make the generic raw data importer support batch import

Location:

trunk/src

Files:

• core/net/sf/basedb/core/DataFileType.java (modified) (2 diffs)
• core/net/sf/basedb/core/UnsupportedFileFormatException.java (added)
• core/net/sf/basedb/core/plugin/AbstractPlugin.java (modified) (1 diff)
• core/net/sf/basedb/core/plugin/GuiContext.java (modified) (2 diffs)
• core/net/sf/basedb/util/AutoDetectFileFormat.java (modified) (3 diffs)
• core/net/sf/basedb/util/ContextUtil.java (modified) (2 diffs)
• core/net/sf/basedb/util/FileUtil.java (modified) (1 diff)
• core/net/sf/basedb/util/filter (added)
• core/net/sf/basedb/util/filter/AllOfFilter.java (added)
• core/net/sf/basedb/util/filter/Filter.java (added)
• core/net/sf/basedb/util/filter/FilterUtil.java (added)
• core/net/sf/basedb/util/filter/InstanceOfFilter.java (added)
• core/net/sf/basedb/util/filter/InverseFilter.java (added)
• core/net/sf/basedb/util/filter/NoneOfFilter.java (added)
• core/net/sf/basedb/util/filter/NotNullFilter.java (added)
• core/net/sf/basedb/util/filter/OneOfFilter.java (added)
• core/net/sf/basedb/util/filter/StaticFilter.java (added)
• plugins/core/net/sf/basedb/plugins/AbstractFlatFileImporter.java (modified) (23 diffs)
• plugins/core/net/sf/basedb/plugins/RawDataFlatFileImporter.java (modified) (20 diffs)
• test/TestRawDataFlatFileImporter.java (modified) (1 diff)

trunk/src/core/net/sf/basedb/core/DataFileType.java

@@ -217 +217 @@
   public static ItemQuery<DataFileType> getQuery(FileStoreEnabled item)
   {
-    ItemQuery<DataFileType> query = new ItemQuery<DataFileType>(DataFileType.class);
-    if (item != null)
-    {
-      // Restrict on itemType
-      Item itemType = item.getType();
+    ItemQuery<DataFileType> query = null;
+    if (item == null)
+    {
+      query = getQuery();
+    }
+    else
+    {
+      query = getQuery(item.getPlatform(), item.getVariant(), item.getType(), null);
+    }
+    return query;   
+  }
+
+  /**
+    Get a query to retrieve <code>DataFileType</code>:s that can be used
+    on specific platform/variant for a given item and/or generic file type.
+    All parameters are optional. If all are null the result is the same
+    as {@link #getQuery()}.
+    
+    @param platform The platform, or null to not restrict by platform (unless
+      a variant is given)
+    @param variant The platform variant, or null to not restrict by variant
+    @param itemType The type of item it should be possible to
+      attach file to, or null to not restrict by item
+    @param genericType The generic file type, or null to not restrict
+      by file type
+    @return A query returning data file types matching the specified
+      restrictions
+    @since 2.9
+   */
+  public static ItemQuery<DataFileType> getQuery(Platform platform, PlatformVariant variant,
+    Item itemType, FileType genericType)
+  {
+    ItemQuery<DataFileType> query = getQuery();
+    // Restrict on itemType
+    if (itemType != null)
+    {
       query.restrictPermanent(
         Restrictions.eq(
@@ -228 +259 @@
         )
       );
-      
-      // Restrict on platform / variant
-      PlatformVariant variant = item.getVariant();
-      Platform platform = variant != null ? variant.getPlatform() : item.getPlatform();
-      if (platform != null)
+    }
+    
+    // Restrict on generic file type
+    if (genericType != null)
+    {
+      query.restrictPermanent(
+        Restrictions.eq(
+          Hql.property("genericType"), 
+          Hql.entity(genericType))
+      );
+    }
+    
+    // Restrict on platform/variant
+    if (variant != null && platform == null) platform = variant.getPlatform();
+    if (platform != null)
+    {
+      query.join(Hql.innerJoin("platforms", "pft"));
+      query.restrict(Restrictions.eq(Hql.property("pft", "platform"), Hql.entity(platform)));
+      if (variant != null)
       {
-        query.join(Hql.innerJoin("platforms", "pft"));
-        query.restrict(Restrictions.eq(Hql.property("pft", "platform"), Hql.entity(platform)));
-        if (variant != null)
-        {
-          query.restrict(Restrictions.or(
-            Restrictions.eq(Hql.property("pft", "variant"), null),
-            Restrictions.eq(Hql.property("pft", "variant"), Hql.entity(variant))
-          ));
-        }
-        else
-        {
-          query.restrict(Restrictions.eq(Hql.property("pft", "variant"), null));
-        }
+        query.restrict(Restrictions.or(
+          Restrictions.eq(Hql.property("pft", "variant"), null),
+          Restrictions.eq(Hql.property("pft", "variant"), Hql.entity(variant))
+        ));
       }
-    }
-    return query;   
-  }
-
+      else
+      {
+        query.restrict(Restrictions.eq(Hql.property("pft", "variant"), null));
+      }
+    }
+    return query;
+  }
+  
+  
   /**
     Creates a new filetype item from the given data.

trunk/src/core/net/sf/basedb/core/plugin/AbstractPlugin.java

@@ -423 +423 @@
   {
     if (logger == null) return;
-    logger.println(message);
-    if (t != null) 
-    {
-      logger.println(t.getMessage());
+    if (t == null)
+    {
+      logger.println(message);
+    }
+    else
+    {
+      logger.println(message + ": " + t.getMessage());
       t.printStackTrace(logger);
     }

trunk/src/core/net/sf/basedb/core/plugin/GuiContext.java

@@ -37 +37 @@
 public class GuiContext
 {
+  /**
+    Shortcut to create a single-item context.
+    @since 2.9
+  */
+  public static GuiContext item(Item item)
+  {
+    return new GuiContext(item, Type.ITEM);
+  }
+
+  /**
+    Shortcut to create a list context.
+    @since 2.9
+  */
+  public static GuiContext list(Item item)
+  {
+    return new GuiContext(item, Type.LIST);
+  }
+
+  
   private final Item item;
   private final Type type;
@@ -74 +93 @@
   {
     return subContext;
+  }
+  
+  /**
+    Checks if the specified object is an instance of the
+    BasicItem subclass used by the {@link Item} in this context.
+    @param o The object to check
+    @since 2.9
+  */
+  public boolean isCorrectBasicItem(Object o)
+  {
+    return item.getItemClass().isInstance(o);
+  }
+  
+  /**
+    Checks if the specified object is an instance of the
+    BasicData subclass used by the {@link Item} in this context.
+    @param o The object to check
+    @since 2.9
+  */
+  public boolean isCorrectBasicData(Object o)
+  {
+    return item.getDataClass().isInstance(o);
   }
   

trunk/src/core/net/sf/basedb/util/AutoDetectFileFormat.java

@@ -40 +40 @@
 import net.sf.basedb.core.plugin.GuiContext;
 import net.sf.basedb.core.plugin.InteractivePlugin;
+import net.sf.basedb.util.ContextUtil.ContextResult;
+import net.sf.basedb.util.filter.Filter;
 
 import java.util.List;
@@ -159 +161 @@
       importer = plugin.newInstance(AutoDetectingImporter.class, null, sc, config, null);
       boolean isInContext = true;
-      if (plugin.isInteractive())
+      if (plugin.isInteractive() && context != null)
       {
         try
@@ -200 +202 @@
   }
   
+  /**
+    Filter implementation that filters a collection of
+    {@link ContextResult}:s by checking each plug-ins ability to
+    import a given file. Eg. the ContextResult is accepted if the
+    {@link AutoDetectingImporter#isImportable(InputStream)} returns
+    true. It is assumed that the collection of ContextResult:s only
+    contains import plug-ins that implements the AutoDetectingImporter 
+    interface.
+  
+    @author Nicklas
+    @version 2.9
+    @base.modified $Date$
+  */
+  public static class IsImportableFilter
+    implements Filter<ContextResult>
+  {
+    private File file;
+    private Filter<? super ContextResult> parent;
+    
+    public IsImportableFilter(File file, Filter<? super ContextResult> parent)
+    {
+      this.file = file;
+      this.parent = parent;
+    }
+    
+    public boolean evaluate(ContextResult result)
+    {
+      if (result == null) return false;
+      if (parent != null && !parent.evaluate(result)) return false;
+      if (!result.isInContext()) return false;
+      return checkImportable(file.getSessionControl(), 
+          result.getPluginDefinition(), result.getPluginConfiguration(), file, 
+          null, null, null);
+    }
+  }
+  
 }

trunk/src/core/net/sf/basedb/util/ContextUtil.java

@@ -36 +36 @@
 import net.sf.basedb.core.plugin.GuiContext;
 import net.sf.basedb.core.plugin.InteractivePlugin;
+import net.sf.basedb.util.filter.Filter;
 
 /**
@@ -249 +250 @@
     
   }
+  
+  /**
+    Filter implementation that works on collections
+    of {@link ContextResult}:s and return only those
+    that return true from {@link ContextResult#isInContext()}.
+  
+    @author Nicklas
+    @version 2.9
+    @base.modified $Date$
+  */
+  public static class IsInContextFilter
+    implements Filter<ContextResult>
+  {
+
+    public IsInContextFilter()
+    {}
+    
+    /*
+      From the Filter interface
+      -------------------------
+    */
+    @Override
+    public boolean evaluate(ContextResult result)
+    {
+      return result != null && result.isInContext();
+    }
+    // -----------------------------
+  }
+  
 }

trunk/src/core/net/sf/basedb/util/FileUtil.java

@@ -139 +139 @@
 
   /**
+    Close an input stream without throwing an exception.
+    @param in The input stream to close
+    @since 2.9
+  */
+  public static void close(InputStream in)
+  {
+    if (in == null) return;
+    try
+    {
+      in.close();
+    }
+    catch (Throwable t)
+    {}
+  }
+  
+  /**
+    Close an output stream without throwing an exception.
+    @param out The output stream to close
+    @since 2.9
+  */
+  public static void close(OutputStream out)
+  {
+    if (out == null) return;
+    try
+    {
+      out.close();
+    }
+    catch (Throwable t)
+    {}
+  }
+  
+  /**
     Get a buffered <code>InputStream</code> object reading from
     the specified file.

trunk/src/plugins/core/net/sf/basedb/plugins/AbstractFlatFileImporter.java

@@ -44 +44 @@
 import net.sf.basedb.core.StringUtil;
 import net.sf.basedb.core.Type;
+import net.sf.basedb.core.UnsupportedFileFormatException;
 
 import net.sf.basedb.core.plugin.AbstractPlugin;
@@ -51 +52 @@
 import net.sf.basedb.core.plugin.Response;
 import net.sf.basedb.core.plugin.Plugin;
+import net.sf.basedb.core.signal.SignalException;
 import net.sf.basedb.core.signal.SignalHandler;
 import net.sf.basedb.core.signal.SignalTarget;
@@ -56 +58 @@
 
 import net.sf.basedb.plugins.util.Parameters;
+import net.sf.basedb.util.FileUtil;
 import net.sf.basedb.util.NumberFormatUtil;
 import net.sf.basedb.util.error.ClassMapErrorHandler;
@@ -68 +71 @@
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
 import java.util.regex.Pattern;
 import java.io.IOException;
@@ -75 +80 @@
 
 /**
-  An abstract base class for all importers that imports data from a single
-  flat file. The implementation in this class uses a {@link FlatFileParser}
-  for parsing the file and uses a callback method {@link #handleData(FlatFileParser.Data)}
+  An abstract base class for all importers that imports data from one or more
+  flat files. The implementation in this class uses a {@link FlatFileParser}
+  for parsing the files and uses a callback method {@link #handleData(FlatFileParser.Data)}
   that lets the subclass do whatever it needs to insert a single line
   of data into the database.
@@ -139 +144 @@
 </pre>
 
+  <p>
+  For multi-file support (added in BASE 2.9) the subclass needs to
+  override the {@link #getFileIterator()} and {@link #getTotalFileSize()}
+  methods.
   
   @base.modified $Date$
@@ -381 +390 @@
     Parameters.numberFormatError(null, null, null);
   
-  private long fileSize;
+  private File currentFile;
+  private long totalFileSize;
+  private long finishedFileSize;
   private int skippedLines;
   private ClassMapErrorHandler errorHandler;
@@ -426 +437 @@
     {
       createLogFile((String)job.getValue(Parameters.LOGFILE_PARAMETER));
-      InputStream in = null;
+      boolean hasCalledFinish = false;
+      int numSuccess = 0;
+      int numTotal = 0;
+      String finalMessage = null;
       try
       {
-        File f = (File)job.getValue("file");
-        log("Parsing file: " + f.getName());
-        in = f.getDownloadStream(0);
-        boolean importable = isImportable(in);
-        in.close();
-        if (!importable)
+        start();
+        String message = null;
+        finishedFileSize = 0;
+        Iterator<File> files = getFileIterator();
+        totalFileSize = getTotalFileSize();
+        
+        while (files.hasNext())
         {
-          throw new InvalidDataException("File '"+f+"' cannot be imported by this plugin.");
-        }
-        fileSize = f.getSize();
-        in = f.getDownloadStream(0);
-        doImport(in, progress);
-        in.close();
-        String message = getSuccessMessage(skippedLines);
-        if (message != null) log(message);
-        response.setDone(message);
-      }
-      catch (Throwable ex)
-      {
-        log("Failed", ex);
-        if (in != null) 
-        {
+          numTotal++;
+          InputStream in = null;
           try
           {
-            in.close();
+            currentFile = files.next();
+            log("Parsing file: " + currentFile.getName());
+            in = currentFile.getDownloadStream(0);
+            boolean importable = isImportable(in);
+            FileUtil.close(in);
+            if (!importable)
+            {
+              throw new UnsupportedFileFormatException("File '"+currentFile.getName()+
+                "' doesn't match the expected file format.");
+            }
+            in = currentFile.getDownloadStream(0);
+            doImport(in, progress);
+            FileUtil.close(in);
+            message = getSuccessMessage(skippedLines);
+            if (message == null) 
+            {
+              message = "File '" + currentFile.getName() + 
+                "' imported successfully. " + 
+                skippedLines + " lines skipped.";
+            }
+            log(message);
+            numSuccess++;
           }
-          catch (IOException iox)
-          {}
+          catch (SignalException s)
+          {
+            // The user aborted the plug-in -> always exit
+            throw s;
+          }
+          catch (Throwable t)
+          {
+            if (!continueWithNextFileAfterError(t)) throw t;
+            log("Failed", t);
+          }
+          finally
+          {
+            if (currentFile != null)
+            {
+              finishedFileSize += currentFile.getSize();
+            }
+            FileUtil.close(in);
+            log("--------------------------------------");
+          }
         }
-        response.setError(ex.getMessage(), Arrays.asList(ex));
+        hasCalledFinish = true;
+        finalMessage = finish(null);
+        if (numSuccess == 0)
+        {
+          if (finalMessage == null)
+          {
+            finalMessage = "No files could be imported";
+          }
+          response.setError(finalMessage + ": " + message, null);
+        }
+        else
+        {
+          if (finalMessage == null && numTotal > 1)
+          {
+            finalMessage = numSuccess + " of " + numTotal + " files could be imported";
+          }
+          response.setDone(finalMessage == null ? message : finalMessage);
+        }
+      }
+      catch (Throwable t)
+      {
+        String message = hasCalledFinish ? null : finish(t);
+        if (message == null) message = t.getMessage();
+        log("Failed: " + message, t);
+        response.setError(message, Arrays.asList(t));
       }
       finally
       {
+        if (finalMessage != null) log(finalMessage);
         closeLogFile();
       }
@@ -536 +601 @@
         if (section != null && section.type() == FlatFileParser.LineType.SECTION)
         {
-          if (progress != null) progress.display(getProgress(ffp), "Parsing section "+section.name()+"...");
+          if (progress != null) 
+          {
+            progress.display(getProgress(ffp), "Parsing section '"+section.name()+
+                "' in file '" + currentFile.getName() + "'...");
+          }
           try
           {
@@ -559 +628 @@
             {
               throw new BaseException(t2.getMessage() + 
-                  " on line " + section.lineNo() + ": " + 
-                  StringUtil.trimString(section.line(), 20), t2); 
+                " on line " + section.lineNo() + 
+                " in file '" + currentFile.getName() + "': " + 
+                StringUtil.trimString(section.line(), 20), t2); 
             }
           }
@@ -566 +636 @@
         else
         {
-          if (progress != null) progress.display(getProgress(ffp), "Parsing headers...");
+          if (progress != null) 
+          {
+            progress.display(getProgress(ffp), 
+              "Parsing headers in file '" + currentFile.getName() + "'...");
+          }
         }
         checkInterrupted();
@@ -602 +676 @@
         {
           throw new BaseException(t.getMessage() + 
-            " on line " + (line == null ? "-1" : line.lineNo() + ": " + StringUtil.trimString(line.line(), 20)), t);      
+            " on line " + (line == null ? "-1" : line.lineNo() + 
+            " in file '" + currentFile.getName() + "': " + 
+             StringUtil.trimString(line.line(), 20)), t);     
         }
         if (result == FlatFileParser.LineType.DATA || result == FlatFileParser.LineType.DATA_HEADER)
         {
-          if (progress != null) progress.display(getProgress(ffp), "Parsing data...");
+          if (progress != null) 
+          {
+            progress.display(getProgress(ffp), 
+              "Parsing data in file '" + currentFile.getName() + "'...");
+          }
           int progressLine = 0;
           
@@ -638 +718 @@
                 if (progress != null)
                 {
-                  progress.display(getProgress(ffp), "Parsing data... (" + ffp.getParsedLines() + " lines so far)");
+                  progress.display(getProgress(ffp), 
+                    "Parsing data in file '" + currentFile.getName() + 
+                    "'... (" + ffp.getParsedLines() + " lines so far)");
                 }
               }
@@ -648 +730 @@
           {
             throw new BaseException(t.getMessage() + 
-              " on line " + (dataline == null ? "-1" : dataline.lineNo() + ": " + StringUtil.trimString(dataline.line(), 20)), t);
+              " on line " + (dataline == null ? "-1" : dataline.lineNo() + 
+              " in file '" + currentFile.getName() + "': " + 
+              StringUtil.trimString(dataline.line(), 20)), t);
           }
         }
@@ -654 +738 @@
       while (ffp.hasMoreSections());
       isSuccess = true;
-      if (progress != null) progress.display(100, "Done parsing "+ffp.getParsedLines()+" lines.\n");
+      if (progress != null) 
+      {
+        progress.display(getProgress(ffp), 
+          "Done parsing file '" + currentFile.getName() + 
+          "' ("+ffp.getParsedLines()+" lines)\n");
+      }
     }
     catch (IOException ex)
@@ -685 +774 @@
 
   /**
+    Called once before starting the import. 
+    The default implementation does nothing.
+    @since 2.9
+  */
+  protected void start()
+  {}
+  
+  /**
+    Get an iterator that returns the files to be imported. The default
+    implementation returns the single file found in the job's "file"
+    parameter. Subclasses that needs multi-file/item import should override
+    this method to provide their own iterator. They should also override
+    the {@link #getTotalFileSize()} method to return sum of all
+    file sizes. Eg. {@link File#getSize()}.
+    @since 2.9
+  */
+  protected Iterator<File> getFileIterator()
+  {
+    return Collections.singleton((File)job.getValue("file")).iterator();
+  } 
+  
+  /**
+    Get the total file size of all files that are going to be imported.
+    A subclass that is going to import from multiple files needs to override
+    this method. The default implementation return the size of the file
+    in the job's "file" parameter.
+    @return The sum of the file sizes
+    @since 2.9
+  */
+  protected long getTotalFileSize()
+  {
+    return ((File)job.getValue("file")).getSize();
+  }
+  
+  /**
     Get the progress of import as a percentage value. The default implementation
-    calls {@link #getNumBytes(FlatFileParser)} and divides it by the file size.
+    calls sums the file size of the completed files and 
+    {@link #getNumBytes(FlatFileParser)}. This values is divided by 
+    {@link #getTotalFileSize()}.
     @param ffp The file parser that is used to parsed the file
     @return A value between 0 and 100
@@ -693 +819 @@
   protected int getProgress(FlatFileParser ffp)
   {
-    return (int) (100 * getNumBytes(ffp) / fileSize);
-  }
-  
+    return (int) (100 * (getNumBytes(ffp) + finishedFileSize) / totalFileSize);
+  }
   
   /**
@@ -715 +840 @@
     return ffp.getParsedBytes();
   }
-
+  
   /**
     This method is called by the {@link #isImportable(InputStream)} method after
@@ -755 +880 @@
     Called just before parsing of the file begins. A subclass
     may override this method if it needs to initialise some
-    resources before the parsing starts.
+    resources before the parsing starts. Note that this method is called
+    once for each file returned by {@link #getFileIterator()}.
     @see #end(boolean)
   */
@@ -797 +923 @@
     or immediately after an error has ocurred. A subclass should clean
     up any resources aquired in the {@link #begin(FlatFileParser)} method here.
+    Note that this metod is called once for every file returned by the
+    {@link #getFileIterator()} iterator.
     @param success TRUE if the file was parsed successfully, FALSE otherwise
     @see #begin(FlatFileParser)
@@ -808 +936 @@
     message that is sent back to the core and user interface. An example
     message might by: <code>178 reporters imported successfully</code>.
-    The default implementation always return null.
+    The default implementation always return null. Note that this method is called
+    once for every file returned by {@link #getFileIterator()}.
     @param skippedLines The number of data lines that were skipped due to errors
   */
   protected String getSuccessMessage(int skippedLines)
+  {
+    return null;
+  }
+  
+  /**
+    Called once when all files has been imported or when exiting due to an error.
+    This method may return a message that is sent back to the core and user interface. 
+    If this method returns null the last message returned by {@link #getSuccessMessage(int)}
+    is used instead.
+    @param t Null if no error has happened
+    @return A message or null
+    @since 2.9
+  */
+  protected String finish(Throwable t)
   {
     return null;
@@ -820 +963 @@
     and other options. This implementation gets all parameters from the 
     {@link #configuration} settings. If a subclass doesn't store the parameters
-    there it must override this method and initialise the parser.
+    there it must override this method and initialise the parser. Note that
+    this method is called once for each file returned by the 
+    {@link #getFileIterator()} and that a new parser is needed for each file.
     @return An intialised flat file parser
     @throws BaseException
@@ -1126 +1271 @@
   
   /**
+    If the importer should continue with the next file after an error.
+    The default implementation of this method return FALSE. A subclass
+    may override this method to let the import continue after an error.
+    This method isn't called until an error has happened, and it is
+    possible to control what should happen on a file-by-file bases. 
+    The importer will of course exit if this method returns FALSE.
+    @param t The error that happened
+    @return TRUE to contine, FALSE to abort
+    @since 2.9
+  */
+  protected boolean continueWithNextFileAfterError(Throwable t)
+  {
+    return false;
+  }
+  
+  /**
     Initialise the error handling system. This method is called just before the 
-    import is starting. A subclass may override this method to add specific 
+    import of a file is starting. A subclass may override this method to add specific 
     error handlers. If <code>super.setUpErrorHandling()</code> isn't called
     error handling in AbstractFlatFileImporter is disabled and the subclass
     must do all it's error handling in it's own code. The subclass may also add 
-    error handlers in the {@link #begin(FlatFileParser)} method.
+    error handlers in the {@link #begin(FlatFileParser)} method. Note that
+    the error handling system is re-initialised for every file returned by
+    {@link #getFileIterator()}.
   */
   protected void setUpErrorHandling()

trunk/src/plugins/core/net/sf/basedb/plugins/RawDataFlatFileImporter.java

@@ -26 +26 @@
 import net.sf.basedb.core.ArrayDesign;
 import net.sf.basedb.core.BaseException;
+import net.sf.basedb.core.BooleanParameterType;
 import net.sf.basedb.core.DataFileType;
 import net.sf.basedb.core.DbControl;
+import net.sf.basedb.core.Experiment;
 import net.sf.basedb.core.FeatureIdentificationMethod;
 import net.sf.basedb.core.File;
 import net.sf.basedb.core.FileParameterType;
+import net.sf.basedb.core.FileSet;
 import net.sf.basedb.core.FileSetMember;
 import net.sf.basedb.core.FileStoreUtil;
 import net.sf.basedb.core.FileType;
+import net.sf.basedb.core.Include;
 import net.sf.basedb.core.Item;
 import net.sf.basedb.core.ItemAlreadyExistsException;
@@ -42 +46 @@
 import net.sf.basedb.core.Permission;
 import net.sf.basedb.core.PermissionDeniedException;
+import net.sf.basedb.core.Platform;
+import net.sf.basedb.core.PlatformVariant;
 import net.sf.basedb.core.PluginConfiguration;
 import net.sf.basedb.core.PluginParameter;
@@ -51 +57 @@
 import net.sf.basedb.core.RequestInformation;
 import net.sf.basedb.core.StringParameterType;
+import net.sf.basedb.core.SystemItems;
+import net.sf.basedb.core.UnsupportedFileFormatException;
 import net.sf.basedb.core.Version;
 import net.sf.basedb.core.data.FeatureData;
@@ -63 +71 @@
 import net.sf.basedb.core.plugin.Request;
 import net.sf.basedb.core.plugin.Response;
+import net.sf.basedb.core.query.Expressions;
+import net.sf.basedb.core.query.Hql;
+import net.sf.basedb.core.query.Orders;
+import net.sf.basedb.core.query.Restrictions;
 import net.sf.basedb.plugins.util.Parameters;
 import net.sf.basedb.util.Enumeration;
@@ -82 +94 @@
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -112 +125 @@
   
   private static final Set<GuiContext> guiContexts = 
-    Collections.singleton(new GuiContext(Item.RAWBIOASSAY, GuiContext.Type.ITEM));
-  
+    Collections.unmodifiableSet(new HashSet<GuiContext>(
+        Arrays.asList(
+          GuiContext.item(Item.RAWBIOASSAY),
+          GuiContext.list(Item.EXPERIMENT)
+        )
+      )); 
   private static final Set<Permissions> permissions = new HashSet<Permissions>();
   
@@ -270 +287 @@
   private PluginParameter<String> rawDataTypeParameter;
   
-  private ItemParameterType<RawBioAssay> rawBioAssayType;
-  private PluginParameter<RawBioAssay> rawBioAssayParameter;
-
   private DbControl dc;
   private FlatFileParser ffp;
   private NumberFormat numberFormat;
   private boolean nullIfException;
+  private boolean continueAfterError;
+  private boolean transactionPerFile;
+  private FileIterator fileIterator;
+  private long totalFileSize;
+  
+  private RawBioAssay rawBioAssay;
   private RawDataBatcher batcher;
-  private RawBioAssay rawBioAssay;
   private FileSetMember rawDataMember;
 
@@ -345 +364 @@
   public String isInContext(GuiContext context, Object item)
   {
-    String message = null;
     if (item == null)
     {
-      message = "The object is null";
-    }
-    else if (!(item instanceof RawBioAssay))
-    {
-      message = "The object is not a RawBioAssay: " + item;
+      return "The object is null";
+    }
+    else if (!context.isCorrectBasicItem(item))
+    {
+      return "The oject is not a " + context.getItem() + ": " + item;
     }
     else
     {
-      RawBioAssay rba = (RawBioAssay)item;
+      Experiment exp = null;
+      RawBioAssay rba = null;
+      RawDataType rdt = null;
+      if (item instanceof RawBioAssay)
+      {
+        rba = (RawBioAssay)item;
+        rdt = rba.getRawDataType();
+        if (rba.getNumDbSpots() > 0)
+        {
+          throw new PermissionDeniedException("The raw bioassay already has data.");
+        }
+        rba.checkPermission(Permission.WRITE);
+      }
+      else
+      {
+        exp = (Experiment)item;
+        rdt = exp.getRawDataType();
+        ItemQuery<RawBioAssay> query = getRawBioAssayQuery(exp);
+        DbControl dc = sc.newDbControl();
+        try
+        {
+          if (query.count(dc) == 0)
+          {
+            return "The experiment doesn't have any raw bioassays with files waiting for import";
+          }
+        }
+        finally
+        {
+          if (dc != null) dc.close();
+        }
+      }
       String rawDataType = (String)configuration.getValue("rawDataType");
-      RawDataType rdt = rba.getRawDataType();
       if (!rdt.getId().equals(rawDataType))
       {
-        message = "Unsupported raw data type: " + rba.getRawDataType().getName();
+        return "Unsupported raw data type: " + rba.getRawDataType().getName();
       }
       else if (!rdt.isStoredInDb())
       {
-        message = "Raw data for raw data type '" + rdt + "' is not stored in the database";
-      }
-      else if (rba.getNumDbSpots() > 0)
-      {
-        throw new PermissionDeniedException("The raw bioassay already has data.");
-      }
-      else
-      {
-        rba.checkPermission(Permission.WRITE);
-      }
-    }
-    return message;   
+        return "Raw data for raw data type '" + rdt + "' is not stored in the database";
+      }
+    }
+    return null;
   }
 
@@ -479 +518 @@
         }
         
-        storeValue(job, request, rawBioAssayParameter);
-        storeValue(job, request, fileParameter);
+        // Single raw bioassay mode
+        storeValue(job, request, ri.getParameter("rawBioAssay"));
+        storeValue(job, request, ri.getParameter("file"));
+        storeValue(job, request, ri.getParameter("featureIdentification"));
+        
+        // Experiment mode
+        storeValue(job, request, ri.getParameter("experiment"));
+        storeValue(job, request, ri.getParameter("dataFileType"));
+        storeValue(job, request, ri.getParameter("transactionPerFile"));
+        storeValue(job, request, ri.getParameter("continueAfterError"));
+        
+        // Common
         storeValue(job, request, ri.getParameter(Parameters.CHARSET_PARAMETER));
         storeValue(job, request, ri.getParameter(Parameters.DECIMAL_SEPARATOR_PARAMETER));
@@ -489 +538 @@
         
         // Error handling parameters
+        storeValue(job, request, ri.getParameter(Parameters.LOGFILE_PARAMETER));
         storeValue(job, request, defaultErrorParameter);
         storeValue(job, request, missingReporterErrorParameter);
@@ -528 +578 @@
   */
   /**
-   Create a {@link DbControl} and a {@link RawDataBatcher}. 
-   Load the {@link RawBioAssay} that has been specified.
+    Prepare for the import. If used from an experiment context
+    prepare the list of files/raw bioassays to import.
+  */
+  @Override 
+  protected void start()
+  {
+    transactionPerFile = Boolean.TRUE.equals(job.getValue("transactionPerFile"));
+    continueAfterError = transactionPerFile && 
+      Boolean.TRUE.equals(job.getValue("continueAfterError"));
+    Experiment exp = (Experiment)job.getValue("experiment");
+    List<RawBioAssay> rawBioAssays = null;
+    List<File> files = null;
+    if (exp == null)
+    {
+      File f = (File)job.getValue("file");
+      RawBioAssay r = (RawBioAssay)job.getValue("rawBioAssay");
+      rawBioAssays = Collections.singletonList(r);
+      files = Collections.singletonList(f);
+      totalFileSize = f.getSize();
+    }
+    else
+    {
+      ItemQuery<RawBioAssay> query = getRawBioAssayQuery(exp);
+      DataFileType fileType = (DataFileType)job.getValue("dataFileType");
+      rawBioAssays = new ArrayList<RawBioAssay>();
+      files = new ArrayList<File>();
+      DbControl dc = sc.newDbControl();
+      try
+      {
+        for (RawBioAssay rba : query.list(dc))
+        {
+          File dataFile = null;
+          if (fileType != null)
+          {
+            FileSet fs = rba.getFileSet();
+            FileSetMember member = fs.getMember(fileType);
+            if (member != null) dataFile = member.getFile();
+          }
+          else
+          {
+            List<File> rawFiles = FileStoreUtil.getGenericDataFiles(dc, rba, FileType.RAW_DATA);
+            if (rawFiles != null && rawFiles.size() > 0) dataFile = rawFiles.get(0);
+          }
+          if (dataFile != null)
+          {
+            rawBioAssays.add(rba);
+            files.add(dataFile);
+            totalFileSize += dataFile.getSize();
+          }
+        }
+      }
+      finally
+      {
+        if (dc != null) dc.close();
+      }
+    }
+    this.fileIterator = new FileIterator(rawBioAssays, files);
+  }
+  
+  /**
+    We will continue if:
+    <ul>
+    <li>The file is not of the correct format, eg. t = {@link UnsupportedFileFormatException}.
+    <li>The 'Continue after error' parameter is true
+    </ul>
+  */
+  protected boolean continueWithNextFileAfterError(Throwable t)
+  {
+    return continueAfterError || t instanceof UnsupportedFileFormatException;
+  }
+  
+  @Override
+  protected Iterator<File> getFileIterator()
+  {
+    return fileIterator;
+  }
+  
+  @Override
+  protected long getTotalFileSize()
+  {
+    return totalFileSize;
+  }
+  
+  /**
+    Prepare for importing the next file. Eg. 
+    create a {@link DbControl} (if needed) and a {@link RawDataBatcher}. 
+    Load the {@link RawBioAssay} that has been specified.
   */
   @Override
@@ -536 +671 @@
   {
     super.begin(ffp);
-    dc = sc.newDbControl();
-    rawBioAssay = (RawBioAssay)job.getValue("rawBioAssay");
-    File rawDataFile = (File)job.getValue("file");
-    rawBioAssay = RawBioAssay.getById(dc, rawBioAssay.getId());
+    if (transactionPerFile || dc == null || dc.isClosed()) 
+    {
+      dc = sc.newDbControl();
+    }
+
+    rawBioAssay = RawBioAssay.getById(dc, fileIterator.getCurrentRawBioAssay().getId());
     rawDataMember = FileStoreUtil.setGenericDataFile(dc, rawBioAssay, 
-      FileType.RAW_DATA, DataFileType.GENERIC_RAW_DATA, rawDataFile);
+      FileType.RAW_DATA, DataFileType.GENERIC_RAW_DATA, fileIterator.getCurrentFile());
     
     PluginConfiguration cfg = getCurrentConfiguration(dc);
@@ -644 +781 @@
       {
         batcher.close();
-        dc.commit();
+        if (transactionPerFile) 
+        {
+          dc.commit();
+          dc = null;
+        }
       }
     }
@@ -654 +795 @@
     finally
     {
+      if (dc != null && transactionPerFile) 
+      {
+        dc.close();
+        dc = null;
+      }
+      super.end(success);
+    }
+  }
+  
+  /**
+    Commit/rollback transaction if needed. If everything
+    went ok, t=null.
+  */
+  @Override
+  public String finish(Throwable t)
+  {
+    try
+    {
+      if (t == null && dc != null && !dc.isClosed()) dc.commit();
+      super.finish(t);
+    }
+    finally
+    {
       if (dc != null) dc.close();
-      super.end(success);
-    }
+    }
+    return null;
   }
   
@@ -770 +934 @@
       List<PluginParameter<?>> parameters = new ArrayList<PluginParameter<?>>();
       
-      // Parameter for getting the current raw bioassay
-      rawBioAssayType = new ItemParameterType<RawBioAssay>(RawBioAssay.class, null, true, 1, null);
-      rawBioAssayParameter = new PluginParameter<RawBioAssay>
-      (
-        "rawBioAssay",
-        "Raw bioassay",
-        "The raw bioassay to import raw data to. It must be of the correct raw data type and shouldn't have any spots already",
-        rawBioAssayType
-      );
-      parameters.add(rawBioAssayParameter);
-
-      RawBioAssay rba = null;
-      int currentRawBioAssayId = sc.getCurrentContext(Item.RAWBIOASSAY).getId();
-      List<File> rawDataFiles = null;
-      dc = sc.newDbControl();
+      RawBioAssay currentRaw = (RawBioAssay)job.getValue("rawBioAssay");
+      Experiment currentExp = (Experiment)job.getValue("experiment");
       boolean configIsAnnotated = false;
-      ArrayDesign design = null;
+      boolean useMissingReporterParameter = true;
+      boolean useFeatureParameters = true;
+      FeatureIdentificationMethod defaultFiMethod = FeatureIdentificationMethod.NONE;
+      DbControl dc = sc.newDbControl();
       try
       {
-        rba = RawBioAssay.getById(dc, currentRawBioAssayId);
-        rawDataFiles = FileStoreUtil.getGenericDataFiles(dc, rba, FileType.RAW_DATA);
         configIsAnnotated = getCurrentConfiguration(dc).isAnnotated();
-        design = rba.getArrayDesign();
-      }
-      catch (Throwable t)
-      {}
-      finally
+        if (GuiContext.item(Item.RAWBIOASSAY).equals(context))
+        {
+          if (currentRaw == null)
+          {
+            int id = sc.getCurrentContext(Item.RAWBIOASSAY).getId();
+            if (id != 0) currentRaw = RawBioAssay.getById(dc, id);
+          }
+          else
+          {
+            currentRaw = RawBioAssay.getById(dc, currentRaw.getId());
+          }
+          if (currentRaw != null)
+          {
+            useFeatureParameters = currentRaw.hasArrayDesign();
+            ArrayDesign design = currentRaw.getArrayDesign();
+            if (design != null) defaultFiMethod = design.getFeatureIdentificationMethod();
+          }
+          useMissingReporterParameter = !useFeatureParameters;
+          addRawBioAssayParameters(dc, currentRaw, parameters);
+        }
+        else
+        {
+          if (currentExp == null)
+          {
+            int id = sc.getCurrentContext(Item.EXPERIMENT).getId();
+            if (id != 0) currentExp = Experiment.getById(dc, id);
+          }
+          addExperimentParameters(dc, currentExp, parameters);
+        }
+      }
+      finally 
       {
         if (dc != null) dc.close();
       }
-      
-      // The raw data file to import from - if a file already hase
-      // been attached to the raw bioassay use it as a default choice
-      PluginParameter<File> fileParameter = new PluginParameter<File>(
-        "file",
-        "Raw data file",
-        "The file that contains the raw data that you want to import",
-        new FileParameterType(rawDataFiles == null || rawDataFiles.isEmpty() ? 
-          null : rawDataFiles.get(0), true, 1)
-      );
-      parameters.add(fileParameter);
-      
-      if (design != null && design.getNumDbFeatures() > 0)
+        
+      if (useFeatureParameters)
       {
         // Parameter for asking which feature identification method to use
         // Only used if the connected array design has features
-        
-        FeatureIdentificationMethod fiMethod = design.getFeatureIdentificationMethod();
         List<String> methods = new ArrayList<String>();
         String description = "Choose which method to use for identifying features: \n";
@@ -843 +1008 @@
               "Identify features by",
               description,
-              new StringParameterType(255, methods.contains(fiMethod.name()) ? fiMethod.name() : null, 
-                true, 1, 0, 0, methods)
+              new StringParameterType(255, 
+                methods.contains(defaultFiMethod.name()) ? defaultFiMethod.name() : null, 
+                false, 1, 0, 0, methods)
             );
           parameters.add(featureIdentificationParameter);
@@ -851 +1017 @@
       
       parameters.add(Parameters.charsetParameter(null, null, 
-          (String)configuration.getValue(Parameters.CHARSET_PARAMETER)));
+          currentExp != null ? null : (String)configuration.getValue(Parameters.CHARSET_PARAMETER)));
       parameters.add(Parameters.decimalSeparatorParameter(null, null, 
           (String)configuration.getValue(Parameters.DECIMAL_SEPARATOR_PARAMETER)));
@@ -867 +1033 @@
       // Error handling parameters
       parameters.add(errorSection);
+      parameters.add(Parameters.logFileParameter(null, null, null));
       parameters.add(defaultErrorParameter);
       
-      RawBioAssay rawBioAssay = (RawBioAssay)job.getValue("rawBioAssay");
-      if (rawBioAssay == null && context != null)
-      {
-        DbControl dc = sc.newDbControl();
-        try
-        {
-          int rawBioAssayId = sc.getCurrentContext(context.getItem()).getId();
-          if (rawBioAssayId != 0) rawBioAssay = RawBioAssay.getById(dc, rawBioAssayId);
-        }
-        finally
-        {
-          if (dc != null) dc.close();
-        }
-      }
-      
-      if (rawBioAssay != null && !rawBioAssay.hasArrayDesign())
+      if (useMissingReporterParameter)
       {
         parameters.add(missingReporterErrorParameter);
       }
-      else
+      if (useFeatureParameters)
       {
         parameters.add(featureMismatchErrorParameter);
@@ -909 +1061 @@
   } 
   
+  private void addRawBioAssayParameters(DbControl dc, RawBioAssay currentRaw, List<PluginParameter<?>> parameters)
+  {
+    // Parameter for getting the current raw bioassay
+    PluginParameter<RawBioAssay> rbaParameter = new PluginParameter<RawBioAssay>
+    (
+      "rawBioAssay",
+      "Raw bioassay",
+      "The raw bioassay to import raw data to. It must be of the correct raw data " +
+      "type and shouldn't have any spots already",
+      new ItemParameterType<RawBioAssay>(RawBioAssay.class, currentRaw, true, 1, null)
+    );
+    parameters.add(rbaParameter);
+  
+    List<File> rawDataFiles = null;
+    if (currentRaw != null)
+    {
+      try
+      {
+        rawDataFiles = FileStoreUtil.getGenericDataFiles(dc, currentRaw, FileType.RAW_DATA);
+      }
+      catch (Throwable t)
+      {}
+    }
+    
+    // The raw data file to import from - if a file already hase
+    // been attached to the raw bioassay use it as a default choice
+    PluginParameter<File> fileParameter = new PluginParameter<File>(
+      "file",
+      "Raw data file",
+      "The file that contains the raw data that you want to import",
+      new FileParameterType(rawDataFiles == null || rawDataFiles.isEmpty() ? 
+        null : rawDataFiles.get(0), true, 1)
+    );
+    parameters.add(fileParameter);
+
+  }
+  
+  private void addExperimentParameters(DbControl dc, Experiment currentExp, List<PluginParameter<?>> parameters)
+  {
+    // Parameter for getting the current raw bioassay
+    PluginParameter<Experiment> expParameter = new PluginParameter<Experiment>
+    (
+      "experiment",
+      "Experiment",
+      "The experiment to import raw data to",
+      new ItemParameterType<Experiment>(Experiment.class, currentExp, true, 1, null)
+    );
+    
+    Platform platform = null;
+    PlatformVariant variant = null;
+    if (currentExp != null)
+    {
+      RawDataType type = currentExp.getRawDataType();
+      platform = type.getPlatform(dc);
+      variant = type.getVariant(dc);
+    }
+    FileType rawData = FileType.getById(dc, SystemItems.getId(FileType.RAW_DATA));
+    ItemQuery<DataFileType> query = DataFileType.getQuery(platform, variant, 
+      Item.RAWBIOASSAY, rawData);
+    query.include(Include.ALL);
+    query.order(Orders.asc(Hql.property("name")));
+    List<DataFileType> fileTypes = query.list(dc);
+
+    PluginParameter<DataFileType> fileTypeParameter = new PluginParameter<DataFileType>
+    (
+      "dataFileType",
+      "File type",
+      "The file type to use when looking for files to import. If not selected " +
+      "a generic lookup for any raw data file type is used. ",
+      new ItemParameterType<DataFileType>(DataFileType.class, null, false, 1, fileTypes)
+    );
+
+    PluginParameter<Boolean> transactionParameter = new PluginParameter<Boolean>
+    (
+      "transactionPerFile",
+      "One transaction per file",
+      "If this option is enabled, a new transaction is used for each " +
+      "raw bioassay to import (recommended). This means that if an import fails, " +
+      "you will not loose what has already been done.",
+      new BooleanParameterType(true, false)
+    );
+    
+    PluginParameter<Boolean> continueParameter = new PluginParameter<Boolean>
+    (
+      "continueAfterError",
+      "Continue with next file after an error",
+      "If this option is enabled the import will continue with the next " +
+      "raw bioassay after an error (recommended). This feature requires that " +
+      "the 'One transaction per file' option is enabled",
+      new BooleanParameterType(true, false)
+    );
+    
+    parameters.add(expParameter);
+    parameters.add(fileTypeParameter);
+    parameters.add(transactionParameter);
+    parameters.add(continueParameter);
+  }
+  
   private RequestInformation getConfigureRawDataTypeParameters()
   {
@@ -984 +1234 @@
   }
   
+  /**
+    Get a query that returns all raw bioassays which it may be possible to
+    import raw data to for the given experiment. The raw bioassays 
+    should have no raw data already and they must have at least one
+    file attached.
+    @param exp The experiment
+  */
+  private ItemQuery<RawBioAssay> getRawBioAssayQuery(Experiment exp)
+  {
+    ItemQuery<RawBioAssay> query = exp.getRawBioAssays();
+    query.include(Include.MINE, Include.SHARED, Include.OTHERS, Include.IN_PROJECT);
+    // Only return raw bioassays with no data...
+    query.restrict(Restrictions.eq(Hql.property("spots"), Expressions.integer(0)));
+    // ... that has files attached
+    query.restrict(Restrictions.neq(Hql.property("fileSet"), null));
+    return query;
+  }
+
+  
+  class FileIterator
+    implements Iterator<File>
+  {
+    private List<RawBioAssay> rawBioAssays;
+    private List<File> files;
+    private int currentIndex;
+    
+    FileIterator(List<RawBioAssay> rawBioAssays, List<File> files)
+    {
+      this.rawBioAssays = rawBioAssays;
+      this.files = files;
+      this.currentIndex = -1;
+    }
+
+    /*
+      From the Iterator interface
+      ---------------------------
+    */
+    @Override
+    public boolean hasNext()
+    {
+      return rawBioAssays.size() - 1 > currentIndex;
+    }
+
+    @Override
+    public File next()
+    {
+      currentIndex++;
+      log("Importing data to raw bioassay: " + getCurrentRawBioAssay().getName());
+      return getCurrentFile();
+    }
+
+    @Override
+    public void remove()
+    {
+      throw new UnsupportedOperationException("remove");
+    }
+    // -------------------------------------
+    
+    File getCurrentFile()
+    {
+      return files.get(currentIndex);
+    }
+    
+    RawBioAssay getCurrentRawBioAssay()
+    {
+      return rawBioAssays.get(currentIndex);
+    }
+    
+  }
+  
 }

trunk/src/test/TestRawDataFlatFileImporter.java

@@ -183 +183 @@
       j.setName(pc.getName());
       
-      PluginConfigurationRequest request = j.configure(null);
+      PluginConfigurationRequest request = j.configure(GuiContext.item(Item.RAWBIOASSAY));
       write_request_information(request.getRequestInformation());
       request.setParameterValue("file", File.getById(dc, fileId));