Changeset 4594


Ignore:
Timestamp:
Oct 21, 2008, 9:20:40 AM (13 years ago)
Author:
Nicklas Nordborg
Message:

Fixes #1145: Make the generic raw data importer support batch import

Location:
trunk/src
Files:
11 added
9 edited

Legend:

Unmodified
Added
Removed
  • trunk/src/core/net/sf/basedb/core/DataFileType.java

    r4517 r4594  
    217217  public static ItemQuery<DataFileType> getQuery(FileStoreEnabled item)
    218218  {
    219     ItemQuery<DataFileType> query = new ItemQuery<DataFileType>(DataFileType.class);
    220     if (item != null)
    221     {
    222       // Restrict on itemType
    223       Item itemType = item.getType();
     219    ItemQuery<DataFileType> query = null;
     220    if (item == null)
     221    {
     222      query = getQuery();
     223    }
     224    else
     225    {
     226      query = getQuery(item.getPlatform(), item.getVariant(), item.getType(), null);
     227    }
     228    return query;   
     229  }
     230
     231  /**
     232    Get a query to retrieve <code>DataFileType</code>:s that can be used
     233    on specific platform/variant for a given item and/or generic file type.
     234    All parameters are optional. If all are null the result is the same
     235    as {@link #getQuery()}.
     236   
     237    @param platform The platform, or null to not restrict by platform (unless
     238      a variant is given)
     239    @param variant The platform variant, or null to not restrict by variant
     240    @param itemType The type of item it should be possible to
     241      attach file to, or null to not restrict by item
     242    @param genericType The generic file type, or null to not restrict
     243      by file type
     244    @return A query returning data file types matching the specified
     245      restrictions
     246    @since 2.9
     247   */
     248  public static ItemQuery<DataFileType> getQuery(Platform platform, PlatformVariant variant,
     249    Item itemType, FileType genericType)
     250  {
     251    ItemQuery<DataFileType> query = getQuery();
     252    // Restrict on itemType
     253    if (itemType != null)
     254    {
    224255      query.restrictPermanent(
    225256        Restrictions.eq(
     
    228259        )
    229260      );
    230      
    231       // Restrict on platform / variant
    232       PlatformVariant variant = item.getVariant();
    233       Platform platform = variant != null ? variant.getPlatform() : item.getPlatform();
    234       if (platform != null)
     261    }
     262   
     263    // Restrict on generic file type
     264    if (genericType != null)
     265    {
     266      query.restrictPermanent(
     267        Restrictions.eq(
     268          Hql.property("genericType"),
     269          Hql.entity(genericType))
     270      );
     271    }
     272   
     273    // Restrict on platform/variant
     274    if (variant != null && platform == null) platform = variant.getPlatform();
     275    if (platform != null)
     276    {
     277      query.join(Hql.innerJoin("platforms", "pft"));
     278      query.restrict(Restrictions.eq(Hql.property("pft", "platform"), Hql.entity(platform)));
     279      if (variant != null)
    235280      {
    236         query.join(Hql.innerJoin("platforms", "pft"));
    237         query.restrict(Restrictions.eq(Hql.property("pft", "platform"), Hql.entity(platform)));
    238         if (variant != null)
    239         {
    240           query.restrict(Restrictions.or(
    241             Restrictions.eq(Hql.property("pft", "variant"), null),
    242             Restrictions.eq(Hql.property("pft", "variant"), Hql.entity(variant))
    243           ));
    244         }
    245         else
    246         {
    247           query.restrict(Restrictions.eq(Hql.property("pft", "variant"), null));
    248         }
     281        query.restrict(Restrictions.or(
     282          Restrictions.eq(Hql.property("pft", "variant"), null),
     283          Restrictions.eq(Hql.property("pft", "variant"), Hql.entity(variant))
     284        ));
    249285      }
    250     }
    251     return query;   
    252   }
    253 
     286      else
     287      {
     288        query.restrict(Restrictions.eq(Hql.property("pft", "variant"), null));
     289      }
     290    }
     291    return query;
     292  }
     293 
     294 
    254295  /**
    255296    Creates a new filetype item from the given data.
  • trunk/src/core/net/sf/basedb/core/plugin/AbstractPlugin.java

    r4523 r4594  
    423423  {
    424424    if (logger == null) return;
    425     logger.println(message);
    426     if (t != null)
    427     {
    428       logger.println(t.getMessage());
     425    if (t == null)
     426    {
     427      logger.println(message);
     428    }
     429    else
     430    {
     431      logger.println(message + ": " + t.getMessage());
    429432      t.printStackTrace(logger);
    430433    }
  • trunk/src/core/net/sf/basedb/core/plugin/GuiContext.java

    r4516 r4594  
    3737public class GuiContext
    3838{
     39  /**
     40    Shortcut to create a single-item context.
     41    @since 2.9
     42  */
     43  public static GuiContext item(Item item)
     44  {
     45    return new GuiContext(item, Type.ITEM);
     46  }
     47
     48  /**
     49    Shortcut to create a list context.
     50    @since 2.9
     51  */
     52  public static GuiContext list(Item item)
     53  {
     54    return new GuiContext(item, Type.LIST);
     55  }
     56
     57 
    3958  private final Item item;
    4059  private final Type type;
     
    7493  {
    7594    return subContext;
     95  }
     96 
     97  /**
     98    Checks if the specified object is an instance of the
     99    BasicItem subclass used by the {@link Item} in this context.
     100    @param o The object to check
     101    @since 2.9
     102  */
     103  public boolean isCorrectBasicItem(Object o)
     104  {
     105    return item.getItemClass().isInstance(o);
     106  }
     107 
     108  /**
     109    Checks if the specified object is an instance of the
     110    BasicData subclass used by the {@link Item} in this context.
     111    @param o The object to check
     112    @since 2.9
     113  */
     114  public boolean isCorrectBasicData(Object o)
     115  {
     116    return item.getDataClass().isInstance(o);
    76117  }
    77118 
  • trunk/src/core/net/sf/basedb/util/AutoDetectFileFormat.java

    r4525 r4594  
    4040import net.sf.basedb.core.plugin.GuiContext;
    4141import net.sf.basedb.core.plugin.InteractivePlugin;
     42import net.sf.basedb.util.ContextUtil.ContextResult;
     43import net.sf.basedb.util.filter.Filter;
    4244
    4345import java.util.List;
     
    159161      importer = plugin.newInstance(AutoDetectingImporter.class, null, sc, config, null);
    160162      boolean isInContext = true;
    161       if (plugin.isInteractive())
     163      if (plugin.isInteractive() && context != null)
    162164      {
    163165        try
     
    200202  }
    201203 
     204  /**
     205    Filter implementation that filters a collection of
     206    {@link ContextResult}:s by checking each plug-ins ability to
     207    import a given file. Eg. the ContextResult is accepted if the
     208    {@link AutoDetectingImporter#isImportable(InputStream)} returns
     209    true. It is assumed that the collection of ContextResult:s only
     210    contains import plug-ins that implements the AutoDetectingImporter
     211    interface.
     212 
     213    @author Nicklas
     214    @version 2.9
     215    @base.modified $Date$
     216  */
     217  public static class IsImportableFilter
     218    implements Filter<ContextResult>
     219  {
     220    private File file;
     221    private Filter<? super ContextResult> parent;
     222   
     223    public IsImportableFilter(File file, Filter<? super ContextResult> parent)
     224    {
     225      this.file = file;
     226      this.parent = parent;
     227    }
     228   
     229    public boolean evaluate(ContextResult result)
     230    {
     231      if (result == null) return false;
     232      if (parent != null && !parent.evaluate(result)) return false;
     233      if (!result.isInContext()) return false;
     234      return checkImportable(file.getSessionControl(),
     235          result.getPluginDefinition(), result.getPluginConfiguration(), file,
     236          null, null, null);
     237    }
     238  }
     239 
    202240}
  • trunk/src/core/net/sf/basedb/util/ContextUtil.java

    r4515 r4594  
    3636import net.sf.basedb.core.plugin.GuiContext;
    3737import net.sf.basedb.core.plugin.InteractivePlugin;
     38import net.sf.basedb.util.filter.Filter;
    3839
    3940/**
     
    249250   
    250251  }
     252 
     253  /**
     254    Filter implementation that works on collections
     255    of {@link ContextResult}:s and return only those
     256    that return true from {@link ContextResult#isInContext()}.
     257 
     258    @author Nicklas
     259    @version 2.9
     260    @base.modified $Date$
     261  */
     262  public static class IsInContextFilter
     263    implements Filter<ContextResult>
     264  {
     265
     266    public IsInContextFilter()
     267    {}
     268   
     269    /*
     270      From the Filter interface
     271      -------------------------
     272    */
     273    @Override
     274    public boolean evaluate(ContextResult result)
     275    {
     276      return result != null && result.isInContext();
     277    }
     278    // -----------------------------
     279  }
     280 
    251281}
  • trunk/src/core/net/sf/basedb/util/FileUtil.java

    r4515 r4594  
    139139
    140140  /**
     141    Close an input stream without throwing an exception.
     142    @param in The input stream to close
     143    @since 2.9
     144  */
     145  public static void close(InputStream in)
     146  {
     147    if (in == null) return;
     148    try
     149    {
     150      in.close();
     151    }
     152    catch (Throwable t)
     153    {}
     154  }
     155 
     156  /**
     157    Close an output stream without throwing an exception.
     158    @param out The output stream to close
     159    @since 2.9
     160  */
     161  public static void close(OutputStream out)
     162  {
     163    if (out == null) return;
     164    try
     165    {
     166      out.close();
     167    }
     168    catch (Throwable t)
     169    {}
     170  }
     171 
     172  /**
    141173    Get a buffered <code>InputStream</code> object reading from
    142174    the specified file.
  • trunk/src/plugins/core/net/sf/basedb/plugins/AbstractFlatFileImporter.java

    r4525 r4594  
    4444import net.sf.basedb.core.StringUtil;
    4545import net.sf.basedb.core.Type;
     46import net.sf.basedb.core.UnsupportedFileFormatException;
    4647
    4748import net.sf.basedb.core.plugin.AbstractPlugin;
     
    5152import net.sf.basedb.core.plugin.Response;
    5253import net.sf.basedb.core.plugin.Plugin;
     54import net.sf.basedb.core.signal.SignalException;
    5355import net.sf.basedb.core.signal.SignalHandler;
    5456import net.sf.basedb.core.signal.SignalTarget;
     
    5658
    5759import net.sf.basedb.plugins.util.Parameters;
     60import net.sf.basedb.util.FileUtil;
    5861import net.sf.basedb.util.NumberFormatUtil;
    5962import net.sf.basedb.util.error.ClassMapErrorHandler;
     
    6871import java.util.ArrayList;
    6972import java.util.Arrays;
     73import java.util.Collections;
     74import java.util.Iterator;
    7075import java.util.regex.Pattern;
    7176import java.io.IOException;
     
    7580
    7681/**
    77   An abstract base class for all importers that imports data from a single
    78   flat file. The implementation in this class uses a {@link FlatFileParser}
    79   for parsing the file and uses a callback method {@link #handleData(FlatFileParser.Data)}
     82  An abstract base class for all importers that imports data from one or more
     83  flat files. The implementation in this class uses a {@link FlatFileParser}
     84  for parsing the files and uses a callback method {@link #handleData(FlatFileParser.Data)}
    8085  that lets the subclass do whatever it needs to insert a single line
    8186  of data into the database.
     
    139144</pre>
    140145
     146  <p>
     147  For multi-file support (added in BASE 2.9) the subclass needs to
     148  override the {@link #getFileIterator()} and {@link #getTotalFileSize()}
     149  methods.
    141150 
    142151  @base.modified $Date$
     
    381390    Parameters.numberFormatError(null, null, null);
    382391 
    383   private long fileSize;
     392  private File currentFile;
     393  private long totalFileSize;
     394  private long finishedFileSize;
    384395  private int skippedLines;
    385396  private ClassMapErrorHandler errorHandler;
     
    426437    {
    427438      createLogFile((String)job.getValue(Parameters.LOGFILE_PARAMETER));
    428       InputStream in = null;
     439      boolean hasCalledFinish = false;
     440      int numSuccess = 0;
     441      int numTotal = 0;
     442      String finalMessage = null;
    429443      try
    430444      {
    431         File f = (File)job.getValue("file");
    432         log("Parsing file: " + f.getName());
    433         in = f.getDownloadStream(0);
    434         boolean importable = isImportable(in);
    435         in.close();
    436         if (!importable)
     445        start();
     446        String message = null;
     447        finishedFileSize = 0;
     448        Iterator<File> files = getFileIterator();
     449        totalFileSize = getTotalFileSize();
     450       
     451        while (files.hasNext())
    437452        {
    438           throw new InvalidDataException("File '"+f+"' cannot be imported by this plugin.");
    439         }
    440         fileSize = f.getSize();
    441         in = f.getDownloadStream(0);
    442         doImport(in, progress);
    443         in.close();
    444         String message = getSuccessMessage(skippedLines);
    445         if (message != null) log(message);
    446         response.setDone(message);
    447       }
    448       catch (Throwable ex)
    449       {
    450         log("Failed", ex);
    451         if (in != null)
    452         {
     453          numTotal++;
     454          InputStream in = null;
    453455          try
    454456          {
    455             in.close();
     457            currentFile = files.next();
     458            log("Parsing file: " + currentFile.getName());
     459            in = currentFile.getDownloadStream(0);
     460            boolean importable = isImportable(in);
     461            FileUtil.close(in);
     462            if (!importable)
     463            {
     464              throw new UnsupportedFileFormatException("File '"+currentFile.getName()+
     465                "' doesn't match the expected file format.");
     466            }
     467            in = currentFile.getDownloadStream(0);
     468            doImport(in, progress);
     469            FileUtil.close(in);
     470            message = getSuccessMessage(skippedLines);
     471            if (message == null)
     472            {
     473              message = "File '" + currentFile.getName() +
     474                "' imported successfully. " +
     475                skippedLines + " lines skipped.";
     476            }
     477            log(message);
     478            numSuccess++;
    456479          }
    457           catch (IOException iox)
    458           {}
     480          catch (SignalException s)
     481          {
     482            // The user aborted the plug-in -> always exit
     483            throw s;
     484          }
     485          catch (Throwable t)
     486          {
     487            if (!continueWithNextFileAfterError(t)) throw t;
     488            log("Failed", t);
     489          }
     490          finally
     491          {
     492            if (currentFile != null)
     493            {
     494              finishedFileSize += currentFile.getSize();
     495            }
     496            FileUtil.close(in);
     497            log("--------------------------------------");
     498          }
    459499        }
    460         response.setError(ex.getMessage(), Arrays.asList(ex));
     500        hasCalledFinish = true;
     501        finalMessage = finish(null);
     502        if (numSuccess == 0)
     503        {
     504          if (finalMessage == null)
     505          {
     506            finalMessage = "No files could be imported";
     507          }
     508          response.setError(finalMessage + ": " + message, null);
     509        }
     510        else
     511        {
     512          if (finalMessage == null && numTotal > 1)
     513          {
     514            finalMessage = numSuccess + " of " + numTotal + " files could be imported";
     515          }
     516          response.setDone(finalMessage == null ? message : finalMessage);
     517        }
     518      }
     519      catch (Throwable t)
     520      {
     521        String message = hasCalledFinish ? null : finish(t);
     522        if (message == null) message = t.getMessage();
     523        log("Failed: " + message, t);
     524        response.setError(message, Arrays.asList(t));
    461525      }
    462526      finally
    463527      {
     528        if (finalMessage != null) log(finalMessage);
    464529        closeLogFile();
    465530      }
     
    536601        if (section != null && section.type() == FlatFileParser.LineType.SECTION)
    537602        {
    538           if (progress != null) progress.display(getProgress(ffp), "Parsing section "+section.name()+"...");
     603          if (progress != null)
     604          {
     605            progress.display(getProgress(ffp), "Parsing section '"+section.name()+
     606                "' in file '" + currentFile.getName() + "'...");
     607          }
    539608          try
    540609          {
     
    559628            {
    560629              throw new BaseException(t2.getMessage() +
    561                   " on line " + section.lineNo() + ": " +
    562                   StringUtil.trimString(section.line(), 20), t2);
     630                " on line " + section.lineNo() +
     631                " in file '" + currentFile.getName() + "': " +
     632                StringUtil.trimString(section.line(), 20), t2);
    563633            }
    564634          }
     
    566636        else
    567637        {
    568           if (progress != null) progress.display(getProgress(ffp), "Parsing headers...");
     638          if (progress != null)
     639          {
     640            progress.display(getProgress(ffp),
     641              "Parsing headers in file '" + currentFile.getName() + "'...");
     642          }
    569643        }
    570644        checkInterrupted();
     
    602676        {
    603677          throw new BaseException(t.getMessage() +
    604             " on line " + (line == null ? "-1" : line.lineNo() + ": " + StringUtil.trimString(line.line(), 20)), t);     
     678            " on line " + (line == null ? "-1" : line.lineNo() +
     679            " in file '" + currentFile.getName() + "': " +
     680             StringUtil.trimString(line.line(), 20)), t);     
    605681        }
    606682        if (result == FlatFileParser.LineType.DATA || result == FlatFileParser.LineType.DATA_HEADER)
    607683        {
    608           if (progress != null) progress.display(getProgress(ffp), "Parsing data...");
     684          if (progress != null)
     685          {
     686            progress.display(getProgress(ffp),
     687              "Parsing data in file '" + currentFile.getName() + "'...");
     688          }
    609689          int progressLine = 0;
    610690         
     
    638718                if (progress != null)
    639719                {
    640                   progress.display(getProgress(ffp), "Parsing data... (" + ffp.getParsedLines() + " lines so far)");
     720                  progress.display(getProgress(ffp),
     721                    "Parsing data in file '" + currentFile.getName() +
     722                    "'... (" + ffp.getParsedLines() + " lines so far)");
    641723                }
    642724              }
     
    648730          {
    649731            throw new BaseException(t.getMessage() +
    650               " on line " + (dataline == null ? "-1" : dataline.lineNo() + ": " + StringUtil.trimString(dataline.line(), 20)), t);
     732              " on line " + (dataline == null ? "-1" : dataline.lineNo() +
     733              " in file '" + currentFile.getName() + "': " +
     734              StringUtil.trimString(dataline.line(), 20)), t);
    651735          }
    652736        }
     
    654738      while (ffp.hasMoreSections());
    655739      isSuccess = true;
    656       if (progress != null) progress.display(100, "Done parsing "+ffp.getParsedLines()+" lines.\n");
     740      if (progress != null)
     741      {
     742        progress.display(getProgress(ffp),
     743          "Done parsing file '" + currentFile.getName() +
     744          "' ("+ffp.getParsedLines()+" lines)\n");
     745      }
    657746    }
    658747    catch (IOException ex)
     
    685774
    686775  /**
     776    Called once before starting the import.
     777    The default implementation does nothing.
     778    @since 2.9
     779  */
     780  protected void start()
     781  {}
     782 
     783  /**
     784    Get an iterator that returns the files to be imported. The default
     785    implementation returns the single file found in the job's "file"
     786    parameter. Subclasses that needs multi-file/item import should override
     787    this method to provide their own iterator. They should also override
     788    the {@link #getTotalFileSize()} method to return sum of all
     789    file sizes. Eg. {@link File#getSize()}.
     790    @since 2.9
     791  */
     792  protected Iterator<File> getFileIterator()
     793  {
     794    return Collections.singleton((File)job.getValue("file")).iterator();
     795  }
     796 
     797  /**
     798    Get the total file size of all files that are going to be imported.
     799    A subclass that is going to import from multiple files needs to override
     800    this method. The default implementation return the size of the file
     801    in the job's "file" parameter.
     802    @return The sum of the file sizes
     803    @since 2.9
     804  */
     805  protected long getTotalFileSize()
     806  {
     807    return ((File)job.getValue("file")).getSize();
     808  }
     809 
     810  /**
    687811    Get the progress of import as a percentage value. The default implementation
    688     calls {@link #getNumBytes(FlatFileParser)} and divides it by the file size.
     812    calls sums the file size of the completed files and
     813    {@link #getNumBytes(FlatFileParser)}. This values is divided by
     814    {@link #getTotalFileSize()}.
    689815    @param ffp The file parser that is used to parsed the file
    690816    @return A value between 0 and 100
     
    693819  protected int getProgress(FlatFileParser ffp)
    694820  {
    695     return (int) (100 * getNumBytes(ffp) / fileSize);
    696   }
    697  
     821    return (int) (100 * (getNumBytes(ffp) + finishedFileSize) / totalFileSize);
     822  }
    698823 
    699824  /**
     
    715840    return ffp.getParsedBytes();
    716841  }
    717 
     842 
    718843  /**
    719844    This method is called by the {@link #isImportable(InputStream)} method after
     
    755880    Called just before parsing of the file begins. A subclass
    756881    may override this method if it needs to initialise some
    757     resources before the parsing starts.
     882    resources before the parsing starts. Note that this method is called
     883    once for each file returned by {@link #getFileIterator()}.
    758884    @see #end(boolean)
    759885  */
     
    797923    or immediately after an error has ocurred. A subclass should clean
    798924    up any resources aquired in the {@link #begin(FlatFileParser)} method here.
     925    Note that this metod is called once for every file returned by the
     926    {@link #getFileIterator()} iterator.
    799927    @param success TRUE if the file was parsed successfully, FALSE otherwise
    800928    @see #begin(FlatFileParser)
     
    808936    message that is sent back to the core and user interface. An example
    809937    message might by: <code>178 reporters imported successfully</code>.
    810     The default implementation always return null.
     938    The default implementation always return null. Note that this method is called
     939    once for every file returned by {@link #getFileIterator()}.
    811940    @param skippedLines The number of data lines that were skipped due to errors
    812941  */
    813942  protected String getSuccessMessage(int skippedLines)
     943  {
     944    return null;
     945  }
     946 
     947  /**
     948    Called once when all files has been imported or when exiting due to an error.
     949    This method may return a message that is sent back to the core and user interface.
     950    If this method returns null the last message returned by {@link #getSuccessMessage(int)}
     951    is used instead.
     952    @param t Null if no error has happened
     953    @return A message or null
     954    @since 2.9
     955  */
     956  protected String finish(Throwable t)
    814957  {
    815958    return null;
     
    820963    and other options. This implementation gets all parameters from the
    821964    {@link #configuration} settings. If a subclass doesn't store the parameters
    822     there it must override this method and initialise the parser.
     965    there it must override this method and initialise the parser. Note that
     966    this method is called once for each file returned by the
     967    {@link #getFileIterator()} and that a new parser is needed for each file.
    823968    @return An intialised flat file parser
    824969    @throws BaseException
     
    11261271 
    11271272  /**
     1273    If the importer should continue with the next file after an error.
     1274    The default implementation of this method return FALSE. A subclass
     1275    may override this method to let the import continue after an error.
     1276    This method isn't called until an error has happened, and it is
     1277    possible to control what should happen on a file-by-file bases.
     1278    The importer will of course exit if this method returns FALSE.
     1279    @param t The error that happened
     1280    @return TRUE to contine, FALSE to abort
     1281    @since 2.9
     1282  */
     1283  protected boolean continueWithNextFileAfterError(Throwable t)
     1284  {
     1285    return false;
     1286  }
     1287 
     1288  /**
    11281289    Initialise the error handling system. This method is called just before the
    1129     import is starting. A subclass may override this method to add specific
     1290    import of a file is starting. A subclass may override this method to add specific
    11301291    error handlers. If <code>super.setUpErrorHandling()</code> isn't called
    11311292    error handling in AbstractFlatFileImporter is disabled and the subclass
    11321293    must do all it's error handling in it's own code. The subclass may also add
    1133     error handlers in the {@link #begin(FlatFileParser)} method.
     1294    error handlers in the {@link #begin(FlatFileParser)} method. Note that
     1295    the error handling system is re-initialised for every file returned by
     1296    {@link #getFileIterator()}.
    11341297  */
    11351298  protected void setUpErrorHandling()
  • trunk/src/plugins/core/net/sf/basedb/plugins/RawDataFlatFileImporter.java

    r4513 r4594  
    2626import net.sf.basedb.core.ArrayDesign;
    2727import net.sf.basedb.core.BaseException;
     28import net.sf.basedb.core.BooleanParameterType;
    2829import net.sf.basedb.core.DataFileType;
    2930import net.sf.basedb.core.DbControl;
     31import net.sf.basedb.core.Experiment;
    3032import net.sf.basedb.core.FeatureIdentificationMethod;
    3133import net.sf.basedb.core.File;
    3234import net.sf.basedb.core.FileParameterType;
     35import net.sf.basedb.core.FileSet;
    3336import net.sf.basedb.core.FileSetMember;
    3437import net.sf.basedb.core.FileStoreUtil;
    3538import net.sf.basedb.core.FileType;
     39import net.sf.basedb.core.Include;
    3640import net.sf.basedb.core.Item;
    3741import net.sf.basedb.core.ItemAlreadyExistsException;
     
    4246import net.sf.basedb.core.Permission;
    4347import net.sf.basedb.core.PermissionDeniedException;
     48import net.sf.basedb.core.Platform;
     49import net.sf.basedb.core.PlatformVariant;
    4450import net.sf.basedb.core.PluginConfiguration;
    4551import net.sf.basedb.core.PluginParameter;
     
    5157import net.sf.basedb.core.RequestInformation;
    5258import net.sf.basedb.core.StringParameterType;
     59import net.sf.basedb.core.SystemItems;
     60import net.sf.basedb.core.UnsupportedFileFormatException;
    5361import net.sf.basedb.core.Version;
    5462import net.sf.basedb.core.data.FeatureData;
     
    6371import net.sf.basedb.core.plugin.Request;
    6472import net.sf.basedb.core.plugin.Response;
     73import net.sf.basedb.core.query.Expressions;
     74import net.sf.basedb.core.query.Hql;
     75import net.sf.basedb.core.query.Orders;
     76import net.sf.basedb.core.query.Restrictions;
    6577import net.sf.basedb.plugins.util.Parameters;
    6678import net.sf.basedb.util.Enumeration;
     
    8294import java.util.HashMap;
    8395import java.util.HashSet;
     96import java.util.Iterator;
    8497import java.util.List;
    8598import java.util.Map;
     
    112125 
    113126  private static final Set<GuiContext> guiContexts =
    114     Collections.singleton(new GuiContext(Item.RAWBIOASSAY, GuiContext.Type.ITEM));
    115  
     127    Collections.unmodifiableSet(new HashSet<GuiContext>(
     128        Arrays.asList(
     129          GuiContext.item(Item.RAWBIOASSAY),
     130          GuiContext.list(Item.EXPERIMENT)
     131        )
     132      ));
    116133  private static final Set<Permissions> permissions = new HashSet<Permissions>();
    117134 
     
    270287  private PluginParameter<String> rawDataTypeParameter;
    271288 
    272   private ItemParameterType<RawBioAssay> rawBioAssayType;
    273   private PluginParameter<RawBioAssay> rawBioAssayParameter;
    274 
    275289  private DbControl dc;
    276290  private FlatFileParser ffp;
    277291  private NumberFormat numberFormat;
    278292  private boolean nullIfException;
     293  private boolean continueAfterError;
     294  private boolean transactionPerFile;
     295  private FileIterator fileIterator;
     296  private long totalFileSize;
     297 
     298  private RawBioAssay rawBioAssay;
    279299  private RawDataBatcher batcher;
    280   private RawBioAssay rawBioAssay;
    281300  private FileSetMember rawDataMember;
    282301
     
    345364  public String isInContext(GuiContext context, Object item)
    346365  {
    347     String message = null;
    348366    if (item == null)
    349367    {
    350       message = "The object is null";
    351     }
    352     else if (!(item instanceof RawBioAssay))
    353     {
    354       message = "The object is not a RawBioAssay: " + item;
     368      return "The object is null";
     369    }
     370    else if (!context.isCorrectBasicItem(item))
     371    {
     372      return "The oject is not a " + context.getItem() + ": " + item;
    355373    }
    356374    else
    357375    {
    358       RawBioAssay rba = (RawBioAssay)item;
     376      Experiment exp = null;
     377      RawBioAssay rba = null;
     378      RawDataType rdt = null;
     379      if (item instanceof RawBioAssay)
     380      {
     381        rba = (RawBioAssay)item;
     382        rdt = rba.getRawDataType();
     383        if (rba.getNumDbSpots() > 0)
     384        {
     385          throw new PermissionDeniedException("The raw bioassay already has data.");
     386        }
     387        rba.checkPermission(Permission.WRITE);
     388      }
     389      else
     390      {
     391        exp = (Experiment)item;
     392        rdt = exp.getRawDataType();
     393        ItemQuery<RawBioAssay> query = getRawBioAssayQuery(exp);
     394        DbControl dc = sc.newDbControl();
     395        try
     396        {
     397          if (query.count(dc) == 0)
     398          {
     399            return "The experiment doesn't have any raw bioassays with files waiting for import";
     400          }
     401        }
     402        finally
     403        {
     404          if (dc != null) dc.close();
     405        }
     406      }
    359407      String rawDataType = (String)configuration.getValue("rawDataType");
    360       RawDataType rdt = rba.getRawDataType();
    361408      if (!rdt.getId().equals(rawDataType))
    362409      {
    363         message = "Unsupported raw data type: " + rba.getRawDataType().getName();
     410        return "Unsupported raw data type: " + rba.getRawDataType().getName();
    364411      }
    365412      else if (!rdt.isStoredInDb())
    366413      {
    367         message = "Raw data for raw data type '" + rdt + "' is not stored in the database";
    368       }
    369       else if (rba.getNumDbSpots() > 0)
    370       {
    371         throw new PermissionDeniedException("The raw bioassay already has data.");
    372       }
    373       else
    374       {
    375         rba.checkPermission(Permission.WRITE);
    376       }
    377     }
    378     return message;   
     414        return "Raw data for raw data type '" + rdt + "' is not stored in the database";
     415      }
     416    }
     417    return null;
    379418  }
    380419
     
    479518        }
    480519       
    481         storeValue(job, request, rawBioAssayParameter);
    482         storeValue(job, request, fileParameter);
     520        // Single raw bioassay mode
     521        storeValue(job, request, ri.getParameter("rawBioAssay"));
     522        storeValue(job, request, ri.getParameter("file"));
     523        storeValue(job, request, ri.getParameter("featureIdentification"));
     524       
     525        // Experiment mode
     526        storeValue(job, request, ri.getParameter("experiment"));
     527        storeValue(job, request, ri.getParameter("dataFileType"));
     528        storeValue(job, request, ri.getParameter("transactionPerFile"));
     529        storeValue(job, request, ri.getParameter("continueAfterError"));
     530       
     531        // Common
    483532        storeValue(job, request, ri.getParameter(Parameters.CHARSET_PARAMETER));
    484533        storeValue(job, request, ri.getParameter(Parameters.DECIMAL_SEPARATOR_PARAMETER));
     
    489538       
    490539        // Error handling parameters
     540        storeValue(job, request, ri.getParameter(Parameters.LOGFILE_PARAMETER));
    491541        storeValue(job, request, defaultErrorParameter);
    492542        storeValue(job, request, missingReporterErrorParameter);
     
    528578  */
    529579  /**
    530    Create a {@link DbControl} and a {@link RawDataBatcher}.
    531    Load the {@link RawBioAssay} that has been specified.
     580    Prepare for the import. If used from an experiment context
     581    prepare the list of files/raw bioassays to import.
     582  */
     583  @Override
     584  protected void start()
     585  {
     586    transactionPerFile = Boolean.TRUE.equals(job.getValue("transactionPerFile"));
     587    continueAfterError = transactionPerFile &&
     588      Boolean.TRUE.equals(job.getValue("continueAfterError"));
     589    Experiment exp = (Experiment)job.getValue("experiment");
     590    List<RawBioAssay> rawBioAssays = null;
     591    List<File> files = null;
     592    if (exp == null)
     593    {
     594      File f = (File)job.getValue("file");
     595      RawBioAssay r = (RawBioAssay)job.getValue("rawBioAssay");
     596      rawBioAssays = Collections.singletonList(r);
     597      files = Collections.singletonList(f);
     598      totalFileSize = f.getSize();
     599    }
     600    else
     601    {
     602      ItemQuery<RawBioAssay> query = getRawBioAssayQuery(exp);
     603      DataFileType fileType = (DataFileType)job.getValue("dataFileType");
     604      rawBioAssays = new ArrayList<RawBioAssay>();
     605      files = new ArrayList<File>();
     606      DbControl dc = sc.newDbControl();
     607      try
     608      {
     609        for (RawBioAssay rba : query.list(dc))
     610        {
     611          File dataFile = null;
     612          if (fileType != null)
     613          {
     614            FileSet fs = rba.getFileSet();
     615            FileSetMember member = fs.getMember(fileType);
     616            if (member != null) dataFile = member.getFile();
     617          }
     618          else
     619          {
     620            List<File> rawFiles = FileStoreUtil.getGenericDataFiles(dc, rba, FileType.RAW_DATA);
     621            if (rawFiles != null && rawFiles.size() > 0) dataFile = rawFiles.get(0);
     622          }
     623          if (dataFile != null)
     624          {
     625            rawBioAssays.add(rba);
     626            files.add(dataFile);
     627            totalFileSize += dataFile.getSize();
     628          }
     629        }
     630      }
     631      finally
     632      {
     633        if (dc != null) dc.close();
     634      }
     635    }
     636    this.fileIterator = new FileIterator(rawBioAssays, files);
     637  }
     638 
     639  /**
     640    We will continue if:
     641    <ul>
     642    <li>The file is not of the correct format, eg. t = {@link UnsupportedFileFormatException}.
     643    <li>The 'Continue after error' parameter is true
     644    </ul>
     645  */
     646  protected boolean continueWithNextFileAfterError(Throwable t)
     647  {
     648    return continueAfterError || t instanceof UnsupportedFileFormatException;
     649  }
     650 
     651  @Override
     652  protected Iterator<File> getFileIterator()
     653  {
     654    return fileIterator;
     655  }
     656 
     657  @Override
     658  protected long getTotalFileSize()
     659  {
     660    return totalFileSize;
     661  }
     662 
     663  /**
     664    Prepare for importing the next file. Eg.
     665    create a {@link DbControl} (if needed) and a {@link RawDataBatcher}.
     666    Load the {@link RawBioAssay} that has been specified.
    532667  */
    533668  @Override
     
    536671  {
    537672    super.begin(ffp);
    538     dc = sc.newDbControl();
    539     rawBioAssay = (RawBioAssay)job.getValue("rawBioAssay");
    540     File rawDataFile = (File)job.getValue("file");
    541     rawBioAssay = RawBioAssay.getById(dc, rawBioAssay.getId());
     673    if (transactionPerFile || dc == null || dc.isClosed())
     674    {
     675      dc = sc.newDbControl();
     676    }
     677
     678    rawBioAssay = RawBioAssay.getById(dc, fileIterator.getCurrentRawBioAssay().getId());
    542679    rawDataMember = FileStoreUtil.setGenericDataFile(dc, rawBioAssay,
    543       FileType.RAW_DATA, DataFileType.GENERIC_RAW_DATA, rawDataFile);
     680      FileType.RAW_DATA, DataFileType.GENERIC_RAW_DATA, fileIterator.getCurrentFile());
    544681   
    545682    PluginConfiguration cfg = getCurrentConfiguration(dc);
     
    644781      {
    645782        batcher.close();
    646         dc.commit();
     783        if (transactionPerFile)
     784        {
     785          dc.commit();
     786          dc = null;
     787        }
    647788      }
    648789    }
     
    654795    finally
    655796    {
     797      if (dc != null && transactionPerFile)
     798      {
     799        dc.close();
     800        dc = null;
     801      }
     802      super.end(success);
     803    }
     804  }
     805 
     806  /**
     807    Commit/rollback transaction if needed. If everything
     808    went ok, t=null.
     809  */
     810  @Override
     811  public String finish(Throwable t)
     812  {
     813    try
     814    {
     815      if (t == null && dc != null && !dc.isClosed()) dc.commit();
     816      super.finish(t);
     817    }
     818    finally
     819    {
    656820      if (dc != null) dc.close();
    657       super.end(success);
    658     }
     821    }
     822    return null;
    659823  }
    660824 
     
    770934      List<PluginParameter<?>> parameters = new ArrayList<PluginParameter<?>>();
    771935     
    772       // Parameter for getting the current raw bioassay
    773       rawBioAssayType = new ItemParameterType<RawBioAssay>(RawBioAssay.class, null, true, 1, null);
    774       rawBioAssayParameter = new PluginParameter<RawBioAssay>
    775       (
    776         "rawBioAssay",
    777         "Raw bioassay",
    778         "The raw bioassay to import raw data to. It must be of the correct raw data type and shouldn't have any spots already",
    779         rawBioAssayType
    780       );
    781       parameters.add(rawBioAssayParameter);
    782 
    783       RawBioAssay rba = null;
    784       int currentRawBioAssayId = sc.getCurrentContext(Item.RAWBIOASSAY).getId();
    785       List<File> rawDataFiles = null;
    786       dc = sc.newDbControl();
     936      RawBioAssay currentRaw = (RawBioAssay)job.getValue("rawBioAssay");
     937      Experiment currentExp = (Experiment)job.getValue("experiment");
    787938      boolean configIsAnnotated = false;
    788       ArrayDesign design = null;
     939      boolean useMissingReporterParameter = true;
     940      boolean useFeatureParameters = true;
     941      FeatureIdentificationMethod defaultFiMethod = FeatureIdentificationMethod.NONE;
     942      DbControl dc = sc.newDbControl();
    789943      try
    790944      {
    791         rba = RawBioAssay.getById(dc, currentRawBioAssayId);
    792         rawDataFiles = FileStoreUtil.getGenericDataFiles(dc, rba, FileType.RAW_DATA);
    793945        configIsAnnotated = getCurrentConfiguration(dc).isAnnotated();
    794         design = rba.getArrayDesign();
    795       }
    796       catch (Throwable t)
    797       {}
    798       finally
     946        if (GuiContext.item(Item.RAWBIOASSAY).equals(context))
     947        {
     948          if (currentRaw == null)
     949          {
     950            int id = sc.getCurrentContext(Item.RAWBIOASSAY).getId();
     951            if (id != 0) currentRaw = RawBioAssay.getById(dc, id);
     952          }
     953          else
     954          {
     955            currentRaw = RawBioAssay.getById(dc, currentRaw.getId());
     956          }
     957          if (currentRaw != null)
     958          {
     959            useFeatureParameters = currentRaw.hasArrayDesign();
     960            ArrayDesign design = currentRaw.getArrayDesign();
     961            if (design != null) defaultFiMethod = design.getFeatureIdentificationMethod();
     962          }
     963          useMissingReporterParameter = !useFeatureParameters;
     964          addRawBioAssayParameters(dc, currentRaw, parameters);
     965        }
     966        else
     967        {
     968          if (currentExp == null)
     969          {
     970            int id = sc.getCurrentContext(Item.EXPERIMENT).getId();
     971            if (id != 0) currentExp = Experiment.getById(dc, id);
     972          }
     973          addExperimentParameters(dc, currentExp, parameters);
     974        }
     975      }
     976      finally
    799977      {
    800978        if (dc != null) dc.close();
    801979      }
    802      
    803       // The raw data file to import from - if a file already hase
    804       // been attached to the raw bioassay use it as a default choice
    805       PluginParameter<File> fileParameter = new PluginParameter<File>(
    806         "file",
    807         "Raw data file",
    808         "The file that contains the raw data that you want to import",
    809         new FileParameterType(rawDataFiles == null || rawDataFiles.isEmpty() ?
    810           null : rawDataFiles.get(0), true, 1)
    811       );
    812       parameters.add(fileParameter);
    813      
    814       if (design != null && design.getNumDbFeatures() > 0)
     980       
     981      if (useFeatureParameters)
    815982      {
    816983        // Parameter for asking which feature identification method to use
    817984        // Only used if the connected array design has features
    818        
    819         FeatureIdentificationMethod fiMethod = design.getFeatureIdentificationMethod();
    820985        List<String> methods = new ArrayList<String>();
    821986        String description = "Choose which method to use for identifying features: \n";
     
    8431008              "Identify features by",
    8441009              description,
    845               new StringParameterType(255, methods.contains(fiMethod.name()) ? fiMethod.name() : null,
    846                 true, 1, 0, 0, methods)
     1010              new StringParameterType(255,
     1011                methods.contains(defaultFiMethod.name()) ? defaultFiMethod.name() : null,
     1012                false, 1, 0, 0, methods)
    8471013            );
    8481014          parameters.add(featureIdentificationParameter);
     
    8511017     
    8521018      parameters.add(Parameters.charsetParameter(null, null,
    853           (String)configuration.getValue(Parameters.CHARSET_PARAMETER)));
     1019          currentExp != null ? null : (String)configuration.getValue(Parameters.CHARSET_PARAMETER)));
    8541020      parameters.add(Parameters.decimalSeparatorParameter(null, null,
    8551021          (String)configuration.getValue(Parameters.DECIMAL_SEPARATOR_PARAMETER)));
     
    8671033      // Error handling parameters
    8681034      parameters.add(errorSection);
     1035      parameters.add(Parameters.logFileParameter(null, null, null));
    8691036      parameters.add(defaultErrorParameter);
    8701037     
    871       RawBioAssay rawBioAssay = (RawBioAssay)job.getValue("rawBioAssay");
    872       if (rawBioAssay == null && context != null)
    873       {
    874         DbControl dc = sc.newDbControl();
    875         try
    876         {
    877           int rawBioAssayId = sc.getCurrentContext(context.getItem()).getId();
    878           if (rawBioAssayId != 0) rawBioAssay = RawBioAssay.getById(dc, rawBioAssayId);
    879         }
    880         finally
    881         {
    882           if (dc != null) dc.close();
    883         }
    884       }
    885      
    886       if (rawBioAssay != null && !rawBioAssay.hasArrayDesign())
     1038      if (useMissingReporterParameter)
    8871039      {
    8881040        parameters.add(missingReporterErrorParameter);
    8891041      }
    890       else
     1042      if (useFeatureParameters)
    8911043      {
    8921044        parameters.add(featureMismatchErrorParameter);
     
    9091061  }
    9101062 
     1063  private void addRawBioAssayParameters(DbControl dc, RawBioAssay currentRaw, List<PluginParameter<?>> parameters)
     1064  {
     1065    // Parameter for getting the current raw bioassay
     1066    PluginParameter<RawBioAssay> rbaParameter = new PluginParameter<RawBioAssay>
     1067    (
     1068      "rawBioAssay",
     1069      "Raw bioassay",
     1070      "The raw bioassay to import raw data to. It must be of the correct raw data " +
     1071      "type and shouldn't have any spots already",
     1072      new ItemParameterType<RawBioAssay>(RawBioAssay.class, currentRaw, true, 1, null)
     1073    );
     1074    parameters.add(rbaParameter);
     1075 
     1076    List<File> rawDataFiles = null;
     1077    if (currentRaw != null)
     1078    {
     1079      try
     1080      {
     1081        rawDataFiles = FileStoreUtil.getGenericDataFiles(dc, currentRaw, FileType.RAW_DATA);
     1082      }
     1083      catch (Throwable t)
     1084      {}
     1085    }
     1086   
     1087    // The raw data file to import from - if a file already hase
     1088    // been attached to the raw bioassay use it as a default choice
     1089    PluginParameter<File> fileParameter = new PluginParameter<File>(
     1090      "file",
     1091      "Raw data file",
     1092      "The file that contains the raw data that you want to import",
     1093      new FileParameterType(rawDataFiles == null || rawDataFiles.isEmpty() ?
     1094        null : rawDataFiles.get(0), true, 1)
     1095    );
     1096    parameters.add(fileParameter);
     1097
     1098  }
     1099 
     1100  private void addExperimentParameters(DbControl dc, Experiment currentExp, List<PluginParameter<?>> parameters)
     1101  {
     1102    // Parameter for getting the current raw bioassay
     1103    PluginParameter<Experiment> expParameter = new PluginParameter<Experiment>
     1104    (
     1105      "experiment",
     1106      "Experiment",
     1107      "The experiment to import raw data to",
     1108      new ItemParameterType<Experiment>(Experiment.class, currentExp, true, 1, null)
     1109    );
     1110   
     1111    Platform platform = null;
     1112    PlatformVariant variant = null;
     1113    if (currentExp != null)
     1114    {
     1115      RawDataType type = currentExp.getRawDataType();
     1116      platform = type.getPlatform(dc);
     1117      variant = type.getVariant(dc);
     1118    }
     1119    FileType rawData = FileType.getById(dc, SystemItems.getId(FileType.RAW_DATA));
     1120    ItemQuery<DataFileType> query = DataFileType.getQuery(platform, variant,
     1121      Item.RAWBIOASSAY, rawData);
     1122    query.include(Include.ALL);
     1123    query.order(Orders.asc(Hql.property("name")));
     1124    List<DataFileType> fileTypes = query.list(dc);
     1125
     1126    PluginParameter<DataFileType> fileTypeParameter = new PluginParameter<DataFileType>
     1127    (
     1128      "dataFileType",
     1129      "File type",
     1130      "The file type to use when looking for files to import. If not selected " +
     1131      "a generic lookup for any raw data file type is used. ",
     1132      new ItemParameterType<DataFileType>(DataFileType.class, null, false, 1, fileTypes)
     1133    );
     1134
     1135    PluginParameter<Boolean> transactionParameter = new PluginParameter<Boolean>
     1136    (
     1137      "transactionPerFile",
     1138      "One transaction per file",
     1139      "If this option is enabled, a new transaction is used for each " +
     1140      "raw bioassay to import (recommended). This means that if an import fails, " +
     1141      "you will not loose what has already been done.",
     1142      new BooleanParameterType(true, false)
     1143    );
     1144   
     1145    PluginParameter<Boolean> continueParameter = new PluginParameter<Boolean>
     1146    (
     1147      "continueAfterError",
     1148      "Continue with next file after an error",
     1149      "If this option is enabled the import will continue with the next " +
     1150      "raw bioassay after an error (recommended). This feature requires that " +
     1151      "the 'One transaction per file' option is enabled",
     1152      new BooleanParameterType(true, false)
     1153    );
     1154   
     1155    parameters.add(expParameter);
     1156    parameters.add(fileTypeParameter);
     1157    parameters.add(transactionParameter);
     1158    parameters.add(continueParameter);
     1159  }
     1160 
    9111161  private RequestInformation getConfigureRawDataTypeParameters()
    9121162  {
     
    9841234  }
    9851235 
     1236  /**
     1237    Get a query that returns all raw bioassays which it may be possible to
     1238    import raw data to for the given experiment. The raw bioassays
     1239    should have no raw data already and they must have at least one
     1240    file attached.
     1241    @param exp The experiment
     1242  */
     1243  private ItemQuery<RawBioAssay> getRawBioAssayQuery(Experiment exp)
     1244  {
     1245    ItemQuery<RawBioAssay> query = exp.getRawBioAssays();
     1246    query.include(Include.MINE, Include.SHARED, Include.OTHERS, Include.IN_PROJECT);
     1247    // Only return raw bioassays with no data...
     1248    query.restrict(Restrictions.eq(Hql.property("spots"), Expressions.integer(0)));
     1249    // ... that has files attached
     1250    query.restrict(Restrictions.neq(Hql.property("fileSet"), null));
     1251    return query;
     1252  }
     1253
     1254 
     1255  class FileIterator
     1256    implements Iterator<File>
     1257  {
     1258    private List<RawBioAssay> rawBioAssays;
     1259    private List<File> files;
     1260    private int currentIndex;
     1261   
     1262    FileIterator(List<RawBioAssay> rawBioAssays, List<File> files)
     1263    {
     1264      this.rawBioAssays = rawBioAssays;
     1265      this.files = files;
     1266      this.currentIndex = -1;
     1267    }
     1268
     1269    /*
     1270      From the Iterator interface
     1271      ---------------------------
     1272    */
     1273    @Override
     1274    public boolean hasNext()
     1275    {
     1276      return rawBioAssays.size() - 1 > currentIndex;
     1277    }
     1278
     1279    @Override
     1280    public File next()
     1281    {
     1282      currentIndex++;
     1283      log("Importing data to raw bioassay: " + getCurrentRawBioAssay().getName());
     1284      return getCurrentFile();
     1285    }
     1286
     1287    @Override
     1288    public void remove()
     1289    {
     1290      throw new UnsupportedOperationException("remove");
     1291    }
     1292    // -------------------------------------
     1293   
     1294    File getCurrentFile()
     1295    {
     1296      return files.get(currentIndex);
     1297    }
     1298   
     1299    RawBioAssay getCurrentRawBioAssay()
     1300    {
     1301      return rawBioAssays.get(currentIndex);
     1302    }
     1303   
     1304  }
     1305 
    9861306}
  • trunk/src/test/TestRawDataFlatFileImporter.java

    r4514 r4594  
    183183      j.setName(pc.getName());
    184184     
    185       PluginConfigurationRequest request = j.configure(null);
     185      PluginConfigurationRequest request = j.configure(GuiContext.item(Item.RAWBIOASSAY));
    186186      write_request_information(request.getRequestInformation());
    187187      request.setParameterValue("file", File.getById(dc, fileId));
Note: See TracChangeset for help on using the changeset viewer.