Changeset 810


Ignore:
Timestamp:
Oct 22, 2008, 12:49:11 PM (14 years ago)
Author:
Nicklas Nordborg
Message:

Fixes #138: Importing data for multiple raw bioassays in one step

Location:
plugins/base2/net.sf.basedb.illumina/trunk
Files:
7 edited

Legend:

Unmodified
Added
Removed
  • plugins/base2/net.sf.basedb.illumina/trunk/COMPILE

    r807 r810  
    12122. Java 1.6
    1313
    14 3. BASE 2.6
     143. BASE 2.9.0
    1515
    1616---------
  • plugins/base2/net.sf.basedb.illumina/trunk/INSTALL

    r699 r810  
    2929== Requirements ==
    3030
    31  1. BASE 2.6.0 or later.
     31 1. BASE 2.9.0 or later.
    3232
    3333
  • plugins/base2/net.sf.basedb.illumina/trunk/META-INF/base-plugins.xml

    r681 r810  
    44<plugins jarname="%%plugins.jar%%">
    55  <pluginclass classname="net.sf.basedb.illumina.install.Install">
    6     <minbaseversion>2.6</minbaseversion>
     6    <minbaseversion>2.9</minbaseversion>
    77    <hasconfigurations/>
    88  </pluginclass>
    99  <pluginclass classname="net.sf.basedb.illumina.plugins.BackgroundCorrection">
    10     <minbaseversion>2.6</minbaseversion>
     10    <minbaseversion>2.9</minbaseversion>
    1111    <hasconfigurations/>
    1212  </pluginclass>
    1313  <pluginclass classname="net.sf.basedb.illumina.plugins.BeadSummaryImporter">
    14     <minbaseversion>2.6</minbaseversion>
     14    <minbaseversion>2.9</minbaseversion>
    1515    <hasconfigurations/>
    1616  </pluginclass>
    1717  <pluginclass classname="net.sf.basedb.illumina.plugins.BgxFeatureImporter">
    18     <minbaseversion>2.6</minbaseversion>
     18    <minbaseversion>2.9</minbaseversion>
    1919    <hasconfigurations>yes</hasconfigurations>
    2020  </pluginclass>
    2121  <pluginclass classname="net.sf.basedb.illumina.plugins.BgxReporterImporter">
    22     <minbaseversion>2.6</minbaseversion>
     22    <minbaseversion>2.9</minbaseversion>
    2323    <hasconfigurations>yes</hasconfigurations>
    2424  </pluginclass>
    2525  <pluginclass classname="net.sf.basedb.illumina.plugins.DetectionPValue">
    26     <minbaseversion>2.6</minbaseversion>
     26    <minbaseversion>2.9</minbaseversion>
    2727    <hasconfigurations/>
    2828  </pluginclass>
    2929  <pluginclass classname="net.sf.basedb.illumina.plugins.SnpIntensityCalculator">
    30     <minbaseversion>2.6</minbaseversion>
     30    <minbaseversion>2.9</minbaseversion>
    3131    <hasconfigurations/>
    3232  </pluginclass>
    3333  <pluginclass classname="net.sf.basedb.illumina.plugins.SnpRawDataImporter">
    34     <minbaseversion>2.6</minbaseversion>
     34    <minbaseversion>2.9</minbaseversion>
    3535    <hasconfigurations/>
    3636  </pluginclass>
    3737  <pluginclass classname="net.sf.basedb.illumina.plugins.SnpReporterImporter">
    38     <minbaseversion>2.6</minbaseversion>
     38    <minbaseversion>2.9</minbaseversion>
    3939    <hasconfigurations>yes</hasconfigurations>
    4040  </pluginclass>
  • plugins/base2/net.sf.basedb.illumina/trunk/README

    r627 r810  
    2626== Requirements ==
    2727
    28  1. BASE 2.6.0 or later.
     28 1. BASE 2.9.0 or later.
    2929 
    3030For expression experiments:
     
    229229in step 3 are shared to the appropriate users, for example, the Everyone group.
    230230
     231Tip! The data import step in (4) above can be done for an entire experiment at a time.
     232
     233
  • plugins/base2/net.sf.basedb.illumina/trunk/build.xml

    r807 r810  
    2424  <property name="javac.encoding" value="ISO-8859-1" />
    2525  <property name="depend.jars"
    26     value="http://base2.thep.lu.se/base/jars/2.6.0"
     26    value="http://base2.thep.lu.se/base/jars/2.9.0"
    2727    description="The location of the BASE core JARs that we depend on"
    2828  />
  • plugins/base2/net.sf.basedb.illumina/trunk/src/net/sf/basedb/illumina/Illumina.java

    r737 r810  
    3636    The current version of this plug-in package.
    3737  */
    38   public static final String VERSION = "pre1.2";
     38  public static final String VERSION = "1.2pre";
    3939 
    4040  /**
  • plugins/base2/net.sf.basedb.illumina/trunk/src/net/sf/basedb/illumina/plugins/BeadSummaryImporter.java

    r646 r810  
    3030import java.util.Collections;
    3131import java.util.HashMap;
     32import java.util.HashSet;
    3233import java.util.List;
    3334import java.util.Map;
     
    3738import net.sf.basedb.core.ArrayDesign;
    3839import net.sf.basedb.core.BaseException;
     40import net.sf.basedb.core.BooleanParameterType;
    3941import net.sf.basedb.core.DataFileType;
    4042import net.sf.basedb.core.DbControl;
     43import net.sf.basedb.core.Experiment;
    4144import net.sf.basedb.core.FeatureIdentificationMethod;
    4245import net.sf.basedb.core.File;
     
    4548import net.sf.basedb.core.FileStoreEnabled;
    4649import net.sf.basedb.core.FileStoreUtil;
     50import net.sf.basedb.core.Include;
    4751import net.sf.basedb.core.InvalidDataException;
    4852import net.sf.basedb.core.InvalidUseOfNullException;
     
    7781import net.sf.basedb.core.query.Hql;
    7882import net.sf.basedb.core.query.Restrictions;
     83import net.sf.basedb.core.signal.SignalException;
    7984import net.sf.basedb.core.signal.SignalHandler;
    8085import net.sf.basedb.core.signal.SignalTarget;
     
    8287import net.sf.basedb.illumina.Illumina;
    8388import net.sf.basedb.plugins.util.Parameters;
     89import net.sf.basedb.util.ChainedProgressReporter;
    8490import net.sf.basedb.util.MD5;
    8591import net.sf.basedb.util.error.ClassMapErrorHandler;
     
    127133 
    128134  private static final Set<GuiContext> guiContexts =
    129     Collections.singleton(new GuiContext(Item.RAWBIOASSAY, GuiContext.Type.ITEM));
    130  
     135    Collections.unmodifiableSet(new HashSet<GuiContext>(
     136        Arrays.asList(
     137          GuiContext.item(Item.RAWBIOASSAY),
     138          GuiContext.item(Item.EXPERIMENT)
     139        )
     140      ));
    131141
    132142  private static final PluginParameter<String> featureMismatchErrorParameter =
     
    134144      "featureMismatchError",
    135145      "Probe not found",
    136       "How to handle errors that are caused by not finding a feature with the given" +
     146      "How to handle errors that are caused by not finding a feature with the given " +
    137147      "Array_Adress_Id on the array design. " +
    138148      "This can for example happen if an incorrect array design is used.\n\n"+
     
    149159 
    150160  /**
    151     The current raw bioassay.
    152   */
    153   private PluginParameter<RawBioAssay> rawBioAssayParameter;
    154  
    155   /**
    156161    The files to import from.
    157162  */
     
    201206  {
    202207    if (signalHandler != null) signalHandler.setWorkerThread(null);
    203     DbControl dc = sc.newDbControl();
     208    createLogFile((String)job.getValue(Parameters.LOGFILE_PARAMETER));
     209    boolean transactionPerFile = Boolean.TRUE.equals(job.getValue("transactionPerFile"));
     210    boolean continueAfterError = transactionPerFile &&
     211      Boolean.TRUE.equals(job.getValue("continueAfterError"));
     212    setUpErrorHandling(job);
     213    Job theJob = job.getJob();
     214
     215    DbControl dc = null;
    204216    try
    205217    {
    206       if (progress != null) progress.display(0, "Loading items...");
    207       RawBioAssay rba = (RawBioAssay)job.getValue("rawBioAssay");
    208       rba = RawBioAssay.getById(dc, rba.getId());
     218      ChainedProgressReporter chainedProgress = null;
     219      if (progress != null)
     220      {
     221        progress.display(0, "Loading items...");
     222        chainedProgress = new ChainedProgressReporter(progress);
     223      }
     224      Experiment exp = (Experiment)job.getValue("experiment");
     225      RawBioAssay raw = (RawBioAssay)job.getValue("rawBioAssay");
    209226     
    210       List<PlatformFileType> fileTypes = getPlatformFileTypes(dc, rba);
    211       List<File> files = new ArrayList<File>(fileTypes.size());
    212       List<FileSetMember> members = new ArrayList<FileSetMember>(fileTypes.size());
    213       for (PlatformFileType pft : fileTypes)
    214       {
    215         DataFileType dft = pft.getDataFileType();
    216         File f = (File)job.getValue("file." + dft.getExternalId());
    217         if (f != null)
    218         {
    219           files.add(f);
    220           members.add(FileStoreUtil.setDataFile(dc, rba, dft.getExternalId(), f));
    221         }
    222       }
    223 
    224       setUpErrorHandling(job);
    225       String msg = importScandata(rba, files, progress);
     227      dc = sc.newDbControl();
     228      List<RawBioAssay> toImport = exp == null ?
     229        Collections.singletonList(raw) :
     230        new ArrayList<RawBioAssay>(getRawBioAssayQuery(exp).list(dc));
    226231     
    227       for (FileSetMember member : members)
    228       {
    229         member.setValid(true, null);
    230       }     
    231       dc.commit();
     232      int numSuccess = 0;
     233      int numTotal = toImport.size();
     234      float chainedProgressStart = 0.0f;
     235      float chainedProgressDelta = 100.0f / numTotal;
     236      String msg = null;
     237      for (RawBioAssay rba : toImport)
     238      {
     239        if (dc == null) dc = sc.newDbControl();
     240        rba = RawBioAssay.getById(dc, rba.getId());
     241        try
     242        {
     243          log("Importing data to raw bioassay: " + rba.getName());
     244          if (chainedProgress != null)
     245          {
     246            chainedProgress.setRange((int)chainedProgressStart, (int)(chainedProgressStart + chainedProgressDelta));
     247            chainedProgressStart += chainedProgressDelta;
     248          }
     249         
     250          // Load files
     251          List<PlatformFileType> fileTypes = getPlatformFileTypes(dc, rba);
     252          List<File> files = new ArrayList<File>(fileTypes.size());
     253          List<FileSetMember> members = new ArrayList<FileSetMember>(fileTypes.size());
     254          for (PlatformFileType pft : fileTypes)
     255          {
     256            DataFileType dft = pft.getDataFileType();
     257            File f = null;
     258            FileSetMember member = null;
     259            if (exp == null)
     260            {
     261              // Get file from job parameters
     262              f = (File)job.getValue("file." + dft.getExternalId());
     263              if (f != null) member = FileStoreUtil.setDataFile(dc, rba, dft.getExternalId(), f);
     264            }
     265            else
     266            {
     267              // Get already stored files
     268              member = rba.getFileSet().getMember(dft);
     269              if (member != null) f = member.getFile();
     270            }
     271            if (f == null && pft.isRequired())
     272            {
     273              throw new InvalidUseOfNullException("Missing required file '" +
     274                dft.getName() + "' for raw bioassay: " + rba.getName());
     275            }
     276            if (f != null && member != null)
     277            {
     278              files.add(f);
     279              members.add(member);
     280            }
     281          }
     282         
     283          // Import data
     284          msg = importScandata(rba, files, theJob, chainedProgress);
     285          for (FileSetMember member : members)
     286          {
     287            member.setValid(true, null);
     288          }
     289         
     290          // Commit and cleanup
     291          if (transactionPerFile) dc.commit();
     292          log(msg);
     293          numSuccess++;
     294        }
     295        catch (SignalException s)
     296        {
     297          // The user aborted the plug-in -> always exit
     298          throw s;
     299        }
     300        catch (Throwable t)
     301        {
     302          if (!continueAfterError) throw t;
     303          msg = t.getMessage();
     304          log("Failed", t);
     305        }
     306        finally
     307        {
     308          if (transactionPerFile)
     309          {
     310            if (dc != null) dc.close();
     311            dc = null;
     312          }
     313          log("--------------------------------------");
     314        }
     315      }
     316     
     317      // When using a single transaction
     318      if (dc != null && !dc.isClosed()) dc.commit();
    232319     
    233320      if (progress != null) progress.display(100, msg);
    234       response.setDone(msg);
     321      if (numSuccess == 0)
     322      {
     323        response.setError("No files could be imported: " + msg, null);
     324      }
     325      else
     326      {
     327        if (numTotal > 1)
     328        {
     329          msg = numSuccess + " of " + numTotal + " files could be imported";
     330        }
     331        response.setDone(msg);
     332      }
    235333    }
    236334    catch (Throwable t)
    237335    {
     336      log("Failed", t);
    238337      response.setError(t.getMessage(), Arrays.asList(t));
    239338    }
     
    241340    {
    242341      if (dc != null) dc.close();
     342      closeLogFile();
    243343    }
    244344  }
     
    266366        }
    267367       
    268         storeValue(job, request, rawBioAssayParameter);
    269         for (PluginParameter<File> fp : fileParameters)
    270         {
    271           storeValue(job, request, fp);
    272         }
     368        // Single raw bioassay mode
     369        storeValue(job, request, ri.getParameter("rawBioAssay"));
     370        if (fileParameters != null)
     371        {
     372          for (PluginParameter<File> fp : fileParameters)
     373          {
     374            storeValue(job, request, fp);
     375          }
     376        }
     377       
     378        // Experiment mode
     379        storeValue(job, request, ri.getParameter("experiment"));
     380        storeValue(job, request, ri.getParameter("transactionPerFile"));
     381        storeValue(job, request, ri.getParameter("continueAfterError"));
     382       
     383        // Common
    273384        storeValue(job, request, ri.getParameter(Parameters.CHARSET_PARAMETER));
    274385        storeValue(job, request, ri.getParameter(Parameters.DECIMAL_SEPARATOR_PARAMETER));
    275            
     386
    276387        // Error handling parameters
     388        storeValue(job, request, ri.getParameter(Parameters.LOGFILE_PARAMETER));
    277389        storeValue(job, request, ri.getParameter(Parameters.DEFAULT_ERROR));
    278390        storeValue(job, request, featureMismatchErrorParameter);
     
    322434  public String isInContext(GuiContext context, Object item)
    323435  {
    324     String message = null;
    325436    if (item == null)
    326437    {
    327       message = "The object is null";
    328     }
    329     else if (!(item instanceof RawBioAssay))
    330     {
    331       message = "The object is not a RawBioAssay: " + item;
     438      return "The object is null";
     439    }
     440    else if (!context.isCorrectBasicItem(item))
     441    {
     442      return "The oject is not a " + context.getItem() + ": " + item;
    332443    }
    333444    else
    334445    {
    335       RawBioAssay rba = (RawBioAssay)item;
    336       ArrayDesign design = rba.getArrayDesign();
     446      Experiment exp = null;
     447      RawBioAssay rba = null;
     448      RawDataType rdt = null;
    337449      RawDataType beadSummary = Illumina.getBeadSummaryType();
    338       RawDataType rdt = rba.getRawDataType();
    339       if (rdt != beadSummary)
    340       {
    341         message = "Unsupported raw data type: " + rdt.getName();
    342       }
    343       else if (rba.getNumDbSpots() > 0)
    344       {
    345         throw new PermissionDeniedException("The raw bioassay already has data.");
    346       }
    347       else if (design == null)
    348       {
    349         throw new ItemNotFoundException("The raw bioassay has no array design: " + rba.getName());
    350       }
    351       else if (design.getNumDbFeatures() == 0)
    352       {
    353         throw new InvalidDataException("The array design has no features: " + design.getName());
     450      if (item instanceof RawBioAssay)
     451      {
     452        rba = (RawBioAssay)item;
     453        ArrayDesign design = rba.getArrayDesign();
     454        rdt = rba.getRawDataType();
     455        if (rba.getNumDbSpots() > 0)
     456        {
     457          throw new PermissionDeniedException("The raw bioassay already has data.");
     458        }
     459        else if (design == null)
     460        {
     461          throw new ItemNotFoundException("The raw bioassay has no array design: " +
     462              rba.getName());
     463        }
     464        else if (design.getNumDbFeatures() == 0)
     465        {
     466          throw new InvalidDataException("The array design has not imported " +
     467              "features into the database: " + design.getName());
     468        }
     469        else
     470        {
     471          rba.checkPermission(Permission.WRITE);
     472        }
    354473      }
    355474      else
    356475      {
    357         rba.checkPermission(Permission.WRITE);
    358       }
    359     }
    360     return message;   
     476        exp = (Experiment)item;
     477        rdt = exp.getRawDataType();
     478        ItemQuery<RawBioAssay> query = getRawBioAssayQuery(exp);
     479        DbControl dc = sc.newDbControl();
     480        try
     481        {
     482          if (query.count(dc) == 0)
     483          {
     484            return "The experiment doesn't have any raw bioassays with files waiting for import";
     485          }
     486        }
     487        finally
     488        {
     489          if (dc != null) dc.close();
     490        }
     491      }
     492      if (!beadSummary.equals(rdt))
     493      {
     494        return "Unsupported raw data type: " +rdt;
     495      }
     496    }
     497    return null;
    361498  }
    362499  // -------------------------------------------
     
    380517      // Parameters for CONFIGURE_JOB
    381518      List<PluginParameter<?>> parameters = new ArrayList<PluginParameter<?>>();
    382      
    383       // Parameter for getting the current raw bioassay
    384       ItemParameterType<RawBioAssay> rawBioAssayType =
    385         new ItemParameterType<RawBioAssay>(RawBioAssay.class, null, true, 1, null);
    386       rawBioAssayParameter = new PluginParameter<RawBioAssay>
    387       (
    388         "rawBioAssay",
    389         "Raw bioassay",
    390         "The raw bioassay to import raw data to. It must be of the Illumina bead summary " +
    391         "raw data type and shouldn't have any spots already.",
    392         rawBioAssayType
    393       );
    394       parameters.add(rawBioAssayParameter);
    395 
    396       RawBioAssay rba = null;
    397       int currentRawBioAssayId = sc.getCurrentContext(Item.RAWBIOASSAY).getId();
     519      RawBioAssay currentRaw = (RawBioAssay)job.getValue("rawBioAssay");
     520      Experiment currentExp = (Experiment)job.getValue("experiment");
    398521      DbControl dc = sc.newDbControl();
    399522      try
    400523      {
    401         rba = RawBioAssay.getById(dc, currentRawBioAssayId);
    402        
    403         fileParameters = new ArrayList<PluginParameter<File>>();
    404         List<PlatformFileType> fileTypes = getPlatformFileTypes(dc, rba);
    405        
    406         for (PlatformFileType pft : fileTypes)
    407         {
    408           DataFileType dft = pft.getDataFileType();
    409           File f = FileStoreUtil.getDataFile(dc, rba, dft.getExternalId());
    410          
    411           PluginParameter<File> fileParameter = new PluginParameter<File>(
    412             "file." + dft.getExternalId(),
    413             dft.getName(),
    414             dft.getDescription(),
    415               new FileParameterType(f, pft.isRequired(), 1)
    416             );
    417           parameters.add(fileParameter);
    418           fileParameters.add(fileParameter);
    419         }
    420       }
    421       /*
    422       catch (Throwable t)
    423       {}
    424       */
     524        if (GuiContext.item(Item.RAWBIOASSAY).equals(context))
     525        {
     526          int id = currentRaw == null ? sc.getCurrentContext(Item.RAWBIOASSAY).getId() : currentRaw.getId();
     527          if (id != 0) currentRaw = RawBioAssay.getById(dc, id);
     528          addRawBioAssayParameters(dc, currentRaw, parameters);
     529        }
     530        else
     531        {
     532          int id = currentExp == null ? sc.getCurrentContext(Item.EXPERIMENT).getId() : currentExp.getId();
     533          if (id != 0) currentExp = Experiment.getById(dc, id);
     534          addExperimentParameters(dc, currentExp, parameters);
     535        }
     536      }
    425537      finally
    426538      {
    427539        if (dc != null) dc.close();
    428540      }
    429      
     541
    430542      parameters.add(Parameters.charsetParameter(null, null, null));
    431543      parameters.add(Parameters.decimalSeparatorParameter(null, null, null));
     
    433545      // Error handling parameters
    434546      parameters.add(Parameters.errorSection(null, null));
     547      parameters.add(Parameters.logFileParameter(null, null, null));
    435548      parameters.add(Parameters.defaultError(null, null, null));
    436549     
     
    456569  }
    457570 
     571  private void addRawBioAssayParameters(DbControl dc, RawBioAssay currentRaw, List<PluginParameter<?>> parameters)
     572  {
     573    // Parameter for getting the current raw bioassay
     574    PluginParameter<RawBioAssay> rbaParameter = new PluginParameter<RawBioAssay>
     575    (
     576      "rawBioAssay",
     577      "Raw bioassay",
     578      "The raw bioassay to import raw data to. It must be of the Illumina bead summary " +
     579      "raw data type and shouldn't have any spots already.",
     580      new ItemParameterType<RawBioAssay>(RawBioAssay.class, currentRaw, true, 1, null)
     581    );
     582    parameters.add(rbaParameter);
     583
     584    fileParameters = new ArrayList<PluginParameter<File>>();
     585    List<PlatformFileType> fileTypes = getPlatformFileTypes(dc, currentRaw);
     586   
     587    for (PlatformFileType pft : fileTypes)
     588    {
     589      DataFileType dft = pft.getDataFileType();
     590      File f = FileStoreUtil.getDataFile(dc, currentRaw, dft.getExternalId());
     591     
     592      PluginParameter<File> fileParameter = new PluginParameter<File>(
     593        "file." + dft.getExternalId(),
     594        dft.getName(),
     595        dft.getDescription(),
     596        new FileParameterType(f, pft.isRequired(), 1)
     597        );
     598      parameters.add(fileParameter);
     599      fileParameters.add(fileParameter);
     600    }
     601  }
     602 
     603  private void addExperimentParameters(DbControl dc, Experiment currentExp, List<PluginParameter<?>> parameters)
     604  {
     605    // Parameter for getting the current raw bioassay
     606    PluginParameter<Experiment> expParameter = new PluginParameter<Experiment>
     607    (
     608      "experiment",
     609      "Experiment",
     610      "The experiment to import raw data to",
     611      new ItemParameterType<Experiment>(Experiment.class, currentExp, true, 1, null)
     612    );
     613    PluginParameter<Boolean> transactionParameter = new PluginParameter<Boolean>
     614    (
     615      "transactionPerFile",
     616      "One transaction per file",
     617      "If this option is enabled, a new transaction is used for each " +
     618      "raw bioassay to import (recommended). This means that if an import fails, " +
     619      "you will not loose what has already been done.",
     620      new BooleanParameterType(true, false)
     621    );
     622   
     623    PluginParameter<Boolean> continueParameter = new PluginParameter<Boolean>
     624    (
     625      "continueAfterError",
     626      "Continue with next file after an error",
     627      "If this option is enabled the import will continue with the next " +
     628      "raw bioassay after an error (recommended). This feature requires that " +
     629      "the 'One transaction per file' option is enabled",
     630      new BooleanParameterType(true, false)
     631    );
     632   
     633    parameters.add(expParameter);
     634    parameters.add(transactionParameter);
     635    parameters.add(continueParameter);
     636  }
     637 
    458638  public FlatFileParser createFlatFileParser()
    459639  {
     
    464644  }
    465645 
    466   public String importScandata(RawBioAssay rba, List<File> files, ProgressReporter progress)
    467   {
    468     // We assume that each file contains rougly the same number of lines
    469     // Progress reporter should be divided into numFile + 1 steps
    470     int offset = 0;
    471     double deltaOffset = 100 / (files.size() + 1);
     646  public String importScandata(RawBioAssay rba, List<File> files, Job job, ProgressReporter progress)
     647  {
     648    long offsetFileSize = 0;
     649    long totalFileSize = 0;
     650    for (File file : files)
     651    {
     652      totalFileSize += file.getSize();
     653    }
     654    // 0-40% = file parsing; 40-60% = batcher preparation; 60-100% = data insertion
     655    double factor = 40.0 / totalFileSize;
    472656   
    473657    // For the final status report
     
    482666    for (File file : files)
    483667    {
    484       double factor =  deltaOffset / file.getSize();
    485       long current = 0;
    486668      InputStream in = file.getDownloadStream(0);
    487669      ffp.setInputStream(in, "ISO-8859-1");
     
    505687          if (progress != null && lines % 100 == 0)
    506688          {
    507             String message = "Parsing file " + file.getName() + "; " + lines + " lines done.";
    508             current = ffp.getParsedBytes();
    509             int percent = (int)(offset + factor * current);
     689            String message = rba.getName() + " - parsing file " + file.getName() +
     690              "; " + lines + " lines done.";
     691            int percent = (int)(factor * (offsetFileSize + ffp.getParsedBytes()));
    510692            progress.display(percent, message);
    511693          }
    512694         
    513695          // Error handling enabled section
     696          FlatFileParser.Data data = null;
    514697          try
    515698          {
    516699            // Get next data line
    517             FlatFileParser.Data data = ffp.nextData();
     700            data = ffp.nextData();
    518701            numLines++;
    519702            String featureId = MD5.leftPad(data.get(0), '0', 10);
     
    544727              errorHandler.handleError(t);
    545728            }
     729            log("Skipped line " + data.lineNo() + ": " + t.getMessage());
    546730            skipped++;
    547731          }
    548732        }
    549733       
    550         offset += deltaOffset;
     734        offsetFileSize += file.getSize();
    551735      }
    552736      catch (Throwable t)
     
    556740      }
    557741    }
     742 
     743    if (progress != null)
     744    {
     745      progress.display(50, rba.getName() + " - parsed " + numLines + " lines in " +
     746        files.size() + " file(s). Preparing database for import...");
     747    }
    558748   
    559  
    560749    // Insert data into db
    561     long current = 0;
    562     RawDataBatcher batcher = rba.getRawDataBatcher(FeatureIdentificationMethod.FEATURE_ID);
    563     double factor =  deltaOffset / rawdata.size();
     750    RawDataBatcher batcher = rba.getRawDataBatcher(FeatureIdentificationMethod.FEATURE_ID, job);
     751    int done = 0;
     752    factor =  40.0 / rawdata.size();
    564753    String currentFeatureId = null;
    565754    try
     
    573762       
    574763        // Progress reporting
    575         if (progress != null && current % 100 == 0)
    576         {
    577           String message = "Saving to database; " + current + " spots done.";
    578           int percent = (int)(offset + factor * current);
     764        if (progress != null && done % 100 == 0)
     765        {
     766          String message = rba.getName() + " - saving to database; " + done + " spots done.";
     767          int percent = (int)(60 + factor * done);
    579768          progress.display(percent, message);
    580769        }
     
    600789            errorHandler.handleError(t);
    601790          }
     791          log("Skipped spot " + currentFeatureId + ": " + t.getMessage());
    602792          skipped++;
    603793        }
    604         current++;
     794        done++;
    605795      }
    606796      batcher.flush();
     
    690880    }
    691881    return handler;
    692   }
     882  }
     883 
     884  /**
     885    Get a query that returns all raw bioassays which it may be possible to
     886    import raw data to for the given experiment. The raw bioassays
     887    should have no raw data already they must have at least one
     888    file attached and be associated with an array design that has features.
     889    @param exp The experiment
     890  */
     891  private ItemQuery<RawBioAssay> getRawBioAssayQuery(Experiment exp)
     892  {
     893    ItemQuery<RawBioAssay> query = exp.getRawBioAssays();
     894    query.include(Include.MINE, Include.SHARED, Include.OTHERS, Include.IN_PROJECT);
     895    // Only return raw bioassays with no data...
     896    query.restrict(Restrictions.eq(Hql.property("spots"), Expressions.integer(0)));
     897    // ... that has files attached
     898    query.restrict(Restrictions.neq(Hql.property("fileSet"), null));
     899    query.restrict(Restrictions.gt(
     900        Hql.property("arrayDesign.numDbFeatures"),
     901        Expressions.integer(0)
     902      ));
     903    return query;
     904  }
    693905}
Note: See TracChangeset for help on using the changeset viewer.