Changeset 7129


Ignore:
Timestamp:
Apr 21, 2016, 9:51:07 AM (6 years ago)
Author:
Nicklas Nordborg
Message:

References #2000: Batch API for annotation handling

Progress reporting now assign the first 50% to file parsing and the last 50% to database actions. It is still a bit unbalanced but better that before when it was 75%/25%.

Removed debug output.

File:
1 edited

Legend:

Unmodified
Added
Removed
  • trunk/src/plugins/core/net/sf/basedb/plugins/AnnotationFlatFileImporter.java

    r7127 r7129  
    2929import java.util.Collection;
    3030import java.util.Collections;
    31 import java.util.Date;
    3231import java.util.EnumSet;
    3332import java.util.HashMap;
     
    722721  protected void beginData()
    723722  {
    724     System.out.println(new Date()+": begin");
    725723    this.dc = sc.newDbControl();
    726724    this.unitCache = new UnitCache(dc);
     
    773771  {
    774772
    775     if (data.dataLineNo() % 100 == 0)
    776     {
    777       System.out.println(new Date()+": line " + data.dataLineNo());
    778     }
    779    
    780773    // Find the item(s) to annotate. Mapper gives us name OR external ID
    781774    String nameOrId = itemMapper.getValue(data);
     
    931924  protected void end(boolean success)
    932925  {
    933     System.out.println(new Date()+": end start");
    934926    try
    935927    {
     
    941933        for (Set<NewAnnotations> n : itemCache.values())
    942934        {
     935          if (progress != null && current % 100 == 0)
     936          {
     937            progress.display(50 + (48 * current) / total, "Saving to database (" + current + " of  " + total + " items done)...");
     938          }
    943939          current++;
    944 
    945           if (progress != null)
    946           {
    947             progress.display(75 + (20 * current) / total, "Saving to database...");
    948           }
    949940          numItems += n.size();
    950941          for (NewAnnotations na : n)
    951942          {
    952943            na.setNewAnnotations(batcher, addToUnlimited, replaceExisting, failIfTooManyValues, removeExisting);
    953             if (na.getNumSet() == 0) numItems--;
    954944            numAnnotations += na.getNumSet();
    955945            numError += na.getNumError();
     
    958948            ThreadSignalHandler.checkInterrupted();
    959949          }
    960          
    961           if (current % 100 == 0)
    962           {
    963             System.out.println(new Date()+": " + current + "; " + numItems + " items; " + numAnnotations + " annotations");
    964           }
    965 
    966950        }
    967951        batcher.close();
    968         System.out.println(new Date()+": commit start");
     952        if (progress != null) progress.display(99, "Committing transaction (" + numItems + " items)");
    969953        dc.commit();
    970         System.out.println(new Date()+": commit end");
    971954        if (progress != null) progress.display(100, "Done");
    972         //if (true) throw new RuntimeException("success!!");
    973955      }
    974956    }
     
    991973  {
    992974    String msg = numItems + " item(s) annotated with " + numAnnotations + " annotation(s)";
    993     if (numReplaced > 0) msg += " (" + numReplaced + " was replaced)";
     975    if (numReplaced > 0) msg += "; " + numReplaced + " was replaced)";
    994976    if (numRemoved > 0) msg += "; " + numRemoved + " annotations removed";
    995977    if (numItemNotFound > 0) msg += "; " + numItemNotFound + " row(s) skipped because no item was found";
     
    999981  /**
    1000982    Overriding default implementation since we want file parsing to
    1001     only use 75% and the remaining 25% for storing to the database.
     983    only use 50% and the remaining 50% for storing to the database.
    1002984   */
    1003985  @Override
    1004986  protected int getProgress(FlatFileParser ffp)
    1005987  {
    1006     return super.getProgress(ffp) * 75 / 100;
     988    return super.getProgress(ffp) * 50 / 100;
    1007989  }
    1008990  // -------------------------------------------
     
    14081390          continue; // Do not try to set too many values
    14091391        }
    1410          
     1392       
    14111393        List<Object> theValues = new ArrayList<Object>(newValues.size());
    14121394        // Make sure all annotation values are converted to the same unit
Note: See TracChangeset for help on using the changeset viewer.