Changeset 7129

Timestamp:

Apr 21, 2016, 9:51:07 AM (7 years ago)

Author:

Nicklas Nordborg

Message:

References #2000: Batch API for annotation handling

Progress reporting now assign the first 50% to file parsing and the last 50% to database actions. It is still a bit unbalanced but better that before when it was 75%/25%.

Removed debug output.

File:

• trunk/src/plugins/core/net/sf/basedb/plugins/AnnotationFlatFileImporter.java (modified) (9 diffs)

trunk/src/plugins/core/net/sf/basedb/plugins/AnnotationFlatFileImporter.java

@@ -29 +29 @@
 import java.util.Collection;
 import java.util.Collections;
-import java.util.Date;
 import java.util.EnumSet;
 import java.util.HashMap;
@@ -722 +721 @@
   protected void beginData()
   {
-    System.out.println(new Date()+": begin");
     this.dc = sc.newDbControl();
     this.unitCache = new UnitCache(dc);
@@ -773 +771 @@
   {
 
-    if (data.dataLineNo() % 100 == 0)
-    {
-      System.out.println(new Date()+": line " + data.dataLineNo());
-    }
-    
     // Find the item(s) to annotate. Mapper gives us name OR external ID
     String nameOrId = itemMapper.getValue(data);
@@ -931 +924 @@
   protected void end(boolean success)
   {
-    System.out.println(new Date()+": end start");
     try
     {
@@ -941 +933 @@
         for (Set<NewAnnotations> n : itemCache.values())
         {
+          if (progress != null && current % 100 == 0) 
+          {
+            progress.display(50 + (48 * current) / total, "Saving to database (" + current + " of  " + total + " items done)...");
+          }
           current++;
-
-          if (progress != null) 
-          {
-            progress.display(75 + (20 * current) / total, "Saving to database...");
-          }
           numItems += n.size();
           for (NewAnnotations na : n)
           {
             na.setNewAnnotations(batcher, addToUnlimited, replaceExisting, failIfTooManyValues, removeExisting);
-            if (na.getNumSet() == 0) numItems--;
             numAnnotations += na.getNumSet();
             numError += na.getNumError();
@@ -958 +948 @@
             ThreadSignalHandler.checkInterrupted();
           }
-          
-          if (current % 100 == 0)
-          {
-            System.out.println(new Date()+": " + current + "; " + numItems + " items; " + numAnnotations + " annotations");
-          }
-
         }
         batcher.close();
-        System.out.println(new Date()+": commit start");
+        if (progress != null) progress.display(99, "Committing transaction (" + numItems + " items)");
         dc.commit();
-        System.out.println(new Date()+": commit end");
         if (progress != null) progress.display(100, "Done");
-        //if (true) throw new RuntimeException("success!!");
       }
     }
@@ -991 +973 @@
   {
     String msg = numItems + " item(s) annotated with " + numAnnotations + " annotation(s)";
-    if (numReplaced > 0) msg += " (" + numReplaced + " was replaced)";
+    if (numReplaced > 0) msg += "; " + numReplaced + " was replaced)";
     if (numRemoved > 0) msg += "; " + numRemoved + " annotations removed";
     if (numItemNotFound > 0) msg += "; " + numItemNotFound + " row(s) skipped because no item was found";
@@ -999 +981 @@
   /**
     Overriding default implementation since we want file parsing to 
-    only use 75% and the remaining 25% for storing to the database.
+    only use 50% and the remaining 50% for storing to the database.
    */
   @Override
   protected int getProgress(FlatFileParser ffp)
   {
-    return super.getProgress(ffp) * 75 / 100;
+    return super.getProgress(ffp) * 50 / 100;
   }
   // -------------------------------------------
@@ -1408 +1390 @@
           continue; // Do not try to set too many values
         }
-          
+        
         List<Object> theValues = new ArrayList<Object>(newValues.size());
         // Make sure all annotation values are converted to the same unit