Changeset 7127

Timestamp:

Apr 20, 2016, 2:11:26 PM (7 years ago)

Author:

Nicklas Nordborg

Message:

References #2000: Batch API for annotation handling

Several bug fixes and improvements.

In AnnotationBatcher:

• Evict items from the second-level cache in Hibernate if we make any changes (eg. create a new annotation set)
• Add support for merging values for annotation types that have unlimited number of annotations.
• Fixes issues with setting the correct unit for annotation types that supports unit. Now the unit should always be set, either to the default unit or the unit that was specified.

In AnnotationFlatFileImporter:

• Merge is supported
• Ignore annotations if there are too many values
• Not removing annotations when no values have been specified

Location:

trunk/src

Files:

• core/net/sf/basedb/core/AnnotationBatcher.java (modified) (12 diffs)
• plugins/core/net/sf/basedb/plugins/AnnotationFlatFileImporter.java (modified) (5 diffs)

trunk/src/core/net/sf/basedb/core/AnnotationBatcher.java

@@ -91 +91 @@
   private final Date batchDate;
 
+  // We need some interaction with Hibernate
   private final org.hibernate.StatelessSession statelessSession;
   private final org.hibernate.dialect.Dialect dialect;
+  private final org.hibernate.Cache secondLevelCache;
 
   // Discriminator values for Type items (eg. Type.INT -> IntegerParameterValueData -> 1)
@@ -135 +137 @@
   private int currentAnnotationSetId;
   private boolean deleteSnapshotIfModifed;
+  private boolean evictFromSecondLevelCacheIfModified;
   
   // Lists for temporary keeping track of value IDs for
@@ -175 +178 @@
     this.statelessSession = dc.getStatelessSession();
     this.dialect = HibernateUtil.getDialect();
+    this.secondLevelCache = HibernateUtil.getSessionFactory().getCache();
 
     Type[] types = Type.values();
@@ -306 +310 @@
   public void flush()
   {
-    System.out.println(new Date() + ": " + numItems + " done; flushing batchers");
+    log.debug(numItems + " done. Flushing batchers.");
     String currentSql = null;
     try
@@ -332 +336 @@
   protected void onBeforeClose()
   {
-    System.out.println(new Date() + "; done!! " + numItems + " items.");
+    log.debug(numItems + " done. Closing.");
     for (UpdateBatcher b : allUpdateBatchers)
     {
@@ -459 +463 @@
     this.currentAnnotationSetId = 0;
     this.deleteSnapshotIfModifed = false;
+    this.evictFromSecondLevelCacheIfModified = secondLevelCache != null && secondLevelCache.containsEntity(itemType.getDataClass(), item.getId());
 
     // Read current values
@@ -549 +554 @@
     more information.
   */
-  public Change setValue(AnnotationType annotationType, Object value, Unit unit)
-  {
-    return setValues(annotationType, value == null ? null : Collections.singletonList(value), unit);
+  public Change setValue(AnnotationType annotationType, Object value, Unit unit, boolean merge)
+  {
+    return setValues(annotationType, value == null ? null : Collections.singletonList(value), unit, merge);
   }
   
@@ -581 +586 @@
     @param values The values to set or null/empty to remove values
     @param unit The unit of the values. Ignored if the annotation type doesn't use units.
-      
+    @param merge If TRUE and the annotation type allows an unlimted number 
+      of annotations (multiplicity=0) merges the new values with the existing values.
+      Ignored if multiplicity > 0.
+    
     @return The resulting change
   */
-  public Change setValues(AnnotationType annotationType, List<?> values, Unit unit)
+  public Change setValues(AnnotationType annotationType, List<?> values, Unit unit, boolean merge)
   {
     if (currentItem == null) throw new IllegalStateException("Must call setCurrentItem() first");
@@ -641 +649 @@
       else
       {
+        // Check multiplicity
+        int multiplicity = annotationType.getMultiplicity();
+        if (multiplicity > 0 && values.size() > multiplicity)
+        {
+          throw new InvalidDataException("Too many values. Max "+multiplicity+" value(s) are allowed for this annotation type: "+annotationType.getName());
+        }
+        
         // Check that values are allowed by annotation type
         // and convert to the default unit if it is needed
         UnitConverter converter = null;
-        if (unit != null && info.defaultUnit != null && !unit.equals(info.defaultUnit))
+        if (info.defaultUnit != null)
         {
-          // Two different units, convert the values to the default unit
-          converter = getUnitConverter(unit, info.defaultUnit);
+          // The annotation use units
+          if (unit == null)
+          {
+            // Assume values use the default unit if none was specifed
+            unit = info.defaultUnit;
+          }
+          else if (!unit.equals(info.defaultUnit))
+          {
+            // Two different units, convert the values to the default unit
+            converter = getUnitConverter(unit, info.defaultUnit);
+          }
+          info.unitId = unit.getId();
         }
         
@@ -679 +704 @@
             changeType = Change.UPDATED;
             
-            // Delete from **Values
-            deleteValues[annotationType.getValueType().ordinal()].addToBatch(info.valueId);
+            // Delete from **Values unless merging
+            if (!merge || multiplicity != 0)
+            {
+              deleteValues[annotationType.getValueType().ordinal()].addToBatch(info.valueId);
+            }
             
             // Insert into **Values
@@ -689 +717 @@
             
             // Update last_update, unit, version in Annotations
-            if (unit != null) info.unitId = unit.getId();
             updateAnnotations.addToBatch(info.version+1, info.unitId, batchDate, info.annotationId, info.version);
           }
@@ -726 +753 @@
             // Update item table, annotationset_id column
             updateItemTable.addToBatch(currentAnnotationSetId, currentItem.getId());
+            if (evictFromSecondLevelCacheIfModified)
+            {
+              // We must evict the entity from the Hibernate second-level 
+              // cache or Hibernate may use the old data
+              secondLevelCache.evictEntity(itemType.getDataClass(), currentItem.getId());
+            }
           }
 
           // Insert into Annotations
-          info.annotationId = insertIntoAnnotations.insert(currentAnnotationSetId, info.annotationTypeId, null, info.valueId, batchDate);
+          info.annotationId = insertIntoAnnotations.insert(currentAnnotationSetId, info.annotationTypeId, info.unitId, info.valueId, batchDate);
         }
       }

trunk/src/plugins/core/net/sf/basedb/plugins/AnnotationFlatFileImporter.java

@@ -41 +41 @@
 
 import net.sf.basedb.core.Annotatable;
-import net.sf.basedb.core.Annotation;
 import net.sf.basedb.core.AnnotationBatcher;
 import net.sf.basedb.core.AnnotationBatcher.Change;
-import net.sf.basedb.core.AnnotationSet;
 import net.sf.basedb.core.AnnotationType;
 import net.sf.basedb.core.BaseException;
@@ -337 +335 @@
     {
       permissions.add(new Permissions(Item.FILE, null, EnumSet.of(Permission.READ)));
-      permissions.add(new Permissions(Item.ANNOTATIONTYPE, null, EnumSet.of(Permission.READ)));
+      permissions.add(new Permissions(Item.ANNOTATIONTYPE, null, EnumSet.of(Permission.USE)));
       permissions.add(new Permissions(Item.UNIT, EnumSet.of(Permission.READ), null));
       permissions.add(new Permissions(Item.QUANTITY, EnumSet.of(Permission.READ), null));
@@ -1362 +1360 @@
       if (value != null) 
       {
+        if (at.supportUnits() && unit == null) unit = at.getDefaultUnit();
         listOfValues.add(new AnnotationValue(value, unit));
         numAnnotations++;
@@ -1389 +1388 @@
       if (values.size() == 0) return;
       int numAnnotations = 0;
-//      AnnotationSet as = item.isAnnotated() ? item.getAnnotationSet() : null;
       
       batcher.setCurrentItem(item);
@@ -1396 +1394 @@
       {
         AnnotationType at = entry.getKey();
+
         List<AnnotationValue> newValues = entry.getValue();
         
-        List<Object> theValues = new ArrayList<Object>(newValues.size());
-        for (AnnotationValue value : newValues)
-        {
-          Object theValue = value.value;
-          theValues.add(theValue);
-        }
-        Change change = batcher.setValues(at, theValues, null);
-        if (change == Change.DELETED)
-        {
-          numRemoved++;
-        }
-        else if (change == Change.ADDED )
-        {
-          numSet += theValues.size();
-        }
-        else if (change == Change.UPDATED)
-        {
-          numSet += theValues.size();
-          numReplaced += theValues.size();
-        }
-        
-        /*
         int size = newValues.size();
         int multiplicity = at.getMultiplicity();
-        boolean hasAnnotation = as != null && as.hasAnnotation(at, Annotation.Source.PRIMARY);
-        boolean merge = addToUnlimited && hasAnnotation && multiplicity == 0;
+        boolean merge = addToUnlimited && multiplicity == 0;
         boolean tooManyValues = multiplicity > 0 && size > multiplicity;
-        if (size == 0)
-        {
-          if (removeAnnotations && hasAnnotation)
-          {
-            as.removeAnnotation(at);
-            numRemoved++;
-          }
-        }
-        else
-        {
-          if (!hasAnnotation || replaceExisting || merge)
-          {
-            if (!tooManyValues || failIfTooManyValues)
-            {
-              if (as == null) as = item.getAnnotationSet();
-              Annotation a = as.getAnnotation(at);
-              
-              List<Object> theValues = new ArrayList<Object>(newValues.size());
-              // Make sure all annotation values are converted to the same unit
-              final Unit unit = merge ? a.getUnit() : newValues.get(0).unit;
-              if (merge)
-              {
-                theValues.addAll(a.getValues(unit));
-              }
-              for (AnnotationValue value : newValues)
-              {
-                Object theValue = value.value;
-                if (unit != null && !unit.equals(value.unit)) 
-                {
-                  theValue = unit.fromOther(((Number)value.value).doubleValue(), value.unit);
-                }
-                theValues.add(theValue);
-              }
-              if (a.setValuesIfDifferent(theValues, unit))
-              {
-                numSet += size;
-                if (hasAnnotation && !merge) numReplaced += size;
-              }
-            }
-            else
-            {
-              numError += size;
-            }
-          }
-        }
-        */
+        
+        if (size == 0 && !removeAnnotations) continue; // Skip removing
+        if (tooManyValues && !failIfTooManyValues) 
+        {
+          numError += size;
+          continue; // Do not try to set too many values
+        }
+          
+        List<Object> theValues = new ArrayList<Object>(newValues.size());
+        // Make sure all annotation values are converted to the same unit
+        Unit unitToUse = size > 0 ? newValues.get(0).unit : at.getDefaultUnit();
+        for (AnnotationValue value : newValues)
+        {
+          Object theValue = value.value;
+          if (unitToUse != null && value.unit != null && !unitToUse.equals(value.unit)) 
+          {
+            theValue = unitToUse.fromOther(((Number)value.value).doubleValue(), value.unit);
+          }
+          theValues.add(theValue);
+        }
+        Change change = batcher.setValues(at, theValues, unitToUse, merge);
+        if (change == Change.DELETED)
+        {
+          numRemoved++;
+        }
+        else if (change == Change.ADDED)
+        {
+          numSet += theValues.size();
+        }
+        else if (change == Change.UPDATED)
+        {
+          numSet += theValues.size();
+          if (!merge) numReplaced += theValues.size();
+        }
       }
     }