Changeset 7127


Ignore:
Timestamp:
Apr 20, 2016, 2:11:26 PM (5 years ago)
Author:
Nicklas Nordborg
Message:

References #2000: Batch API for annotation handling

Several bug fixes and improvements.

In AnnotationBatcher:

  • Evict items from the second-level cache in Hibernate if we make any changes (eg. create a new annotation set)
  • Add support for merging values for annotation types that have unlimited number of annotations.
  • Fixes issues with setting the correct unit for annotation types that supports unit. Now the unit should always be set, either to the default unit or the unit that was specified.


In AnnotationFlatFileImporter:

  • Merge is supported
  • Ignore annotations if there are too many values
  • Not removing annotations when no values have been specified


Location:
trunk/src
Files:
2 edited

Legend:

Unmodified
Added
Removed
  • trunk/src/core/net/sf/basedb/core/AnnotationBatcher.java

    r7125 r7127  
    9191  private final Date batchDate;
    9292
     93  // We need some interaction with Hibernate
    9394  private final org.hibernate.StatelessSession statelessSession;
    9495  private final org.hibernate.dialect.Dialect dialect;
     96  private final org.hibernate.Cache secondLevelCache;
    9597
    9698  // Discriminator values for Type items (eg. Type.INT -> IntegerParameterValueData -> 1)
     
    135137  private int currentAnnotationSetId;
    136138  private boolean deleteSnapshotIfModifed;
     139  private boolean evictFromSecondLevelCacheIfModified;
    137140 
    138141  // Lists for temporary keeping track of value IDs for
     
    175178    this.statelessSession = dc.getStatelessSession();
    176179    this.dialect = HibernateUtil.getDialect();
     180    this.secondLevelCache = HibernateUtil.getSessionFactory().getCache();
    177181
    178182    Type[] types = Type.values();
     
    306310  public void flush()
    307311  {
    308     System.out.println(new Date() + ": " + numItems + " done; flushing batchers");
     312    log.debug(numItems + " done. Flushing batchers.");
    309313    String currentSql = null;
    310314    try
     
    332336  protected void onBeforeClose()
    333337  {
    334     System.out.println(new Date() + "; done!! " + numItems + " items.");
     338    log.debug(numItems + " done. Closing.");
    335339    for (UpdateBatcher b : allUpdateBatchers)
    336340    {
     
    459463    this.currentAnnotationSetId = 0;
    460464    this.deleteSnapshotIfModifed = false;
     465    this.evictFromSecondLevelCacheIfModified = secondLevelCache != null && secondLevelCache.containsEntity(itemType.getDataClass(), item.getId());
    461466
    462467    // Read current values
     
    549554    more information.
    550555  */
    551   public Change setValue(AnnotationType annotationType, Object value, Unit unit)
    552   {
    553     return setValues(annotationType, value == null ? null : Collections.singletonList(value), unit);
     556  public Change setValue(AnnotationType annotationType, Object value, Unit unit, boolean merge)
     557  {
     558    return setValues(annotationType, value == null ? null : Collections.singletonList(value), unit, merge);
    554559  }
    555560 
     
    581586    @param values The values to set or null/empty to remove values
    582587    @param unit The unit of the values. Ignored if the annotation type doesn't use units.
    583      
     588    @param merge If TRUE and the annotation type allows an unlimted number
     589      of annotations (multiplicity=0) merges the new values with the existing values.
     590      Ignored if multiplicity > 0.
     591   
    584592    @return The resulting change
    585593  */
    586   public Change setValues(AnnotationType annotationType, List<?> values, Unit unit)
     594  public Change setValues(AnnotationType annotationType, List<?> values, Unit unit, boolean merge)
    587595  {
    588596    if (currentItem == null) throw new IllegalStateException("Must call setCurrentItem() first");
     
    641649      else
    642650      {
     651        // Check multiplicity
     652        int multiplicity = annotationType.getMultiplicity();
     653        if (multiplicity > 0 && values.size() > multiplicity)
     654        {
     655          throw new InvalidDataException("Too many values. Max "+multiplicity+" value(s) are allowed for this annotation type: "+annotationType.getName());
     656        }
     657       
    643658        // Check that values are allowed by annotation type
    644659        // and convert to the default unit if it is needed
    645660        UnitConverter converter = null;
    646         if (unit != null && info.defaultUnit != null && !unit.equals(info.defaultUnit))
     661        if (info.defaultUnit != null)
    647662        {
    648           // Two different units, convert the values to the default unit
    649           converter = getUnitConverter(unit, info.defaultUnit);
     663          // The annotation use units
     664          if (unit == null)
     665          {
     666            // Assume values use the default unit if none was specifed
     667            unit = info.defaultUnit;
     668          }
     669          else if (!unit.equals(info.defaultUnit))
     670          {
     671            // Two different units, convert the values to the default unit
     672            converter = getUnitConverter(unit, info.defaultUnit);
     673          }
     674          info.unitId = unit.getId();
    650675        }
    651676       
     
    679704            changeType = Change.UPDATED;
    680705           
    681             // Delete from **Values
    682             deleteValues[annotationType.getValueType().ordinal()].addToBatch(info.valueId);
     706            // Delete from **Values unless merging
     707            if (!merge || multiplicity != 0)
     708            {
     709              deleteValues[annotationType.getValueType().ordinal()].addToBatch(info.valueId);
     710            }
    683711           
    684712            // Insert into **Values
     
    689717           
    690718            // Update last_update, unit, version in Annotations
    691             if (unit != null) info.unitId = unit.getId();
    692719            updateAnnotations.addToBatch(info.version+1, info.unitId, batchDate, info.annotationId, info.version);
    693720          }
     
    726753            // Update item table, annotationset_id column
    727754            updateItemTable.addToBatch(currentAnnotationSetId, currentItem.getId());
     755            if (evictFromSecondLevelCacheIfModified)
     756            {
     757              // We must evict the entity from the Hibernate second-level
     758              // cache or Hibernate may use the old data
     759              secondLevelCache.evictEntity(itemType.getDataClass(), currentItem.getId());
     760            }
    728761          }
    729762
    730763          // Insert into Annotations
    731           info.annotationId = insertIntoAnnotations.insert(currentAnnotationSetId, info.annotationTypeId, null, info.valueId, batchDate);
     764          info.annotationId = insertIntoAnnotations.insert(currentAnnotationSetId, info.annotationTypeId, info.unitId, info.valueId, batchDate);
    732765        }
    733766      }
  • trunk/src/plugins/core/net/sf/basedb/plugins/AnnotationFlatFileImporter.java

    r7121 r7127  
    4141
    4242import net.sf.basedb.core.Annotatable;
    43 import net.sf.basedb.core.Annotation;
    4443import net.sf.basedb.core.AnnotationBatcher;
    4544import net.sf.basedb.core.AnnotationBatcher.Change;
    46 import net.sf.basedb.core.AnnotationSet;
    4745import net.sf.basedb.core.AnnotationType;
    4846import net.sf.basedb.core.BaseException;
     
    337335    {
    338336      permissions.add(new Permissions(Item.FILE, null, EnumSet.of(Permission.READ)));
    339       permissions.add(new Permissions(Item.ANNOTATIONTYPE, null, EnumSet.of(Permission.READ)));
     337      permissions.add(new Permissions(Item.ANNOTATIONTYPE, null, EnumSet.of(Permission.USE)));
    340338      permissions.add(new Permissions(Item.UNIT, EnumSet.of(Permission.READ), null));
    341339      permissions.add(new Permissions(Item.QUANTITY, EnumSet.of(Permission.READ), null));
     
    13621360      if (value != null)
    13631361      {
     1362        if (at.supportUnits() && unit == null) unit = at.getDefaultUnit();
    13641363        listOfValues.add(new AnnotationValue(value, unit));
    13651364        numAnnotations++;
     
    13891388      if (values.size() == 0) return;
    13901389      int numAnnotations = 0;
    1391 //      AnnotationSet as = item.isAnnotated() ? item.getAnnotationSet() : null;
    13921390     
    13931391      batcher.setCurrentItem(item);
     
    13961394      {
    13971395        AnnotationType at = entry.getKey();
     1396
    13981397        List<AnnotationValue> newValues = entry.getValue();
    13991398       
    1400         List<Object> theValues = new ArrayList<Object>(newValues.size());
    1401         for (AnnotationValue value : newValues)
    1402         {
    1403           Object theValue = value.value;
    1404           theValues.add(theValue);
    1405         }
    1406         Change change = batcher.setValues(at, theValues, null);
    1407         if (change == Change.DELETED)
    1408         {
    1409           numRemoved++;
    1410         }
    1411         else if (change == Change.ADDED )
    1412         {
    1413           numSet += theValues.size();
    1414         }
    1415         else if (change == Change.UPDATED)
    1416         {
    1417           numSet += theValues.size();
    1418           numReplaced += theValues.size();
    1419         }
    1420        
    1421         /*
    14221399        int size = newValues.size();
    14231400        int multiplicity = at.getMultiplicity();
    1424         boolean hasAnnotation = as != null && as.hasAnnotation(at, Annotation.Source.PRIMARY);
    1425         boolean merge = addToUnlimited && hasAnnotation && multiplicity == 0;
     1401        boolean merge = addToUnlimited && multiplicity == 0;
    14261402        boolean tooManyValues = multiplicity > 0 && size > multiplicity;
    1427         if (size == 0)
    1428         {
    1429           if (removeAnnotations && hasAnnotation)
    1430           {
    1431             as.removeAnnotation(at);
    1432             numRemoved++;
    1433           }
    1434         }
    1435         else
    1436         {
    1437           if (!hasAnnotation || replaceExisting || merge)
    1438           {
    1439             if (!tooManyValues || failIfTooManyValues)
    1440             {
    1441               if (as == null) as = item.getAnnotationSet();
    1442               Annotation a = as.getAnnotation(at);
    1443              
    1444               List<Object> theValues = new ArrayList<Object>(newValues.size());
    1445               // Make sure all annotation values are converted to the same unit
    1446               final Unit unit = merge ? a.getUnit() : newValues.get(0).unit;
    1447               if (merge)
    1448               {
    1449                 theValues.addAll(a.getValues(unit));
    1450               }
    1451               for (AnnotationValue value : newValues)
    1452               {
    1453                 Object theValue = value.value;
    1454                 if (unit != null && !unit.equals(value.unit))
    1455                 {
    1456                   theValue = unit.fromOther(((Number)value.value).doubleValue(), value.unit);
    1457                 }
    1458                 theValues.add(theValue);
    1459               }
    1460               if (a.setValuesIfDifferent(theValues, unit))
    1461               {
    1462                 numSet += size;
    1463                 if (hasAnnotation && !merge) numReplaced += size;
    1464               }
    1465             }
    1466             else
    1467             {
    1468               numError += size;
    1469             }
    1470           }
    1471         }
    1472         */
     1403       
     1404        if (size == 0 && !removeAnnotations) continue; // Skip removing
     1405        if (tooManyValues && !failIfTooManyValues)
     1406        {
     1407          numError += size;
     1408          continue; // Do not try to set too many values
     1409        }
     1410         
     1411        List<Object> theValues = new ArrayList<Object>(newValues.size());
     1412        // Make sure all annotation values are converted to the same unit
     1413        Unit unitToUse = size > 0 ? newValues.get(0).unit : at.getDefaultUnit();
     1414        for (AnnotationValue value : newValues)
     1415        {
     1416          Object theValue = value.value;
     1417          if (unitToUse != null && value.unit != null && !unitToUse.equals(value.unit))
     1418          {
     1419            theValue = unitToUse.fromOther(((Number)value.value).doubleValue(), value.unit);
     1420          }
     1421          theValues.add(theValue);
     1422        }
     1423        Change change = batcher.setValues(at, theValues, unitToUse, merge);
     1424        if (change == Change.DELETED)
     1425        {
     1426          numRemoved++;
     1427        }
     1428        else if (change == Change.ADDED)
     1429        {
     1430          numSet += theValues.size();
     1431        }
     1432        else if (change == Change.UPDATED)
     1433        {
     1434          numSet += theValues.size();
     1435          if (!merge) numReplaced += theValues.size();
     1436        }
    14731437      }
    14741438    }
Note: See TracChangeset for help on using the changeset viewer.