Changeset 7121

Timestamp:

Apr 19, 2016, 8:56:43 AM (7 years ago)

Author:

Nicklas Nordborg

Message:

References #2000: Batch API for annotation handling

First version of the new AnnotationBatcher implementation. It should have support for creating, updating and deleting annotation values.

It lacks a lot functionality:

• It doesn't check annotation values (eg. if a String is a String, if a values is allowed by the an enumerated annotation, etc.)
• It doesn't handle units
• Changes are not recorded in the change log

The AnnotationFlatFileImporter has been modified to use the batcher. All possible options for updating are supported (eg. merge).

Location:

trunk

Files:

• config/dist/mysql-queries.xml (modified) (1 diff)
• src/core/common-queries.xml (modified) (1 diff)
• src/core/net/sf/basedb/core/AnnotationBatcher.java (added)
• src/core/net/sf/basedb/core/BasicItem.java (modified) (1 diff)
• src/plugins/core/net/sf/basedb/plugins/AnnotationFlatFileImporter.java (modified) (7 diffs)

trunk/config/dist/mysql-queries.xml

@@ -38 +38 @@
   </query>
 
+  <query id="AB_INSERT_INTO_ANNOTATIONS" type="SQL">
+    <sql>
+      INSERT INTO [Annotations] ([version], [annotationset_id], [annotationtype_id], [unit_id], [value_id], [last_update], [source]) 
+      VALUES (0, ?, ?, ?, ?, ?, 0)
+    </sql>
+    <description>
+      SQL query for inserting rows into the Annotations table.
+      The primary key (id) must be auto-generated.
+      Parameters: annotationset_id, annotationtype_id, unit_id, value_id, last_updated
+    </description>
+  </query>
+  
+  <query id="AB_INSERT_INTO_ANNOTATIONSETS" type="SQL">
+    <sql>
+      INSERT INTO [AnnotationSets] ([version], [item_type], [item_id]) 
+      VALUES (0, {1}, ?)
+    </sql>
+    <description>
+      SQL query for inserting rows into the AnnotationSets table.
+      The primary key (id) must be auto-generated.
+      Parameters: item_id
+    </description>
+  </query>
+  
+
+
 </predefined-queries>

trunk/src/core/common-queries.xml

@@ -3582 +3582 @@
   </query>
 
+  <query id="AB_LOAD_ANNOTATION_INFO" type="SQL">
+    <sql>
+      SELECT [annotationtype_id], [id], [version], [unit_id], [value_id], [last_update]
+      FROM [Annotations] 
+      WHERE [annotationset_id] = :annotationSet AND [source] = 0
+    </sql>
+    <description>
+      SQL query for loading current (primary) annotations for 
+      a given item. 
+      Used by the AnnotationBatcher as a SQL query via Hibernate.
+    </description>
+  </query>
+
+  <query id="AB_LOAD_ANNOTATION_VALUES" type="SQL">
+    <sql>
+      SELECT [id], [value]
+      FROM [{1}] 
+      WHERE [id] IN (:listOfIds)
+    </sql>
+    <description>
+      SQL query for loading current annotation values
+      for a given list of annotation value IDs.
+      Used by the AnnotationBatcher as a SQL query via Hibernate.
+    </description>
+  </query>
+
+  <query id="AB_INSERT_INTO_PARAMETERVALUES" type="SQL">
+    <sql>
+      INSERT INTO [ParameterValues] ([id], [discriminator], [version]) 
+      VALUES (nextval('hibernate_sequence'), ?, 0)
+    </sql>
+    <description>
+      SQL query for inserting rows into the ParameterValues table.
+      The primary key (id) must be auto-generated.
+      Parameters: discriminator
+    </description>
+  </query>
+
+  <query id="AB_INSERT_INTO_ANNOTATIONS" type="SQL">
+    <sql>
+      INSERT INTO [Annotations] ([id], [version], [annotationset_id], [annotationtype_id], [unit_id], [value_id], [last_update], [source]) 
+      VALUES (nextval('hibernate_sequence'), 0, ?, ?, ?, ?, ?, 0)
+    </sql>
+    <description>
+      SQL query for inserting rows into the Annotations table.
+      The primary key (id) must be auto-generated.
+      Parameters: annotationset_id, annotationtype_id, unit_id, value_id, last_updated
+    </description>
+  </query>
+
+  <query id="AB_UPDATE_ANNOTATIONS" type="SQL">
+    <sql>
+      UPDATE [Annotations] 
+      SET [version] = ?, [unit_id] = ?, [last_update] = ?
+      WHERE [id] = ? AND [version] = ?
+    </sql>
+    <description>
+      SQL query for updating rows into the Annotations table.
+      Parameters: version (new), unit_id, last_updated, id, version (old)
+    </description>
+  </query>
+
+  <query id="AB_INSERT_INTO_ANNOTATIONSETS" type="SQL">
+    <sql>
+      INSERT INTO [AnnotationSets] ([id], [version], [item_type], [item_id]) 
+      VALUES (nextval('hibernate_sequence'), 0, {1}, ?)
+    </sql>
+    <description>
+      SQL query for inserting rows into the AnnotationSets table.
+      The primary key (id) must be auto-generated.
+      Parameters: item_id
+    </description>
+  </query>
+  
+  <query id="AB_UPDATE_ITEMTABLE" type="SQL">
+    <sql>
+      UPDATE [{1}] 
+      SET [annotationset_id] = ?
+      WHERE [id] = ?
+    </sql>
+    <description>
+      SQL query for updating the annotationset_id column in any
+      table it exists in.
+      Parameters: annotationset_id, item_id
+    </description>
+  </query>
+
+  <query id="AB_NULLIFY_CLONED_ANNOTATIONS" type="SQL">
+    <sql>
+      UPDATE [Annotations] 
+      SET [inherited_id] = null 
+      WHERE [inherited_id] = ? and [source] = 2
+    </sql>
+    <description>
+      SQL query for setting the 'inherited_id' column to null
+      for cloned annotations that reference the annotation that
+      is about to be deleted.
+      Parameters: inherited_id
+    </description>
+  </query>
+
+  <query id="AB_DELETE_INHERITED_ANNOTATIONS" type="SQL">
+    <sql>
+      DELETE FROM [Annotations] 
+      WHERE [inherited_id] = ? and [source] = 1
+    </sql>
+    <description>
+      SQL query for deleting inherited annotations that reference 
+      the annotation that is about to be deleted.
+      Parameters: inherited_id
+    </description>
+  </query>
+
+  <query id="AB_DELETE_FROM_ANNOTATIONS" type="SQL">
+    <sql>
+      DELETE FROM [Annotations] 
+      WHERE [id] = ?
+    </sql>
+    <description>
+      SQL query for deleting an annotation with a given id.
+      Used by the AnnotationBatcher as a PreparedStatement.
+      Parameters: id
+    </description>
+  </query>
+
+  <query id="AB_DELETE_FROM_PARAMETERVALUES" type="SQL">
+    <sql>
+      DELETE FROM [ParameterValues] WHERE [id] = ?
+    </sql>
+    <description>
+      SQL query for deleting a parameter value with a given id.
+      Used by the AnnotationBatcher as a PreparedStatement.
+    </description>
+  </query>
+
+  <query id="AB_DELETE_FROM_VALUES" type="SQL">
+    <sql>
+      DELETE FROM [{1}] WHERE [id] = ?
+    </sql>
+    <description>
+      SQL query for deleting all actual annotation values with a given id.
+      The {1} parameter is the table the value are stored in, eg. StringValues,
+      IntegerValues, FloatValues, etc.
+      Used by the AnnotationBatcher as a PreparedStatement.
+    </description>
+  </query>
+  
+  <query id="AB_INSERT_INTO_VALUES" type="SQL">
+    <sql>
+      INSERT INTO [{1}] ([id], [value]) VALUES (?, ?)
+    </sql>
+    <description>
+      SQL query for inserting annotation values in their respective 
+      table (eg. StringValues, IntegerValues, FloatValues, etc.).
+      Used by the AnnotationBatcher as a PreparedStatement.
+    </description>
+  </query>
   
 </predefined-queries>

trunk/src/core/net/sf/basedb/core/BasicItem.java

@@ -281 +281 @@
   {
     return 0;
+  }
+  
+  private boolean hasBeenCalledFromGetAnnotationSet = false;
+  private boolean hasBeenCalledFromAnnotationBatcher = false;
+  
+  /**
+    Annotatable items that are used with the {@link AnnotationBatcher}
+    must not call {@link Annotatable#getAnnotationSet()} in the same
+    transaction. This method is a helper method for keeping track of
+    which method that has been called.
+    @param annotationSet TRUE when called from the getAnnotationSet method
+    @param batcher TRUE when called from the batcher
+    @since 3.8
+  */
+  void checkBatchAnnotatableUsage(boolean annotationSet, boolean batcher)
+  {
+    if (annotationSet)
+    {
+      if (hasBeenCalledFromAnnotationBatcher)
+      {
+        throw new IllegalStateException("Can't call getAnnotationSet() after the item has been used with an AnnotationBatcher: " + this);
+      }
+      hasBeenCalledFromGetAnnotationSet = true;
+    }
+    if (batcher)
+    {
+      if (hasBeenCalledFromGetAnnotationSet)
+      {
+        throw new IllegalStateException("Can't use with AnnotationBatcher after getAnnotationSet() has been called: " + this);
+      }
+      if (hasBeenCalledFromAnnotationBatcher)
+      {
+        throw new IllegalStateException("Can't use an item more than once with an AnnotationBatcher: " + this);
+      }
+      hasBeenCalledFromAnnotationBatcher = true;
+    }
   }
   

trunk/src/plugins/core/net/sf/basedb/plugins/AnnotationFlatFileImporter.java

@@ -42 +42 @@
 import net.sf.basedb.core.Annotatable;
 import net.sf.basedb.core.Annotation;
+import net.sf.basedb.core.AnnotationBatcher;
+import net.sf.basedb.core.AnnotationBatcher.Change;
 import net.sf.basedb.core.AnnotationSet;
 import net.sf.basedb.core.AnnotationType;
@@ -594 +596 @@
   private List<String> columnHeaders;
   private DbControl dc;
+  private AnnotationBatcher batcher;
+  
   private UnitCache unitCache;
   private FlatFileParser ffp;
@@ -746 +750 @@
       ++i;
     }
+    batcher = new AnnotationBatcher(dc, itemType);
+    batcher.addAnnotationTypes(mappers.values());
     
     // Include options
@@ -946 +952 @@
           for (NewAnnotations na : n)
           {
-            na.setNewAnnotations(addToUnlimited, replaceExisting, failIfTooManyValues, removeExisting);
+            na.setNewAnnotations(batcher, addToUnlimited, replaceExisting, failIfTooManyValues, removeExisting);
             if (na.getNumSet() == 0) numItems--;
             numAnnotations += na.getNumSet();
@@ -961 +967 @@
 
         }
+        batcher.close();
         System.out.println(new Date()+": commit start");
         dc.commit();
         System.out.println(new Date()+": commit end");
         if (progress != null) progress.display(100, "Done");
+        //if (true) throw new RuntimeException("success!!");
       }
     }
@@ -1376 +1384 @@
         the multiplicity settings allows, TRUE to throw an exception
     */
-    void setNewAnnotations(boolean addToUnlimited, boolean replaceExisting, 
+    void setNewAnnotations(AnnotationBatcher batcher, boolean addToUnlimited, boolean replaceExisting, 
       boolean failIfTooManyValues, boolean removeAnnotations)
     {
       if (values.size() == 0) return;
       int numAnnotations = 0;
-      AnnotationSet as = item.isAnnotated() ? item.getAnnotationSet() : null;
+//      AnnotationSet as = item.isAnnotated() ? item.getAnnotationSet() : null;
+      
+      batcher.setCurrentItem(item);
+      
       for (Map.Entry<AnnotationType, List<AnnotationValue>> entry : values.entrySet())
       {
         AnnotationType at = entry.getKey();
         List<AnnotationValue> newValues = entry.getValue();
+        
+        List<Object> theValues = new ArrayList<Object>(newValues.size());
+        for (AnnotationValue value : newValues)
+        {
+          Object theValue = value.value;
+          theValues.add(theValue);
+        }
+        Change change = batcher.setValues(at, theValues, null);
+        if (change == Change.DELETED)
+        {
+          numRemoved++;
+        }
+        else if (change == Change.ADDED )
+        {
+          numSet += theValues.size();
+        }
+        else if (change == Change.UPDATED)
+        {
+          numSet += theValues.size();
+          numReplaced += theValues.size();
+        }
+        
+        /*
         int size = newValues.size();
         int multiplicity = at.getMultiplicity();
@@ -1436 +1470 @@
           }
         }
+        */
       }
     }