Changeset 3588

Timestamp:

Jul 23, 2007, 11:28:10 AM (16 years ago)

Author:

Nicklas Nordborg

Message:

Fixes #326: Mass annotation plug-in

Location:

trunk

Files:

• doc/src/docbook/userdoc/annotations.xml (modified) (1 diff)
• src/plugins/core/net/sf/basedb/plugins/AbstractFlatFileImporter.java (modified) (4 diffs)
• src/plugins/core/net/sf/basedb/plugins/AnnotationFlatFileImporter.java (modified) (32 diffs)
• www/admin/plugindefinitions/edit_plugin.jsp (modified) (5 diffs)

trunk/doc/src/docbook/userdoc/annotations.xml

@@ -692 +692 @@
     </sect2>
     
+    <sect2 id="annotations.massimport">
+      <title>Mass annotation import plug-in</title>
+      
+      <para>
+        BASE includes a plug-in for importing annotations to multiple items
+        in one go. The plug-in read annotation values from a simple column-based
+        text file. Ususally, a tab is used as the delimiter between columns.
+        The first row should contain the column headers. One column should contain
+        the name or the external ID of the item. The rest of the columns can each be
+        mapped to an annotation type and contains the annotation values. If a column 
+        header exactly match the name of an annotation type, the plug-in will automatically 
+        create the mapping, otherwise you must do it manually. You don't have to map
+        all columns if you don't want to.
+      </para>
+      
+      <para>
+        Each column can only contain a single annotation value for each row. 
+        If you have annotation types that accept multiple values you can map
+        two or more columns to the same annotation type, or you can add an
+        extra row only giving the name and the extra annotation value.
+        Here is a simple example of a valid file with comma as column separator:
+      </para>
+      
+      <programlisting>
+# 'Time' and 'Age' are integer types
+# 'Subtype' is a string enumeration
+# 'Comment' is a text type that accept multiple values
+Name,Time (hours),Age (years),Subtype,Comment
+Sample #1,0,0,alfa,Very good
+Sample #2,24,0,beta,Not so bad
+Sample #2,,,,Yet another comment
+</programlisting>
+      
+      <para>
+        The plug-in can be used with or without a configuration. The configuration
+        keeps the regular expressions and other settings used to parse the file. If
+        you often import annotations from the same file format, we recommend that
+        you use a configuration. The mapping from file columns to annotation types 
+        is not part of the configuration, it must be done each time the plug-in is used.
+      </para>
+      
+      <para>
+        The plug-in can be used from the list view of all annotatable items.
+        Using the plug-in is a three-step wizard:
+      </para>
+      
+      <orderedlist>
+      <listitem>
+        <para>
+        Select a file to import from and the regular expressions and other
+        settings used to parse the file. In this step you also select the column
+        that contains the name or external ID the items. If a configuration is used
+        all settings on this page, except the file to import from, already has values.
+        </para>
+      </listitem>
+      
+      <listitem>
+        <para>
+        The plug-in will start parsing the file until it finds the column headers.
+        You are asked to select an annotation type for each column.
+        </para>
+      </listitem>
+      
+      <listitem>
+        <para>
+        Set error handling options and some other import options.
+        </para>
+      </listitem>
+      
+      </orderedlist>
+      
+    </sect2>
+    
   </sect1>
 

trunk/src/plugins/core/net/sf/basedb/plugins/AbstractFlatFileImporter.java

@@ -534 +534 @@
             try
             {
-              errorHandler.handleError(t);
+              if (errorHandler == null)
+              {
+                throw t;
+              }
+              else
+              {
+                errorHandler.handleError(t);
+              }
               skippedLines++;
             }
@@ -565 +572 @@
               catch (Throwable t)
               {
-                errorHandler.handleError(t);
+                if (errorHandler == null)
+                {
+                  throw t;
+                }
+                else
+                {
+                  errorHandler.handleError(t);
+                }
                 skippedLines++;
               }
@@ -593 +607 @@
             catch (Throwable t)
             {
-              errorHandler.handleError(t);
+              if (errorHandler == null)
+              {
+                throw t;
+              }
+              else
+              {
+                errorHandler.handleError(t);
+              }
               skippedLines++;
             }
@@ -987 +1008 @@
     Initialise the error handling system. This method is called just before the 
     import is starting. A subclass may override this method to add specific 
-    error handlers, but it is important to call <code>super.setUpErrorHandling()</code>
-    or the error handling will not work. The subclass may also add error handlers
-    in the {@link #begin(FlatFileParser)} method.
+    error handlers. If <code>super.setUpErrorHandling()</code> isn't called
+    error handling in AbstractFlatFileImporter is disabled and the subclass
+    must do all it's error handling in it's own code. The subclass may also add 
+    error handlers in the {@link #begin(FlatFileParser)} method.
   */
   protected void setUpErrorHandling()

trunk/src/plugins/core/net/sf/basedb/plugins/AnnotationFlatFileImporter.java

@@ -29 +29 @@
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.EnumSet;
 import java.util.HashMap;
@@ -60 +61 @@
 import net.sf.basedb.core.ParameterType;
 import net.sf.basedb.core.Permission;
+import net.sf.basedb.core.PermissionDeniedException;
 import net.sf.basedb.core.PluginParameter;
 import net.sf.basedb.core.RequestInformation;
@@ -71 +73 @@
 import net.sf.basedb.core.plugin.ParameterValues;
 import net.sf.basedb.core.plugin.ParameterValuesWrapper;
+import net.sf.basedb.core.plugin.Permissions;
 import net.sf.basedb.core.plugin.Request;
 import net.sf.basedb.core.plugin.Response;
@@ -88 +91 @@
 
 /**
+  Plug-in for importing annotations from simple text files. The plug-in supports
+  all files that can be parsed with the {@link FlatFileParser} class. This plug-in
+  works without a configuration but can use a configuration to store regular
+  expressions and other settings for the flat file parser. In both cases,
+  the job configuration is a three-step process:
+  
+  <ol>
+  <li>Setup regular expressions and other options for the flat file parser. If a
+    configuration is used all values should already be filled in. In this step
+    a file to import from must also be selected.
+  <li>Map file columns to annotation types. Annotation types that support multiple
+    values may be mapped to more than one column.
+  <li>Setup error handling options and other settings for the plug-in (for example
+    if existing annotations should be replaced or not).
+  </ol>
 
   @author nicklas
@@ -112 +130 @@
   
   private static Set<GuiContext> guiContexts;
-  
+  private static final Set<Permissions> permissions = new HashSet<Permissions>();
+  
+  // second step in the wizard
   private static final String CONFIGURE_MAPPING = "configure_mapping";
+  // third step in the wizard
   private static final String CONFIGURE_IMPORT = "configure_import";
   
@@ -147 +168 @@
 
   /**
-    Section definition for grouping all mappings of columns to annotation types
+    Section definition for grouping all mappings of columns to annotation types.
+    Parameters in this section will be generated on the fly based on the
+    column headers from the file to import from.
   */
   private static final PluginParameter<String> annotationTypeSection = new PluginParameter<String>(
@@ -199 +222 @@
     "skip = Skip the current annotation and continue\n"+
     "fail = Stop with an error message (default)",
-    new StringParameterType(255, "fail", true, 1, 0, 0, 
+    new StringParameterType(255, "skip", true, 1, 0, 0, 
       Arrays.asList( new String[] { "skip", "fail"} ))
   );
@@ -214 +237 @@
 
   private static final PluginParameter<String> multipleItemsFoundErrorParameter = new PluginParameter<String>(
-      "multipleItemsFoundError",
-      "Multiple items found",
-      "What to do if the plug-in finds more than one item with the same name or external ID.\n\n"+
-      "all = Annotate all items\n"+
-      "fail = Stop with an error message",
-      new StringParameterType(255, "all", false, 1, 0, 0, 
-        Arrays.asList( new String[] { "all", "fail"} ))
-    );
+    "multipleItemsFoundError",
+    "Multiple items found",
+    "What to do if the plug-in finds more than one item with the same name or external ID.\n\n"+
+    "all = Annotate all items\n"+
+    "fail = Stop with an error message",
+    new StringParameterType(255, "all", false, 1, 0, 0, 
+      Arrays.asList( new String[] { "all", "fail"} ))
+  );
 
   
@@ -232 +255 @@
     "skip = Skip the current annotation and continue\n"+
     "fail = Stop with an error message",
-    new StringParameterType(255, null, false, 1, 0, 0, 
+    new StringParameterType(255, "crop", false, 1, 0, 0, 
       Arrays.asList( new String[] { "crop", "skip", "fail"} ))
   );
@@ -304 +327 @@
     return true;
   }
+  /**
+    Request read access to File:s, read access to annotation types
+    and write access to all annotatable items.
+  */
+  public Collection<Permissions> getPermissions()
+  {
+    if (permissions.size() == 0)
+    {
+      permissions.add(new Permissions(Item.FILE, null, EnumSet.of(Permission.READ)));
+      permissions.add(new Permissions(Item.ANNOTATIONTYPE, null, EnumSet.of(Permission.READ)));
+      
+      for (GuiContext ctx : getGuiContexts())
+      {
+        permissions.add(new Permissions(ctx.getItem(), null, EnumSet.of(Permission.WRITE)));
+      }
+    }
+    return permissions;
+  }
   // -------------------------------------------
 
@@ -310 +351 @@
     -------------------------------------------
   */
+  /**
+    This plug-in works in list context of all {@link Annotatable} items,
+    except bioassay sets, bioassays and wells because they are not standalone
+    items. We use {@link Metadata#getAnnotatableItems()} to create the contexts.
+  */
   public Set<GuiContext> getGuiContexts()
   {
@@ -319 +365 @@
         for (Item item : Metadata.getAnnotatableItems())
         {
-          guiContexts.add(new GuiContext(item, GuiContext.Type.LIST));
+          if (item.getDefinedPermissions() != null)
+          {
+            guiContexts.add(new GuiContext(item, GuiContext.Type.LIST));
+          }
         }
       }
@@ -327 +376 @@
   public String isInContext(GuiContext context, Object item)
   {
+    if (!sc.hasPermission(Permission.WRITE, context.getItem()))
+    {
+      throw new PermissionDeniedException(Permission.WRITE, context.getItem().toString());
+    }
     return null;
   } 
@@ -522 +575 @@
   private boolean failIfTooManyValues;
   
+  // Messaging
+  private int numItems;
+  private int numItemNotFound;
+  private int numAnnotations;
+  private int numReplaced;
+  private int numError;
+  
   @Override
+  /**
+    The flat file parser is initialised with settings from either the job or configuration.
+  */
   protected FlatFileParser getInitializedFlatFileParser()
     throws BaseException
@@ -530 +593 @@
   }
   
+  /**
+    Don't use AbstractFlatFileImporter to handle errors.
+  */
+  @Override
+  protected void setUpErrorHandling()
+  {}
+  
+  /**
+    Setup error handling and pre-load some of the configuration options.
+  */
   @Override
   protected void begin(FlatFileParser ffp)
@@ -558 +631 @@
   }
   
+  /**
+    Setup column mapping. Creates DbControl and query to find items.
+  */
   @Override
   protected void beginData()
@@ -597 +673 @@
   }
   
+  /**
+    Read annotations from a single data line. Errors are handled internally. 
+    Errors thrown from this method should be reported back to client (ie. error
+    handling in AbstractFlatFileParser) must be disabled. This method will
+    load items and annotation values and put them in an internal cache. No
+    items are annotated until the entire file has been parsed.
+  */
   @Override
   protected void handleData(Data data) 
@@ -602 +685 @@
   {
 
-    // Find the item(s) to annotate
+    // Find the item(s) to annotate; mapper gives us name OR external ID
     String name = itemMapper.getValue(data);
+    // 1. check the cache
     Set<NewAnnotations> items = itemCache.get(name);
     if (items == null)
     {
       items = new HashSet<NewAnnotations>();
+      // The 'name' parameter can also query against the external ID
       itemQuery.setParameter("name", name, Type.STRING);
       List<?> result = itemQuery.list(dc);
-      if (result.isEmpty() && !ignoreNotFoundItems)
-      {
-        throw new ItemNotFoundException(itemType + 
-          "[" + (searchExternalId ? "externalId" : "name") + "=" + name + "]");
+      if (result.isEmpty())
+      {
+        // No item with specified name/external ID was found
+        if (!ignoreNotFoundItems)
+        {
+          throw new ItemNotFoundException(itemType + 
+            "[" + (searchExternalId ? "externalId" : "name") + "=" + name + "]");
+        }
+        else
+        {
+          numItemNotFound++;
+        }
       }
       if (result.size() > 1 && failIfMultipleFoundItems)
@@ -622 +715 @@
       for (Object item : result)
       {
+        // Add each item that was found to the cache
         items.add(new NewAnnotations((Annotatable)item));
       }
@@ -629 +723 @@
     if (!items.isEmpty())
     {
-      // Load annotation values
+      // Load annotation values from column mappers
       for (Map.Entry<Mapper, AnnotationType> entry : mappers.entrySet())
       {
@@ -641 +735 @@
           if (at.isEnumeration() && at.getValueType() == Type.STRING)
           {
+            // Case and leading/trailing whitespace are ignore for enumerated string annotations
             String enumValue = at.findValue(sValue, true, true);
             if (enumValue != null) sValue = enumValue;
@@ -650 +745 @@
             for (NewAnnotations item : items)
             {
+              // Add annotation value to the cache
               item.addValue(at, annotationValue);
             }
@@ -656 +752 @@
         catch (Throwable t)
         {
+          // In case the annotation value is not valid...
+          numError++;
           try
           {
             errorHandler.handleError(t);
+            // The error should be ignored if we get passed the above line
           }
           catch (Throwable t2)
           {
+            // handleData doesn't allow us to throw any Throwable
             if (t2 instanceof RuntimeException)
             {
@@ -676 +776 @@
   }
   
+  /**
+    Now it's time to update the items in the cache with the new annotation
+    values.
+  */
   @Override
   protected void end(boolean success)
@@ -685 +789 @@
         for (Set<NewAnnotations> n : itemCache.values())
         {
+          numItems += n.size();
           for (NewAnnotations na : n)
           {
             na.setNewAnnotations(addToUnlimited, replaceExisting, failIfTooManyValues);
+            if (na.getNumSet() == 0) numItems--;
+            numAnnotations += na.getNumSet();
+            numError += na.getNumError();
+            numReplaced += na.getNumReplaced();
           }
         }
@@ -700 +809 @@
     finally
     {
-      dc.close();
+      if (dc != null) dc.close();
       super.end(success);
       ffp = null;
       mappers.clear();
       itemCache.clear();
-      
-    }
+    }
+  }
+  @Override
+  protected String getSuccessMessage(int skippedLines)
+  {
+    String msg = numItems + " item(s) annotated with " + numAnnotations + " annotation(s)";
+    if (numReplaced > 0) msg += " (" + numReplaced + " was replaced)";
+    if (numItemNotFound > 0) msg += "; " + numItemNotFound + " row(s) skipped because no item was found";
+    if (numError > 0) msg += "; " + numError + " annotation(s) skipped because of invalid values";
+    return msg;
   }
   // -------------------------------------------
@@ -936 +1053 @@
   
   /**
-    Check if the current item has an 'externalId' property.
+    Check if the current item has an 'externalId' property. We use reflection to
+    look for the 'getExternalId' method in the item's data class.
   */
   private boolean hasExternalId(Item item)
@@ -951 +1069 @@
   }
   
+  /**
+    Create a query that return items of the specified type. We use reflection
+    to call the static method 'getQuery' on the item's item class, for
+    example {@link net.sf.basedb.core.Sample#getQuery()}. We add a restriction
+    to either the 'name' or the 'externalId' property but the parameter is 
+    always called 'name' to make it easier to implement the code that is
+    using the query.
+    
+    @param itemType The type of items to search
+    @param searchExternalId If we should search by name or external ID
+    @param includes Include options passed to {@link ItemQuery#include(java.util.Collection)}
+    @return A query that searches items of the specified type.
+  */
   private ItemQuery<?> createQuery(Item itemType, boolean searchExternalId, Set<Include> includes)
   {
@@ -989 +1120 @@
   }
   
+  /**
+    Internal cache for storing annotation values from the file 
+    for a single item. The items are not annotated until the entire
+    file has been parsed.
+  */
   private static class NewAnnotations
   {
     private final Annotatable item;
     private final Map<AnnotationType, List<Object>> values;
-    
+    private int numAnnotations;
+    private int numSet;
+    private int numError;
+    private int numReplaced;
+    
+    /**
+      Create a new cache for the specified item.
+    */
     private NewAnnotations(Annotatable item)
     {
       this.item = item;
       this.values = new HashMap<AnnotationType, List<Object>>();
-    }
-    
+      this.numAnnotations = 0;
+    }
+    
+    /**
+      Add an annotation value. The value should have been checked for
+      error before calling this method.
+    */
     private void addValue(AnnotationType at, Object value)
     {
@@ -1009 +1157 @@
       }
       listOfValues.add(value);
-    }
-    
+      numAnnotations++;
+    }
+    
+    /**
+      The total numer of annotation values that has been added to this 
+      cache.
+    */
+    private int getNumAnnotations()
+    {
+      return numAnnotations;
+    }
+    
+    /**
+      Annotate the item with the annotation values from the file.
+      @param addToUnlimited TRUE to append to annotation types with multiplicity==0, FALSE
+        to replace
+      @param replaceExisting TRUE to replace existing annotation values, FALSE to skip
+      @param failIfTooManyValues FALSE to skip annotations with more values than
+        the multiplicity settings allows, TRUE to throw an exception
+    */
     private void setNewAnnotations(boolean addToUnlimited, boolean replaceExisting, boolean failIfTooManyValues)
     {
       if (values.size() == 0) return;
+      int numAnnotations = 0;
       AnnotationSet as = item.getAnnotationSet();
       for (Map.Entry<AnnotationType, List<Object>> entry : values.entrySet())
@@ -1019 +1186 @@
         AnnotationType at = entry.getKey();
         List<Object> newValues = entry.getValue();
+        int size = newValues.size();
         int multiplicity = at.getMultiplicity();
         boolean hasAnnotation = as.hasAnnotation(at);
@@ -1028 +1196 @@
           {
             Annotation a = as.getAnnotation(at);
-            newValues.addAll(0, a.getValues());
-            a.setValues(newValues); 
+            if (merge) newValues.addAll(0, a.getValues());
+            a.setValues(newValues);
+            numSet += size;
+            if (hasAnnotation && !merge) numReplaced += size;
           }
-        }
-      }
-    }
-  }
-  
-  
+          else
+          {
+            numError += size;
+          }
+        }
+      }
+    }
+    
+    /**
+      Number of annotations that was set (=added or replaced) by the call to the 
+      {@link #setNewAnnotations(boolean, boolean, boolean)} method.
+    */
+    private int getNumSet()
+    {
+      return numSet;
+    }
+
+    /**
+      Number of annotations that was replaced by the call to the
+      {@link #setNewAnnotations(boolean, boolean, boolean)} method.
+    */
+    private int getNumReplaced()
+    {
+      return numReplaced;
+    }     
+    
+    /**
+      Number of annotations that was couldn't be set by the call to the 
+      {@link #setNewAnnotations(boolean, boolean, boolean)} method because of an
+      error (too many values).
+    */
+    private int getNumError()
+    {
+      return numError;
+    }
+    
+  }
 }

trunk/www/admin/plugindefinitions/edit_plugin.jsp

@@ -85 +85 @@
   Set<Permission> defined, Set<Permission> always, Set<Permission> maybe)
 {
-  if (defined.contains(p))
+  if (defined != null && defined.contains(p))
   {
     if (always != null && always.contains(p))
@@ -688 +688 @@
         {
           Item ppType = pp.getItemType();
-          Set<Permission> always = Permission.expand(pp.getAlwaysGranted());
-          Set<Permission> maybe = Permission.expand(pp.getMaybeGranted());
-          %>
-          frm['<%=ppType.name()%>_granted'].value = <%=always == null ? 0 : PermissionUtil.getPermissionCode(always)%>;
-          frm['<%=ppType.name()%>_denied'].value = <%=maybe == null ? 255 : 255 & ~PermissionUtil.getPermissionCode(maybe)%>;
-          <%
+          if (ppType.getDefinedPermissions() != null)
+          {
+            Set<Permission> always = Permission.expand(pp.getAlwaysGranted());
+            Set<Permission> maybe = Permission.expand(pp.getMaybeGranted());
+            %>
+            frm['<%=ppType.name()%>_granted'].value = <%=always == null ? 0 : PermissionUtil.getPermissionCode(always)%>;
+            frm['<%=ppType.name()%>_denied'].value = <%=maybe == null ? 255 : 255 & ~PermissionUtil.getPermissionCode(maybe)%>;
+            <%
+          }
         }
         %>
@@ -770 +773 @@
       {
         %>
-
         <div class="error"><%=warning%></div>
         <%
@@ -827 +829 @@
     <td>
       <table>
-      <tr><td>
+      <tr valign="top"><td>
         <b>Always grant</b><br>
         <input type="checkbox" name="grant_create" onClick="permissionOnClick(this)">Create<br>
@@ -861 +863 @@
           Set<Permission> maybe = Permission.expand(pp.getMaybeGranted());
           Set<Permission> defined = ppType.getDefinedPermissions();
-          StringBuilder sb = new StringBuilder();
-          sb.append("[");
-          appendPermissionLetter(sb, Permission.CREATE, "C", defined, always, maybe);
-          appendPermissionLetter(sb, Permission.READ, "R", defined, always, maybe);
-          appendPermissionLetter(sb, Permission.USE, "U", defined, always, maybe);
-          appendPermissionLetter(sb, Permission.WRITE, "W", defined, always, maybe);
-          appendPermissionLetter(sb, Permission.DELETE, "D", defined, always, maybe);
-          appendPermissionLetter(sb, Permission.SET_OWNER, "O", defined, always, maybe);
-          appendPermissionLetter(sb, Permission.SET_PERMISSION, "P", defined, always, maybe);
-          sb.append("]");
-          %>
-          <tr>
-            <td><%=ppType%></td>
-            <td><%=sb.toString()%></td>
-          </tr>
-          <%
+          if (defined != null)
+          {
+            StringBuilder sb = new StringBuilder();
+            sb.append("[");
+            appendPermissionLetter(sb, Permission.CREATE, "C", defined, always, maybe);
+            appendPermissionLetter(sb, Permission.READ, "R", defined, always, maybe);
+            appendPermissionLetter(sb, Permission.USE, "U", defined, always, maybe);
+            appendPermissionLetter(sb, Permission.WRITE, "W", defined, always, maybe);
+            appendPermissionLetter(sb, Permission.DELETE, "D", defined, always, maybe);
+            appendPermissionLetter(sb, Permission.SET_OWNER, "O", defined, always, maybe);
+            appendPermissionLetter(sb, Permission.SET_PERMISSION, "P", defined, always, maybe);
+            sb.append("]");
+            %>
+            <tr>
+              <td><%=ppType%></td>
+              <td><%=sb.toString()%></td>
+            </tr>
+            <%
+          }
         }
         %>