Changeset 3588


Ignore:
Timestamp:
Jul 23, 2007, 11:28:10 AM (14 years ago)
Author:
Nicklas Nordborg
Message:

Fixes #326: Mass annotation plug-in

Location:
trunk
Files:
4 edited

Legend:

Unmodified
Added
Removed
  • trunk/doc/src/docbook/userdoc/annotations.xml

    r3487 r3588  
    692692    </sect2>
    693693   
     694    <sect2 id="annotations.massimport">
     695      <title>Mass annotation import plug-in</title>
     696     
     697      <para>
     698        BASE includes a plug-in for importing annotations to multiple items
     699        in one go. The plug-in read annotation values from a simple column-based
     700        text file. Ususally, a tab is used as the delimiter between columns.
     701        The first row should contain the column headers. One column should contain
     702        the name or the external ID of the item. The rest of the columns can each be
     703        mapped to an annotation type and contains the annotation values. If a column
     704        header exactly match the name of an annotation type, the plug-in will automatically
     705        create the mapping, otherwise you must do it manually. You don't have to map
     706        all columns if you don't want to.
     707      </para>
     708     
     709      <para>
     710        Each column can only contain a single annotation value for each row.
     711        If you have annotation types that accept multiple values you can map
     712        two or more columns to the same annotation type, or you can add an
     713        extra row only giving the name and the extra annotation value.
     714        Here is a simple example of a valid file with comma as column separator:
     715      </para>
     716     
     717      <programlisting>
     718# 'Time' and 'Age' are integer types
     719# 'Subtype' is a string enumeration
     720# 'Comment' is a text type that accept multiple values
     721Name,Time (hours),Age (years),Subtype,Comment
     722Sample #1,0,0,alfa,Very good
     723Sample #2,24,0,beta,Not so bad
     724Sample #2,,,,Yet another comment
     725</programlisting>
     726     
     727      <para>
     728        The plug-in can be used with or without a configuration. The configuration
     729        keeps the regular expressions and other settings used to parse the file. If
     730        you often import annotations from the same file format, we recommend that
     731        you use a configuration. The mapping from file columns to annotation types
     732        is not part of the configuration, it must be done each time the plug-in is used.
     733      </para>
     734     
     735      <para>
     736        The plug-in can be used from the list view of all annotatable items.
     737        Using the plug-in is a three-step wizard:
     738      </para>
     739     
     740      <orderedlist>
     741      <listitem>
     742        <para>
     743        Select a file to import from and the regular expressions and other
     744        settings used to parse the file. In this step you also select the column
     745        that contains the name or external ID the items. If a configuration is used
     746        all settings on this page, except the file to import from, already has values.
     747        </para>
     748      </listitem>
     749     
     750      <listitem>
     751        <para>
     752        The plug-in will start parsing the file until it finds the column headers.
     753        You are asked to select an annotation type for each column.
     754        </para>
     755      </listitem>
     756     
     757      <listitem>
     758        <para>
     759        Set error handling options and some other import options.
     760        </para>
     761      </listitem>
     762     
     763      </orderedlist>
     764     
     765    </sect2>
     766   
    694767  </sect1>
    695768
  • trunk/src/plugins/core/net/sf/basedb/plugins/AbstractFlatFileImporter.java

    r3586 r3588  
    534534            try
    535535            {
    536               errorHandler.handleError(t);
     536              if (errorHandler == null)
     537              {
     538                throw t;
     539              }
     540              else
     541              {
     542                errorHandler.handleError(t);
     543              }
    537544              skippedLines++;
    538545            }
     
    565572              catch (Throwable t)
    566573              {
    567                 errorHandler.handleError(t);
     574                if (errorHandler == null)
     575                {
     576                  throw t;
     577                }
     578                else
     579                {
     580                  errorHandler.handleError(t);
     581                }
    568582                skippedLines++;
    569583              }
     
    593607            catch (Throwable t)
    594608            {
    595               errorHandler.handleError(t);
     609              if (errorHandler == null)
     610              {
     611                throw t;
     612              }
     613              else
     614              {
     615                errorHandler.handleError(t);
     616              }
    596617              skippedLines++;
    597618            }
     
    9871008    Initialise the error handling system. This method is called just before the
    9881009    import is starting. A subclass may override this method to add specific
    989     error handlers, but it is important to call <code>super.setUpErrorHandling()</code>
    990     or the error handling will not work. The subclass may also add error handlers
    991     in the {@link #begin(FlatFileParser)} method.
     1010    error handlers. If <code>super.setUpErrorHandling()</code> isn't called
     1011    error handling in AbstractFlatFileImporter is disabled and the subclass
     1012    must do all it's error handling in it's own code. The subclass may also add
     1013    error handlers in the {@link #begin(FlatFileParser)} method.
    9921014  */
    9931015  protected void setUpErrorHandling()
  • trunk/src/plugins/core/net/sf/basedb/plugins/AnnotationFlatFileImporter.java

    r3586 r3588  
    2929import java.util.ArrayList;
    3030import java.util.Arrays;
     31import java.util.Collection;
    3132import java.util.EnumSet;
    3233import java.util.HashMap;
     
    6061import net.sf.basedb.core.ParameterType;
    6162import net.sf.basedb.core.Permission;
     63import net.sf.basedb.core.PermissionDeniedException;
    6264import net.sf.basedb.core.PluginParameter;
    6365import net.sf.basedb.core.RequestInformation;
     
    7173import net.sf.basedb.core.plugin.ParameterValues;
    7274import net.sf.basedb.core.plugin.ParameterValuesWrapper;
     75import net.sf.basedb.core.plugin.Permissions;
    7376import net.sf.basedb.core.plugin.Request;
    7477import net.sf.basedb.core.plugin.Response;
     
    8891
    8992/**
     93  Plug-in for importing annotations from simple text files. The plug-in supports
     94  all files that can be parsed with the {@link FlatFileParser} class. This plug-in
     95  works without a configuration but can use a configuration to store regular
     96  expressions and other settings for the flat file parser. In both cases,
     97  the job configuration is a three-step process:
     98 
     99  <ol>
     100  <li>Setup regular expressions and other options for the flat file parser. If a
     101    configuration is used all values should already be filled in. In this step
     102    a file to import from must also be selected.
     103  <li>Map file columns to annotation types. Annotation types that support multiple
     104    values may be mapped to more than one column.
     105  <li>Setup error handling options and other settings for the plug-in (for example
     106    if existing annotations should be replaced or not).
     107  </ol>
    90108
    91109  @author nicklas
     
    112130 
    113131  private static Set<GuiContext> guiContexts;
    114  
     132  private static final Set<Permissions> permissions = new HashSet<Permissions>();
     133 
     134  // second step in the wizard
    115135  private static final String CONFIGURE_MAPPING = "configure_mapping";
     136  // third step in the wizard
    116137  private static final String CONFIGURE_IMPORT = "configure_import";
    117138 
     
    147168
    148169  /**
    149     Section definition for grouping all mappings of columns to annotation types
     170    Section definition for grouping all mappings of columns to annotation types.
     171    Parameters in this section will be generated on the fly based on the
     172    column headers from the file to import from.
    150173  */
    151174  private static final PluginParameter<String> annotationTypeSection = new PluginParameter<String>(
     
    199222    "skip = Skip the current annotation and continue\n"+
    200223    "fail = Stop with an error message (default)",
    201     new StringParameterType(255, "fail", true, 1, 0, 0,
     224    new StringParameterType(255, "skip", true, 1, 0, 0,
    202225      Arrays.asList( new String[] { "skip", "fail"} ))
    203226  );
     
    214237
    215238  private static final PluginParameter<String> multipleItemsFoundErrorParameter = new PluginParameter<String>(
    216       "multipleItemsFoundError",
    217       "Multiple items found",
    218       "What to do if the plug-in finds more than one item with the same name or external ID.\n\n"+
    219       "all = Annotate all items\n"+
    220       "fail = Stop with an error message",
    221       new StringParameterType(255, "all", false, 1, 0, 0,
    222         Arrays.asList( new String[] { "all", "fail"} ))
    223     );
     239    "multipleItemsFoundError",
     240    "Multiple items found",
     241    "What to do if the plug-in finds more than one item with the same name or external ID.\n\n"+
     242    "all = Annotate all items\n"+
     243    "fail = Stop with an error message",
     244    new StringParameterType(255, "all", false, 1, 0, 0,
     245      Arrays.asList( new String[] { "all", "fail"} ))
     246  );
    224247
    225248 
     
    232255    "skip = Skip the current annotation and continue\n"+
    233256    "fail = Stop with an error message",
    234     new StringParameterType(255, null, false, 1, 0, 0,
     257    new StringParameterType(255, "crop", false, 1, 0, 0,
    235258      Arrays.asList( new String[] { "crop", "skip", "fail"} ))
    236259  );
     
    304327    return true;
    305328  }
     329  /**
     330    Request read access to File:s, read access to annotation types
     331    and write access to all annotatable items.
     332  */
     333  public Collection<Permissions> getPermissions()
     334  {
     335    if (permissions.size() == 0)
     336    {
     337      permissions.add(new Permissions(Item.FILE, null, EnumSet.of(Permission.READ)));
     338      permissions.add(new Permissions(Item.ANNOTATIONTYPE, null, EnumSet.of(Permission.READ)));
     339     
     340      for (GuiContext ctx : getGuiContexts())
     341      {
     342        permissions.add(new Permissions(ctx.getItem(), null, EnumSet.of(Permission.WRITE)));
     343      }
     344    }
     345    return permissions;
     346  }
    306347  // -------------------------------------------
    307348
     
    310351    -------------------------------------------
    311352  */
     353  /**
     354    This plug-in works in list context of all {@link Annotatable} items,
     355    except bioassay sets, bioassays and wells because they are not standalone
     356    items. We use {@link Metadata#getAnnotatableItems()} to create the contexts.
     357  */
    312358  public Set<GuiContext> getGuiContexts()
    313359  {
     
    319365        for (Item item : Metadata.getAnnotatableItems())
    320366        {
    321           guiContexts.add(new GuiContext(item, GuiContext.Type.LIST));
     367          if (item.getDefinedPermissions() != null)
     368          {
     369            guiContexts.add(new GuiContext(item, GuiContext.Type.LIST));
     370          }
    322371        }
    323372      }
     
    327376  public String isInContext(GuiContext context, Object item)
    328377  {
     378    if (!sc.hasPermission(Permission.WRITE, context.getItem()))
     379    {
     380      throw new PermissionDeniedException(Permission.WRITE, context.getItem().toString());
     381    }
    329382    return null;
    330383  }
     
    522575  private boolean failIfTooManyValues;
    523576 
     577  // Messaging
     578  private int numItems;
     579  private int numItemNotFound;
     580  private int numAnnotations;
     581  private int numReplaced;
     582  private int numError;
     583 
    524584  @Override
     585  /**
     586    The flat file parser is initialised with settings from either the job or configuration.
     587  */
    525588  protected FlatFileParser getInitializedFlatFileParser()
    526589    throws BaseException
     
    530593  }
    531594 
     595  /**
     596    Don't use AbstractFlatFileImporter to handle errors.
     597  */
     598  @Override
     599  protected void setUpErrorHandling()
     600  {}
     601 
     602  /**
     603    Setup error handling and pre-load some of the configuration options.
     604  */
    532605  @Override
    533606  protected void begin(FlatFileParser ffp)
     
    558631  }
    559632 
     633  /**
     634    Setup column mapping. Creates DbControl and query to find items.
     635  */
    560636  @Override
    561637  protected void beginData()
     
    597673  }
    598674 
     675  /**
     676    Read annotations from a single data line. Errors are handled internally.
     677    Errors thrown from this method should be reported back to client (ie. error
     678    handling in AbstractFlatFileParser) must be disabled. This method will
     679    load items and annotation values and put them in an internal cache. No
     680    items are annotated until the entire file has been parsed.
     681  */
    599682  @Override
    600683  protected void handleData(Data data)
     
    602685  {
    603686
    604     // Find the item(s) to annotate
     687    // Find the item(s) to annotate; mapper gives us name OR external ID
    605688    String name = itemMapper.getValue(data);
     689    // 1. check the cache
    606690    Set<NewAnnotations> items = itemCache.get(name);
    607691    if (items == null)
    608692    {
    609693      items = new HashSet<NewAnnotations>();
     694      // The 'name' parameter can also query against the external ID
    610695      itemQuery.setParameter("name", name, Type.STRING);
    611696      List<?> result = itemQuery.list(dc);
    612       if (result.isEmpty() && !ignoreNotFoundItems)
    613       {
    614         throw new ItemNotFoundException(itemType +
    615           "[" + (searchExternalId ? "externalId" : "name") + "=" + name + "]");
     697      if (result.isEmpty())
     698      {
     699        // No item with specified name/external ID was found
     700        if (!ignoreNotFoundItems)
     701        {
     702          throw new ItemNotFoundException(itemType +
     703            "[" + (searchExternalId ? "externalId" : "name") + "=" + name + "]");
     704        }
     705        else
     706        {
     707          numItemNotFound++;
     708        }
    616709      }
    617710      if (result.size() > 1 && failIfMultipleFoundItems)
     
    622715      for (Object item : result)
    623716      {
     717        // Add each item that was found to the cache
    624718        items.add(new NewAnnotations((Annotatable)item));
    625719      }
     
    629723    if (!items.isEmpty())
    630724    {
    631       // Load annotation values
     725      // Load annotation values from column mappers
    632726      for (Map.Entry<Mapper, AnnotationType> entry : mappers.entrySet())
    633727      {
     
    641735          if (at.isEnumeration() && at.getValueType() == Type.STRING)
    642736          {
     737            // Case and leading/trailing whitespace are ignore for enumerated string annotations
    643738            String enumValue = at.findValue(sValue, true, true);
    644739            if (enumValue != null) sValue = enumValue;
     
    650745            for (NewAnnotations item : items)
    651746            {
     747              // Add annotation value to the cache
    652748              item.addValue(at, annotationValue);
    653749            }
     
    656752        catch (Throwable t)
    657753        {
     754          // In case the annotation value is not valid...
     755          numError++;
    658756          try
    659757          {
    660758            errorHandler.handleError(t);
     759            // The error should be ignored if we get passed the above line
    661760          }
    662761          catch (Throwable t2)
    663762          {
     763            // handleData doesn't allow us to throw any Throwable
    664764            if (t2 instanceof RuntimeException)
    665765            {
     
    676776  }
    677777 
     778  /**
     779    Now it's time to update the items in the cache with the new annotation
     780    values.
     781  */
    678782  @Override
    679783  protected void end(boolean success)
     
    685789        for (Set<NewAnnotations> n : itemCache.values())
    686790        {
     791          numItems += n.size();
    687792          for (NewAnnotations na : n)
    688793          {
    689794            na.setNewAnnotations(addToUnlimited, replaceExisting, failIfTooManyValues);
     795            if (na.getNumSet() == 0) numItems--;
     796            numAnnotations += na.getNumSet();
     797            numError += na.getNumError();
     798            numReplaced += na.getNumReplaced();
    690799          }
    691800        }
     
    700809    finally
    701810    {
    702       dc.close();
     811      if (dc != null) dc.close();
    703812      super.end(success);
    704813      ffp = null;
    705814      mappers.clear();
    706815      itemCache.clear();
    707      
    708     }
     816    }
     817  }
     818  @Override
     819  protected String getSuccessMessage(int skippedLines)
     820  {
     821    String msg = numItems + " item(s) annotated with " + numAnnotations + " annotation(s)";
     822    if (numReplaced > 0) msg += " (" + numReplaced + " was replaced)";
     823    if (numItemNotFound > 0) msg += "; " + numItemNotFound + " row(s) skipped because no item was found";
     824    if (numError > 0) msg += "; " + numError + " annotation(s) skipped because of invalid values";
     825    return msg;
    709826  }
    710827  // -------------------------------------------
     
    9361053 
    9371054  /**
    938     Check if the current item has an 'externalId' property.
     1055    Check if the current item has an 'externalId' property. We use reflection to
     1056    look for the 'getExternalId' method in the item's data class.
    9391057  */
    9401058  private boolean hasExternalId(Item item)
     
    9511069  }
    9521070 
     1071  /**
     1072    Create a query that return items of the specified type. We use reflection
     1073    to call the static method 'getQuery' on the item's item class, for
     1074    example {@link net.sf.basedb.core.Sample#getQuery()}. We add a restriction
     1075    to either the 'name' or the 'externalId' property but the parameter is
     1076    always called 'name' to make it easier to implement the code that is
     1077    using the query.
     1078   
     1079    @param itemType The type of items to search
     1080    @param searchExternalId If we should search by name or external ID
     1081    @param includes Include options passed to {@link ItemQuery#include(java.util.Collection)}
     1082    @return A query that searches items of the specified type.
     1083  */
    9531084  private ItemQuery<?> createQuery(Item itemType, boolean searchExternalId, Set<Include> includes)
    9541085  {
     
    9891120  }
    9901121 
     1122  /**
     1123    Internal cache for storing annotation values from the file
     1124    for a single item. The items are not annotated until the entire
     1125    file has been parsed.
     1126  */
    9911127  private static class NewAnnotations
    9921128  {
    9931129    private final Annotatable item;
    9941130    private final Map<AnnotationType, List<Object>> values;
    995    
     1131    private int numAnnotations;
     1132    private int numSet;
     1133    private int numError;
     1134    private int numReplaced;
     1135   
     1136    /**
     1137      Create a new cache for the specified item.
     1138    */
    9961139    private NewAnnotations(Annotatable item)
    9971140    {
    9981141      this.item = item;
    9991142      this.values = new HashMap<AnnotationType, List<Object>>();
    1000     }
    1001    
     1143      this.numAnnotations = 0;
     1144    }
     1145   
     1146    /**
     1147      Add an annotation value. The value should have been checked for
     1148      error before calling this method.
     1149    */
    10021150    private void addValue(AnnotationType at, Object value)
    10031151    {
     
    10091157      }
    10101158      listOfValues.add(value);
    1011     }
    1012    
     1159      numAnnotations++;
     1160    }
     1161   
     1162    /**
     1163      The total numer of annotation values that has been added to this
     1164      cache.
     1165    */
     1166    private int getNumAnnotations()
     1167    {
     1168      return numAnnotations;
     1169    }
     1170   
     1171    /**
     1172      Annotate the item with the annotation values from the file.
     1173      @param addToUnlimited TRUE to append to annotation types with multiplicity==0, FALSE
     1174        to replace
     1175      @param replaceExisting TRUE to replace existing annotation values, FALSE to skip
     1176      @param failIfTooManyValues FALSE to skip annotations with more values than
     1177        the multiplicity settings allows, TRUE to throw an exception
     1178    */
    10131179    private void setNewAnnotations(boolean addToUnlimited, boolean replaceExisting, boolean failIfTooManyValues)
    10141180    {
    10151181      if (values.size() == 0) return;
     1182      int numAnnotations = 0;
    10161183      AnnotationSet as = item.getAnnotationSet();
    10171184      for (Map.Entry<AnnotationType, List<Object>> entry : values.entrySet())
     
    10191186        AnnotationType at = entry.getKey();
    10201187        List<Object> newValues = entry.getValue();
     1188        int size = newValues.size();
    10211189        int multiplicity = at.getMultiplicity();
    10221190        boolean hasAnnotation = as.hasAnnotation(at);
     
    10281196          {
    10291197            Annotation a = as.getAnnotation(at);
    1030             newValues.addAll(0, a.getValues());
    1031             a.setValues(newValues);
     1198            if (merge) newValues.addAll(0, a.getValues());
     1199            a.setValues(newValues);
     1200            numSet += size;
     1201            if (hasAnnotation && !merge) numReplaced += size;
    10321202          }
    1033         }
    1034       }
    1035     }
    1036   }
    1037  
    1038  
     1203          else
     1204          {
     1205            numError += size;
     1206          }
     1207        }
     1208      }
     1209    }
     1210   
     1211    /**
     1212      Number of annotations that was set (=added or replaced) by the call to the
     1213      {@link #setNewAnnotations(boolean, boolean, boolean)} method.
     1214    */
     1215    private int getNumSet()
     1216    {
     1217      return numSet;
     1218    }
     1219
     1220    /**
     1221      Number of annotations that was replaced by the call to the
     1222      {@link #setNewAnnotations(boolean, boolean, boolean)} method.
     1223    */
     1224    private int getNumReplaced()
     1225    {
     1226      return numReplaced;
     1227    }     
     1228   
     1229    /**
     1230      Number of annotations that was couldn't be set by the call to the
     1231      {@link #setNewAnnotations(boolean, boolean, boolean)} method because of an
     1232      error (too many values).
     1233    */
     1234    private int getNumError()
     1235    {
     1236      return numError;
     1237    }
     1238   
     1239  }
    10391240}
  • trunk/www/admin/plugindefinitions/edit_plugin.jsp

    r3501 r3588  
    8585  Set<Permission> defined, Set<Permission> always, Set<Permission> maybe)
    8686{
    87   if (defined.contains(p))
     87  if (defined != null && defined.contains(p))
    8888  {
    8989    if (always != null && always.contains(p))
     
    688688        {
    689689          Item ppType = pp.getItemType();
    690           Set<Permission> always = Permission.expand(pp.getAlwaysGranted());
    691           Set<Permission> maybe = Permission.expand(pp.getMaybeGranted());
    692           %>
    693           frm['<%=ppType.name()%>_granted'].value = <%=always == null ? 0 : PermissionUtil.getPermissionCode(always)%>;
    694           frm['<%=ppType.name()%>_denied'].value = <%=maybe == null ? 255 : 255 & ~PermissionUtil.getPermissionCode(maybe)%>;
    695           <%
     690          if (ppType.getDefinedPermissions() != null)
     691          {
     692            Set<Permission> always = Permission.expand(pp.getAlwaysGranted());
     693            Set<Permission> maybe = Permission.expand(pp.getMaybeGranted());
     694            %>
     695            frm['<%=ppType.name()%>_granted'].value = <%=always == null ? 0 : PermissionUtil.getPermissionCode(always)%>;
     696            frm['<%=ppType.name()%>_denied'].value = <%=maybe == null ? 255 : 255 & ~PermissionUtil.getPermissionCode(maybe)%>;
     697            <%
     698          }
    696699        }
    697700        %>
     
    770773      {
    771774        %>
    772 
    773775        <div class="error"><%=warning%></div>
    774776        <%
     
    827829    <td>
    828830      <table>
    829       <tr><td>
     831      <tr valign="top"><td>
    830832        <b>Always grant</b><br>
    831833        <input type="checkbox" name="grant_create" onClick="permissionOnClick(this)">Create<br>
     
    861863          Set<Permission> maybe = Permission.expand(pp.getMaybeGranted());
    862864          Set<Permission> defined = ppType.getDefinedPermissions();
    863           StringBuilder sb = new StringBuilder();
    864           sb.append("[");
    865           appendPermissionLetter(sb, Permission.CREATE, "C", defined, always, maybe);
    866           appendPermissionLetter(sb, Permission.READ, "R", defined, always, maybe);
    867           appendPermissionLetter(sb, Permission.USE, "U", defined, always, maybe);
    868           appendPermissionLetter(sb, Permission.WRITE, "W", defined, always, maybe);
    869           appendPermissionLetter(sb, Permission.DELETE, "D", defined, always, maybe);
    870           appendPermissionLetter(sb, Permission.SET_OWNER, "O", defined, always, maybe);
    871           appendPermissionLetter(sb, Permission.SET_PERMISSION, "P", defined, always, maybe);
    872           sb.append("]");
    873           %>
    874           <tr>
    875             <td><%=ppType%></td>
    876             <td><%=sb.toString()%></td>
    877           </tr>
    878           <%
     865          if (defined != null)
     866          {
     867            StringBuilder sb = new StringBuilder();
     868            sb.append("[");
     869            appendPermissionLetter(sb, Permission.CREATE, "C", defined, always, maybe);
     870            appendPermissionLetter(sb, Permission.READ, "R", defined, always, maybe);
     871            appendPermissionLetter(sb, Permission.USE, "U", defined, always, maybe);
     872            appendPermissionLetter(sb, Permission.WRITE, "W", defined, always, maybe);
     873            appendPermissionLetter(sb, Permission.DELETE, "D", defined, always, maybe);
     874            appendPermissionLetter(sb, Permission.SET_OWNER, "O", defined, always, maybe);
     875            appendPermissionLetter(sb, Permission.SET_PERMISSION, "P", defined, always, maybe);
     876            sb.append("]");
     877            %>
     878            <tr>
     879              <td><%=ppType%></td>
     880              <td><%=sb.toString()%></td>
     881            </tr>
     882            <%
     883          }
    879884        }
    880885        %>
Note: See TracChangeset for help on using the changeset viewer.