Changeset 7588


Ignore:
Timestamp:
Feb 18, 2019, 10:54:49 AM (5 years ago)
Author:
Nicklas Nordborg
Message:

References #2149: Batch item importers should be able to use an annotation for item identification

Added the AnnotationIdMethod implementation that makes it possible to specify an annotation type to use for item identification. It has been implemented in the biosource and sample importers so far.

While it seems to work as expected the numer of possible annotation types that can be used probably need to be limited in some way. The plug-ins will currently list all annotation types that have STRING, INT or LONG values that are not enumerations. Maybe we should add a flag option to the annotation type?

Location:
trunk/src/plugins/core/net/sf/basedb/plugins/batchimport
Files:
1 added
3 edited

Legend:

Unmodified
Added
Removed

  • trunk/src/plugins/core/net/sf/basedb/plugins/batchimport/AbstractItemImporter.java

    r7330 r7588  
    3434import java.util.regex.Pattern;
    3535
     36import net.sf.basedb.core.AnnotationType;
    3637import net.sf.basedb.core.AnnotationTypeCategory;
    3738import net.sf.basedb.core.ArrayBatch;
    … …  
    105106import net.sf.basedb.core.query.Expressions;
    106107import net.sf.basedb.core.query.Hql;
     108import net.sf.basedb.core.query.Orders;
    107109import net.sf.basedb.core.query.Restrictions;
    108110import net.sf.basedb.plugins.AbstractFlatFileImporter;
    … …  
    188190      );
    189191 
     192  /**
     193    Column mapping parameter for the annotation column used for item identification.
     194    @since 3.15
     195  */
     196  protected static final PluginParameter<String> idAnnotationColumnMapping = new PluginParameter<String>(
     197      "idAnnotationColumnMapping",
     198      "Annotation ID",
     199      "Mapping that picks the annotation value to use for item identification from the data columns. This " +
     200      "parameter is required if the selected identification method is based on an annotation, " +
     201      "otherwise it is ignored. Example: \\Sample-ID\\",
     202      optionalColumnMapping
     203      );
     204 
    190205  protected static final PluginParameter<String> subtypeColumnMapping = new PluginParameter<String>(
    191206      "subtypeColumnMapping",
    … …  
    379394        }
    380395
     396        dc = sc.newDbControl();
    381397        ParameterValues wrapper = new ParameterValuesWrapper(request, job, configuration, forJob);
    382398        if (forJob)
    383399        {
    384400          // Check that file can be parsed with given regular expressions
    385           dc = sc.newDbControl();
    386401          FlatFileParser ffp = getInitializedFlatFileParser(wrapper);
    387402          File f = File.getById(dc, ((File)request.getParameterValue("file")).getId());
    … …  
    400415       
    401416        // Check that a mapping has been provided for the selected idMethod
    402         IdMethod idMethod = getIdMethod((String)request.getParameterValue("idMethod"));
     417        IdMethod idMethod = getIdMethod(dc, (String)request.getParameterValue("idMethod"));
    403418        if (idMethod == null)
    404419        {
    … …  
    407422          return;
    408423        }
     424
    409425        PluginParameter<?> idMapping = ri.getParameter(idMethod.getColumnMappingParameterName());
    410426        if (request.getParameterValue(idMapping.getName()) == null)
    … …  
    568584  {
    569585    this.ffp = ffp;
     586    this.dc = sc.newDbControl();
    570587    String dateFormat = Values.getString((String)job.getValue(Parameters.DATE_FORMAT_PARAMETER), "yyyy-MM-dd");
    571588    this.dateFormatter = new DateFormatter(dateFormat);
    572589    this.dateFormatter.getDateFormat().setLenient(false);
    573     this.idMethod = getIdMethod((String)job.getValue("idMethod"));
     590    this.idMethod = getIdMethod(dc, (String)job.getValue("idMethod"));
    574591    this.idMapping = (String)job.getValue(idMethod.getColumnMappingParameterName());
    575592    this.itemCache = new HashMap<String, BasicItem>();
    … …  
    612629  protected void beginData()
    613630  {
    614     this.dc = sc.newDbControl();
    615631    if (sc.getActiveProjectId() != 0)
    616632    {
    … …  
    893909 
    894910  /**
     911    Get identification methods that are based on annotations. The default implementation
     912    first calls the {@link #getItemForAnnotationTypes()}. If this returns a non-null value it will
     913    make a query for all annotation types that are suitable as identifiers:
     914   
     915     * The value type must be one of STRING, INT or LONG
     916     * It must not be an enumeration
     917   
     918    @since 3.15
     919  */
     920  protected List<IdMethod> getAnnotationIdMethods(DbControl dc)
     921  {
     922    List<IdMethod> idMethods = new ArrayList<>();
     923    Item itemType = getItemForAnnotationTypes();
     924    if (itemType != null)
     925    {
     926      ItemQuery<AnnotationType> query = AnnotationType.getQuery(itemType);
     927      query.include(Include.MINE, Include.IN_PROJECT, Include.SHARED);
     928      // The annotation type must be STRING, INTEGER or LONG
     929      query.restrict(Restrictions.in(Hql.property("valueType"), Expressions.parameter("valueTypes", Type.INT)));
     930      query.setParameter("valueTypes", Arrays.asList(Type.STRING.getValue(), Type.INT.getValue(), Type.LONG.getValue()), Type.INT);
     931      // Enumerations can't be used
     932      query.restrict(Restrictions.eq(Hql.property("enumeration"), Expressions.bool(false)));
     933     
     934      query.order(Orders.asc(Hql.property("name")));
     935      // Get a list of possible annotation types
     936      List<AnnotationType> annotationTypes = new ArrayList<AnnotationType>(query.list(dc));
     937   
     938      for (AnnotationType at : annotationTypes)
     939      {
     940        idMethods.add(new AnnotationIdMethod(at));
     941      }
     942    }
     943    return idMethods;
     944  }
     945 
     946  /**
    895947    Add extra job parameters that the subclass needs. This
    896948    method is called during configuration of the plug-in and
    … …  
    9581010  */
    9591011  protected Item getItemForSubtypes()
     1012  {
     1013    return null;
     1014  }
     1015 
     1016  /**
     1017    Does the importer want to use annotation types as identifiers importing?
     1018    If this method returns a non-null, this class will add all annotation
     1019    types from the {@link #getAnnotationIdMethods()} to the ID method
     1020    parameter. The subclass must also add the {@link #idAnnotationColumnMapping}
     1021    to the {@link #addMoreColumnMappingParameters(List)} method.
     1022    The default implementation returns null.
     1023    @return An Item or null
     1024    @since 3.15
     1025  */
     1026  protected Item getItemForAnnotationTypes()
    9601027  {
    9611028    return null;
    … …  
    11391206  /**
    11401207    Get the id method to use for finding items. This implementation
    1141     searches the methods returned by {@link #getIdMethods()} for a method
    1142     with the given name. If no method can be found {@link PropertyIdMethod#NAME}
     1208    searches the methods returned by {@link #getIdMethods()} and {@link #getAnnotationIdMethods()}
     1209    for a method with the given name. If no method can be found {@link PropertyIdMethod#NAME}
    11431210    is returned.
    11441211  */
    1145   protected IdMethod getIdMethod(String method)
     1212  protected IdMethod getIdMethod(DbControl dc, String method)
    11461213  {
    11471214    for (IdMethod im : getIdMethods())
     1215    {
     1216      if (im.getMethod().equals(method))
     1217      {
     1218        return im;
     1219      }
     1220    }
     1221    for (IdMethod im : getAnnotationIdMethods(dc))
    11481222    {
    11491223      if (im.getMethod().equals(method))
    … …  
    23052379    if (configureParser == null)
    23062380    {
    2307       List<PluginParameter<?>> parameters = new ArrayList<PluginParameter<?>>();
    2308 
    2309       if (forJob)
     2381      DbControl dc = null;
     2382      try
    23102383      {
    2311         // The file to import from
    2312         parameters.add(fileParameter);
    2313       }
    2314       else
    2315       {
    2316         parameters.add(defaultsSection);
    2317       }
    2318       List<String> allowedModes = new ArrayList<String>();
    2319       String defaultMode = "update";
    2320       if (context == null || sc.hasPermission(Permission.CREATE, context.getItem()))
    2321       {
    2322         allowedModes.add("create+update");
    2323         allowedModes.add("create");
    2324         defaultMode = "create+update";
    2325       }
    2326       allowedModes.add("update");
    2327       PluginParameter<String> modeParameter = new PluginParameter<String>(
    2328           "mode",
    2329           "Mode",
    2330           "Select an import mode: \n" +
    2331           "create+update = create missing items, update existing items\n" +
    2332           "create = create missing items only\n" +
    2333           "update = update existing items only\n\n" +
    2334           "NOTE! Not all modes may be available due to lack of permissions.",
    2335           new StringParameterType(255, defaultMode, forJob, 1, 0, 0, allowedModes)
    2336           );
    2337       parameters.add(modeParameter);
    2338       if (forJob)
    2339       {
    2340         addMoreJobParameters(parameters);
    2341       }
    2342       else
    2343       {
    2344         parameters.add(Parameters.dryRunParameter(null, null, null));
    2345       }
    2346 
    2347       // Identification method
    2348       if (forJob)
    2349       {
    2350         parameters.add(identificationSection);
    2351       }
    2352       Enumeration<String, String> idMethods = new Enumeration<String, String>();
    2353       for (IdMethod idm : getIdMethods())
    2354       {
    2355         idMethods.add(idm.getMethod(), idm.getTitle());
    2356       }
    2357       PluginParameter<String> idMethodParameter = new PluginParameter<String>(
    2358         "idMethod",
    2359         "Identification method",
    2360         "The method to use to find already existing items. You can use either the " +
    2361         "internal BASE ID, the name or, for some items, some another appropriate identifer " +
    2362         "property. You need to specify a column mapping for " +
    2363         "the selected method.",
    2364         new StringParameterType(null, idMethods.getKey(0), forJob, 1, 0, 0, idMethods)
    2365       );
    2366       parameters.add(idMethodParameter);
     2384        dc = sc.newDbControl();
     2385        List<PluginParameter<?>> parameters = new ArrayList<PluginParameter<?>>();
     2386 
     2387        if (forJob)
     2388        {
     2389          // The file to import from
     2390          parameters.add(fileParameter);
     2391        }
     2392        else
     2393        {
     2394          parameters.add(defaultsSection);
     2395        }
     2396        List<String> allowedModes = new ArrayList<String>();
     2397        String defaultMode = "update";
     2398        if (context == null || sc.hasPermission(Permission.CREATE, context.getItem()))
     2399        {
     2400          allowedModes.add("create+update");
     2401          allowedModes.add("create");
     2402          defaultMode = "create+update";
     2403        }
     2404        allowedModes.add("update");
     2405        PluginParameter<String> modeParameter = new PluginParameter<String>(
     2406            "mode",
     2407            "Mode",
     2408            "Select an import mode: \n" +
     2409            "create+update = create missing items, update existing items\n" +
     2410            "create = create missing items only\n" +
     2411            "update = update existing items only\n\n" +
     2412            "NOTE! Not all modes may be available due to lack of permissions.",
     2413            new StringParameterType(255, defaultMode, forJob, 1, 0, 0, allowedModes)
     2414            );
     2415        parameters.add(modeParameter);
     2416        if (forJob)
     2417        {
     2418          addMoreJobParameters(parameters);
     2419        }
     2420        else
     2421        {
     2422          parameters.add(Parameters.dryRunParameter(null, null, null));
     2423        }
     2424 
     2425        // Identification method
     2426        if (forJob)
     2427        {
     2428          parameters.add(identificationSection);
     2429        }
     2430        Enumeration<String, String> idMethods = new Enumeration<String, String>();
     2431        for (IdMethod idm : getIdMethods())
     2432        {
     2433          idMethods.add(idm.getMethod(), idm.getTitle());
     2434        }
     2435        for (IdMethod idm : getAnnotationIdMethods(dc))
     2436        {
     2437          idMethods.add(idm.getMethod(), idm.getTitle());
     2438        }
     2439        PluginParameter<String> idMethodParameter = new PluginParameter<String>(
     2440          "idMethod",
     2441          "Identification method",
     2442          "The method to use for finding existing items. You can use either the " +
     2443          "internal BASE ID, the name or, for some items, some another appropriate identifer " +
     2444          "property or annotation. You need to specify a column mapping for " +
     2445          "the selected method.",
     2446          new StringParameterType(null, idMethods.getKey(0), forJob, 1, 0, 0, idMethods)
     2447        );
     2448        parameters.add(idMethodParameter);
    23672449     
    2368       if (getItemForSubtypes() != null)
    2369       {
    2370         DbControl dc = sc.newDbControl();
    2371         try
     2450        if (getItemForSubtypes() != null)
    23722451        {
    23732452          // Get a list of possible subtypes
    … …  
    23932472          }
    23942473        }
    2395         finally
    2396         {
    2397           if (dc != null) dc.close();
    2398         }
    2399 
    2400       }
    2401      
    2402       if (forJob)
    2403       {
    2404         BooleanParameterType optionalBooleanType = new BooleanParameterType(null, false);
    2405         ItemContext cc = sc.getCurrentContext(context.getItem());
    2406         Set<Include> include = cc.getInclude();
    2407         // Should we search in MINE, SHARED, IN_PROJECT, etc.
    2408         parameters.add(new PluginParameter<Boolean>(
    2409             "includeMine",
    2410             "Owned by me",
    2411             "Search items that are owned by yourself.",
    2412             include.contains(Include.MINE) ? Boolean.TRUE : null,
    2413             optionalBooleanType)
    2414           );
    2415         parameters.add(new PluginParameter<Boolean>(
    2416             "includeShared",
    2417             "Shared to me",
    2418             "Search items that have been shared to you by other users.",
    2419             include.contains(Include.SHARED) ? Boolean.TRUE : null,
    2420             optionalBooleanType)
    2421           );
    2422          
    2423         if (sc.getActiveProjectId() != 0)
    2424         {
     2474       
     2475        if (forJob)
     2476        {
     2477          BooleanParameterType optionalBooleanType = new BooleanParameterType(null, false);
     2478          ItemContext cc = sc.getCurrentContext(context.getItem());
     2479          Set<Include> include = cc.getInclude();
     2480          // Should we search in MINE, SHARED, IN_PROJECT, etc.
    24252481          parameters.add(new PluginParameter<Boolean>(
    2426               "includeInProject",
    2427               "In current project",
    2428               "Search items that are shared to the current project.",
    2429               include.contains(Include.IN_PROJECT) ? Boolean.TRUE : null,
     2482              "includeMine",
     2483              "Owned by me",
     2484              "Search items that are owned by yourself.",
     2485              include.contains(Include.MINE) ? Boolean.TRUE : null,
    24302486              optionalBooleanType)
    24312487            );
    2432         }
    2433         if (sc.hasPermission(Permission.READ, context.getItem()))
    2434         {
    24352488          parameters.add(new PluginParameter<Boolean>(
    2436               "includeOthers",
    2437               "Owned by others",
    2438               "Search items that are owned by other users.",
    2439               include.contains(Include.OTHERS) ? Boolean.TRUE : null,
     2489              "includeShared",
     2490              "Shared to me",
     2491              "Search items that have been shared to you by other users.",
     2492              include.contains(Include.SHARED) ? Boolean.TRUE : null,
    24402493              optionalBooleanType)
    24412494            );
     2495           
     2496          if (sc.getActiveProjectId() != 0)
     2497          {
     2498            parameters.add(new PluginParameter<Boolean>(
     2499                "includeInProject",
     2500                "In current project",
     2501                "Search items that are shared to the current project.",
     2502                include.contains(Include.IN_PROJECT) ? Boolean.TRUE : null,
     2503                optionalBooleanType)
     2504              );
     2505          }
     2506          if (sc.hasPermission(Permission.READ, context.getItem()))
     2507          {
     2508            parameters.add(new PluginParameter<Boolean>(
     2509                "includeOthers",
     2510                "Owned by others",
     2511                "Search items that are owned by other users.",
     2512                include.contains(Include.OTHERS) ? Boolean.TRUE : null,
     2513                optionalBooleanType)
     2514              );
     2515          }
     2516        }
     2517 
     2518        // Parser regular expressions
     2519        parameters.add(parserSection);
     2520        parameters.add(dataHeaderRegexpParameter);
     2521        parameters.add(dataSplitterRegexpParameter);
     2522        parameters.add(trimQuotesParameter);
     2523        parameters.add(ignoreRegexpParameter);
     2524        parameters.add(dataFooterRegexpParameter);
     2525        parameters.add(minDataColumnsParameter);
     2526        parameters.add(maxDataColumnsParameter);
     2527        parameters.add(Parameters.charsetParameter(null, null, getCharset()));
     2528        parameters.add(Parameters.decimalSeparatorParameter(null, null, getDecimalSeparator()));
     2529        parameters.add(Parameters.dateFormatParameter(null, null,
     2530          sc.getUserClientSetting("formatter.date.formatstring")));
     2531        addMoreParsingParameters(parameters);
     2532       
     2533        // Column mappings
     2534        parameters.add(mappingSection);
     2535        parameters.add(complexMappings);
     2536        addMoreColumnMappingParameters(parameters);
     2537       
     2538        if (forJob)
     2539        {
     2540          configureParser = new RequestInformation
     2541          (
     2542            Request.COMMAND_CONFIGURE_JOB,
     2543            "Select a file from which to import item data",
     2544            "Here you select which file to import data from, and the " +
     2545            "regular expressions and other settings used to parse it. Step 1 of 2.",
     2546            parameters
     2547          );
     2548        }
     2549        else
     2550        {
     2551          configureParser = new RequestInformation
     2552          (
     2553            Request.COMMAND_CONFIGURE_PLUGIN,
     2554            "Specify parser regular expressions and settings",
     2555            "This plug-in can be configured with the regular expressions and " +
     2556            "settings used to parse files.",
     2557            parameters
     2558          );       
    24422559        }
    24432560      }
    2444 
    2445       // Parser regular expressions
    2446       parameters.add(parserSection);
    2447       parameters.add(dataHeaderRegexpParameter);
    2448       parameters.add(dataSplitterRegexpParameter);
    2449       parameters.add(trimQuotesParameter);
    2450       parameters.add(ignoreRegexpParameter);
    2451       parameters.add(dataFooterRegexpParameter);
    2452       parameters.add(minDataColumnsParameter);
    2453       parameters.add(maxDataColumnsParameter);
    2454       parameters.add(Parameters.charsetParameter(null, null, getCharset()));
    2455       parameters.add(Parameters.decimalSeparatorParameter(null, null, getDecimalSeparator()));
    2456       parameters.add(Parameters.dateFormatParameter(null, null,
    2457         sc.getUserClientSetting("formatter.date.formatstring")));
    2458       addMoreParsingParameters(parameters);
    2459      
    2460       // Column mappings
    2461       parameters.add(mappingSection);
    2462       parameters.add(complexMappings);
    2463       addMoreColumnMappingParameters(parameters);
    2464      
    2465       if (forJob)
     2561      finally
    24662562      {
    2467         configureParser = new RequestInformation
    2468         (
    2469           Request.COMMAND_CONFIGURE_JOB,
    2470           "Select a file from which to import item data",
    2471           "Here you select which file to import data from, and the " +
    2472           "regular expressions and other settings used to parse it. Step 1 of 2.",
    2473           parameters
    2474         );
    2475       }
    2476       else
    2477       {
    2478         configureParser = new RequestInformation
    2479         (
    2480           Request.COMMAND_CONFIGURE_PLUGIN,
    2481           "Specify parser regular expressions and settings",
    2482           "This plug-in can be configured with the regular expressions and " +
    2483           "settings used to parse files.",
    2484           parameters
    2485         );       
     2563        if (dc != null) dc.close();
    24862564      }
    24872565    }

  • trunk/src/plugins/core/net/sf/basedb/plugins/batchimport/BioSourceImporter.java

    r7330 r7588  
    9797 
    9898  /**
     99    Enable annotations as item identifier.
     100  */
     101  @Override
     102  protected Item getItemForAnnotationTypes()
     103  {
     104    return Item.BIOSOURCE;
     105  }
     106 
     107  /**
    99108    Adds column mappings for name, externalId and description.
    100109  */
    … …  
    106115    parameters.add(subtypeColumnMapping);
    107116    parameters.add(externalIdColumnMapping);
     117    parameters.add(idAnnotationColumnMapping);
    108118    parameters.add(descriptionColumnMapping);
    109119    parameters.add(registeredColumnMapping);

  • trunk/src/plugins/core/net/sf/basedb/plugins/batchimport/SampleImporter.java

    r7330 r7588  
    227227    return Item.SAMPLE;
    228228  }
     229 
     230  /**
     231    Enable annotations as item identifier.
     232  */
     233  @Override
     234  protected Item getItemForAnnotationTypes()
     235  {
     236    return Item.SAMPLE;
     237  }
    229238
    230239  /**
    … …  
    240249    parameters.add(subtypeColumnMapping);
    241250    parameters.add(externalIdColumnMapping);
     251    parameters.add(idAnnotationColumnMapping);
    242252    parameters.add(descriptionColumnMapping);
    243253    parameters.add(protocolColumnMapping);
Note: See TracChangeset for help on using the changeset viewer.