Changeset 4372


Ignore:
Timestamp:
Jul 4, 2008, 11:22:35 AM (15 years ago)
Author:
Nicklas Nordborg
Message:

References #1028: Plug-in for batch upload/modification of items

Added importers for extract and labeled extract.

Location:
trunk/src
Files:
5 added
10 edited

Legend:

Unmodified
Added
Removed

  • trunk/src/core/net/sf/basedb/core/Install.java

    r4366 r4372  
    642642      createPluginDefinition("net.sf.basedb.plugins.batchimport.BioSourceImporter", null, keyEveryoneUse, true, null, false);
    643643      createPluginDefinition("net.sf.basedb.plugins.batchimport.SampleImporter", null, keyEveryoneUse, true, null, false);
    644      
     644      createPluginDefinition("net.sf.basedb.plugins.batchimport.ExtractImporter", null, keyEveryoneUse, true, null, false);
     645      createPluginDefinition("net.sf.basedb.plugins.batchimport.LabeledExtractImporter", null, keyEveryoneUse, true, null, false);
     646
    645647     
    646648      // Plugin configurations

  • trunk/src/plugins/core/net/sf/basedb/plugins/batchimport/AbstractItemImporter.java

    r4366 r4372  
    4141import net.sf.basedb.core.BooleanParameterType;
    4242import net.sf.basedb.core.DbControl;
     43import net.sf.basedb.core.Extract;
    4344import net.sf.basedb.core.File;
     45import net.sf.basedb.core.Hardware;
     46import net.sf.basedb.core.HardwareType;
    4447import net.sf.basedb.core.Include;
    4548import net.sf.basedb.core.IntegerUtil;
    … …  
    5053import net.sf.basedb.core.ItemQuery;
    5154import net.sf.basedb.core.Job;
     55import net.sf.basedb.core.Label;
     56import net.sf.basedb.core.LabeledExtract;
    5257import net.sf.basedb.core.NumberOutOfRangeException;
    5358import net.sf.basedb.core.Permission;
    … …  
    5964import net.sf.basedb.core.StringTooLongException;
    6065import net.sf.basedb.core.StringUtil;
     66import net.sf.basedb.core.SystemItems;
    6167import net.sf.basedb.core.plugin.GuiContext;
    6268import net.sf.basedb.core.plugin.InteractivePlugin;
    … …  
    6571import net.sf.basedb.core.plugin.Request;
    6672import net.sf.basedb.core.plugin.Response;
     73import net.sf.basedb.core.query.Expressions;
     74import net.sf.basedb.core.query.Hql;
     75import net.sf.basedb.core.query.Restrictions;
    6776import net.sf.basedb.plugins.AbstractFlatFileImporter;
    6877import net.sf.basedb.plugins.util.Parameters;
    … …  
    393402  private String lastIdentifier;
    394403  private int numMultiLines;
     404  private boolean skippedLast;
    395405 
    396406  // Error handling
    … …  
    483493    if (Boolean.TRUE.equals((Boolean)job.getValue("includeOthers"))) includes.add(Include.OTHERS);
    484494    itemQuery = idMethod.prepareQuery(dc, createItemQuery());
     495    itemQuery.setItemPermission(Permission.WRITE);
    485496    itemQuery.include(includes);
    486497  }
    … …  
    498509    lastIdentifier = identifier;
    499510
     511    if (sameAsLast && skippedLast)
     512    {
     513      // If the first line in a multi-line entry was skipped, skip the remaining lines also
     514      log("Skipped: " + idMethod + "=" + identifier, data);
     515      return;
     516    }
     517   
    500518    I item = null;
    501519    if (identifier != null)
    … …  
    545563          itemCache.put(cacheKey, item);
    546564          log("Created: " + idMethod + "=" + identifier, data);
     565          skippedLast = false;
    547566          numCreated++;
    548567        }
    … …  
    553572            errorHandler.handleError(t);
    554573            log("Skipped: " + idMethod + "=" + identifier, data, t);
     574            skippedLast = true;
    555575            numError++;
    556576            // The error should be ignored if we get passed the above line
    … …  
    567587      {
    568588        log("Not found: " + idMethod + "=" + identifier, data);
     589        skippedLast = true;
    569590        numNotFound++;
    570591      }
    … …  
    586607        }
    587608        log("Updated: " + idMethod + "=" + identifier, data);
     609        skippedLast = false;
    588610      }
    589611      catch (Throwable t)
    … …  
    593615          errorHandler.handleError(t);
    594616          log("Skipped: " + idMethod + "=" + identifier, data);
     617          skippedLast = true;
    595618          numError++;
    596619          // The error should be ignored if we get passed the above line
    … …  
    798821    @param dc The DbControl to use for database access
    799822    @param identifier The identifier protocol
    800     @return A protocol, or null if no protocol could be found
     823    @return A protocol, or null if no item could be found
    801824  */
    802825  protected Protocol findProtocol(DbControl dc, IdMethod idMetod, String identifier)
    … …  
    807830  }
    808831
     832  private ItemQuery<Label> labelQuery;
     833  /**
     834    Find a label with a given identifier. This is a utility method that
     835    subclasses can use when creating or updating items.
     836    <p>
     837    NOTE! The first time this method is called a query object is initialised
     838    using the {@link IdMethod#prepareQuery(DbControl, ItemQuery)} method.
     839    Subsequent calls uses the same query. Thus, this method should always be
     840    called with the same id method object, otherwise the result is undefined.
     841   
     842    @param dc The DbControl to use for database access
     843    @param identifier The identifier protocol
     844    @return A label, or null if no item could be found
     845  */
     846  protected Label findLabel(DbControl dc, IdMethod idMetod, String identifier)
     847  {
     848    if (identifier == null) return null;
     849    if (labelQuery == null) labelQuery = initReferenceQuery(dc, idMethod, Label.getQuery());
     850    return findReferencedItem(dc, idMethod, labelQuery, identifier);
     851  }
     852
     853  private ItemQuery<Hardware> scannerQuery;
     854  /**
     855    Find a scanner with a given identifier. This is a utility method that
     856    subclasses can use when creating or updating items.
     857   
     858    <p>
     859    NOTE! A scanner is a {@link Hardware} item with it type set to
     860    {@link HardwareType#SCANNER}.
     861   
     862    <p>
     863    NOTE! The first time this method is called a query object is initialised
     864    using the {@link IdMethod#prepareQuery(DbControl, ItemQuery)} method.
     865    Subsequent calls uses the same query. Thus, this method should always be
     866    called with the same id method object, otherwise the result is undefined.
     867   
     868    @param dc The DbControl to use for database access
     869    @param identifier The identifier protocol
     870    @return A scanner, or null if no item could be found
     871  */
     872  protected Hardware findScanner(DbControl dc, IdMethod idMetod, String identifier)
     873  {
     874    if (identifier == null) return null;
     875    if (scannerQuery == null)
     876    {
     877      scannerQuery = initReferenceQuery(dc, idMethod, Hardware.getQuery());
     878      scannerQuery.restrictPermanent(
     879        Restrictions.eq(Hql.property("hardwareType"),
     880        Expressions.integer(SystemItems.getId(HardwareType.SCANNER)))
     881      );
     882    }
     883    return findReferencedItem(dc, idMethod, scannerQuery, identifier);
     884  }
     885 
    809886  private ItemQuery<BioSource> bioSourceQuery;
    810887  /**
    … …  
    849926  }
    850927
     928  private ItemQuery<Extract> extractQuery;
     929  /**
     930    Find an extract with a given identifier. This is a utility method that
     931    subclasses can use when creating or updating items.
     932    <p>
     933    NOTE! The first time this method is called a query object is initialised
     934    using the {@link IdMethod#prepareQuery(DbControl, ItemQuery)} method.
     935    Subsequent calls uses the same query. Thus, this method should always be
     936    called with the same id method object, otherwise the result is undefined.
     937   
     938    @param dc The DbControl to use for database access
     939    @param identifier The identifier protocol
     940    @return An extract, or null if no item could be found
     941  */
     942  protected Extract findExtract(DbControl dc, IdMethod idMetod, String identifier)
     943  {
     944    if (identifier == null) return null;
     945    if (extractQuery == null) extractQuery = initReferenceQuery(dc, idMethod, Extract.getQuery());
     946    return findReferencedItem(dc, idMethod, extractQuery, identifier);
     947  }
     948
     949  private ItemQuery<LabeledExtract> labeledExtractQuery;
     950  /**
     951    Find a labled extract with a given identifier. This is a utility method that
     952    subclasses can use when creating or updating items.
     953    <p>
     954    NOTE! The first time this method is called a query object is initialised
     955    using the {@link IdMethod#prepareQuery(DbControl, ItemQuery)} method.
     956    Subsequent calls uses the same query. Thus, this method should always be
     957    called with the same id method object, otherwise the result is undefined.
     958   
     959    @param dc The DbControl to use for database access
     960    @param identifier The identifier protocol
     961    @return A labeled extract, or null if no item could be found
     962  */
     963  protected LabeledExtract findLabeledExtract(DbControl dc, IdMethod idMetod, String identifier)
     964  {
     965    if (identifier == null) return null;
     966    if (labeledExtractQuery == null)
     967    {
     968      labeledExtractQuery = initReferenceQuery(dc, idMethod, LabeledExtract.getQuery());
     969    }
     970    return findReferencedItem(dc, idMethod, labeledExtractQuery, identifier);
     971  }
     972
    851973  /**
    852974    Cache that holds loaded/created items. The cache key is item type + identifier,
    … …  
    860982    Initialise a query that is used to find referenced items. This method
    861983    delegates to {@link IdMethod#prepareQuery(DbControl, ItemQuery)} and
    862     the adds {@link Include} options: MINE, IN_PROJECT, SHARED and OTHERS.
     984    then adds {@link Include} options: MINE, IN_PROJECT, SHARED and OTHERS.
     985    The query will only search for items that the logged in user has
     986    USE permission for.
    863987   
    864988    @param dc A DbControl used for database access
    … …  
    871995    query = idMethod.prepareQuery(dc, query);
    872996    query.include(Include.MINE, Include.IN_PROJECT, Include.SHARED, Include.OTHERS);
     997    query.setItemPermission(Permission.USE);
    873998    return query;
    874999  }

  • trunk/src/plugins/core/net/sf/basedb/plugins/batchimport/BioSourceImporter.java

    r4366 r4372  
    108108    <li>{@link PropertyIdMethod#NAME}
    109109    <li>{@link PropertyIdMethod#EXTERNAL_ID}
    110     <li>{@link PropertyIdMethod#INTERNAL_ID}
     110    <li>{@link InternalIdMethod#INTERNAL_ID}
    111111    <li>{@link FallbackIdMethod#NAME_OR_ID}
    112112    </ul>
    … …  
    116116  {
    117117    return new IdMethod[] { PropertyIdMethod.NAME, PropertyIdMethod.EXTERNAL_ID,
    118         PropertyIdMethod.INTERNAL_ID, FallbackIdMethod.NAME_OR_ID};
     118        InternalIdMethod.INTERNAL_ID, FallbackIdMethod.NAME_OR_ID};
    119119  }
    120120 

  • trunk/src/plugins/core/net/sf/basedb/plugins/batchimport/IdMethod.java

    r4366 r4372  
    6565    other modifications.
    6666    @param query The query to prepare
     67    @return The query that is used for item lookup
    6768  */
    6869  public <I extends BasicItem> ItemQuery<I> prepareQuery(DbControl dc, ItemQuery<I> query);

  • trunk/src/plugins/core/net/sf/basedb/plugins/batchimport/PropertyIdMethod.java

    r4366 r4372  
    4747
    4848  /**
    49     Use the internal ID for finding items.
    50   */
    51   public static final PropertyIdMethod INTERNAL_ID =
    52     new PropertyIdMethod("internalId", "Internal ID", "id", true, Type.INT);
    53  
    54   /**
    5549    Use the name for finding items.
    5650  */
    … …  
    7165
    7266  public static final IdMethod[] DEFAULT_METHODS =
    73     new IdMethod[] { NAME, INTERNAL_ID, FallbackIdMethod.NAME_OR_ID };
     67    new IdMethod[] { NAME, InternalIdMethod.INTERNAL_ID, FallbackIdMethod.NAME_OR_ID };
    7468
    7569  private final String method;

  • trunk/src/plugins/core/net/sf/basedb/plugins/batchimport/SampleImporter.java

    r4366 r4372  
    180180    <li>{@link PropertyIdMethod#NAME}
    181181    <li>{@link PropertyIdMethod#EXTERNAL_ID}
    182     <li>{@link PropertyIdMethod#INTERNAL_ID}
     182    <li>{@link InternalIdMethod#INTERNAL_ID}
    183183    <li>{@link FallbackIdMethod#NAME_OR_ID}
    184184    </ul>
    … …  
    188188  {
    189189    return new IdMethod[] { PropertyIdMethod.NAME, PropertyIdMethod.EXTERNAL_ID,
    190         PropertyIdMethod.INTERNAL_ID, FallbackIdMethod.NAME_OR_ID};
     190        InternalIdMethod.INTERNAL_ID, FallbackIdMethod.NAME_OR_ID};
    191191  }
    192192 

  • trunk/src/test/TestAll.java

    r4239 r4372  
    138138    results.put("TestJob", TestJob.test_all());
    139139    results.put("TestJobAgent", TestJobAgent.test_all());
     140    results.put("TestItemImporter", TestItemImporter.test_all());
    140141    results.put("TestReporterFlatFileImporter", TestReporterFlatFileImporter.test_all());
    141142    results.put("TestReporterMapFlatFileImporter", TestReporterMapFlatFileImporter.test_all());

  • trunk/src/test/TestExtract.java

    r3679 r4372  
    2525*/
    2626import net.sf.basedb.core.*;
     27
    2728import java.util.Date;
    2829
    … …  
    234235  }
    235236
     237  static void test_delete_all()
     238  {
     239    DbControl dc = null;
     240    try
     241    {
     242      dc = TestUtil.getDbControl();
     243      ItemResultList<Extract> l = Extract.getQuery().list(dc);
     244      Trashcan.delete(dc.getSessionControl(), l, true, null);
     245      dc.commit();
     246      write("--Delete all extracts OK ("+l.size()+")");
     247    }
     248    catch (Throwable ex)
     249    {
     250      write("--Delete all extracts FAILED");
     251      ex.printStackTrace();
     252      ok = false;
     253    }
     254    finally
     255    {
     256      if (dc != null) dc.close();
     257    }
     258  }
     259
     260 
    236261  static void write_header()
    237262  {

  • trunk/src/test/TestItemImporter.java

    r4366 r4372  
    5858    // Create referenced items
    5959    int samplingProtocolId = TestProtocol.test_create(SystemItems.getId(ProtocolType.SAMPLING), "Sampling", false);
     60    int extractionProtocolId = TestProtocol.test_create(SystemItems.getId(ProtocolType.EXTRACTION), "Extraction", false);
     61    int labelingProtocolId = TestProtocol.test_create(SystemItems.getId(ProtocolType.LABELING), "Labeling", false);
    6062    int poolingProtocolId = TestProtocol.test_create(SystemItems.getId(ProtocolType.POOLING), "Pooling", false);
    6163   
    … …  
    7375    ok &= TestJob.test_execute(sampleJobId);
    7476    TestSample.test_list(4);
     77
     78    // Import extracts
     79    int extractFileId = TestFile.test_create("data/test.batchimport.extracts.txt", false, false);
     80    int extractImporterId  = TestPluginDefinition.test_get("net.sf.basedb.plugins.batchimport.ExtractImporter");
     81    int extractJobId = test_create_item_import_job(Item.EXTRACT, extractImporterId, extractFileId);
     82    ok &= TestJob.test_execute(extractJobId);
     83    TestExtract.test_list(4);
     84
     85    // Import labeled extracts
     86    int labledExtractFileId = TestFile.test_create("data/test.batchimport.labeledextracts.txt", false, false);
     87    int labledExtractImporterId  = TestPluginDefinition.test_get("net.sf.basedb.plugins.batchimport.LabeledExtractImporter");
     88    int labledExtractJobId = test_create_item_import_job(Item.LABELEDEXTRACT, labledExtractImporterId, labledExtractFileId);
     89    ok &= TestJob.test_execute(labledExtractJobId);
     90    TestLabeledExtract.test_list(7);
    7591   
    7692    if (TestUtil.waitBeforeDelete()) TestUtil.waitForEnter();
    7793
     94    // Delete labeled extracts
     95    TestLabeledExtract.test_delete_all();
     96    TestJob.test_delete(labledExtractJobId);
     97    TestFile.test_delete(labledExtractFileId);
     98
     99    // Delete extracts
     100    TestExtract.test_delete_all();
     101    TestJob.test_delete(extractJobId);
     102    TestFile.test_delete(extractFileId);
     103   
    78104    // Delete samples
    79105    TestSample.test_delete_all();
    … …  
    88114    // References items
    89115    TestProtocol.test_delete(samplingProtocolId);
     116    TestProtocol.test_delete(extractionProtocolId);
     117    TestProtocol.test_delete(labelingProtocolId);
    90118    TestProtocol.test_delete(poolingProtocolId);
    91119
    … …  
    180208    request.setParameterValue("idMethod", "name");
    181209    request.setParameterValue("idColumnMapping", "\\Name\\");
    182    
    183210  }
    184211 
    … …  
    189216    request.setParameterValue("externalIdColumnMapping", "\\External ID\\");
    190217   
    191     if (itemType == Item.SAMPLE)
     218    if (itemType == Item.SAMPLE || itemType == Item.EXTRACT || itemType == Item.LABELEDEXTRACT)
    192219    {
    193220      request.setParameterValue("protocolColumnMapping", "\\Protocol\\");
    … …  
    198225      request.setParameterValue("usedQuantityColumnMapping", "\\Used quantity\\");
    199226    }
     227    if (itemType == Item.LABELEDEXTRACT)
     228    {
     229      request.setParameterValue("labelColumnMapping", "\\Label\\");
     230    }
    200231  }
    201232 

  • trunk/src/test/TestLabeledExtract.java

    r3679 r4372  
    2424*/
    2525import net.sf.basedb.core.*;
     26
    2627import java.util.Date;
    2728
    … …  
    237238    }
    238239  }
     240 
     241  static void test_delete_all()
     242  {
     243    DbControl dc = null;
     244    try
     245    {
     246      dc = TestUtil.getDbControl();
     247      ItemResultList<LabeledExtract> l = LabeledExtract.getQuery().list(dc);
     248      Trashcan.delete(dc.getSessionControl(), l, true, null);
     249      dc.commit();
     250      write("--Delete all labeled extracts OK ("+l.size()+")");
     251    }
     252    catch (Throwable ex)
     253    {
     254      write("--Delete all labeled extracts FAILED");
     255      ex.printStackTrace();
     256      ok = false;
     257    }
     258    finally
     259    {
     260      if (dc != null) dc.close();
     261    }
     262  }
    239263
    240264  static void write_header()
Note: See TracChangeset for help on using the changeset viewer.