Changeset 3596


Ignore:
Timestamp:
Jul 24, 2007, 2:14:09 PM (14 years ago)
Author:
Nicklas Nordborg
Message:

Fixes #654: Batch deletion of reporters

Location:
trunk
Files:
6 edited

Legend:

Unmodified
Added
Removed
  • trunk/doc/src/docbook/userdoc/reporters.xml

    r3487 r3596  
    322322     
    323323      <para>
    324         Reporters which has been referenced to from raw data, array
     324        Reporters which has been referenced to from reporter lists, raw data, array
    325325        designs, plates or any other item cannot be deleted.
    326326      </para>
     327     
     328      <sect3 id="reporter.delete.batch">
     329        <title>Batch deletion</title>
     330        <para>
     331          A common problem is to delete reporters that has been accidentaly
     332          created. The regular web interface is usually no good since it
     333          only allows you to select at most 99 reporters at a time. To solve
     334          this problem the reporter import plug-in can be used in delete mode.
     335          You can use the same file as you used when importing. Just select
     336          the <userinput>delete</userinput> option for the <guilabel>mode</guilabel>
     337          parameter in the configuration wizard and continue as usual.
     338          If the plug-in is used in delete mode from a reporter list it will
     339          only remove the reporters from the reporter list. The reporters are not
     340          deleted from the database.
     341        </para>
     342       
     343        <note>
     344          <para>
     345          It may be a bit confusing to delete things from an import plug-in. But
     346          since plug-ins can only belong to one category and we wanted to re-use
     347          existing file format definitions this was our only option.
     348          </para>
     349        </note>
     350      </sect3>
     351     
    327352    </sect2>
    328353
  • trunk/src/core/net/sf/basedb/core/Install.java

    r3594 r3596  
    19231923      {
    19241924        int version = schemaVersion.getSchemaVersion();
    1925         pd.loadPluginInformation(jarPath, className, version < 20);
     1925        pd.loadPluginInformation(jarPath, className, true);
    19261926        if (version < 21) pd.setAllowImmediateExecution(allowImmediateExecution);
    19271927        dc.commit();
  • trunk/src/core/net/sf/basedb/core/ItemInUseException.java

    r2304 r3596  
    4848    super("The item "+what+" is used by another item.");
    4949  }
     50 
     51  /**
     52    Creates a new <code>ItemInUseException</code> with a cause. The error
     53    message produced will look like:
     54    <code>Permission denied. The item User[ID=325] is used by another item.</code>
     55 
     56    @param what A description of what already exists, for
     57      example User[Id=325]
     58    @param cause The cause of the error
     59    @since 2.4
     60  */
     61  public ItemInUseException(String what, Throwable cause)
     62  {
     63    super("The item "+what+" is used by another item.", cause);
     64  }
     65 
    5066}
  • trunk/src/plugins/core/net/sf/basedb/plugins/ReporterFlatFileImporter.java

    r3495 r3596  
    2525
    2626import net.sf.basedb.core.FileType;
     27import net.sf.basedb.core.HibernateUtil;
    2728import net.sf.basedb.core.InvalidUseOfNullException;
    2829import net.sf.basedb.core.ItemContext;
     30import net.sf.basedb.core.ItemInUseException;
    2931import net.sf.basedb.core.Permission;
    3032import net.sf.basedb.core.ReporterList;
     
    3739import net.sf.basedb.core.ItemParameterType;
    3840import net.sf.basedb.core.StringParameterType;
    39 import net.sf.basedb.core.BooleanParameterType;
    4041import net.sf.basedb.core.BaseException;
    4142import net.sf.basedb.core.ItemNotFoundException;
     
    6162import net.sf.basedb.core.plugin.GuiContext;
    6263
     64import net.sf.basedb.util.error.SimpleErrorHandler;
    6365import net.sf.basedb.util.parser.ConfigureByExample;
    6466import net.sf.basedb.util.parser.ConstantMapper;
     
    7678import java.util.HashMap;
    7779import java.util.Set;
     80import java.util.TreeSet;
    7881
    7982/**
     
    112115  private static final StringParameterType requiredColumnMapping = new StringParameterType(255, null, true);
    113116  private static final StringParameterType optionalColumnMapping = new StringParameterType(255, null, false);
    114   private static final BooleanParameterType updateExistingType = new BooleanParameterType(false, true);
    115117 
    116118  private static final PluginParameter<String> nameColumnMapping = new PluginParameter<String>(
     
    161163    );
    162164
    163   private static final PluginParameter<Boolean> updateExistingParameter = new PluginParameter<Boolean>(
    164     "updateExisting",
    165     "Update existing reporters",
    166     "If this option is selected, already existing reporters will be updated with the information "+
    167     "in the file. If this option isn't selected existing reporters are left untouched.",
    168     updateExistingType
    169     );
     165  protected static final PluginParameter<String> reporterIsUsedErrorParameter = new PluginParameter<String>(
     166      "reporterIsUsedError",
     167      "Reporter is used",
     168      "How to handle errors that are caused by a reporter beeing used by other items" +
     169      "and can't be deleted. This option is used in DELETE mode only. " +
     170      "If not specified the default error handling is used.\n\n" +
     171      "skip = Skip the current data line and continue\n"+
     172      "fail = Stop with an error message",
     173      new StringParameterType(255, "skip", false, 1, 0, 0,
     174          Arrays.asList( new String[] { "skip", "fail"} ))
     175      ); 
    170176 
    171177  private PluginParameter<ReporterType> reporterTypeParameter;
     
    204210  }
    205211  /**
    206     Request create and write access to Reporter:s, write access to ReporterList:s
     212    Request create, write and delete access to Reporter:s, write access to ReporterList:s
    207213    and read access to File:s and ReporterType:s.
    208214  */
     
    211217    if (permissions.size() == 0)
    212218    {
    213       permissions.add(new Permissions(Item.REPORTER, null, EnumSet.of(Permission.CREATE, Permission.WRITE)));
     219      permissions.add(new Permissions(Item.REPORTER, null, EnumSet.of(Permission.CREATE, Permission.DELETE)));
    214220      permissions.add(new Permissions(Item.REPORTERLIST, null, EnumSet.of(Permission.WRITE)));
    215221      permissions.add(new Permissions(Item.REPORTERTYPE, EnumSet.of(Permission.CREATE, Permission.USE), null));
     
    227233    Return a set containing the context:s [REPORTER, LIST],
    228234    [REPORTERLIST, ITEM], [REPORTERSCORE, LIST]. The first context
    229     is for reporter reporters only, and the last two for importing
     235    is for importing reporters only, and the last two for importing
    230236    reporters to a reporter list.
    231237  */
     
    346352        storeValue(job, request, ri.getParameter(CHARSET));
    347353        storeValue(job, request, ri.getParameter(DECIMAL_SEPARATOR));
    348         storeValue(job, request, updateExistingParameter);
     354        storeValue(job, request, ri.getParameter("mode"));
    349355       
    350356        // Error handling parameters
     
    354360        storeValue(job, request, numberFormatErrorParameter);
    355361        storeValue(job, request, numberOutOfRangeErrorParameter);
     362        storeValue(job, request, reporterIsUsedErrorParameter);
    356363       
    357364        response.setDone("Job configuration complete", Job.ExecutionTime.SHORT);
     
    381388  private ReporterList reporterList;
    382389  private boolean updateExisting;
     390  private boolean deleteMode;
    383391  private int numInserted;
    384392  private int numUpdated;
    385393  private int numExists;
    386394  private int numAddedToList;
     395  private int numNotFound;
     396  private int numUsed;
     397  private int numDeleted;
    387398  private Map<String, Float> deferred;
     399  private Set<String> deleted;
    388400  private FlatFileParser ffp;
    389401  private NumberFormat numberFormat;
    390402  private boolean nullIfException;
     403  private boolean failIfUsed;
    391404 
    392405  /**
     
    410423      numAddedToList = 0;
    411424    }
    412     updateExisting = (Boolean)job.getValue("updateExisting");
     425    String mode = (String)job.getValue("mode");
     426    updateExisting = "update".equals(mode);
     427    deleteMode = "delete".equals(mode);
    413428    this.ffp = ffp;
    414429    this.numberFormat = ffp.getDefaultNumberFormat();
    415430    this.nullIfException = "null".equals(job.getValue("numberFormatError"));
     431    String reporterIsUsedError = (String)getErrorOption("reporterIsUsedError");
     432    this.failIfUsed = "fail".equals(reporterIsUsedError);
     433    if (reporterIsUsedError != null)
     434    {
     435      addErrorHandler(ItemInUseException.class, new SimpleErrorHandler(!failIfUsed));
     436    }
     437    if (deleteMode)
     438    {
     439      // Keep track of already deleted reporters
     440      deleted = HibernateUtil.getDbEngine().caseInsensitiveComparison() ?
     441        new TreeSet<String>(String.CASE_INSENSITIVE_ORDER) : new HashSet<String>();
     442    }
    416443    numInserted = 0;
    417444    numUpdated = 0;
     
    430457    idMapper = getMapper(ffp, (String)configuration.getValue("reporterIdColumnMapping"),
    431458      cropStrings ? ReporterData.MAX_EXTERNAL_ID_LENGTH : null, nullMapper);
    432     nameMapper = getMapper(ffp, (String)configuration.getValue("nameColumnMapping"),
    433       cropStrings ? ReporterData.MAX_NAME_LENGTH : null, null);
    434     symbolMapper = getMapper(ffp, (String)configuration.getValue("symbolColumnMapping"),
    435       cropStrings ? ReporterData.MAX_SYMBOL_LENGTH : null, null);
    436     descriptionMapper = getMapper(ffp, (String)configuration.getValue("descriptionColumnMapping"),
    437       cropStrings ? ReporterData.MAX_DESCRIPTION_LENGTH : null, null);
    438     reporterTypeMapper = getMapper(ffp, (String)configuration.getValue("reporterTypeColumnMapping"),
    439       cropStrings ? ReporterType.MAX_NAME_LENGTH : null, null);
    440     scoreMapper = getMapper(ffp, (String)configuration.getValue("scoreColumnMapping"), null, null);
    441    
    442     reporterTypes = new HashMap<String, ReporterType>();
    443     extendedMappers = new HashMap<ExtendedProperty, Mapper>();
    444     List<ExtendedProperty> extendedProperties = ExtendedProperties.getProperties("ReporterData");
    445     if (extendedProperties != null)
    446     {
    447       for (ExtendedProperty ep : extendedProperties)
    448       {
    449         String name = "extendedColumnMapping." + ep.getName();
    450         Mapper m = getMapper(ffp, (String)configuration.getValue(name),
    451           cropStrings && ep.getLength() > 0 ? ep.getLength() : null, null);
    452         if (m != null) extendedMappers.put(ep, m);
     459    if (!deleteMode)
     460    {
     461      nameMapper = getMapper(ffp, (String)configuration.getValue("nameColumnMapping"),
     462        cropStrings ? ReporterData.MAX_NAME_LENGTH : null, null);
     463      symbolMapper = getMapper(ffp, (String)configuration.getValue("symbolColumnMapping"),
     464        cropStrings ? ReporterData.MAX_SYMBOL_LENGTH : null, null);
     465      descriptionMapper = getMapper(ffp, (String)configuration.getValue("descriptionColumnMapping"),
     466        cropStrings ? ReporterData.MAX_DESCRIPTION_LENGTH : null, null);
     467      reporterTypeMapper = getMapper(ffp, (String)configuration.getValue("reporterTypeColumnMapping"),
     468        cropStrings ? ReporterType.MAX_NAME_LENGTH : null, null);
     469      scoreMapper = getMapper(ffp, (String)configuration.getValue("scoreColumnMapping"), null, null);
     470     
     471      reporterTypes = new HashMap<String, ReporterType>();
     472      extendedMappers = new HashMap<ExtendedProperty, Mapper>();
     473      List<ExtendedProperty> extendedProperties = ExtendedProperties.getProperties("ReporterData");
     474      if (extendedProperties != null)
     475      {
     476        for (ExtendedProperty ep : extendedProperties)
     477        {
     478          String name = "extendedColumnMapping." + ep.getName();
     479          Mapper m = getMapper(ffp, (String)configuration.getValue(name),
     480            cropStrings && ep.getLength() > 0 ? ep.getLength() : null, null);
     481          if (m != null) extendedMappers.put(ep, m);
     482        }
    453483      }
    454484    }
     
    504534    ReporterData reporter =  null;
    505535   
    506     // Is the reporter already queued for insert?
    507     if (!batcher.isInInsertQueue(externalId))
    508     {
    509       // No, it's not... try to load it from the database
    510       try
    511       {
    512         reporter = batcher.getByExternalId(externalId);
    513       }
    514       catch (ItemNotFoundException ex)
    515       {
    516         // It wasn't in the database either, create a new reporter
    517         reporter = Reporter.getNew(externalId);
    518       }
    519     }
    520 
    521     // If we have a reporter object, we must set the properties or add it to a reporter list
    522     if (reporter != null)
    523     {
    524       int currentId = reporter.getId();
    525       if (reporterList != null)
    526       {
    527         // Add to reporter list
    528         Float score = scoreMapper == null ?
    529             reporterList.getScore(reporter) : scoreMapper.getFloat(data);
    530 //            (Float)Type.FLOAT.parseString(scoreMapper.getValue(data), numberFormat);
    531         if (currentId == 0)
    532         {
    533           // It is a new reporter, we must wait to add it until the batcher has been flushed
    534           deferred.put(externalId, score);
     536    if (deleteMode)
     537    {
     538      if (deleted.add(externalId))
     539      {
     540        try
     541        {
     542          reporter = batcher.getByExternalId(externalId);
     543        }
     544        catch (ItemNotFoundException ex)
     545        {
     546          numNotFound++;
     547        }
     548        if (reporter != null)
     549        {
     550          if (reporterList != null)
     551          {
     552            reporterList.removeReporter(reporter);
     553            numDeleted++;
     554          }
     555          else
     556          {
     557            try
     558            {
     559              batcher.delete(reporter);
     560              batcher.flushDelete();
     561              numDeleted++;
     562            }
     563            catch (BaseException ex)
     564            {
     565              if (failIfUsed) throw new ItemInUseException("Reporter[externalId=" + externalId + "]", ex);
     566              numUsed++;
     567            }
     568          }
     569        }
     570      }
     571    }
     572    else
     573    {
     574      // Is the reporter already queued for insert?
     575      if (!batcher.isInInsertQueue(externalId))
     576      {
     577        // No, it's not... try to load it from the database
     578        try
     579        {
     580          reporter = batcher.getByExternalId(externalId);
     581        }
     582        catch (ItemNotFoundException ex)
     583        {
     584          // It wasn't in the database either, create a new reporter
     585          reporter = Reporter.getNew(externalId);
     586        }
     587      }
     588 
     589      // If we have a reporter object, we must set the properties or add it to a reporter list
     590      if (reporter != null)
     591      {
     592        int currentId = reporter.getId();
     593        if (reporterList != null)
     594        {
     595          // Add to reporter list
     596          Float score = scoreMapper == null ?
     597              reporterList.getScore(reporter) : scoreMapper.getFloat(data);
     598  //            (Float)Type.FLOAT.parseString(scoreMapper.getValue(data), numberFormat);
     599          if (currentId == 0)
     600          {
     601            // It is a new reporter, we must wait to add it until the batcher has been flushed
     602            deferred.put(externalId, score);
     603          }
     604          else
     605          {
     606            reporterList.addReporter(reporter, score);
     607            numAddedToList++;
     608          }
     609        }
     610       
     611        // The actual reporter needs updating or it is a new one
     612        if ((updateExisting && currentId != 0) || currentId == 0)
     613        {
     614         
     615          if (reporterType != null)
     616          {
     617            Reporter.setReporterType(reporter, reporterType);
     618          }
     619          else if (reporterTypeMapper != null)
     620          {
     621            Reporter.setReporterType(reporter, getReporterType(reporterTypeMapper.getValue(data)));
     622          }
     623          String name = nameMapper == null ? reporter.getName() : nameMapper.getValue(data);
     624          reporter.setName(name == null ? externalId : name);
     625         
     626          if (symbolMapper != null) reporter.setSymbol(symbolMapper.getValue(data));
     627          if (descriptionMapper != null) reporter.setDescription(descriptionMapper.getValue(data));
     628          for (Map.Entry<ExtendedProperty, Mapper> entry : extendedMappers.entrySet())
     629          {
     630            ExtendedProperty ep = entry.getKey();
     631            Mapper m = entry.getValue();
     632            reporter.setExtended(ep.getName(), ep.parseString(m.getValue(data),
     633              numberFormat, nullIfException));
     634          }
     635          if (currentId != 0)
     636          {
     637            batcher.update(reporter);
     638            numUpdated++;
     639          }
     640          else
     641          {
     642            batcher.insert(reporter);
     643            numInserted++;
     644          }
    535645        }
    536646        else
    537647        {
    538           reporterList.addReporter(reporter, score);
    539           numAddedToList++;
    540         }
    541       }
    542      
    543       // The actual reporter needs updating or it is a new one
    544       if ((updateExisting && currentId != 0) || currentId == 0)
    545       {
    546        
    547         if (reporterType != null)
    548         {
    549           Reporter.setReporterType(reporter, reporterType);
    550         }
    551         else if (reporterTypeMapper != null)
    552         {
    553           Reporter.setReporterType(reporter, getReporterType(reporterTypeMapper.getValue(data)));
    554         }
    555         String name = nameMapper == null ? reporter.getName() : nameMapper.getValue(data);
    556         reporter.setName(name == null ? externalId : name);
    557        
    558         if (symbolMapper != null) reporter.setSymbol(symbolMapper.getValue(data));
    559         if (descriptionMapper != null) reporter.setDescription(descriptionMapper.getValue(data));
    560         for (Map.Entry<ExtendedProperty, Mapper> entry : extendedMappers.entrySet())
    561         {
    562           ExtendedProperty ep = entry.getKey();
    563           Mapper m = entry.getValue();
    564           reporter.setExtended(ep.getName(), ep.parseString(m.getValue(data),
    565             numberFormat, nullIfException));
    566         }
    567         if (currentId != 0)
    568         {
    569           batcher.update(reporter);
    570           numUpdated++;
    571         }
    572         else
    573         {
    574           batcher.insert(reporter);
    575           numInserted++;
    576         }
    577       }
    578       else
    579       {
    580         numExists++;
     648          numExists++;
     649        }
    581650      }
    582651    }
     
    587656  protected String getSuccessMessage(int skippedLines)
    588657  {
    589     String msg = numInserted + " new reporter(s)";
    590     if (numUpdated > 0) msg += "; " + numUpdated + " updated reporter(s)";
    591     if (numExists > 0) msg += "; " + numExists + " reporter(s) skipped (already existed)"; 
    592     if (reporterList != null) msg += "; " + numAddedToList + " reporter(s) added to list";
     658    String msg;
     659    if (deleteMode)
     660    {
     661      msg = numDeleted + " deleted reporter(s)";
     662      if (numUsed > 0) msg += "; " + numUsed + " reporter(s) are used and could not be deleted";
     663      if (numNotFound > 0) msg += "; " + numNotFound + " reporter(s) ignored because they didn't exist";
     664    }
     665    else
     666    {
     667      msg = numInserted + " new reporter(s)";
     668      if (numUpdated > 0) msg += "; " + numUpdated + " updated reporter(s)";
     669      if (numExists > 0) msg += "; " + numExists + " reporter(s) skipped (already existed)"; 
     670      if (reporterList != null) msg += "; " + numAddedToList + " reporter(s) added to list";
     671    }
    593672    if (skippedLines > 0) msg += "; " + skippedLines + " line(s) skipped due to errors";
    594673    return msg;
     
    645724      }
    646725      parameters.add(fileParameter);
    647       parameters.add(updateExistingParameter);
     726     
     727      List<String> allowedModes = new ArrayList<String>();
     728      String defaultMode = "update";
     729      allowedModes.add("create");
     730      allowedModes.add("update");
     731      if (context.getItem() != Item.REPORTER || sc.hasPermission(Permission.DELETE, Item.REPORTER))
     732      {
     733        allowedModes.add("delete");
     734      }
     735      StringParameterType modeType = new StringParameterType(255, defaultMode, false, 1, 0, 0, allowedModes);
     736      PluginParameter<String> modeParameter = new PluginParameter<String>(
     737          "mode",
     738          "Mode",
     739          "Select the operating mode of the plug-in. Depending on permissions not all modes " +
     740          "may be available\n\n" +
     741          "create = Only create missing reporters; existing ones are not updated\n" +
     742          "update = Update existing reporters and create missing ones\n" +
     743          "delete = Delete existing reporters",
     744          modeType
     745          );
     746     
     747      parameters.add(modeParameter);
    648748      parameters.add(getCharsetParameter(null, null, (String)configuration.getValue(CHARSET)));
    649749      parameters.add(getDecimalSeparatorParameter(null, null, (String)configuration.getValue(DECIMAL_SEPARATOR)));
     
    655755      parameters.add(numberFormatErrorParameter);
    656756      parameters.add(numberOutOfRangeErrorParameter);
     757      parameters.add(reporterIsUsedErrorParameter);
    657758     
    658759      configureJob = new RequestInformation
  • trunk/src/test/TestReporterFlatFileImporter.java

    r2959 r3596  
    1313import net.sf.basedb.core.ReporterType;
    1414import net.sf.basedb.core.RequestInformation;
     15import net.sf.basedb.core.plugin.GuiContext;
    1516import net.sf.basedb.core.plugin.Response;
    1617
     
    6263    int pluginConfigurationId = test_create_configuration(pluginDefinitionId, reporterTypeId);
    6364    TestPluginConfiguration.test_load(pluginConfigurationId);
    64     int jobId = test_create_job(pluginConfigurationId, fileId);
     65    int jobId = test_create_job(pluginConfigurationId, fileId, "update");
    6566   
    6667    // Execute job
     
    7273   
    7374    if (TestUtil.waitBeforeDelete()) TestUtil.waitForEnter();
    74     TestReporter.test_delete();
     75    int jobId2 = test_create_job(pluginConfigurationId, fileId, "delete");
     76    ok &= TestJob.test_execute(jobId2);
     77   
     78//    TestReporter.test_delete();
    7579
    7680    TestJob.test_delete(jobId);
     81    TestJob.test_delete(jobId2);
    7782    TestPluginConfiguration.test_delete(pluginConfigurationId);
    7883    TestFile.test_delete(fileId);
     
    151156  }
    152157
    153   static int test_create_job(int pluginConfigurationId, int fileId)
     158  static int test_create_job(int pluginConfigurationId, int fileId, String mode)
    154159  {
    155160    if (pluginConfigurationId == 0 || fileId == 0 || !TestUtil.hasPermission(Permission.CREATE, Item.JOB)) return 0;
     
    164169      j.setName(pc.getName());
    165170     
    166       PluginConfigurationRequest request = j.configure(null);
     171      PluginConfigurationRequest request = j.configure(new GuiContext(Item.REPORTER, GuiContext.Type.LIST));
    167172      write_request_information(request.getRequestInformation());
    168173      request.setParameterValue("file", File.getById(dc, fileId));
    169       request.setParameterValue("updateExisting", true);
     174      request.setParameterValue("mode", mode);
    170175      request.setParameterValue("invalidUseOfNullError", "skip");
    171176      PluginResponse response = request.invoke();
     
    181186      }
    182187      id = j.getId();
    183       write("--Create job for reporter import OK");
     188      write("--Create job for reporter import (mode="+mode+") OK");
    184189    }
    185190    catch (Throwable ex)
    186191    {
    187       write("--Create job for reporter import FAILED");
     192      write("--Create job for reporter import (mode="+mode+") FAILED");
    188193      ex.printStackTrace();
    189194      ok = false;
  • trunk/src/test/TestReporterMapFlatFileImporter.java

    r2959 r3596  
    6565    int reporterPluginDefinitionId  = TestPluginDefinition.test_get("net.sf.basedb.plugins.ReporterFlatFileImporter");
    6666    int reporterPluginConfigurationId = test_create_configuration_reporter(reporterPluginDefinitionId, reporterTypeId);
    67     int jobId = TestReporterFlatFileImporter.test_create_job(reporterPluginConfigurationId, fileId);
     67    int jobId = TestReporterFlatFileImporter.test_create_job(reporterPluginConfigurationId, fileId, "create");
    6868   
    6969    // Execute job
Note: See TracChangeset for help on using the changeset viewer.