Changeset 2154


Ignore:
Timestamp:
Apr 7, 2006, 11:35:26 AM (17 years ago)
Author:
Nicklas Nordborg
Message:

Fixes #137: Write documentation 5b) Plug-ins for importing data
Added source code for example plugins
Added ant target: exampleplugins

Location:
trunk
Files:
13 added
6 edited

Legend:

Unmodified
Added
Removed

  • trunk/build.xml

    r2060 r2154  
    629629  </target>
    630630
    631   <target name="uml"
    632     description="generate uml for core package using UmlGraph" >
    633     <javadoc
    634       packagenames="net.sf.basedb.*"
    635       sourcepath="${core.src}"
    636       >
    637       <doclet name="UmlGraph" path="${lib}/umldoclet/UmlGraph.jar">
    638         <param name="-hide" value="java.lang.Exception"/>
    639       </doclet>
    640     </javadoc>
    641     <apply executable="dot">
    642       <arg value="-Tps"/>
    643       <arg value="-o${doc}/core.ps"/>
    644       <fileset file="graph.dot"/>
    645     </apply>
    646   </target>
    647 
     631  <target
     632    name="exampleplugins"
     633    description="Tar the code for example plugins"
     634    >
     635    <property name="exampledir" location="${src}/examples/plugins" />
     636    <javac
     637      encoding="ISO-8859-1"
     638      srcdir="${exampledir}/src"
     639      destdir="${exampledir}/bin"
     640      >
     641      <classpath>
     642        <fileset dir="${exampledir}/lib">
     643          <include name="**/*.jar"/>
     644        </fileset>
     645        <pathelement path="${core.build}" />
     646      </classpath>
     647    </javac>
     648    <jar
     649      jarfile="${exampledir}/ExamplePlugins.jar"
     650      basedir="${exampledir}/bin"
     651      manifest="${exampledir}/MANIFEST.MF"
     652    />
     653    <tar
     654      destfile="${doc}/development/plugins/exampleplugins.tar.gz"
     655      compression="gzip"
     656      >
     657      <tarfileset
     658        dir="${src}/examples/plugins"
     659        preserveLeadingSlashes="true"
     660      />
     661    </tar>
     662  </target>
     663 
    648664  <target
    649665    name="javadoc"

  • trunk/doc/development/index.html

    r2151 r2154  
    262262  </dl>
    263263 
    264   <a name="plugins">
     264  <a name="plugins"></a>
    265265  <h2>6. Plug-ins</h2>
    266   </a>
    267266 
    268267  <dl>
    … …  
    272271  </dd>
    273272
    274   <dt>b) <a href="plugins/import/index.html">Plug-ins for importing data</a> - TODO</dt>
     273  <dt>b) <a href="plugins/import/index.html">Plug-ins for importing data</a></dt>
    275274  <dd>
    276275    How to create a plug-in that imports data.
    … …  
    291290  </dd>
    292291 
     292  <dt>f) Example plugins with source code
     293  <dd>
     294    <p>
     295    Contains the following example plugins:
     296    </p>
     297   
     298    <ul>
     299    <li>ExampleImporter: Pretends to import samples. It will ask for a file
     300      and if existing samples should be updated or not, but doesn't
     301      actually import anything.
     302    </ul>
     303    <br>
     304    <p>
     305    <a href="plugins/exampleplugins.tar.gz">Download</a> a tar file with the source
     306    and compiled code.
     307    </p>
     308   
     309  </dd>
    293310 
    294311  </dl>

  • trunk/doc/development/plugins/import/index.html

    r734 r2154  
    33  $Id$
    44
    5   BioArray Software Environment (BASE) - http://base.thep.lu.se/
    6   Copyright (C) 2002-2004 Lao Saal, Carl Troein,
    7   Johan Vallon-Christersson, Jari Häkkinen, Nicklas Nordborg
    8 
    9   This file is part of BASE.
     5  Copyright (C) 2006 Nicklas Nordborg
     6
     7  This file is part of BASE - BioArray Software Environment.
     8  Available at http://base.thep.lu.se/
    109
    1110  BASE is free software; you can redistribute it and/or
    … …  
    4443  <div class="abstract">
    4544 
    46     TODO
     45    <p>
     46    This document contains information specific for writing import plugins.
     47    Notably, it has some information about the <code>AbstractFlatFileImporter</code>
     48    class which is useful if you are importing things from text files.
     49    </p>
    4750 
    4851    <p>
    … …  
    5053    </p>
    5154    <ol>
    52       <li>
    53 
     55      <li><a href="#import">Import plugins</a>
     56      <li><a href="#abstract">AbstractFlatFileImporter</a>
     57      <li><a href="#autodetect">Autodetecting file formats</a>
    5458    </ol>
     59   
    5560    <p>
    5661    <b>See also</b>
     62    </p>
     63   
    5764    <ul>
    58     <li>
     65    <li><a href="../index.html">How to write plug-ins</a>
    5966    </ul>
    60     </p>
    6167   
    6268    <p class="authors">
    63     <b>Last updated:</b> $Date$
     69    <b>Last updated:</b> $Date$<br>
     70    <b>Copyright &copy;</b> 2006 The respective authors. All rights reserved.
    6471    </p>
    6572  </div>
     73
     74  <a name="import"></a>
     75  <h2>1. Import plugins</h2>
     76 
     77  <p>
     78  A plugin becoms an import plugin simply by returning <code>Plugin.MainType.IMPORT</code>
     79  from the <code>Plugin.getMainType()</code> method.
     80  </p>
     81
     82  <a name="abstract"></a>
     83  <h2>2. AbstractFlatFileImporter</h2> 
     84 
     85  <p>
     86  The <code>AbstractFlatFileImporter</code> is a very useful abstract class to inherit from
     87  if your plugin uses regular text files that can be parsed by an instance of the
     88  <code>net.sf.basedb.util.FlatFileParser</code> class. This class parses a file
     89  by checking each line against a few regular expressions. Depending on which regular
     90  expression matches the line, it is classified as a header line, a section line, a comment,
     91  a data line, a footer line or unknown. Header lines are inspected in a group, but data
     92  lines individually, meaning that it consumes very little memory since only a few lines
     93  at a time needs to be loaded.
     94  </p>
     95 
     96  <p>
     97  The <code>AbstractFlatFileImporter</code> defines <code>PluginParameter</code> objects
     98  for each of the regular expressions and other parameters used by the parser. It also
     99  implements the <code>Plugin.run()</code> method and does most of the ground work
     100  for instantiating a <code>FlatFileParser</code> and parsing the file. What you have to
     101  do in your plugin is to put together the <code>RequestInformation</code> objects
     102  for configuring the plugin and creating a job and implement the
     103  <code>InteractivePlugin.configure()</code> method for validating and storing the
     104  parameteters. You should also implement some abstract methods like <code>handleHeader()</code>
     105  and <code>handleData()</code> but more of that later.
     106  </p>
     107 
     108  <p>
     109  Here is what you need to do:
     110  </p>
     111 
     112  <dl>
     113  <dt>Implement <code>getAbout()</code> and <code>getMainType()</code></dt>
     114  <dd>
     115    See <a href="../index.html#plugin">The Plugin interface</a> for more information
     116  </dd>
     117
     118  <dt>Implement the <code>InteractivePlugin</code> methods</dt>
     119  <dd>
     120    See <a href="../index.html#interactive">The InteractivePlugin interface</a>
     121    for more information. Note that the <code>AbstractFlatFileImporter</code>
     122    has defined many parameters for regular expressions used by the parser
     123    already. You should just pick them and put in your <code>RequestInformation</code>
     124    object.
     125   
     126    <pre class="code">
     127// Parameter that maps the items name from a column
     128private PluginParameter&lt;String&gt; nameColumnMapping;
     129
     130// Parameter that maps the items description from a column
     131private PluginParameter&lt;String&gt; descriptionColumnMapping;
     132
     133private RequestInformation getConfigurePluginParameters(GuiContext context)
     134{
     135   if (configurePlugin == null)
     136   {
     137      // RequestInformation object for CONFIGURE_PLUGIN
     138      List&lt;PluginParameter&lt;?&gt;&gt; parameters = new ArrayList&lt;PluginParameter&lt;?&gt;&gt;();
     139
     140      // Parser regular expressions - from AbstractFlatFileParser
     141      parameters.add(parserSection);
     142      parameters.add(headerRegexpParameter);
     143      parameters.add(dataHeaderRegexpParameter);
     144      parameters.add(dataSplitterRegexpParameter);
     145      parameters.add(ignoreRegexpParameter);
     146      parameters.add(dataFooterRegexpParameter);
     147      parameters.add(minDataColumnsParameter);
     148      parameters.add(maxDataColumnsParameter);
     149
     150      // Column mappings
     151      nameColumnMapping = new PluginParameter&lt;String&gt;(
     152         "nameColumnMapping",
     153         "Name",
     154         "Mapping that picks the items name from the data columns",
     155         new StringParameterType(255, null, true)
     156      );
     157   
     158      descriptionColumnMapping = new PluginParameter&lt;String&gt;(
     159        "descriptionColumnMapping",
     160        "Description",
     161        "Mapping that picks the items description from the data columns",
     162        new StringParameterType(255, null, false)
     163      );
     164
     165      parameters.add(mappingSection);
     166      parameters.add(nameColumnMapping);
     167      parameters.add(descriptionColumnMapping);
     168     
     169      configurePlugin = new RequestInformation
     170      (
     171         Request.COMMAND_CONFIGURE_PLUGIN,
     172         "File parser settings",
     173         "TODO - description",
     174         parameters
     175      );
     176
     177   }
     178   return configurePlugin;
     179}
     180</pre>
     181  </dd>
     182 
     183  <dt>Implement/override some of the methods defined by <code>AbstractFlatFileParser</code></dt>
     184 
     185  <dd>
     186    <dl>
     187    <dt class="method">protected void begin()</dt>
     188    <dd>
     189      This method is called just before the parsing of the file
     190      begins. Override this emthod if you need to initialise some
     191      internal state. This is, for example, a good place to open
     192      a <code>DbControl</code> object, read parameters from the job and configuration and
     193      put them into more useful variables. The default implementation
     194      does nothing, but we recommend that <code>super.begin()</code> is
     195      always called.
     196     
     197      <pre class="code">
     198// Snippets from the RawDataFlatFileImporter class
     199private DbControl dc;
     200private RawDataBatcher batcher;
     201private RawBioAssay rawBioAssay;
     202private Map&lt;String, String&gt; columnMappings;
     203private int numInserted;
     204
     205@Override
     206protected void begin()
     207   throws BaseException
     208{
     209   super.begin();
     210
     211   // Get DbControl
     212   dc = sc.newDbControl();
     213   rawBioAssay = (RawBioAssay)job.getValue(rawBioAssayParameter.getName());
     214
     215   // Reload raw bioassay using current DbControl
     216   rawBioAssay = RawBioAssay.getById(dc, rawBioAssay.getId());
     217   
     218   // Create a batcher for inserting spots
     219   batcher = rawBioAssay.getRawDataBatcher();
     220
     221   // Cache columns mappings in map
     222   columnMappings = new HashMap&lt;String, String&gt;();
     223   for (PluginParameter&lt;?&gt; pp : getAllColumnMappings(rawBioAssay.getRawDataType()))
     224   {
     225      columnMappings.put(pp.getName(),
     226         (String)configuration.getValue(pp.getName()));
     227   }
     228   
     229   // For progress reporting
     230   numInserted = 0;
     231}
     232</pre>
     233 
     234    </dd>
     235   
     236    <dt class="method">protected void handleHeader(FlatFileParser.Line line)</dt>
     237    <dd>
     238      <p>
     239      This method is called once for every header line that is found in
     240      the file. The <code>line</code> parameter contains information
     241      about the header. The default implementation of this method does
     242      nothing.
     243      </p>
     244     
     245    <pre class="code">
     246@Override
     247protected void handleHeader(Line line)
     248   throws BaseException
     249{
     250   super.handleHeader(line);
     251   if (line.name() != null && line.value() != null)
     252   {
     253      rawBioAssay.setHeader(line.name(), line.value());
     254   }
     255}
     256</pre>
     257    </dd>
     258   
     259    <dt class="method">protected void handleSection(FlatFileParser.Line line)</dt>
     260    <dd>
     261      <p>
     262      This method is called once for each section that is found in the file.
     263      The <code>line</code> parameter contains information
     264      about the header. The default implementation of this method does
     265      nothing. Currently, we have no plugins using this feature and can't show any
     266      example code.
     267      </p>
     268    </dd>
     269   
     270    <dt class="method">protected abstract void handleData(FlatFileParser.Data data)
     271      throws BaseException;</dt>
     272    <dd>
     273      <p>
     274      This method is abstract and must be implemented by all subclasses.
     275      It follows the same pattern as the other methods, and is called
     276      once for every data line in the the file.
     277      </p>
     278     
     279      <pre class="code">
     280// Snippets from the RawDataFlatFileImporter class
     281@Override
     282protected void handleData(Data data)
     283   throws BaseException
     284{
     285   // Create new RawData object
     286   RawData raw = batcher.newRawData();
     287
     288   // External ID for the reporter
     289   String externalId = data.map(columnMappings.get("reporterIdColumnMapping"));
     290   
     291   // Block, row and column numbers
     292   String block = data.map(columnMappings.get(blockColumnMapping.getName()));
     293   String column = data.map(columnMappings.get(columnColumnMapping.getName()));
     294   String row = data.map(columnMappings.get(rowColumnMapping.getName()));
     295   // ... more: metaGrid coordinate, X-Y coordinate
     296
     297   if (block != null) raw.setBlock(Integer.valueOf(block));
     298   if (column != null) raw.setColumn(Integer.valueOf(column));
     299   if (row != null) raw.setRow(Integer.valueOf(row));
     300   // ... more: metaGrid coordinate, X-Y coordinate
     301
     302   // Other properties
     303   for (RawDataProperty rdp : rawBioAssay.getRawDataType().getProperties())
     304   {
     305      String extendedData = data.map(
     306         columnMappings.get("propertyMapping."+rdp.getName()));
     307      raw.setExtended(rdp.getName(), rdp.parseString(extendedData));
     308   }
     309   
     310   // Insert raw data to the database
     311   batcher.insert(raw, externalId);
     312   numInserted++;
     313}
     314</pre>
     315    </dd>
     316   
     317    <dt class="method">protected void end(boolean success)
     318      throws BaseException</dt>
     319     
     320    <dd>
     321      <p>
     322      Called when the parsing has ended, either because the end of
     323      file was reached or because an error has occurred. The subclass
     324      should close any open resources, ie. the <code>DbControl</code>
     325      object. The <code>success</code> parameter is <code>true</code>
     326      if the parsing was successful, <code>false</code> otherwise.
     327      The default implementation does nothing.
     328      </p>
     329     
     330      <pre class="code">
     331@Override
     332protected void end(boolean success)
     333   throws BaseException
     334{
     335   try
     336   {
     337      // Commit if the parsing was successful
     338      if (success)
     339      {
     340         batcher.close();
     341         dc.commit();
     342      }
     343   }
     344   catch (BaseException ex)
     345   {
     346      // Well, now we got an exception
     347      success = false;
     348      throw ex;
     349   }
     350   finally
     351   {
     352      // Always close... and call super.end()
     353      if (dc != null) dc.close();
     354      super.end(success);
     355   }
     356}     
     357</pre>
     358      </dd>
     359     
     360      <dt class="method">protected String getSuccessMessage()</dt>
     361      <dd>
     362        <p>
     363        This is the last method that is called, and it is only called if
     364        everything went suceessfully. This method allows a subclass to generate
     365        a short message that is sent back to the database as a final progress
     366        report. The default implementation returns null, which means that no
     367        message will be generated.
     368        </p>
     369       
     370        <pre class="code">
     371@Override
     372protected String getSuccessMessage()
     373{
     374   return numInserted + (numInserted == 1 ? " spot inserted" : " spots inserted");
     375}
     376</pre>
     377      </dd>
     378
     379    </dl>
     380 
     381    </dd>
     382  </dl>
     383 
     384  <a name="autodetect"></a>
     385  <h2>3. Autodetecting file formats</h2>
     386 
     387  <p>
     388  Base has built-in functionality for autodetecting file formats. If your import plugin
     389  wants to participate in that feature it must implement the <code>AutoDetectingImporter</code>
     390  interface. This interface has two methods:
     391  </p>
     392 
     393  <dl>
     394  <dt class="method">public boolean isImportable(InputStream in)
     395    throws BaseException;</dt>
     396  <dd>
     397    <p>
     398    Check the input stream if it seems to contain data that can be imported by
     399    the plugin. Usually it means scanning a few lines for some header
     400    mathing a predefined string or a regexp.
     401    </p>
     402    <p>
     403    The <code>AbstractFlatFileImporter</code> implements this method
     404    by checking reading the headers from the input stream and checking if
     405    it stopped at an unknown type of line or not:
     406    <pre class="code">
     407public final boolean isImportable(InputStream in)
     408   throws BaseException
     409{
     410   FlatFileParser ffp = getInitializedFlatFileParser();
     411   ffp.setInputStream(in);
     412   try
     413   {
     414      FlatFileParser.LineType result = ffp.parseHeaders();
     415      return result != FlatFileParser.LineType.UNKNOWN;
     416   }
     417   catch (IOException ex)
     418   {
     419      throw new BaseException(ex);
     420   }
     421}
     422</pre>
     423    <p>
     424    Note that the input stream doesn't have to be a text file. It could be any type
     425    of file, for example a binary or xml file. In the case of an xml file you would need
     426    to validate the entiry input stream in order to be a 100% sure that it is a valid
     427    xml file, but we recommend that you only check the first few xml tags, ie.
     428    the &lt;!DOCTYPE &gt; declaration and/or the root element tag.
     429    </p>
     430   
     431  </dd>
     432 
     433  <dt class="method">public void doImport(InputStream in, ProgressReporter progress)
     434    throws BaseException;</dt>
     435  <dd>
     436    <p>
     437    Parse the input stream and import all data that is found. This method is of
     438    cource only called if the <code>isImportable</code> has returned true. Note
     439    however that the input stream is reopened at the start of the file. It may even
     440    be the case that the <code>isImportable</code> method is called on one instance
     441    of the plugin and the <code>doImport</code> method is called on another.
     442    Thus, the <code>doImport</code> can't rely on any state set by the <code>isImportable</code>
     443    method.
     444    </p>
     445 
     446  </dl>
     447
    66448
    67449</body>
    68450</html>
     451   

  • trunk/doc/development/plugins/index.html

    r2151 r2154  
    5959    <li><a href="#packaging">Packaging and installing the plugin</a>
    6060    <li><a href="#organize">How to organize your plugin project</a>
     61    <li><a href="#jsp">Using a custom JSP page for parameter input</a>
    6162    </ol>
    6263   
    … …  
    556557  <p>
    557558  Note! Most parameter types include support for suppying a predefined list
    558   of options to select from. In that case the list will be displayed as drop-down
     559  of options to select from. In that case the list will be displayed as a drop-down
    559560  list for the user, otherwise a free input field is used.
    560561  </p>
    … …  
    578579    where the output files should be placed.
    579580  </ul>
    580 
     581 
     582  <p>
     583  You can also create a <code>PluginParameter</code> with a null name and
     584  <code>ParameterType</code>. In that case, the core will not ask for input from
     585  the user, instead it is used as a section header, allowing you to group parameters
     586  into different sections which increase the readability of the input parameters page.
     587  </p>
     588 
     589  <pre class="code">
     590PluginParameter firstSection = new PluginParameter(null, "First section", null, null);
     591PluginParameter secondSection = new PluginParameter(null, "First section", null, null);
     592// ...
     593
     594parameters.add(firstSection);
     595parameters.add(firstParameterInFirstSection);
     596parameters.add(secondParameteInFirstSection);
     597
     598parameters.add(secondSection);
     599parameters.add(firstParameterInSecondSection);
     600parameters.add(secondParameteInSecondSection);
     601</pre>
     602 
    581603  </dd>
    582604 
    … …  
    822844  </p>
    823845 
     846  <a name="jsp"></a>
     847  <h2>4. Using a custom JSP page for parameter input</h2>
     848 
     849  <p>
     850  This is an advanced option for plugins that require a different interface
     851  for specifying plugin parameters than the default list showing each parameter
     852  at a time. This feature is used by settin the <code>RequestInformation.getJspPage()</code>
     853  property when construction the request information object. If this property has a non-null
     854  value, the web client will send the browser to the specified JSP page instead of
     855  to the generic parameter input page.
     856  </p>
     857
     858  <p>
     859  When setting the JSP page you should not specify any path information. The web client
     860  has a special location for these JSP pages, generated from the package name of
     861  your plugin and the returned values. If the plugin is located in the package
     862  <code>org.company</code> the JSP page must be located in
     863  <code>&lt;www-root&gt;/plugins/org/company/</code>. Please note that the browser
     864  still thinks that it is showing the regular page at the usual location:
     865  <code>&lt;www-root&gt;/common/plugin/index.jsp</code>, so all links in your JSP page
     866  should be relative to that directory.
     867  </p>
     868 
     869  <p>
     870  Even if you use your own JSP page we recommend that you use the built-in
     871  facility for passing the parameters back to the plugin. For this to work
     872  you must:
     873  </p>
     874 
     875  <ul>
     876  <li>Generate the list of <code>PluginParameter</code> objects as usual
     877  <li>Name all your input fields like: <code>parameter:&lt;name-of-parameter&gt;</code>
     878    for example:
     879    <pre class="code">
     880// Plugin generate PluginParameter
     881StringParameterType stringPT = new StringParameterType(255, null, true);
     882PluginParameter one = new PluginParameter("one", "One", "First string", stringPT);
     883PluginParameter two = new PluginParameter("two", "Two", "Second string", stringPT);
     884
     885// JSP should name fiels as:
     886First string: &lt;input type="text" name="parameter:one"&gt;&lt;br&gt;
     887Second stirng: &lt;input type="text" name="parameter:two"&gt;
     888</pre>
     889 
     890  <li>Send the form to <code>index.jsp</code> with some parameters:
     891 
     892  <pre class="code">
     893&lt;form action="index.jsp" method="post"&gt;
     894&lt;input type="hidden" name="ID" value="<%=ID%>"&gt;
     895&lt;input type="hidden" name="cmd" value="SetParameters"&gt;
     896...
     897&lt;/form&gt;
     898</pre> 
     899  </ul>
     900   
     901  <p>
     902  In your JSP page you will probably need to access some information
     903  like the <code>SessionControl</code> and possible even the
     904  <code>RequestInformation</code> object created by your plugin.
     905  </p>
     906   
     907  <pre class="code">
     908// Get session control and it's ID (required to post to index.jsp)
     909final SessionControl sc = Base.getExistingSessionControl(pageContext, true);
     910final String ID = sc.getId();
     911
     912// Get information about the current request to the plugin
     913PluginConfigurationRequest pcRequest =
     914   (PluginConfigurationRequest)sc.getSessionSetting("plugin.configure.request");
     915PluginDefinition plugin =
     916   (PluginDefinition)sc.getSessionSetting("plugin.configure.plugin");
     917PluginConfiguration pluginConfig =
     918   (PluginConfiguration)sc.getSessionSetting("plugin.configure.config");
     919PluginDefinition job =
     920   (PluginDefinition)sc.getSessionSetting("plugin.configure.job");
     921RequestInformation ri = pcRequest.getRequestInformation();
     922</pre>
     923 
     924 
    824925</body>
    825926</html>

  • trunk/doc/index.html

    r2145 r2154  
    398398</dd>
    399399
     400<dt>Plugin examples</dt>
     401<dd>
     402  <p>
     403  We have compiled some small example plugins which might be of interest
     404  if you are going to develop plugins yourself. The tar file contains
     405  the source code and compiled classes.
     406  </p>
     407  <p>
     408  <a href="development/plugins/exampleplugins.tar.gz">Download</a>
     409  </p>
     410 
     411  <p>
     412  For more information about developing plugins see:
     413  <a href="development/index.html#plugins">Development information - Plug-ins</a>
     414 
     415</dd>
    400416<!--
    401417<dt>Tar ball</dt>

  • trunk/src/core/net/sf/basedb/plugins/RawDataFlatFileImporter.java

    r2151 r2154  
    418418  protected void end(boolean success)
    419419    throws BaseException
    420     {
     420  {
    421421    try
    422422    {
    423       batcher.close();
    424423      if (success)
    425424      {
     425        batcher.close();
    426426        dc.commit();
    427427      }
    428       else
    429       {
    430         dc.close();
    431       }
    432428    }
    433429    catch (BaseException ex)
    434430    {
    435       dc.close();
     431      success = false;
    436432      throw ex;
    437433    }
    438434    finally
    439435    {
     436      if (dc != null) dc.close();
    440437      super.end(success);
    441438    }
Note: See TracChangeset for help on using the changeset viewer.