Changeset 5764

Timestamp:

Sep 27, 2011, 2:35:28 PM (12 years ago)

Author:

Nicklas Nordborg

Message:

Fixes #1624: Create array design (feature) importer for GTF files

• Created GtfReporterMapImporter and two configurations that uses transcript_id/gene_id for reporter ids. The GTF importer builds on the ReporterMapFlatFileImporter but has less configuration options. Eg. it is locked to use FEATURE_ID for identifying features.
• Added a new parameter to the regular ReporterMapFlatFileImporter that makes it possible to limit a configuration to certain platforms. This is used to hide genepix, illumina, etc. when using the sequencing platform.
• Added SEQUENCING platform and SEQUENCING_EXPRESSION variant.
• Created a GTF validator that hooks into the fileset validation extension point for array designs and extracts the number of unique transcript_id:s from the GTF file.

Location:

trunk

Files:

• data/illumina.configurations.xml (modified) (1 diff)
• data/plugin_configfile.xml (modified) (2 diffs)
• src/core/core-extensions.xml (modified) (1 diff)
• src/core/net/sf/basedb/core/DataFileType.java (modified) (1 diff)
• src/core/net/sf/basedb/core/FileStoreUtil.java (modified) (2 diffs)
• src/core/net/sf/basedb/core/Install.java (modified) (4 diffs)
• src/core/net/sf/basedb/core/Platform.java (modified) (1 diff)
• src/core/net/sf/basedb/core/PlatformVariant.java (modified) (1 diff)
• src/core/net/sf/basedb/util/gtf/GtfInputStream.java (modified) (4 diffs)
• src/core/net/sf/basedb/util/gtf/GtfValidationAction.java (added)
• src/core/net/sf/basedb/util/gtf/GtfValidationFactory.java (added)
• src/plugins/core/core-plugins.xml (modified) (2 diffs)
• src/plugins/core/net/sf/basedb/plugins/ReporterMapFlatFileImporter.java (modified) (18 diffs)
• src/plugins/core/net/sf/basedb/plugins/gtf/DefaultConfigurationValues.java (added)
• src/plugins/core/net/sf/basedb/plugins/gtf/GtfReporterImporter.java (modified) (3 diffs)
• src/plugins/core/net/sf/basedb/plugins/gtf/GtfReporterMapImporter.java (added)
• src/test/TestAll.java (modified) (1 diff)
• src/test/TestArrayDesign.java (modified) (3 diffs)
• src/test/TestGtfImporters.java (moved) (moved from trunk/src/test/TestGtfReporterImporter.java) (5 diffs)

trunk/data/illumina.configurations.xml

@@ -479 +479 @@
 The line number starts at 1 for the first data line (headers are ignored). This is the same as the IlluminaRawDataImporter plug-in does so it should be possible to connect the raw bioassays with an array design if needed.</description>
     <parameter>
+      <name>platforms</name>
+      <label>Platforms/variants</label>
+      <description>Select all platforms/variants where this configuration can be used. If not selected, the configuration can be used on all except file-only platforms.</description>
+      <class>java.lang.String</class>
+      <value>P:generic</value>
+    </parameter>
+    <parameter>
       <name>dataHeaderRegexp</name>
       <label>Data header</label>

trunk/data/plugin_configfile.xml

@@ -293 +293 @@
 Block mapping is used in this configuration.</description>
     <parameter>
+      <name>platforms</name>
+      <label>Platforms/variants</label>
+      <description>Select all platforms/variants where this configuration can be used. If not selected, the configuration can be used on all except file-only platforms.</description>
+      <class>java.lang.String</class>
+      <value>P:generic</value>
+    </parameter>
+    <parameter>
       <name>dataHeaderRegexp</name>
       <label>dataHeaderRegexp</label>
@@ -1061 +1068 @@
     </parameter>
   </configuration>
+  <configuration pluginClassName="net.sf.basedb.plugins.gtf.GtfReporterMapImporter">
+    <configname>transcript_id (no prefix)</configname>
+    <description>A configuration that uses the transcript_id (no prefix) as reporter and feature id.</description>
+    <parameter>
+      <name>reporterIdColumnMapping</name>
+      <label>Reporter ID</label>
+      <description>Mapping that picks the reporter's external ID from the data columns. For example: \ID\</description>
+      <class>java.lang.String</class>
+      <value>\&lt;transcript_id&gt;\</value>
+    </parameter>
+    <parameter>
+      <name>trimQuotes</name>
+      <label>Remove quotes</label>
+      <description>If true quotes (" or ') around data value will be removed.</description>
+      <class>java.lang.Boolean</class>
+      <value>true</value>
+    </parameter>
+    <parameter>
+      <name>dataHeaderRegexp</name>
+      <label>Data header</label>
+      <description>A regular expression that matches the header line just before the data begins. For example: Block\tRow\tColumn.*</description>
+      <class>java.lang.String</class>
+      <value>&lt;seqname&gt;\t.*&lt;transcript_id&gt;.*</value>
+    </parameter>
+    <parameter>
+      <name>featureIdentification</name>
+      <label />
+      <description />
+      <class>java.lang.String</class>
+      <value>FEATURE_ID</value>
+    </parameter>
+    <parameter>
+      <name>complexExpressions</name>
+      <label>Complex column mappings</label>
+      <description>disallow = Only allow simple mappings that are constant value or pick the value from one column only, for example, '1.6' or '\Row\'
+allow = Allow expression and complex mappings, for example, '\Row\, \Column\' or '=2*col('radius')'</description>
+      <class>java.lang.String</class>
+      <value>disallow</value>
+    </parameter>
+    <parameter>
+      <name>charset</name>
+      <label>Character set</label>
+      <description>The character set to use when reading the file. This setting overrides the character set specified by the file. If neither this parameter nor the file specifies a character set, the system default is used (ISO-8859-1).</description>
+      <class>java.lang.String</class>
+      <value>ISO-8859-1</value>
+    </parameter>
+    <parameter>
+      <name>featureIdColumnMapping</name>
+      <label>Feature ID</label>
+      <description>Mapping that picks the feature's ID from the data columns. This column is only used when the array design uses the FEATURE_ID method for identifying features. In the other cases, the value is just stored as it is.For example: \Feature ID\</description>
+      <class>java.lang.String</class>
+      <value>\&lt;transcript_id&gt;\</value>
+    </parameter>
+    <parameter>
+      <name>dataSplitterRegexp</name>
+      <label>Data splitter</label>
+      <description>A regular expression that splits each data line into individual columns. For example, split on tabs: \t</description>
+      <class>java.lang.String</class>
+      <value>\t</value>
+    </parameter>
+  </configuration>
+  <configuration pluginClassName="net.sf.basedb.plugins.gtf.GtfReporterMapImporter">
+    <configname>gene_id (no prefix)</configname>
+    <description>A configuration that uses the gene_id (no prefix) as reporter id and transcript_id as feature id. Note that gene_id may not be unique so it is not recommended to use that as feature id.</description>
+    <parameter>
+      <name>reporterIdColumnMapping</name>
+      <label>Reporter ID</label>
+      <description>Mapping that picks the reporter's external ID from the data columns. For example: \ID\</description>
+      <class>java.lang.String</class>
+      <value>\&lt;gene_id&gt;\</value>
+    </parameter>
+    <parameter>
+      <name>trimQuotes</name>
+      <label>Remove quotes</label>
+      <description>If true quotes (" or ') around data value will be removed.</description>
+      <class>java.lang.Boolean</class>
+      <value>true</value>
+    </parameter>
+    <parameter>
+      <name>dataHeaderRegexp</name>
+      <label>Data header</label>
+      <description>A regular expression that matches the header line just before the data begins. For example: Block\tRow\tColumn.*</description>
+      <class>java.lang.String</class>
+      <value>&lt;seqname&gt;\t.*&lt;gene_id&gt;.*</value>
+    </parameter>
+    <parameter>
+      <name>featureIdentification</name>
+      <label />
+      <description />
+      <class>java.lang.String</class>
+      <value>FEATURE_ID</value>
+    </parameter>
+    <parameter>
+      <name>complexExpressions</name>
+      <label>Complex column mappings</label>
+      <description>disallow = Only allow simple mappings that are constant value or pick the value from one column only, for example, '1.6' or '\Row\'
+allow = Allow expression and complex mappings, for example, '\Row\, \Column\' or '=2*col('radius')'</description>
+      <class>java.lang.String</class>
+      <value>disallow</value>
+    </parameter>
+    <parameter>
+      <name>charset</name>
+      <label>Character set</label>
+      <description>The character set to use when reading the file. This setting overrides the character set specified by the file. If neither this parameter nor the file specifies a character set, the system default is used (ISO-8859-1).</description>
+      <class>java.lang.String</class>
+      <value>ISO-8859-1</value>
+    </parameter>
+    <parameter>
+      <name>featureIdColumnMapping</name>
+      <label>Feature ID</label>
+      <description>Mapping that picks the feature's ID from the data columns. This column is only used when the array design uses the FEATURE_ID method for identifying features. In the other cases, the value is just stored as it is.For example: \Feature ID\</description>
+      <class>java.lang.String</class>
+      <value>\&lt;transcript_id&gt;\</value>
+    </parameter>
+    <parameter>
+      <name>dataSplitterRegexp</name>
+      <label>Data splitter</label>
+      <description>A regular expression that splits each data line into individual columns. For example, split on tabs: \t</description>
+      <class>java.lang.String</class>
+      <value>\t</value>
+    </parameter>
+  </configuration>
 </configfile>

trunk/src/core/core-extensions.xml

@@ -141 +141 @@
   </extension>
 
+  <extension 
+    id="net.sf.basedb.util.gtf.gtf-validator"
+    extends="net.sf.basedb.core.filehandler.validator"
+    >
+    <about>
+      <name>GTF file validator</name>
+      <description>
+        Validator for GTF reference files. The validation checks that
+        the first few lines follow the GTF specification. 
+        See http://mblab.wustl.edu/GTF22.html
+      </description>
+    </about>
+    <index>1</index>
+    <action-factory>
+      <factory-class>net.sf.basedb.util.gtf.GtfValidationFactory</factory-class>
+    </action-factory>
+  </extension>
+
   <!-- 
   <extension 

trunk/src/core/net/sf/basedb/core/DataFileType.java

@@ -150 +150 @@
   public static final String MICROARRAY_IMAGE = "microarray.image";
 
+  /**
+    The external ID for the file type representing a GTF
+    reference file that define the features on an array design
+    that we are interested in.
+    @since 3.0
+  */
+  public static final String REF_SEQ_GTF = "refseq.gtf";
+  
   /**
     The maximum length of the file extension that can be stored in the database.

trunk/src/core/net/sf/basedb/core/FileStoreUtil.java

@@ -24 +24 @@
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
@@ -201 +202 @@
   
   /**
+    Get all file members with a specific generic {@link ItemSubtype} in a given item's file
+    set. Note! This method will only return members that the logged in user
+    has permission to read the file. Other members in the file set will be ignored. 
+    Note! This method checks the file type from {@link DataFileType#getGenericType()},
+    not the file type from {@link File#getItemSubtype()}.
+    
+    @param dc A DbControl to use for database access
+    @param item The {@link FileStoreEnabled} item to check
+    @param genericType The system ID of the {@link ItemSubtype} to check for
+    @return A list of member items that matches
+    @since 3.0
+  */
+  public static List<FileSetMember> getGenericMembers(DbControl dc, FileStoreEnabled item, 
+    String genericType)
+  {
+    List<FileSetMember> members = new LinkedList<FileSetMember>();
+    if (item.hasFileSet())
+    {
+      ItemQuery<FileSetMember> query = item.getFileSet().getMembers(genericType);
+      query.order(Orders.asc(Hql.property("file.name")));
+      for (FileSetMember member : query.list(dc))
+      {
+        try
+        {
+          member.getFile();
+          members.add(member);
+        }
+        catch (Throwable t)
+        {}
+      }
+    }
+    return members;
+  }
+
+  /**
+    Get a list of all platform file types that has the specified generic {@link ItemSubtype}.
+    If the item doesn't have any associated platform or variant, no file types are returned.
+    
+    @param dc A DbControl to use for database access
+    @param item The {@link FileStoreEnabled} item to check
+    @param genericType The system ID of the {@link ItemSubtype} to check for
+    @return  A list of platform file types that matches
+    @since 3.0
+  */
+  public static List<PlatformFileType> getGenericPlatformFileTypes(DbControl dc, FileStoreEnabled item,
+      String genericType)
+  {
+    Platform platform = item.getPlatform();
+    PlatformVariant variant = item.getVariant();
+    if (platform == null && variant == null) 
+    {
+      return Collections.emptyList();
+    }
+    
+    ItemQuery<PlatformFileType> query = 
+      PlatformFileType.getQuery(platform, variant, variant == null);
+    query.restrict(
+      Restrictions.eq(
+        Hql.property("dataFileType.genericType.systemId"), 
+        Expressions.string(genericType)
+      )
+    );
+    query.order(Orders.asc(Hql.property("dataFileType.name")));
+    return query.list(dc);
+  }
+  
+  /**
     Set or remove a file to {@link FileStoreEnabled} item based on
     the file's {@link DataFileType}.

trunk/src/core/net/sf/basedb/core/Install.java

@@ -37 +37 @@
 import net.sf.basedb.core.data.PlatformData;
 import net.sf.basedb.core.data.PlatformFileTypeData;
+import net.sf.basedb.core.data.PlatformVariantData;
 import net.sf.basedb.core.data.PluginConfigurationData;
 import net.sf.basedb.core.data.ProjectKeyData;
@@ -878 +879 @@
         Item.DERIVEDBIOASSAY, "tif", imageType, new ItemSubtypeFT(scan, false, true)
       );
+      DataFileTypeData gtfDesignFile = createDataFileType(
+        DataFileType.REF_SEQ_GTF, "GTF ref-seq file",
+        "A GTF reference file that contains all features that we are interested in. " +
+        "The default settings use <transcript_id> to identify genes.", 
+        Item.ARRAYDESIGN, "gtf", reporterMapType);
       
       // Platforms and variants
       progressStep++;
       if (progress != null) progress.display((int)(progressStep*progress_factor), "--Creating platforms...");
-      createPlatform("generic", "Generic", "Generic platform which expects data to be imported into the database",
+      createPlatform(Platform.GENERIC, "Generic microarray", 
+        "Generic platform which expects data to be imported into the database",
         false, null, 0, 
         new PlatformFT(reporterMapFile, false, false),
@@ -888 +895 @@
         new PlatformFT(rawDataFile, false, true)
       );
-      createPlatform("affymetrix", "Affymetrix", "Affymetrix platform", true, null, 1, 
+      createPlatform(Platform.AFFYMETRIX, "Affymetrix", "Affymetrix platform", true, null, 1, 
         new PlatformFT(celFile, true, false), 
         new PlatformFT(cdfFile, true, false)
+      );
+      PlatformData sequencing = createPlatform(Platform.SEQUENCING, "Sequencing", 
+        "Generic platform for sequence data. We do not specify any more details here, but " +
+        "use variants.", 
+        false, null, 0
+      );
+      createPlatformVariant(sequencing, PlatformVariant.SEQUENCING_EXPRESSION, "Expression-like",
+        "Variant for sequence data which has been pre-processed enough to make it possible " +
+        "for importing into the database. ", false, null, 0, 
+        new PlatformFT(gtfDesignFile, true, false)
       );
       
@@ -2630 +2647 @@
 
   /**
+    Create a {@link PlatformVariant}.
+  */
+  private static PlatformVariantData createPlatformVariant(PlatformData platform, String externalId, 
+    String name, String description, boolean fileOnly, String rawDataType, 
+    int channels, PlatformFT... fileTypes)
+    throws BaseException
+  {
+    org.hibernate.Transaction tx = null;
+    PlatformVariantData variant = null;
+    try
+    {
+      tx = HibernateUtil.newTransaction(session);
+    
+      if (variant == null)
+      {
+        org.hibernate.Query query = HibernateUtil.getPredefinedQuery(session, 
+          "GET_PLATFORMVARIANT_FOR_EXTERNAL_ID");
+        query.setString("externalId", externalId);
+        variant = HibernateUtil.loadData(PlatformVariantData.class, query);
+      }
+      
+      if (variant != null)
+      {
+        log.info("createPlatformVariant: EXISTS [externalId="+externalId+"]");
+        
+        for (PlatformFileTypeData fileType : platform.getFileTypes().values())
+        {
+          if (variant.equals(fileType.getVariant()))
+          {
+            for (PlatformFT ft : fileTypes)
+            {
+              if (ft.fileType.equals(fileType.getDataFileType()))
+              {
+                fileType.setAllowMultiple(ft.multiple);
+                fileType.setRequired(ft.required);
+              }
+            }
+          }
+        }
+        
+        HibernateUtil.commit(tx);
+      }
+      else
+      {
+        variant = new PlatformVariantData();
+        variant.setPlatform(platform);
+        variant.setExternalId(externalId);
+        variant.setName(name);
+        variant.setDescription(description);
+        variant.setFileOnly(fileOnly);
+        if (fileOnly)
+        {
+          variant.setChannels(channels);
+          variant.setRawDataType("variant." + externalId);
+        }
+        else
+        {
+          variant.setRawDataType(rawDataType);
+        }
+        HibernateUtil.saveData(session, variant);
+  
+        if (fileTypes != null)
+        {
+          for (PlatformFT pft : fileTypes)
+          {
+            PlatformFileTypeData platformFile = new PlatformFileTypeData();
+            platformFile.setPlatform(platform);
+            platformFile.setVariant(variant);
+            platformFile.setDataFileType(pft.fileType);
+            platformFile.setRequired(pft.required);
+            platformFile.setAllowMultiple(pft.multiple);
+            HibernateUtil.saveData(session, platformFile);
+          }
+        }
+        
+        HibernateUtil.commit(tx);
+        log.info("createPlatformVariant: OK [externalId="+externalId+"]");
+      }
+    }
+    catch (BaseException ex)
+    {
+      if(tx != null) HibernateUtil.rollback(tx);
+      log.error("createPlatformVariant: FAILED [externalId="+externalId+"]", ex);
+      throw ex;
+    }
+    return variant;
+  }
+
+  /**
     Create a {@link DataFileType}.
   */

trunk/src/core/net/sf/basedb/core/Platform.java

@@ -76 +76 @@
   */
   public static final String AFFYMETRIX = "affymetrix";
+  
+  /**
+    The external id for the generic <code>Sequencing</code> platform.
+    We don't defined any specific details at this level, but use
+    variants for this. 
+    @see PlatformVariant#SEQUENCING_EXPRESSION
+    @since 3.0
+  */
+  public static final String SEQUENCING = "sequencing";
   
   /**

trunk/src/core/net/sf/basedb/core/PlatformVariant.java

@@ -38 +38 @@
   implements Nameable, Removable
 {
+  
+  /**
+    The external id for the expression-like sequencing data.
+    @since 3.0
+  */
+  public static final String SEQUENCING_EXPRESSION = "sequencing.expression";
+
   
   /**

trunk/src/core/net/sf/basedb/util/gtf/GtfInputStream.java

@@ -100 +100 @@
     this.charset = Charset.forName(charset == null ? "ISO-8859-1" : charset);
     this.skipRepeatedTranscriptIds = skipRepeatedTranscriptIds;
-    this.transcriptIds = skipRepeatedTranscriptIds ? new HashSet<String>() : null;
-    this.reader = new BufferedReader(new InputStreamReader(master, charset));
+    this.transcriptIds = new HashSet<String>();
+    this.reader = new BufferedReader(new InputStreamReader(master, this.charset));
     this.ATTRIBUTE_PATTERN = Pattern.compile("([^ ]+) (([^\";]*)|(\"[^\"]*\"));");
     init();
@@ -184 +184 @@
 
   /**
+    Get the number of lines parsed so far.
+  */
+  public int getNumLines()
+  {
+    return lineNum;
+  }
+  
+  /**
+    Get the number of unique transcript ids found so far.
+  */
+  public int getNumUniqueTranscriptIds()
+  {
+    return transcriptIds.size();
+  }
+  
+  /**
     Initialize the converter by reading the first line from the GTF
     file. The attributes will be extracted and a header row is created
@@ -201 +217 @@
     // Generate header line
     StringBuffer sb = new StringBuffer();
+    transcriptIds.add(attributes[transcriptIdIndex].value);
     if (skipRepeatedTranscriptIds)
     {
-      transcriptIds.add(attributes[transcriptIdIndex].value);
       sb.append("<seqname>\t<source>");
     }
@@ -244 +260 @@
       parseAttributes(line[8]);
       
-      if (skipRepeatedTranscriptIds)
-      {
-        String id = attributes[transcriptIdIndex].value;
-        if (!transcriptIds.contains(id))
-        {
-          transcriptIds.add(id);
-          break;
-        }
+      String id = attributes[transcriptIdIndex].value;
+      if (transcriptIds.add(id) && skipRepeatedTranscriptIds)
+      {
+        break;
       }
     } while (skipRepeatedTranscriptIds);

trunk/src/plugins/core/core-plugins.xml

@@ -766 +766 @@
       <name>GTF reporter importer</name>
       <description>
-        Creater reporters and reporter lists from GTF (Gene transfer format) 
+        Creates reporters and reporter lists from GTF (Gene transfer format) 
         files. The default configuration is to use the transcript_id value
         as the reporter id and name. No other fields are used, but this can
@@ -781 +781 @@
   </plugin-definition>
 
+  <plugin-definition id="GtfReporterMapImporter">
+    <about>
+      <name>GTF reporter map importer</name>
+      <description>
+        Maps reporters to array design features from GTF files. The default
+        implementation use the transcript_id as both the feature id and
+        reporter id, but this can be changed by user configuration. For
+        example, to use the gene_id instead or to add prefixes to the id
+        values. The importer build on the regular reporter map importer
+        and perform on-the-fly transformation of GTF attributes to a
+        completely column-based format.
+      </description>
+    </about>
+    <plugin-class>net.sf.basedb.plugins.gtf.GtfReporterMapImporter</plugin-class>
+    <settings>
+      <property name="everyone-use">1</property>
+    </settings>
+  </plugin-definition>
+
   <!-- 
   <plugin-definition id="">

trunk/src/plugins/core/net/sf/basedb/plugins/ReporterMapFlatFileImporter.java

@@ -41 +41 @@
 import net.sf.basedb.core.ItemContext;
 import net.sf.basedb.core.ItemNotFoundException;
+import net.sf.basedb.core.ItemQuery;
 import net.sf.basedb.core.Permission;
+import net.sf.basedb.core.Platform;
+import net.sf.basedb.core.PlatformFileType;
+import net.sf.basedb.core.PlatformVariant;
 import net.sf.basedb.core.ReporterBatcher;
 import net.sf.basedb.core.ItemParameterType;
@@ -57 +61 @@
 import net.sf.basedb.core.plugin.Request;
 import net.sf.basedb.core.plugin.Response;
+import net.sf.basedb.core.query.Expressions;
+import net.sf.basedb.core.query.Hql;
+import net.sf.basedb.core.query.Orders;
+import net.sf.basedb.core.query.Restrictions;
 import net.sf.basedb.plugins.util.Parameters;
+import net.sf.basedb.util.Enumeration;
 import net.sf.basedb.util.error.SimpleErrorHandler;
 import net.sf.basedb.util.parser.ConfigureByExample;
@@ -96 +105 @@
   private static final StringParameterType optionalColumnMapping = new StringParameterType(255, null, false);
   
-  private static final PluginParameter<String> featureIdentificationParameter = 
+  protected static final PluginParameter<String> featureIdentificationParameter = 
     new PluginParameter<String>(
       "featureIdentification",
@@ -109 +118 @@
     );
   
-  private static final PluginParameter<String> reporterIdColumnMapping = new PluginParameter<String>(
+  protected static final PluginParameter<String> reporterIdColumnMapping = new PluginParameter<String>(
     "reporterIdColumnMapping",
     "Reporter ID",
@@ -117 +126 @@
     );
   
-  private static final PluginParameter<String> featureIdColumnMapping = new PluginParameter<String>(
+  protected static final PluginParameter<String> featureIdColumnMapping = new PluginParameter<String>(
     "featureIdColumnMapping",
     "Feature ID",
@@ -127 +136 @@
     );
 
-  private static final PluginParameter<String> positionColumnMapping = new PluginParameter<String>(
+  protected static final PluginParameter<String> positionColumnMapping = new PluginParameter<String>(
     "positionColumnMapping",
     "Position",
@@ -139 +148 @@
   
   
-  private static final PluginParameter<String> blockColumnMapping = new PluginParameter<String>(
+  protected static final PluginParameter<String> blockColumnMapping = new PluginParameter<String>(
     "blockColumnMapping",
     "Block",
@@ -151 +160 @@
     );
 
-  private static final PluginParameter<String> metaGridXColumnMapping = new PluginParameter<String>(
+  protected static final PluginParameter<String> metaGridXColumnMapping = new PluginParameter<String>(
     "metaGridXColumnMapping",
     "Meta grid X",
@@ -162 +171 @@
     );
 
-  private static final PluginParameter<String> metaGridYColumnMapping = new PluginParameter<String>(
+  protected static final PluginParameter<String> metaGridYColumnMapping = new PluginParameter<String>(
     "metaGridYColumnMapping",
     "Meta grid Y",
@@ -173 +182 @@
     );
   
-  private static final PluginParameter<String> columnColumnMapping = new PluginParameter<String>(
+  protected static final PluginParameter<String> columnColumnMapping = new PluginParameter<String>(
     "columnColumnMapping",
     "Column",
@@ -183 +192 @@
     );
   
-  private static final PluginParameter<String> rowColumnMapping = new PluginParameter<String>(
+  protected static final PluginParameter<String> rowColumnMapping = new PluginParameter<String>(
     "rowColumnMapping",
     "Row",
@@ -275 +284 @@
       permissions.add(new Permissions(Item.REPORTER, null, EnumSet.of(Permission.READ)));
       permissions.add(new Permissions(Item.FILE, null, EnumSet.of(Permission.READ)));
-      permissions.add(new Permissions(Item.PLATFORM, null, EnumSet.of(Permission.READ)));
-      permissions.add(new Permissions(Item.DATAFILETYPE, null, EnumSet.of(Permission.READ)));
+      permissions.add(new Permissions(Item.PLATFORM, EnumSet.of(Permission.READ), null));
+      permissions.add(new Permissions(Item.DATAFILETYPE, EnumSet.of(Permission.READ), null));
     }
     return permissions;
@@ -296 +305 @@
   /**
     Returns null if the item is a {@link ArrayDesign} that doesn't already have 
-    any features and isn't an affy design.
+    any features and is using a supported platform/variant.
     @throws PermissionDeniedException If the array design already has features
       or if the logged in user doesn't have write permission
   */
+  @SuppressWarnings("unchecked")
   public String isInContext(GuiContext context, Object item)
   {
@@ -324 +334 @@
       else
       {
+        List<String> platforms = (List<String>)configuration.getValues("platforms");
+        if (platforms != null && platforms.size() > 0)
+        {
+          Platform platform = ad.getPlatform();
+          if (platforms.indexOf("P:" + platform.getExternalId()) == -1)
+          {
+            PlatformVariant variant = ad.getVariant();
+            if (variant == null || platforms.indexOf("V:" + variant.getExternalId()) == -1)
+            {
+              message = "Unsupported platform/variant: " + platform.getName();
+              if (variant != null) message += "/" + variant.getName();
+            }
+          }
+        }
         ad.checkPermission(Permission.WRITE);
       }
@@ -415 +439 @@
         }
         
+        storeValues(configuration, request, ri.getParameter("platforms"));
         // Parser settings
         storeValue(configuration, request, headerRegexpParameter);
@@ -665 +690 @@
       ArrayDesign design = null;
       int currentArrayDesignId = sc.getCurrentContext(Item.ARRAYDESIGN).getId();
-      List<File> reporterMapFiles = null;
+      File reporterMapFile = null;
+      DataFileType reporterMapType = null;
       dc = sc.newDbControl();
       try
       {
         design = ArrayDesign.getById(dc, currentArrayDesignId);
-        reporterMapFiles = FileStoreUtil.getGenericDataFiles(dc, design, File.REPORTER_MAP);
+        // Check for an attached file of the 'reporter map' generic type
+        List<FileSetMember> tmp = FileStoreUtil.getGenericMembers(dc, design, File.REPORTER_MAP);
+        if (tmp.size() == 1) 
+        {
+          reporterMapFile = tmp.get(0).getFile();
+          reporterMapType = tmp.get(0).getDataFileType();
+        }
+        else if (tmp.size() == 0)
+        {
+          // Check if the platform defines a 'reporter map' generic file type
+          List<PlatformFileType> tmp2 = FileStoreUtil.getGenericPlatformFileTypes(dc, design, File.REPORTER_MAP);
+          if (tmp2.size() == 1)
+          {
+            reporterMapType = tmp2.get(0).getDataFileType();
+          }         
+        }
       }
       catch (Throwable t)
@@ -683 +724 @@
       PluginParameter<File> fileParameter = new PluginParameter<File>(
         "file",
-        "Reporter map file",
+        reporterMapType == null ? "Reporter map file" : reporterMapType.getName(),
         "The file with reporter information to import features from",
-        new FileParameterType(reporterMapFiles == null || reporterMapFiles.isEmpty() ? 
-          null : reporterMapFiles.get(0), true, 1)
+        new FileParameterType(reporterMapFile == null ? 
+          null : reporterMapFile, true, 1)
       );
       parameters.add(fileParameter);
@@ -719 +760 @@
       List<PluginParameter<?>> parameters = new ArrayList<PluginParameter<?>>();
 
+      Enumeration<String, String> platforms = new Enumeration<String, String>();
+      try
+      {
+        dc = sc.newDbControl();
+        ItemQuery<Platform> query = Platform.getQuery();
+        query.order(Orders.asc(Hql.property("name")));    
+        for (Platform platform : query.list(dc))
+        {
+          ItemQuery<PlatformVariant> variantQuery = platform.getVariants();
+          variantQuery.order(Orders.asc(Hql.property("name")));
+          variantQuery.restrict(
+            Restrictions.eq(
+                Hql.property("fileOnly"), Expressions.parameter("fileOnly", false)
+            ));
+          List<PlatformVariant> variants = variantQuery.list(dc);
+          if (variants.size() > 0 || !platform.isFileOnly())
+          {
+            platforms.add("P:" + platform.getExternalId(), platform.getName());
+          }
+          int i = 0;
+          int last = variants.size() - 1;
+          for (PlatformVariant variant : variants)
+          {
+            String prefix = i == last ? " └" : " ├";
+            platforms.add("V:"+variant.getExternalId(), prefix + variant.getName());
+          }
+        }
+      }
+      finally
+      {
+        if (dc != null) dc.close();
+      }
+      
+      if (platforms.size() > 0)
+      {
+        parameters.add(new PluginParameter<String>(
+            "platforms", "Platforms/variants", 
+            "Select all platforms/variants where this configuration can be used. If " +
+            "not selected, the configuration can be used on all except file-only platforms.",
+            new StringParameterType(Platform.MAX_EXTERNAL_ID_LENGTH, null, false, 0, 0, 0, platforms)
+          ));
+      }
+      
       // Parser regular expressions
       parameters.add(parserSection);

trunk/src/plugins/core/net/sf/basedb/plugins/gtf/GtfReporterImporter.java

@@ -24 +24 @@
 import java.io.IOException;
 import java.io.InputStream;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
 
 import net.sf.basedb.core.BaseException;
-import net.sf.basedb.core.InvalidDataException;
-import net.sf.basedb.core.Job;
-import net.sf.basedb.core.ParameterType;
-import net.sf.basedb.core.Permission;
-import net.sf.basedb.core.PermissionDeniedException;
-import net.sf.basedb.core.PluginConfiguration;
-import net.sf.basedb.core.PluginDefinition;
 import net.sf.basedb.core.SessionControl;
 import net.sf.basedb.core.plugin.ParameterValues;
@@ -89 +79 @@
     super.init(sc, configuration, job);
   }
-
+  // -------------------------------------------
+  
   /*
     From the WrappedConfigureByExample interface
@@ -115 +106 @@
   }
 
-  /**
-    Provides a default configuration when none has been provided by the user.
-  */
-  static class DefaultConfigurationValues
-    implements ParameterValues
-  {
-
-    private final ParameterValues configuration;
-    private Map<String, Object> defaultValues;
-    
-    DefaultConfigurationValues(ParameterValues configuration)
-    {
-      this.configuration = configuration;
-    }
-    
-    @Override
-    public PluginDefinition getPluginDefinition()
-    {
-      return configuration == null ? null : configuration.getPluginDefinition();
-    }
-
-    @Override
-    public Job getJob()
-    {
-      return configuration == null ? null : configuration.getJob();
-    }
-
-
-    @Override
-    public PluginConfiguration getPluginConfiguration()
-    {
-      return configuration == null ? null : configuration.getPluginConfiguration();
-    }
-
-
-    @Override
-    public int getId()
-    {
-      return configuration == null ? 0 : configuration.getId();
-    }
-
-    @Override
-    public Object getValue(String name)
-      throws PermissionDeniedException, BaseException
-    {
-      Object value = configuration == null ? null : configuration.getValue(name);
-      if (value == null)
-      {
-        if (defaultValues == null)
-        {
-          defaultValues = new HashMap<String, Object>();
-          defaultValues.put("dataSplitterRegexp", "\\t");
-          defaultValues.put("dataHeaderRegexp", "<seqname>\\t.*<transcript_id>.*");
-          defaultValues.put("minDataColumns", 4);
-          defaultValues.put("nameColumnMapping", "\\<transcript_id>\\");
-          defaultValues.put("reporterIdColumnMapping", "\\<transcript_id>\\");
-        }
-        value = defaultValues.get(name);
-      }
-      return value;
-    }
-
-    @Override
-    public List<?> getValues(String name)
-      throws PermissionDeniedException, BaseException
-    {
-      return configuration == null ? null : configuration.getValues(name);
-    }
-
-    @Override
-    public <T> void setValue(String name, ParameterType<T> type, T value)
-      throws PermissionDeniedException, InvalidDataException, BaseException
-    {
-      if (configuration != null)
-      {
-        configuration.setValue(name, type, value);
-      }
-      else
-      {
-        throw new PermissionDeniedException(Permission.WRITE, "Parameter: "+name);
-      }
-    }
-
-    @Override
-    public <T> void setValues(String name, ParameterType<T> type, List<T> values)
-      throws PermissionDeniedException, InvalidDataException, BaseException
-    {
-      if (configuration != null)
-      {
-        configuration.setValues(name, type, values);
-      }
-      else
-      {
-        throw new PermissionDeniedException(Permission.WRITE, "Parameter: "+name);
-      }
-    }
-    
-  }
 }

trunk/src/test/TestAll.java

@@ -154 +154 @@
     results.put("TestIlluminaImporter", TestIlluminaImporter.test_all());
     results.put("TestGtfInputStream", TestGtfInputStream.test_all());
-    results.put("TestGtfReporterImporter", TestGtfReporterImporter.test_all());
+    results.put("TestGtfImporters", TestGtfImporters.test_all());
     
     // Experiments

trunk/src/test/TestArrayDesign.java

@@ -36 +36 @@
 import net.sf.basedb.core.Item;
 import net.sf.basedb.core.ItemAlreadyExistsException;
+import net.sf.basedb.core.ItemNotFoundException;
 import net.sf.basedb.core.ItemProxy;
 import net.sf.basedb.core.Permission;
@@ -42 +43 @@
 import net.sf.basedb.core.Plate;
 import net.sf.basedb.core.Platform;
+import net.sf.basedb.core.PlatformVariant;
 import net.sf.basedb.core.Reporter;
 import net.sf.basedb.core.ReporterBatcher;
@@ -142 +144 @@
     {
       dc = TestUtil.getDbControl();
-      Platform platform = Platform.getByExternalId(dc, platformId);
-      ArrayDesign ad = ArrayDesign.getNew(dc, platform);
+      ArrayDesign ad = null;
+      try
+      {
+        Platform platform = Platform.getByExternalId(dc, platformId);
+        ad = ArrayDesign.getNew(dc, platform);
+      }
+      catch (ItemNotFoundException ex)
+      {
+        PlatformVariant variant = PlatformVariant.getByExternalId(dc, platformId);
+        ad = ArrayDesign.getNew(dc, variant);
+      }
       if (setAll)
       {

trunk/src/test/TestGtfImporters.java

@@ +1 @@
+import net.sf.basedb.core.ArrayDesign;
 import net.sf.basedb.core.BaseException;
 import net.sf.basedb.core.DbControl;
@@ -6 +7 @@
 import net.sf.basedb.core.ParameterType;
 import net.sf.basedb.core.Permission;
+import net.sf.basedb.core.PlatformVariant;
+import net.sf.basedb.core.PluginConfiguration;
 import net.sf.basedb.core.PluginConfigurationRequest;
 import net.sf.basedb.core.PluginDefinition;
@@ -36 +39 @@
 */
 
-public class TestGtfReporterImporter
+public class TestGtfImporters
 {
   static boolean ok = true;
@@ -50 +53 @@
   static boolean test_all()
   {
-    write("++Testing GTF reporter import using plugin");
-    // Create reporter type and upload file
+    write("++Testing GTF imports using plugin");
+    // Upload GTF file
     int fileId = TestFile.test_create("data/test.gtf", false, false);
     
-    // Create plugin configuration and job
-    int pluginDefinitionId  = TestPluginDefinition.test_get("net.sf.basedb.plugins.gtf.GtfReporterImporter");
-    ok = pluginDefinitionId != 0;
-    int jobId = test_create_job(pluginDefinitionId, fileId, "update");
+    // Test reporter importer
+    int gtfReporterImporterId  = TestPluginDefinition.test_get("net.sf.basedb.plugins.gtf.GtfReporterImporter");
+    ok = gtfReporterImporterId != 0;
+    int reporterJobId = test_create_reporter_job(gtfReporterImporterId, fileId, "update");
+    ok &= TestJob.test_execute(reporterJobId, false);
+    TestReporter.write_header();
+    TestReporter.test_list(35);
     
-    // Execute job
-    ok &= TestJob.test_execute(jobId, false);
+
+    // Test reporter map importer
+    int arrayDesignId = TestArrayDesign.test_create(PlatformVariant.SEQUENCING_EXPRESSION, false);
+    int gtfFeatureImporterId  = TestPluginDefinition.test_get("net.sf.basedb.plugins.gtf.GtfReporterMapImporter");
+    ok = gtfFeatureImporterId != 0;
+    int featureConfigurationId = test_create_feature_configuration(gtfFeatureImporterId);
+    int featureJobId = test_create_feature_job(featureConfigurationId, fileId, arrayDesignId);
+    ok &= TestJob.test_execute(featureJobId, false);
+    TestArrayDesign.write_feature_header();
+    TestArrayDesign.test_list_features(arrayDesignId, 35);
+
+    if (TestUtil.waitBeforeDelete()) TestUtil.waitForEnter();
+    TestArrayDesign.test_delete(arrayDesignId);
+    // Delete reporters
+    int deleteReporterJobId = test_create_reporter_job(gtfReporterImporterId, fileId, "delete");
+    ok &= TestJob.test_execute(deleteReporterJobId, false);
     
-    // Test: list the result
-    //TestReporter.write_header();
-    //TestReporter.test_list(-1);
-    
-    if (TestUtil.waitBeforeDelete()) TestUtil.waitForEnter();
-    int jobId2 = test_create_job(pluginDefinitionId, fileId, "delete");
-    ok &= TestJob.test_execute(jobId2, false);
-    
-    TestJob.test_delete(jobId);
-    TestJob.test_delete(jobId2);
+    TestJob.test_delete(reporterJobId);
+    TestJob.test_delete(deleteReporterJobId);
+    TestJob.test_delete(featureJobId);
+    TestPluginConfiguration.test_delete(featureConfigurationId);
     TestFile.test_delete(fileId);
 
-    write("++Testing GTF reporter import using plugin "+(ok ? "OK" : "Failed")+"\n");
+    write("++Testing GTF importers "+(ok ? "OK" : "Failed")+"\n");
     return ok;
   }
 
 
-  static int test_create_job(int pluginId, int fileId, String mode)
+  static int test_create_reporter_job(int pluginId, int fileId, String mode)
   {
     if (pluginId == 0 || fileId == 0 || !TestUtil.hasPermission(Permission.CREATE, Item.JOB)) return 0;
@@ -149 +163 @@
     }
   }
+  
+  static int test_create_feature_configuration(int pluginId)
+  {
+    if (pluginId == 0 || !TestUtil.hasPermission(Permission.CREATE, Item.PLUGINCONFIGURATION)) return 0;
+    int id = 0;
+    DbControl dc = null;
+    try
+    {
+      dc = TestUtil.getDbControl();
+
+      PluginDefinition pd = PluginDefinition.getById(dc, pluginId);
+      PluginConfiguration pc = pd.newPluginConfiguration();
+      pc.setName(pd.getName());
+      
+      PluginConfigurationRequest request = pc.configure();
+      RequestInformation ri = request.getRequestInformation();
+
+      write_request_information(ri);
+      
+      for (PluginParameter parameter : ri.getParameters())
+      {
+        request.setParameterValue(parameter.getName(), parameter.getDefaultValue());
+      }
+      
+      PluginResponse response = request.invoke();
+      if (response.getStatus() == Response.Status.DONE)
+      {
+        response.saveParameters(dc);
+        dc.saveItem(pc);
+        dc.commit();
+      }
+      else if (response.getStatus() == Response.Status.ERROR)
+      {
+        throw new BaseException(response.getMessage(), response.getErrorList().get(0));
+      }
+      else
+      {
+        throw new BaseException("Unknown status on response: "+response.getStatus());
+      }
+      id = pc.getId();
+      write("--Create configuration for GTF reporter map import OK");
+    }
+    catch (Throwable ex)
+    {
+      write("--Create configuration for GTF reporter map import FAILED");
+      ex.printStackTrace();
+      ok = false;
+    }
+    finally
+    {
+      if (dc != null) dc.close();
+    }
+    return id;
+  }
+
+  static int test_create_feature_job(int pluginConfigurationId, int fileId, int arrayDesignId)
+  {
+    if (pluginConfigurationId == 0 || fileId == 0 || arrayDesignId == 0 || !TestUtil.hasPermission(Permission.CREATE, Item.JOB)) return 0;
+    int id = 0;
+    DbControl dc = null;
+    try
+    {
+      dc = TestUtil.getDbControl();
+
+      PluginConfiguration pc = PluginConfiguration.getById(dc, pluginConfigurationId);
+      Job j = pc.newJob(null);
+      j.setName(pc.getName());
+      
+      PluginConfigurationRequest request = j.configure(null);
+      write_request_information(request.getRequestInformation());
+      request.setParameterValue("file", File.getById(dc, fileId));
+      request.setParameterValue("arrayDesign", ArrayDesign.getById(dc, arrayDesignId));
+      
+      PluginResponse response = request.invoke();
+      if (response.getStatus() == Response.Status.DONE)
+      {
+        response.saveParameters(dc);
+        dc.saveItem(j);
+        dc.commit();
+      }
+      else
+      {
+        throw new BaseException(response.getMessage(), response.getErrorList().get(0));
+      }
+      id = j.getId();
+      write("--Create job for GTF reporter map import OK");
+    }
+    catch (Throwable ex)
+    {
+      write("--Create job for GTF reporter map import FAILED");
+      ex.printStackTrace();
+      ok = false;
+    }
+    finally
+    {
+      if (dc != null) dc.close();
+    }
+    return id;
+  }
+
 }