Changeset 5764


Ignore:
Timestamp:
Sep 27, 2011, 2:35:28 PM (10 years ago)
Author:
Nicklas Nordborg
Message:

Fixes #1624: Create array design (feature) importer for GTF files

  • Created GtfReporterMapImporter and two configurations that uses transcript_id/gene_id for reporter ids. The GTF importer builds on the ReporterMapFlatFileImporter but has less configuration options. Eg. it is locked to use FEATURE_ID for identifying features.
  • Added a new parameter to the regular ReporterMapFlatFileImporter that makes it possible to limit a configuration to certain platforms. This is used to hide genepix, illumina, etc. when using the sequencing platform.
  • Added SEQUENCING platform and SEQUENCING_EXPRESSION variant.
  • Created a GTF validator that hooks into the fileset validation extension point for array designs and extracts the number of unique transcript_id:s from the GTF file.
Location:
trunk
Files:
4 added
14 edited
1 moved

Legend:

Unmodified
Added
Removed
  • trunk/data/illumina.configurations.xml

    r3626 r5764  
    479479The line number starts at 1 for the first data line (headers are ignored). This is the same as the IlluminaRawDataImporter plug-in does so it should be possible to connect the raw bioassays with an array design if needed.</description>
    480480    <parameter>
     481      <name>platforms</name>
     482      <label>Platforms/variants</label>
     483      <description>Select all platforms/variants where this configuration can be used. If not selected, the configuration can be used on all except file-only platforms.</description>
     484      <class>java.lang.String</class>
     485      <value>P:generic</value>
     486    </parameter>
     487    <parameter>
    481488      <name>dataHeaderRegexp</name>
    482489      <label>Data header</label>
  • trunk/data/plugin_configfile.xml

    r5761 r5764  
    293293Block mapping is used in this configuration.</description>
    294294    <parameter>
     295      <name>platforms</name>
     296      <label>Platforms/variants</label>
     297      <description>Select all platforms/variants where this configuration can be used. If not selected, the configuration can be used on all except file-only platforms.</description>
     298      <class>java.lang.String</class>
     299      <value>P:generic</value>
     300    </parameter>
     301    <parameter>
    295302      <name>dataHeaderRegexp</name>
    296303      <label>dataHeaderRegexp</label>
     
    10611068    </parameter>
    10621069  </configuration>
     1070  <configuration pluginClassName="net.sf.basedb.plugins.gtf.GtfReporterMapImporter">
     1071    <configname>transcript_id (no prefix)</configname>
     1072    <description>A configuration that uses the transcript_id (no prefix) as reporter and feature id.</description>
     1073    <parameter>
     1074      <name>reporterIdColumnMapping</name>
     1075      <label>Reporter ID</label>
     1076      <description>Mapping that picks the reporter's external ID from the data columns. For example: \ID\</description>
     1077      <class>java.lang.String</class>
     1078      <value>\&lt;transcript_id&gt;\</value>
     1079    </parameter>
     1080    <parameter>
     1081      <name>trimQuotes</name>
     1082      <label>Remove quotes</label>
     1083      <description>If true quotes (" or ') around data value will be removed.</description>
     1084      <class>java.lang.Boolean</class>
     1085      <value>true</value>
     1086    </parameter>
     1087    <parameter>
     1088      <name>dataHeaderRegexp</name>
     1089      <label>Data header</label>
     1090      <description>A regular expression that matches the header line just before the data begins. For example: Block\tRow\tColumn.*</description>
     1091      <class>java.lang.String</class>
     1092      <value>&lt;seqname&gt;\t.*&lt;transcript_id&gt;.*</value>
     1093    </parameter>
     1094    <parameter>
     1095      <name>featureIdentification</name>
     1096      <label />
     1097      <description />
     1098      <class>java.lang.String</class>
     1099      <value>FEATURE_ID</value>
     1100    </parameter>
     1101    <parameter>
     1102      <name>complexExpressions</name>
     1103      <label>Complex column mappings</label>
     1104      <description>disallow = Only allow simple mappings that are constant value or pick the value from one column only, for example, '1.6' or '\Row\'
     1105allow = Allow expression and complex mappings, for example, '\Row\, \Column\' or '=2*col('radius')'</description>
     1106      <class>java.lang.String</class>
     1107      <value>disallow</value>
     1108    </parameter>
     1109    <parameter>
     1110      <name>charset</name>
     1111      <label>Character set</label>
     1112      <description>The character set to use when reading the file. This setting overrides the character set specified by the file. If neither this parameter nor the file specifies a character set, the system default is used (ISO-8859-1).</description>
     1113      <class>java.lang.String</class>
     1114      <value>ISO-8859-1</value>
     1115    </parameter>
     1116    <parameter>
     1117      <name>featureIdColumnMapping</name>
     1118      <label>Feature ID</label>
     1119      <description>Mapping that picks the feature's ID from the data columns. This column is only used when the array design uses the FEATURE_ID method for identifying features. In the other cases, the value is just stored as it is.For example: \Feature ID\</description>
     1120      <class>java.lang.String</class>
     1121      <value>\&lt;transcript_id&gt;\</value>
     1122    </parameter>
     1123    <parameter>
     1124      <name>dataSplitterRegexp</name>
     1125      <label>Data splitter</label>
     1126      <description>A regular expression that splits each data line into individual columns. For example, split on tabs: \t</description>
     1127      <class>java.lang.String</class>
     1128      <value>\t</value>
     1129    </parameter>
     1130  </configuration>
     1131  <configuration pluginClassName="net.sf.basedb.plugins.gtf.GtfReporterMapImporter">
     1132    <configname>gene_id (no prefix)</configname>
     1133    <description>A configuration that uses the gene_id (no prefix) as reporter id and transcript_id as feature id. Note that gene_id may not be unique so it is not recommended to use that as feature id.</description>
     1134    <parameter>
     1135      <name>reporterIdColumnMapping</name>
     1136      <label>Reporter ID</label>
     1137      <description>Mapping that picks the reporter's external ID from the data columns. For example: \ID\</description>
     1138      <class>java.lang.String</class>
     1139      <value>\&lt;gene_id&gt;\</value>
     1140    </parameter>
     1141    <parameter>
     1142      <name>trimQuotes</name>
     1143      <label>Remove quotes</label>
     1144      <description>If true quotes (" or ') around data value will be removed.</description>
     1145      <class>java.lang.Boolean</class>
     1146      <value>true</value>
     1147    </parameter>
     1148    <parameter>
     1149      <name>dataHeaderRegexp</name>
     1150      <label>Data header</label>
     1151      <description>A regular expression that matches the header line just before the data begins. For example: Block\tRow\tColumn.*</description>
     1152      <class>java.lang.String</class>
     1153      <value>&lt;seqname&gt;\t.*&lt;gene_id&gt;.*</value>
     1154    </parameter>
     1155    <parameter>
     1156      <name>featureIdentification</name>
     1157      <label />
     1158      <description />
     1159      <class>java.lang.String</class>
     1160      <value>FEATURE_ID</value>
     1161    </parameter>
     1162    <parameter>
     1163      <name>complexExpressions</name>
     1164      <label>Complex column mappings</label>
     1165      <description>disallow = Only allow simple mappings that are constant value or pick the value from one column only, for example, '1.6' or '\Row\'
     1166allow = Allow expression and complex mappings, for example, '\Row\, \Column\' or '=2*col('radius')'</description>
     1167      <class>java.lang.String</class>
     1168      <value>disallow</value>
     1169    </parameter>
     1170    <parameter>
     1171      <name>charset</name>
     1172      <label>Character set</label>
     1173      <description>The character set to use when reading the file. This setting overrides the character set specified by the file. If neither this parameter nor the file specifies a character set, the system default is used (ISO-8859-1).</description>
     1174      <class>java.lang.String</class>
     1175      <value>ISO-8859-1</value>
     1176    </parameter>
     1177    <parameter>
     1178      <name>featureIdColumnMapping</name>
     1179      <label>Feature ID</label>
     1180      <description>Mapping that picks the feature's ID from the data columns. This column is only used when the array design uses the FEATURE_ID method for identifying features. In the other cases, the value is just stored as it is.For example: \Feature ID\</description>
     1181      <class>java.lang.String</class>
     1182      <value>\&lt;transcript_id&gt;\</value>
     1183    </parameter>
     1184    <parameter>
     1185      <name>dataSplitterRegexp</name>
     1186      <label>Data splitter</label>
     1187      <description>A regular expression that splits each data line into individual columns. For example, split on tabs: \t</description>
     1188      <class>java.lang.String</class>
     1189      <value>\t</value>
     1190    </parameter>
     1191  </configuration>
    10631192</configfile>
  • trunk/src/core/core-extensions.xml

    r5623 r5764  
    141141  </extension>
    142142
     143  <extension
     144    id="net.sf.basedb.util.gtf.gtf-validator"
     145    extends="net.sf.basedb.core.filehandler.validator"
     146    >
     147    <about>
     148      <name>GTF file validator</name>
     149      <description>
     150        Validator for GTF reference files. The validation checks that
     151        the first few lines follow the GTF specification.
     152        See http://mblab.wustl.edu/GTF22.html
     153      </description>
     154    </about>
     155    <index>1</index>
     156    <action-factory>
     157      <factory-class>net.sf.basedb.util.gtf.GtfValidationFactory</factory-class>
     158    </action-factory>
     159  </extension>
     160
    143161  <!--
    144162  <extension
  • trunk/src/core/net/sf/basedb/core/DataFileType.java

    r5698 r5764  
    150150  public static final String MICROARRAY_IMAGE = "microarray.image";
    151151
     152  /**
     153    The external ID for the file type representing a GTF
     154    reference file that define the features on an array design
     155    that we are interested in.
     156    @since 3.0
     157  */
     158  public static final String REF_SEQ_GTF = "refseq.gtf";
     159 
    152160  /**
    153161    The maximum length of the file extension that can be stored in the database.
  • trunk/src/core/net/sf/basedb/core/FileStoreUtil.java

    r5719 r5764  
    2424
    2525import java.util.ArrayList;
     26import java.util.Collections;
    2627import java.util.LinkedList;
    2728import java.util.List;
     
    201202 
    202203  /**
     204    Get all file members with a specific generic {@link ItemSubtype} in a given item's file
     205    set. Note! This method will only return members that the logged in user
     206    has permission to read the file. Other members in the file set will be ignored.
     207    Note! This method checks the file type from {@link DataFileType#getGenericType()},
     208    not the file type from {@link File#getItemSubtype()}.
     209   
     210    @param dc A DbControl to use for database access
     211    @param item The {@link FileStoreEnabled} item to check
     212    @param genericType The system ID of the {@link ItemSubtype} to check for
     213    @return A list of member items that matches
     214    @since 3.0
     215  */
     216  public static List<FileSetMember> getGenericMembers(DbControl dc, FileStoreEnabled item,
     217    String genericType)
     218  {
     219    List<FileSetMember> members = new LinkedList<FileSetMember>();
     220    if (item.hasFileSet())
     221    {
     222      ItemQuery<FileSetMember> query = item.getFileSet().getMembers(genericType);
     223      query.order(Orders.asc(Hql.property("file.name")));
     224      for (FileSetMember member : query.list(dc))
     225      {
     226        try
     227        {
     228          member.getFile();
     229          members.add(member);
     230        }
     231        catch (Throwable t)
     232        {}
     233      }
     234    }
     235    return members;
     236  }
     237
     238  /**
     239    Get a list of all platform file types that has the specified generic {@link ItemSubtype}.
     240    If the item doesn't have any associated platform or variant, no file types are returned.
     241   
     242    @param dc A DbControl to use for database access
     243    @param item The {@link FileStoreEnabled} item to check
     244    @param genericType The system ID of the {@link ItemSubtype} to check for
     245    @return  A list of platform file types that matches
     246    @since 3.0
     247  */
     248  public static List<PlatformFileType> getGenericPlatformFileTypes(DbControl dc, FileStoreEnabled item,
     249      String genericType)
     250  {
     251    Platform platform = item.getPlatform();
     252    PlatformVariant variant = item.getVariant();
     253    if (platform == null && variant == null)
     254    {
     255      return Collections.emptyList();
     256    }
     257   
     258    ItemQuery<PlatformFileType> query =
     259      PlatformFileType.getQuery(platform, variant, variant == null);
     260    query.restrict(
     261      Restrictions.eq(
     262        Hql.property("dataFileType.genericType.systemId"),
     263        Expressions.string(genericType)
     264      )
     265    );
     266    query.order(Orders.asc(Hql.property("dataFileType.name")));
     267    return query.list(dc);
     268  }
     269 
     270  /**
    203271    Set or remove a file to {@link FileStoreEnabled} item based on
    204272    the file's {@link DataFileType}.
  • trunk/src/core/net/sf/basedb/core/Install.java

    r5759 r5764  
    3737import net.sf.basedb.core.data.PlatformData;
    3838import net.sf.basedb.core.data.PlatformFileTypeData;
     39import net.sf.basedb.core.data.PlatformVariantData;
    3940import net.sf.basedb.core.data.PluginConfigurationData;
    4041import net.sf.basedb.core.data.ProjectKeyData;
     
    878879        Item.DERIVEDBIOASSAY, "tif", imageType, new ItemSubtypeFT(scan, false, true)
    879880      );
     881      DataFileTypeData gtfDesignFile = createDataFileType(
     882        DataFileType.REF_SEQ_GTF, "GTF ref-seq file",
     883        "A GTF reference file that contains all features that we are interested in. " +
     884        "The default settings use <transcript_id> to identify genes.",
     885        Item.ARRAYDESIGN, "gtf", reporterMapType);
    880886     
    881887      // Platforms and variants
    882888      progressStep++;
    883889      if (progress != null) progress.display((int)(progressStep*progress_factor), "--Creating platforms...");
    884       createPlatform("generic", "Generic", "Generic platform which expects data to be imported into the database",
     890      createPlatform(Platform.GENERIC, "Generic microarray",
     891        "Generic platform which expects data to be imported into the database",
    885892        false, null, 0,
    886893        new PlatformFT(reporterMapFile, false, false),
     
    888895        new PlatformFT(rawDataFile, false, true)
    889896      );
    890       createPlatform("affymetrix", "Affymetrix", "Affymetrix platform", true, null, 1,
     897      createPlatform(Platform.AFFYMETRIX, "Affymetrix", "Affymetrix platform", true, null, 1,
    891898        new PlatformFT(celFile, true, false),
    892899        new PlatformFT(cdfFile, true, false)
     900      );
     901      PlatformData sequencing = createPlatform(Platform.SEQUENCING, "Sequencing",
     902        "Generic platform for sequence data. We do not specify any more details here, but " +
     903        "use variants.",
     904        false, null, 0
     905      );
     906      createPlatformVariant(sequencing, PlatformVariant.SEQUENCING_EXPRESSION, "Expression-like",
     907        "Variant for sequence data which has been pre-processed enough to make it possible " +
     908        "for importing into the database. ", false, null, 0,
     909        new PlatformFT(gtfDesignFile, true, false)
    893910      );
    894911     
     
    26302647
    26312648  /**
     2649    Create a {@link PlatformVariant}.
     2650  */
     2651  private static PlatformVariantData createPlatformVariant(PlatformData platform, String externalId,
     2652    String name, String description, boolean fileOnly, String rawDataType,
     2653    int channels, PlatformFT... fileTypes)
     2654    throws BaseException
     2655  {
     2656    org.hibernate.Transaction tx = null;
     2657    PlatformVariantData variant = null;
     2658    try
     2659    {
     2660      tx = HibernateUtil.newTransaction(session);
     2661   
     2662      if (variant == null)
     2663      {
     2664        org.hibernate.Query query = HibernateUtil.getPredefinedQuery(session,
     2665          "GET_PLATFORMVARIANT_FOR_EXTERNAL_ID");
     2666        query.setString("externalId", externalId);
     2667        variant = HibernateUtil.loadData(PlatformVariantData.class, query);
     2668      }
     2669     
     2670      if (variant != null)
     2671      {
     2672        log.info("createPlatformVariant: EXISTS [externalId="+externalId+"]");
     2673       
     2674        for (PlatformFileTypeData fileType : platform.getFileTypes().values())
     2675        {
     2676          if (variant.equals(fileType.getVariant()))
     2677          {
     2678            for (PlatformFT ft : fileTypes)
     2679            {
     2680              if (ft.fileType.equals(fileType.getDataFileType()))
     2681              {
     2682                fileType.setAllowMultiple(ft.multiple);
     2683                fileType.setRequired(ft.required);
     2684              }
     2685            }
     2686          }
     2687        }
     2688       
     2689        HibernateUtil.commit(tx);
     2690      }
     2691      else
     2692      {
     2693        variant = new PlatformVariantData();
     2694        variant.setPlatform(platform);
     2695        variant.setExternalId(externalId);
     2696        variant.setName(name);
     2697        variant.setDescription(description);
     2698        variant.setFileOnly(fileOnly);
     2699        if (fileOnly)
     2700        {
     2701          variant.setChannels(channels);
     2702          variant.setRawDataType("variant." + externalId);
     2703        }
     2704        else
     2705        {
     2706          variant.setRawDataType(rawDataType);
     2707        }
     2708        HibernateUtil.saveData(session, variant);
     2709 
     2710        if (fileTypes != null)
     2711        {
     2712          for (PlatformFT pft : fileTypes)
     2713          {
     2714            PlatformFileTypeData platformFile = new PlatformFileTypeData();
     2715            platformFile.setPlatform(platform);
     2716            platformFile.setVariant(variant);
     2717            platformFile.setDataFileType(pft.fileType);
     2718            platformFile.setRequired(pft.required);
     2719            platformFile.setAllowMultiple(pft.multiple);
     2720            HibernateUtil.saveData(session, platformFile);
     2721          }
     2722        }
     2723       
     2724        HibernateUtil.commit(tx);
     2725        log.info("createPlatformVariant: OK [externalId="+externalId+"]");
     2726      }
     2727    }
     2728    catch (BaseException ex)
     2729    {
     2730      if(tx != null) HibernateUtil.rollback(tx);
     2731      log.error("createPlatformVariant: FAILED [externalId="+externalId+"]", ex);
     2732      throw ex;
     2733    }
     2734    return variant;
     2735  }
     2736
     2737  /**
    26322738    Create a {@link DataFileType}.
    26332739  */
  • trunk/src/core/net/sf/basedb/core/Platform.java

    r5713 r5764  
    7676  */
    7777  public static final String AFFYMETRIX = "affymetrix";
     78 
     79  /**
     80    The external id for the generic <code>Sequencing</code> platform.
     81    We don't defined any specific details at this level, but use
     82    variants for this.
     83    @see PlatformVariant#SEQUENCING_EXPRESSION
     84    @since 3.0
     85  */
     86  public static final String SEQUENCING = "sequencing";
    7887 
    7988  /**
  • trunk/src/core/net/sf/basedb/core/PlatformVariant.java

    r5713 r5764  
    3838  implements Nameable, Removable
    3939{
     40 
     41  /**
     42    The external id for the expression-like sequencing data.
     43    @since 3.0
     44  */
     45  public static final String SEQUENCING_EXPRESSION = "sequencing.expression";
     46
    4047 
    4148  /**
  • trunk/src/core/net/sf/basedb/util/gtf/GtfInputStream.java

    r5759 r5764  
    100100    this.charset = Charset.forName(charset == null ? "ISO-8859-1" : charset);
    101101    this.skipRepeatedTranscriptIds = skipRepeatedTranscriptIds;
    102     this.transcriptIds = skipRepeatedTranscriptIds ? new HashSet<String>() : null;
    103     this.reader = new BufferedReader(new InputStreamReader(master, charset));
     102    this.transcriptIds = new HashSet<String>();
     103    this.reader = new BufferedReader(new InputStreamReader(master, this.charset));
    104104    this.ATTRIBUTE_PATTERN = Pattern.compile("([^ ]+) (([^\";]*)|(\"[^\"]*\"));");
    105105    init();
     
    184184
    185185  /**
     186    Get the number of lines parsed so far.
     187  */
     188  public int getNumLines()
     189  {
     190    return lineNum;
     191  }
     192 
     193  /**
     194    Get the number of unique transcript ids found so far.
     195  */
     196  public int getNumUniqueTranscriptIds()
     197  {
     198    return transcriptIds.size();
     199  }
     200 
     201  /**
    186202    Initialize the converter by reading the first line from the GTF
    187203    file. The attributes will be extracted and a header row is created
     
    201217    // Generate header line
    202218    StringBuffer sb = new StringBuffer();
     219    transcriptIds.add(attributes[transcriptIdIndex].value);
    203220    if (skipRepeatedTranscriptIds)
    204221    {
    205       transcriptIds.add(attributes[transcriptIdIndex].value);
    206222      sb.append("<seqname>\t<source>");
    207223    }
     
    244260      parseAttributes(line[8]);
    245261     
    246       if (skipRepeatedTranscriptIds)
    247       {
    248         String id = attributes[transcriptIdIndex].value;
    249         if (!transcriptIds.contains(id))
    250         {
    251           transcriptIds.add(id);
    252           break;
    253         }
     262      String id = attributes[transcriptIdIndex].value;
     263      if (transcriptIds.add(id) && skipRepeatedTranscriptIds)
     264      {
     265        break;
    254266      }
    255267    } while (skipRepeatedTranscriptIds);
  • trunk/src/plugins/core/core-plugins.xml

    r5759 r5764  
    766766      <name>GTF reporter importer</name>
    767767      <description>
    768         Creater reporters and reporter lists from GTF (Gene transfer format)
     768        Creates reporters and reporter lists from GTF (Gene transfer format)
    769769        files. The default configuration is to use the transcript_id value
    770770        as the reporter id and name. No other fields are used, but this can
     
    781781  </plugin-definition>
    782782
     783  <plugin-definition id="GtfReporterMapImporter">
     784    <about>
     785      <name>GTF reporter map importer</name>
     786      <description>
     787        Maps reporters to array design features from GTF files. The default
     788        implementation use the transcript_id as both the feature id and
     789        reporter id, but this can be changed by user configuration. For
     790        example, to use the gene_id instead or to add prefixes to the id
     791        values. The importer build on the regular reporter map importer
     792        and perform on-the-fly transformation of GTF attributes to a
     793        completely column-based format.
     794      </description>
     795    </about>
     796    <plugin-class>net.sf.basedb.plugins.gtf.GtfReporterMapImporter</plugin-class>
     797    <settings>
     798      <property name="everyone-use">1</property>
     799    </settings>
     800  </plugin-definition>
     801
    783802  <!--
    784803  <plugin-definition id="">
  • trunk/src/plugins/core/net/sf/basedb/plugins/ReporterMapFlatFileImporter.java

    r5630 r5764  
    4141import net.sf.basedb.core.ItemContext;
    4242import net.sf.basedb.core.ItemNotFoundException;
     43import net.sf.basedb.core.ItemQuery;
    4344import net.sf.basedb.core.Permission;
     45import net.sf.basedb.core.Platform;
     46import net.sf.basedb.core.PlatformFileType;
     47import net.sf.basedb.core.PlatformVariant;
    4448import net.sf.basedb.core.ReporterBatcher;
    4549import net.sf.basedb.core.ItemParameterType;
     
    5761import net.sf.basedb.core.plugin.Request;
    5862import net.sf.basedb.core.plugin.Response;
     63import net.sf.basedb.core.query.Expressions;
     64import net.sf.basedb.core.query.Hql;
     65import net.sf.basedb.core.query.Orders;
     66import net.sf.basedb.core.query.Restrictions;
    5967import net.sf.basedb.plugins.util.Parameters;
     68import net.sf.basedb.util.Enumeration;
    6069import net.sf.basedb.util.error.SimpleErrorHandler;
    6170import net.sf.basedb.util.parser.ConfigureByExample;
     
    96105  private static final StringParameterType optionalColumnMapping = new StringParameterType(255, null, false);
    97106 
    98   private static final PluginParameter<String> featureIdentificationParameter =
     107  protected static final PluginParameter<String> featureIdentificationParameter =
    99108    new PluginParameter<String>(
    100109      "featureIdentification",
     
    109118    );
    110119 
    111   private static final PluginParameter<String> reporterIdColumnMapping = new PluginParameter<String>(
     120  protected static final PluginParameter<String> reporterIdColumnMapping = new PluginParameter<String>(
    112121    "reporterIdColumnMapping",
    113122    "Reporter ID",
     
    117126    );
    118127 
    119   private static final PluginParameter<String> featureIdColumnMapping = new PluginParameter<String>(
     128  protected static final PluginParameter<String> featureIdColumnMapping = new PluginParameter<String>(
    120129    "featureIdColumnMapping",
    121130    "Feature ID",
     
    127136    );
    128137
    129   private static final PluginParameter<String> positionColumnMapping = new PluginParameter<String>(
     138  protected static final PluginParameter<String> positionColumnMapping = new PluginParameter<String>(
    130139    "positionColumnMapping",
    131140    "Position",
     
    139148 
    140149 
    141   private static final PluginParameter<String> blockColumnMapping = new PluginParameter<String>(
     150  protected static final PluginParameter<String> blockColumnMapping = new PluginParameter<String>(
    142151    "blockColumnMapping",
    143152    "Block",
     
    151160    );
    152161
    153   private static final PluginParameter<String> metaGridXColumnMapping = new PluginParameter<String>(
     162  protected static final PluginParameter<String> metaGridXColumnMapping = new PluginParameter<String>(
    154163    "metaGridXColumnMapping",
    155164    "Meta grid X",
     
    162171    );
    163172
    164   private static final PluginParameter<String> metaGridYColumnMapping = new PluginParameter<String>(
     173  protected static final PluginParameter<String> metaGridYColumnMapping = new PluginParameter<String>(
    165174    "metaGridYColumnMapping",
    166175    "Meta grid Y",
     
    173182    );
    174183 
    175   private static final PluginParameter<String> columnColumnMapping = new PluginParameter<String>(
     184  protected static final PluginParameter<String> columnColumnMapping = new PluginParameter<String>(
    176185    "columnColumnMapping",
    177186    "Column",
     
    183192    );
    184193 
    185   private static final PluginParameter<String> rowColumnMapping = new PluginParameter<String>(
     194  protected static final PluginParameter<String> rowColumnMapping = new PluginParameter<String>(
    186195    "rowColumnMapping",
    187196    "Row",
     
    275284      permissions.add(new Permissions(Item.REPORTER, null, EnumSet.of(Permission.READ)));
    276285      permissions.add(new Permissions(Item.FILE, null, EnumSet.of(Permission.READ)));
    277       permissions.add(new Permissions(Item.PLATFORM, null, EnumSet.of(Permission.READ)));
    278       permissions.add(new Permissions(Item.DATAFILETYPE, null, EnumSet.of(Permission.READ)));
     286      permissions.add(new Permissions(Item.PLATFORM, EnumSet.of(Permission.READ), null));
     287      permissions.add(new Permissions(Item.DATAFILETYPE, EnumSet.of(Permission.READ), null));
    279288    }
    280289    return permissions;
     
    296305  /**
    297306    Returns null if the item is a {@link ArrayDesign} that doesn't already have
    298     any features and isn't an affy design.
     307    any features and is using a supported platform/variant.
    299308    @throws PermissionDeniedException If the array design already has features
    300309      or if the logged in user doesn't have write permission
    301310  */
     311  @SuppressWarnings("unchecked")
    302312  public String isInContext(GuiContext context, Object item)
    303313  {
     
    324334      else
    325335      {
     336        List<String> platforms = (List<String>)configuration.getValues("platforms");
     337        if (platforms != null && platforms.size() > 0)
     338        {
     339          Platform platform = ad.getPlatform();
     340          if (platforms.indexOf("P:" + platform.getExternalId()) == -1)
     341          {
     342            PlatformVariant variant = ad.getVariant();
     343            if (variant == null || platforms.indexOf("V:" + variant.getExternalId()) == -1)
     344            {
     345              message = "Unsupported platform/variant: " + platform.getName();
     346              if (variant != null) message += "/" + variant.getName();
     347            }
     348          }
     349        }
    326350        ad.checkPermission(Permission.WRITE);
    327351      }
     
    415439        }
    416440       
     441        storeValues(configuration, request, ri.getParameter("platforms"));
    417442        // Parser settings
    418443        storeValue(configuration, request, headerRegexpParameter);
     
    665690      ArrayDesign design = null;
    666691      int currentArrayDesignId = sc.getCurrentContext(Item.ARRAYDESIGN).getId();
    667       List<File> reporterMapFiles = null;
     692      File reporterMapFile = null;
     693      DataFileType reporterMapType = null;
    668694      dc = sc.newDbControl();
    669695      try
    670696      {
    671697        design = ArrayDesign.getById(dc, currentArrayDesignId);
    672         reporterMapFiles = FileStoreUtil.getGenericDataFiles(dc, design, File.REPORTER_MAP);
     698        // Check for an attached file of the 'reporter map' generic type
     699        List<FileSetMember> tmp = FileStoreUtil.getGenericMembers(dc, design, File.REPORTER_MAP);
     700        if (tmp.size() == 1)
     701        {
     702          reporterMapFile = tmp.get(0).getFile();
     703          reporterMapType = tmp.get(0).getDataFileType();
     704        }
     705        else if (tmp.size() == 0)
     706        {
     707          // Check if the platform defines a 'reporter map' generic file type
     708          List<PlatformFileType> tmp2 = FileStoreUtil.getGenericPlatformFileTypes(dc, design, File.REPORTER_MAP);
     709          if (tmp2.size() == 1)
     710          {
     711            reporterMapType = tmp2.get(0).getDataFileType();
     712          }         
     713        }
    673714      }
    674715      catch (Throwable t)
     
    683724      PluginParameter<File> fileParameter = new PluginParameter<File>(
    684725        "file",
    685         "Reporter map file",
     726        reporterMapType == null ? "Reporter map file" : reporterMapType.getName(),
    686727        "The file with reporter information to import features from",
    687         new FileParameterType(reporterMapFiles == null || reporterMapFiles.isEmpty() ?
    688           null : reporterMapFiles.get(0), true, 1)
     728        new FileParameterType(reporterMapFile == null ?
     729          null : reporterMapFile, true, 1)
    689730      );
    690731      parameters.add(fileParameter);
     
    719760      List<PluginParameter<?>> parameters = new ArrayList<PluginParameter<?>>();
    720761
     762      Enumeration<String, String> platforms = new Enumeration<String, String>();
     763      try
     764      {
     765        dc = sc.newDbControl();
     766        ItemQuery<Platform> query = Platform.getQuery();
     767        query.order(Orders.asc(Hql.property("name")));   
     768        for (Platform platform : query.list(dc))
     769        {
     770          ItemQuery<PlatformVariant> variantQuery = platform.getVariants();
     771          variantQuery.order(Orders.asc(Hql.property("name")));
     772          variantQuery.restrict(
     773            Restrictions.eq(
     774                Hql.property("fileOnly"), Expressions.parameter("fileOnly", false)
     775            ));
     776          List<PlatformVariant> variants = variantQuery.list(dc);
     777          if (variants.size() > 0 || !platform.isFileOnly())
     778          {
     779            platforms.add("P:" + platform.getExternalId(), platform.getName());
     780          }
     781          int i = 0;
     782          int last = variants.size() - 1;
     783          for (PlatformVariant variant : variants)
     784          {
     785            String prefix = i == last ? " └" : " ├";
     786            platforms.add("V:"+variant.getExternalId(), prefix + variant.getName());
     787          }
     788        }
     789      }
     790      finally
     791      {
     792        if (dc != null) dc.close();
     793      }
     794     
     795      if (platforms.size() > 0)
     796      {
     797        parameters.add(new PluginParameter<String>(
     798            "platforms", "Platforms/variants",
     799            "Select all platforms/variants where this configuration can be used. If " +
     800            "not selected, the configuration can be used on all except file-only platforms.",
     801            new StringParameterType(Platform.MAX_EXTERNAL_ID_LENGTH, null, false, 0, 0, 0, platforms)
     802          ));
     803      }
     804     
    721805      // Parser regular expressions
    722806      parameters.add(parserSection);
  • trunk/src/plugins/core/net/sf/basedb/plugins/gtf/GtfReporterImporter.java

    r5759 r5764  
    2424import java.io.IOException;
    2525import java.io.InputStream;
    26 import java.util.HashMap;
    27 import java.util.List;
    28 import java.util.Map;
    2926
    3027import net.sf.basedb.core.BaseException;
    31 import net.sf.basedb.core.InvalidDataException;
    32 import net.sf.basedb.core.Job;
    33 import net.sf.basedb.core.ParameterType;
    34 import net.sf.basedb.core.Permission;
    35 import net.sf.basedb.core.PermissionDeniedException;
    36 import net.sf.basedb.core.PluginConfiguration;
    37 import net.sf.basedb.core.PluginDefinition;
    3828import net.sf.basedb.core.SessionControl;
    3929import net.sf.basedb.core.plugin.ParameterValues;
     
    8979    super.init(sc, configuration, job);
    9080  }
    91 
     81  // -------------------------------------------
     82 
    9283  /*
    9384    From the WrappedConfigureByExample interface
     
    115106  }
    116107
    117   /**
    118     Provides a default configuration when none has been provided by the user.
    119   */
    120   static class DefaultConfigurationValues
    121     implements ParameterValues
    122   {
    123 
    124     private final ParameterValues configuration;
    125     private Map<String, Object> defaultValues;
    126    
    127     DefaultConfigurationValues(ParameterValues configuration)
    128     {
    129       this.configuration = configuration;
    130     }
    131    
    132     @Override
    133     public PluginDefinition getPluginDefinition()
    134     {
    135       return configuration == null ? null : configuration.getPluginDefinition();
    136     }
    137 
    138     @Override
    139     public Job getJob()
    140     {
    141       return configuration == null ? null : configuration.getJob();
    142     }
    143 
    144 
    145     @Override
    146     public PluginConfiguration getPluginConfiguration()
    147     {
    148       return configuration == null ? null : configuration.getPluginConfiguration();
    149     }
    150 
    151 
    152     @Override
    153     public int getId()
    154     {
    155       return configuration == null ? 0 : configuration.getId();
    156     }
    157 
    158     @Override
    159     public Object getValue(String name)
    160       throws PermissionDeniedException, BaseException
    161     {
    162       Object value = configuration == null ? null : configuration.getValue(name);
    163       if (value == null)
    164       {
    165         if (defaultValues == null)
    166         {
    167           defaultValues = new HashMap<String, Object>();
    168           defaultValues.put("dataSplitterRegexp", "\\t");
    169           defaultValues.put("dataHeaderRegexp", "<seqname>\\t.*<transcript_id>.*");
    170           defaultValues.put("minDataColumns", 4);
    171           defaultValues.put("nameColumnMapping", "\\<transcript_id>\\");
    172           defaultValues.put("reporterIdColumnMapping", "\\<transcript_id>\\");
    173         }
    174         value = defaultValues.get(name);
    175       }
    176       return value;
    177     }
    178 
    179     @Override
    180     public List<?> getValues(String name)
    181       throws PermissionDeniedException, BaseException
    182     {
    183       return configuration == null ? null : configuration.getValues(name);
    184     }
    185 
    186     @Override
    187     public <T> void setValue(String name, ParameterType<T> type, T value)
    188       throws PermissionDeniedException, InvalidDataException, BaseException
    189     {
    190       if (configuration != null)
    191       {
    192         configuration.setValue(name, type, value);
    193       }
    194       else
    195       {
    196         throw new PermissionDeniedException(Permission.WRITE, "Parameter: "+name);
    197       }
    198     }
    199 
    200     @Override
    201     public <T> void setValues(String name, ParameterType<T> type, List<T> values)
    202       throws PermissionDeniedException, InvalidDataException, BaseException
    203     {
    204       if (configuration != null)
    205       {
    206         configuration.setValues(name, type, values);
    207       }
    208       else
    209       {
    210         throw new PermissionDeniedException(Permission.WRITE, "Parameter: "+name);
    211       }
    212     }
    213    
    214   }
    215108}
  • trunk/src/test/TestAll.java

    r5759 r5764  
    154154    results.put("TestIlluminaImporter", TestIlluminaImporter.test_all());
    155155    results.put("TestGtfInputStream", TestGtfInputStream.test_all());
    156     results.put("TestGtfReporterImporter", TestGtfReporterImporter.test_all());
     156    results.put("TestGtfImporters", TestGtfImporters.test_all());
    157157   
    158158    // Experiments
  • trunk/src/test/TestArrayDesign.java

    r5696 r5764  
    3636import net.sf.basedb.core.Item;
    3737import net.sf.basedb.core.ItemAlreadyExistsException;
     38import net.sf.basedb.core.ItemNotFoundException;
    3839import net.sf.basedb.core.ItemProxy;
    3940import net.sf.basedb.core.Permission;
     
    4243import net.sf.basedb.core.Plate;
    4344import net.sf.basedb.core.Platform;
     45import net.sf.basedb.core.PlatformVariant;
    4446import net.sf.basedb.core.Reporter;
    4547import net.sf.basedb.core.ReporterBatcher;
     
    142144    {
    143145      dc = TestUtil.getDbControl();
    144       Platform platform = Platform.getByExternalId(dc, platformId);
    145       ArrayDesign ad = ArrayDesign.getNew(dc, platform);
     146      ArrayDesign ad = null;
     147      try
     148      {
     149        Platform platform = Platform.getByExternalId(dc, platformId);
     150        ad = ArrayDesign.getNew(dc, platform);
     151      }
     152      catch (ItemNotFoundException ex)
     153      {
     154        PlatformVariant variant = PlatformVariant.getByExternalId(dc, platformId);
     155        ad = ArrayDesign.getNew(dc, variant);
     156      }
    146157      if (setAll)
    147158      {
  • trunk/src/test/TestGtfImporters.java

    r5759 r5764  
     1import net.sf.basedb.core.ArrayDesign;
    12import net.sf.basedb.core.BaseException;
    23import net.sf.basedb.core.DbControl;
     
    67import net.sf.basedb.core.ParameterType;
    78import net.sf.basedb.core.Permission;
     9import net.sf.basedb.core.PlatformVariant;
     10import net.sf.basedb.core.PluginConfiguration;
    811import net.sf.basedb.core.PluginConfigurationRequest;
    912import net.sf.basedb.core.PluginDefinition;
     
    3639*/
    3740
    38 public class TestGtfReporterImporter
     41public class TestGtfImporters
    3942{
    4043  static boolean ok = true;
     
    5053  static boolean test_all()
    5154  {
    52     write("++Testing GTF reporter import using plugin");
    53     // Create reporter type and upload file
     55    write("++Testing GTF imports using plugin");
     56    // Upload GTF file
    5457    int fileId = TestFile.test_create("data/test.gtf", false, false);
    5558   
    56     // Create plugin configuration and job
    57     int pluginDefinitionId  = TestPluginDefinition.test_get("net.sf.basedb.plugins.gtf.GtfReporterImporter");
    58     ok = pluginDefinitionId != 0;
    59     int jobId = test_create_job(pluginDefinitionId, fileId, "update");
     59    // Test reporter importer
     60    int gtfReporterImporterId  = TestPluginDefinition.test_get("net.sf.basedb.plugins.gtf.GtfReporterImporter");
     61    ok = gtfReporterImporterId != 0;
     62    int reporterJobId = test_create_reporter_job(gtfReporterImporterId, fileId, "update");
     63    ok &= TestJob.test_execute(reporterJobId, false);
     64    TestReporter.write_header();
     65    TestReporter.test_list(35);
    6066   
    61     // Execute job
    62     ok &= TestJob.test_execute(jobId, false);
     67
     68    // Test reporter map importer
     69    int arrayDesignId = TestArrayDesign.test_create(PlatformVariant.SEQUENCING_EXPRESSION, false);
     70    int gtfFeatureImporterId  = TestPluginDefinition.test_get("net.sf.basedb.plugins.gtf.GtfReporterMapImporter");
     71    ok = gtfFeatureImporterId != 0;
     72    int featureConfigurationId = test_create_feature_configuration(gtfFeatureImporterId);
     73    int featureJobId = test_create_feature_job(featureConfigurationId, fileId, arrayDesignId);
     74    ok &= TestJob.test_execute(featureJobId, false);
     75    TestArrayDesign.write_feature_header();
     76    TestArrayDesign.test_list_features(arrayDesignId, 35);
     77
     78    if (TestUtil.waitBeforeDelete()) TestUtil.waitForEnter();
     79    TestArrayDesign.test_delete(arrayDesignId);
     80    // Delete reporters
     81    int deleteReporterJobId = test_create_reporter_job(gtfReporterImporterId, fileId, "delete");
     82    ok &= TestJob.test_execute(deleteReporterJobId, false);
    6383   
    64     // Test: list the result
    65     //TestReporter.write_header();
    66     //TestReporter.test_list(-1);
    67    
    68     if (TestUtil.waitBeforeDelete()) TestUtil.waitForEnter();
    69     int jobId2 = test_create_job(pluginDefinitionId, fileId, "delete");
    70     ok &= TestJob.test_execute(jobId2, false);
    71    
    72     TestJob.test_delete(jobId);
    73     TestJob.test_delete(jobId2);
     84    TestJob.test_delete(reporterJobId);
     85    TestJob.test_delete(deleteReporterJobId);
     86    TestJob.test_delete(featureJobId);
     87    TestPluginConfiguration.test_delete(featureConfigurationId);
    7488    TestFile.test_delete(fileId);
    7589
    76     write("++Testing GTF reporter import using plugin "+(ok ? "OK" : "Failed")+"\n");
     90    write("++Testing GTF importers "+(ok ? "OK" : "Failed")+"\n");
    7791    return ok;
    7892  }
    7993
    8094
    81   static int test_create_job(int pluginId, int fileId, String mode)
     95  static int test_create_reporter_job(int pluginId, int fileId, String mode)
    8296  {
    8397    if (pluginId == 0 || fileId == 0 || !TestUtil.hasPermission(Permission.CREATE, Item.JOB)) return 0;
     
    149163    }
    150164  }
     165 
     166  static int test_create_feature_configuration(int pluginId)
     167  {
     168    if (pluginId == 0 || !TestUtil.hasPermission(Permission.CREATE, Item.PLUGINCONFIGURATION)) return 0;
     169    int id = 0;
     170    DbControl dc = null;
     171    try
     172    {
     173      dc = TestUtil.getDbControl();
     174
     175      PluginDefinition pd = PluginDefinition.getById(dc, pluginId);
     176      PluginConfiguration pc = pd.newPluginConfiguration();
     177      pc.setName(pd.getName());
     178     
     179      PluginConfigurationRequest request = pc.configure();
     180      RequestInformation ri = request.getRequestInformation();
     181
     182      write_request_information(ri);
     183     
     184      for (PluginParameter parameter : ri.getParameters())
     185      {
     186        request.setParameterValue(parameter.getName(), parameter.getDefaultValue());
     187      }
     188     
     189      PluginResponse response = request.invoke();
     190      if (response.getStatus() == Response.Status.DONE)
     191      {
     192        response.saveParameters(dc);
     193        dc.saveItem(pc);
     194        dc.commit();
     195      }
     196      else if (response.getStatus() == Response.Status.ERROR)
     197      {
     198        throw new BaseException(response.getMessage(), response.getErrorList().get(0));
     199      }
     200      else
     201      {
     202        throw new BaseException("Unknown status on response: "+response.getStatus());
     203      }
     204      id = pc.getId();
     205      write("--Create configuration for GTF reporter map import OK");
     206    }
     207    catch (Throwable ex)
     208    {
     209      write("--Create configuration for GTF reporter map import FAILED");
     210      ex.printStackTrace();
     211      ok = false;
     212    }
     213    finally
     214    {
     215      if (dc != null) dc.close();
     216    }
     217    return id;
     218  }
     219
     220  static int test_create_feature_job(int pluginConfigurationId, int fileId, int arrayDesignId)
     221  {
     222    if (pluginConfigurationId == 0 || fileId == 0 || arrayDesignId == 0 || !TestUtil.hasPermission(Permission.CREATE, Item.JOB)) return 0;
     223    int id = 0;
     224    DbControl dc = null;
     225    try
     226    {
     227      dc = TestUtil.getDbControl();
     228
     229      PluginConfiguration pc = PluginConfiguration.getById(dc, pluginConfigurationId);
     230      Job j = pc.newJob(null);
     231      j.setName(pc.getName());
     232     
     233      PluginConfigurationRequest request = j.configure(null);
     234      write_request_information(request.getRequestInformation());
     235      request.setParameterValue("file", File.getById(dc, fileId));
     236      request.setParameterValue("arrayDesign", ArrayDesign.getById(dc, arrayDesignId));
     237     
     238      PluginResponse response = request.invoke();
     239      if (response.getStatus() == Response.Status.DONE)
     240      {
     241        response.saveParameters(dc);
     242        dc.saveItem(j);
     243        dc.commit();
     244      }
     245      else
     246      {
     247        throw new BaseException(response.getMessage(), response.getErrorList().get(0));
     248      }
     249      id = j.getId();
     250      write("--Create job for GTF reporter map import OK");
     251    }
     252    catch (Throwable ex)
     253    {
     254      write("--Create job for GTF reporter map import FAILED");
     255      ex.printStackTrace();
     256      ok = false;
     257    }
     258    finally
     259    {
     260      if (dc != null) dc.close();
     261    }
     262    return id;
     263  }
     264
    151265}
    152266
Note: See TracChangeset for help on using the changeset viewer.