Changeset 6459

Timestamp:

May 27, 2014, 10:46:05 AM (9 years ago)

Author:

Nicklas Nordborg

Message:

Fixes #1808: Support for comment lines in GTF parser

Location:

trunk

Files:

• data/plugin_configfile.xml (modified) (4 diffs)
• src/core/net/sf/basedb/util/gtf/GtfInputStream.java (modified) (9 diffs)
• src/plugins/core/net/sf/basedb/plugins/gtf/DefaultConfigurationValues.java (modified) (1 diff)
• src/test/TestGtfInputStream.java (modified) (1 diff)
• src/test/data/test.gtf (modified) (1 diff)

trunk/data/plugin_configfile.xml

@@ -957 +957 @@
     </parameter>
     <parameter>
+      <name>ignoreRegexp</name>
+      <label>Ignore</label>
+      <description>A regular expression that matches any line that should be ignored. For example, ignore lines starting with #: ^#.*</description>
+      <class>java.lang.String</class>
+      <value>^#.*</value>
+    </parameter>
+    <parameter>
       <name>minDataColumns</name>
       <label>Min data columns</label>
@@ -1039 +1046 @@
     </parameter>
     <parameter>
+      <name>ignoreRegexp</name>
+      <label>Ignore</label>
+      <description>A regular expression that matches any line that should be ignored. For example, ignore lines starting with #: ^#.*</description>
+      <class>java.lang.String</class>
+      <value>^#.*</value>
+    </parameter>
+    <parameter>
       <name>minDataColumns</name>
       <label>Min data columns</label>
@@ -1121 +1135 @@
     </parameter>
     <parameter>
+      <name>ignoreRegexp</name>
+      <label>Ignore</label>
+      <description>A regular expression that matches any line that should be ignored. For example, ignore lines starting with #: ^#.*</description>
+      <class>java.lang.String</class>
+      <value>^#.*</value>
+    </parameter>
+    <parameter>
       <name>minDataColumns</name>
       <label>Min data columns</label>
@@ -1182 +1203 @@
     </parameter>
     <parameter>
+      <name>ignoreRegexp</name>
+      <label>Ignore</label>
+      <description>A regular expression that matches any line that should be ignored. For example, ignore lines starting with #: ^#.*</description>
+      <class>java.lang.String</class>
+      <value>^#.*</value>
+    </parameter>
+    <parameter>
       <name>trimQuotes</name>
       <label>Remove quotes</label>

trunk/src/core/net/sf/basedb/util/gtf/GtfInputStream.java

@@ -52 +52 @@
   in most cases we are not interested in this when importing data to BASE. 
   This option also remove the feature, start, end, score, strand and frame
-  columns from the output.
+  columns from the output. Lines that can't be split into at least 9 columns 
+  (eg. comment lines starting with #) are ignored and forwarded without modification.
 
   @author Nicklas
@@ -103 +104 @@
     this.reader = new BufferedReader(new InputStreamReader(master, this.charset));
     this.ATTRIBUTE_PATTERN = Pattern.compile("([^ ]+) (([^\";]*)|(\"[^\"]*\"));");
-    init();
+    this.buffer = readMore();
   }
 
@@ -197 +198 @@
   {
     return transcriptIds.size();
-  }
-  
-  /**
-    Initialize the converter by reading the first line from the GTF
-    file. The attributes will be extracted and a header row is created
-    with the 8 required columns + new colums for each attribute.
-    @throws IOException
-  */
-  private void init()
-    throws IOException
-  {
-    // Read first line
-    String[] line = getNextLine();
-    if (line == null) return;
-    
-    // Parse the attributes in 9th column
-    parseAttributes(line[8]);
-    
-    // Generate header line
-    StringBuffer sb = new StringBuffer();
-    transcriptIds.add(attributes[transcriptIdIndex].value+ '@' + line[0]);
-    if (skipRepeatedTranscriptIds)
-    {
-      sb.append("<seqname>\t<source>");
-    }
-    else
-    {
-      sb.append("<seqname>\t<source>\t<feature>\t<start>\t<end>\t<score>\t<strand>\t<frame>");
-    }
-    for (Attribute attribute : attributes)
-    {
-      sb.append("\t<").append(attribute.key).append(">");
-    }
-    sb.append("\n");
-
-    // Append first line of data to the buffer
-    appendLine(sb, line, attributes);
-    buffer = sb.toString().getBytes(charset);
   }
   
@@ -248 +211 @@
   {
     String[] line;
+    StringBuffer sb = new StringBuffer();
     do
     {
@@ -258 +222 @@
       
       // Parse attributes in 9th column
-      parseAttributes(line[8]);
-      
-      String id = attributes[transcriptIdIndex].value + '@' + line[0];
-      if (transcriptIds.add(id) && skipRepeatedTranscriptIds)
-      {
+      if (line.length >= 9)
+      {
+        // Generate headers if this is the first line with data
+        boolean generateHeaders = attributes == null;
+        parseAttributes(line[8]);
+        if (generateHeaders)
+        {
+          if (skipRepeatedTranscriptIds)
+          {
+            sb.append("<seqname>\t<source>");
+          }
+          else
+          {
+            sb.append("<seqname>\t<source>\t<feature>\t<start>\t<end>\t<score>\t<strand>\t<frame>");
+          }
+          for (Attribute attribute : attributes)
+          {
+            sb.append("\t<").append(attribute.key).append(">");
+          }
+          sb.append("\n");
+        }
+        
+        // Break out of the loop as soon as we see a new transcript id or if we are including repeated ids
+        String id = attributes[transcriptIdIndex].value + '@' + line[0];
+        if (transcriptIds.add(id) || !skipRepeatedTranscriptIds)
+        {
+          appendLine(sb, line, attributes);
+          break;
+        }
+      }
+      else
+      {
+        // Append line as it is
+        appendLine(sb, line, null);
         break;
       }
@@ -268 +261 @@
     
     // Convert to byte[]
-    return appendLine(new StringBuffer(), line, attributes).toString().getBytes(charset);
-  }
-  
-  /**
-    Read the next line from the GTF file and split on tab character
-    into 9 or 10 columns. If the line doesn't contain at least 9 columns an 
-    exception is thrown. If the 10th column exists it must be a comment 
-    column that starts with a # character.
-
+    return sb.toString().getBytes(charset);
+  }
+  
+  /**
+    Read the next line from the GTF file and split on tab character.
   */
   private String[] getNextLine()
@@ -285 +274 @@
     lineNum++;
     String[] columns = line.split("\\t", 10);
-    if (columns.length < 9) 
-    {
-      throw new IOException("A line must have at least 9 columns, at line: " + lineNum);
-    }
     return columns;
   }
@@ -357 +342 @@
   
   /**
-    Append the first 8 columns to the buffer and then add all values
-    from the attributes.
+    Append columns to the buffer and separate each with a tab.
+    If attributes are given, the first 8 (or 2 if skipRepeatedTranscriptIds=true)
+    columns are appended, then each of the attributes are appended.
+    If no attributes are given, all columns are copied as they are.
+    
     @param sb The buffer to append to
     @param columns The regular columns (must be at least 8)
@@ -367 +355 @@
     // First 8 columns are copied with tab separator
     sb.append(columns[0]);
-    int end = skipRepeatedTranscriptIds ? 2 : 8;
+    int end = attr == null ? columns.length : (skipRepeatedTranscriptIds ? 2 : 8);
     for (int i = 1; i < end; ++i)
     {
       sb.append("\t").append(columns[i]);
     }
-    // Then follows the attributes
-    for (Attribute attribute : attr)
-    {
-      sb.append("\t").append(attribute.value);
-      attribute.value = null;
-    }
-    /*
-    TODO - Handle Comments 
-    if (line.length == 10)
-    {
-      sb.append("\t").append(line[9]);
-    }
-    */
+    if (attr != null)
+    {
+      // Then follows the attributes
+      for (Attribute attribute : attr)
+      {
+        sb.append("\t").append(attribute.value);
+        attribute.value = null;
+      }
+      if (columns.length == 10)
+      {
+        sb.append("\t").append(columns[9]);
+      }
+    }
     sb.append("\n");
     return sb;

trunk/src/plugins/core/net/sf/basedb/plugins/gtf/DefaultConfigurationValues.java

@@ -94 +94 @@
         defaultValues.put("dataSplitterRegexp", "\\t");
         defaultValues.put("dataHeaderRegexp", "<seqname>\\t.*<transcript_id>.*");
+        defaultValues.put("ignoreRegexp", "^#.*");
         defaultValues.put("minDataColumns", 4);
         defaultValues.put("complexExpressions", "allow");

trunk/src/test/TestGtfInputStream.java

@@ -62 +62 @@
       ffp.setDataHeaderRegexp(Pattern.compile("\\<seqname\\>.*"));
       ffp.setDataSplitterRegexp(Pattern.compile("\\t"));
+      ffp.setIgnoreRegexp(Pattern.compile("^#.*"));
       ffp.setMinDataColumns(4);
       ffp.setMaxDataColumns(-1);

trunk/src/test/data/test.gtf

@@ +1 @@
+# comment
 chr1  hg19_refGene  start_codon 67000042  67000044  0 + . gene_id "NM_032291"; transcript_id "NM_032291"; gene_name2 "SGIP1"; 
 chr1  hg19_refGene  CDS 67000042  67000051  0 + 0 gene_id "NM_032291"; transcript_id "NM_032291"; gene_name2 "SGIP1";