Changeset 569

Timestamp:

Feb 5, 2008, 4:51:36 PM (15 years ago)

Author:

Martin Svensson

Message:

References #98 The validator and meta data reader is done. The installation routine needs to be updated before this ticket is finished.

Location:

trunk/net/sf/basedb/illumina/src/net/sf/basedb/illumina

Files:

• Illumina.java (modified) (1 diff)
• filehandler/BgxFileHandler.java (modified) (11 diffs)
• install/Install.java (modified) (4 diffs)

trunk/net/sf/basedb/illumina/src/net/sf/basedb/illumina/Illumina.java

@@ -80 +80 @@
   */
   public static final String BGX_FILE_ID = "illumina.bgx";
+
+  /**
+    Class name of the BGX validator and meta data extractor
+   */
+  public static final String BGX_VALIDATOR_METADATAREADER = "net.sf.basedb.illumina.filehandler.BgxFileHandler";  
+  
   
   /**

trunk/net/sf/basedb/illumina/src/net/sf/basedb/illumina/filehandler/BgxFileHandler.java

@@ -1 @@
-/*
+/**
   $Id$
 
@@ -24 +24 @@
 package net.sf.basedb.illumina.filehandler;
 
-import net.sf.basedb.core.Config;
-import net.sf.basedb.core.DataFileType;
+import net.sf.basedb.core.ArrayDesign;
 import net.sf.basedb.core.DbControl;
 import net.sf.basedb.core.File;
@@ -33 +32 @@
 import net.sf.basedb.core.InvalidUseOfNullException;
 import net.sf.basedb.core.Location;
-import net.sf.basedb.core.RawBioAssay;
 import net.sf.basedb.core.filehandler.AbstractDataFileHandler;
 import net.sf.basedb.core.filehandler.DataFileMetadataReader;
@@ -39 +37 @@
 import net.sf.basedb.illumina.Illumina;
 import net.sf.basedb.util.FileUtil;
-import net.sf.basedb.util.InputStreamTracker;
 
 import java.io.BufferedReader;
@@ -47 +44 @@
 import java.io.PushbackInputStream;
 import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
 import java.util.regex.Matcher;
@@ -55 +50 @@
 
 /**
+  A tool to validate and extract meta data from a BGX-file.
+  The validator parse through the headings in a given file and
+  fails if it does not find the columns 'Probe_Id' and 'Array_Address_Id' 
+  in the section that comes first of '[Probes]' and '[Controls]' 
+  after a certain number of lines.
+  The extracted meta data is the number of controls given by one of the headings.
     @author Martin
     @version 2.6
-  
  */
 public class BgxFileHandler 
@@ -63 +63 @@
   implements DataFileValidator, DataFileMetadataReader
 {
+  //Max number of lines to parse
+  private final int linesToParse = 100;
+  //The start of the heading that holds the number of controls
+  private String controlsHeading = "Number of Controls\t";
+  
+  //List to hold the parsed lines.
+  private List<String> parsedHeadings = null;
+  
+  //Column names
+  private String[] requiredColumns = {"Probe_Id", "Array_Address_Id"};
   
   public BgxFileHandler()
   {}
   
-  /**
-    Checks that it is really a bgx file 
-    that is used.
+  /*
+      @see net.sf.basedb.core.filehandler.DataFileValidator#validate(net.sf.basedb.core.DbControl)
    */
   @Override
@@ -75 +84 @@
     throws InvalidDataException, InvalidRelationException
   {
+    ArrayDesign design = (ArrayDesign)getItem();
+    if (design.getPlatform() == null || !(Illumina.PLATFORM_ID.equals(design.getPlatform().getExternalId())))
+    {
+      throw new InvalidDataException("Array design '" + design.getName() + "' is not using Illumina platform;");
+    }
     FileSetMember bgxMember = getMember(Illumina.BGX_FILE_ID);
-    InputStream in = null;
     if (bgxMember != null)
     {
       File bgxFile = bgxMember.getFile();
-      if (bgxFile == null)
-      {
-        throw new InvalidUseOfNullException("BGX-file");
-      }
-      if (bgxFile.getLocation() != Location.PRIMARY)
-      {
-        throw new InvalidUseOfNullException("Data is not online for file " +
-            bgxFile.getName() + " location = '" + bgxFile.getLocation() + "'");
-      }
-      try
-      {
-        in = wrapInputStream(bgxFile.getDownloadStream(0));
-      }
-      catch(IOException ioex)
-      {
-        throw new InvalidDataException("BGX-file could not be read properly.");
-      }
-      validateHeaders(in, null);
-    }
-  }
-
-  /*  (non-Javadoc)
+      parsedHeadings = parseHeadings(bgxFile, linesToParse);
+    }   
+  }
+
+  /*  
       @see net.sf.basedb.core.filehandler.DataFileMetadataReader#extractMetadata(net.sf.basedb.core.DbControl)
    */
@@ -107 +103 @@
   public void extractMetadata(DbControl dc)
   {
-    // TODO Auto-generated method stub
-
-  }
-
-  /*  (non-Javadoc)
+    if (parsedHeadings == null)
+    {
+      FileSetMember bgxMember = getMember(Illumina.BGX_FILE_ID);
+      if (bgxMember != null)
+      {
+        parsedHeadings = parseHeadings(bgxMember.getFile(), linesToParse);
+      }     
+    }
+    if (parsedHeadings != null)
+    {
+      ArrayDesign design = (ArrayDesign)getItem();
+      int numFeatures = getNumFeatures(parsedHeadings);
+      design.setNumFileFeatures(numFeatures);
+    }
+  }
+
+  /*
       @see net.sf.basedb.core.filehandler.DataFileMetadataReader#resetMetadata(net.sf.basedb.core.DbControl)
    */
@@ -117 +125 @@
   public void resetMetadata(DbControl dc)
   {
-    // TODO Auto-generated method stub
-
-  }
-  
+    ArrayDesign design = (ArrayDesign)getItem();
+    design.setNumFileFeatures(0);
+  }
+  
+  /*
+    Gets a right type of input stream depending on if 
+    it comes from a packed or unpacked file.
+   */
   private InputStream wrapInputStream (InputStream in)
     throws IOException
@@ -137 +149 @@
   }
   
-  /**
-    Looks for defined header lines in an input stream
-    from a bgx-file. The defined lines must come in the
-    same order as they are expected from the input stream. 
-    Each line is matched through a regexp.
-    @param in The input stream to read and validate.
-    @param headerLines Strings to match with the lines
-      from the input stream.
-   */
-  private void validateHeaders(InputStream in, List<String> headerLines)
-  {
-    if (in == null) throw new InvalidUseOfNullException("in");
-    if (headerLines == null) throw new InvalidUseOfNullException("headerLines");
+  /*
+    Takes a file, packed(gzip) or not packed and parse certain amount of lines.
+    Parsed heading lines will be stored in a list.
+    If one of [Probes] or [Controls] section is found, the next line in file will 
+    be checked. 
+    It must contain required (by the feature importer and reporter importer) columns.
+   */
+  private List<String> parseHeadings(File bgxFile, int range)
+  {
+    if (bgxFile == null)
+    {
+      throw new InvalidUseOfNullException("bgxFile");
+    }
+    if (bgxFile.getLocation() != Location.PRIMARY)
+    {
+      throw new InvalidUseOfNullException("Data is not online for file " +
+          bgxFile.getName() + " location = '" + bgxFile.getLocation() + "'");
+    }
+    InputStream in = null;
     try
     {
-      BufferedReader reader = new BufferedReader(new InputStreamReader(in, Config.getCharset()));
-      String line = null;
-      for (String header : headerLines)
+      in = wrapInputStream(bgxFile.getDownloadStream(0));
+    }
+    catch(IOException ioex)
+    {
+      throw new InvalidDataException("BGX-file could not be read properly.");
+    }
+    BufferedReader reader = new BufferedReader(new InputStreamReader(in));
+    List<String> headings = new ArrayList<String>();
+    boolean valid = false;
+    String line = null;
+    do
+    {
+      try
       {
         line = reader.readLine();
-        Pattern pattern = Pattern.compile(header);
-        Matcher m = pattern.matcher(line);
-        if (!m.matches()) 
+        headings.add(line);
+        if (line.matches("\\[Probes\\]") || line.matches("\\[Controls\\]"))
         {
-          throw new InvalidDataException("Line " + (headerLines.indexOf(header)+1) + " in file does not match with BGX-header:" + header);
+          line = reader.readLine();
+          valid = true;
+          for (String column : requiredColumns)
+          {
+            valid = line.contains("\t" + column + "\t") ? valid : false;
+          }
         }
       }
-      
-    }
-    catch (IOException ioex)
-    {
-      throw new InvalidDataException("Headers could not be validated due to error on when reading file");
-    }
-    
-  }
+      catch(IOException ioex)
+      {
+        line = null;
+      }     
+    }while(!valid && (headings.size() < range) && line != null);
+    if (!valid)
+    {
+      throw new InvalidDataException("Could not find required parts in the file");
+    }   
+    return headings;
+  }
+  
+  private int getNumFeatures(List<String> headings)
+  {
+    int numControls = 0;
+    if (headings != null && !(headings.size()<1))
+    {   
+      for (String s : headings)
+      {
+        if (s.startsWith(controlsHeading))
+        {
+          try
+          {
+            numControls = Integer.parseInt(s.substring((controlsHeading.length()-1)).trim());
+          }
+          catch (NumberFormatException nex)
+          {
+            throw new InvalidDataException(nex);
+          }
+        }
+      }
+    }
+    return numControls;
+  } 
 }

trunk/net/sf/basedb/illumina/src/net/sf/basedb/illumina/install/Install.java

@@ -346 +346 @@
         Illumina.BGX_FILE_ID, "BGX file", "Illumina BGX file", 
         Item.ARRAYDESIGN, BGX_FILEYTPE_EXTENSION, reporterMapType, ignoreExistingItems);
+      bgx.setValidatorClass(Illumina.BGX_VALIDATOR_METADATAREADER);
+      bgx.setMetadataReaderClass(Illumina.BGX_VALIDATOR_METADATAREADER);
 
       for (int i = 1; i <= largestNumOfStrips; i++ )
@@ -427 +429 @@
   
     DataFileType dft = null;
+    String jarPath = PluginDefinition.getByClassName(dc, this.getClass().getName()).getJarPath();
     try
     {
@@ -437 +440 @@
       dft = DataFileType.getByExternalId(dc, externalId);
       if (!dft.getExtension().equals(extension)) message = dft.getName() + " has invalid extension";
-      if (!dft.getItemType().equals(itemType)) message = dft.getName() + " has invalid item type";
-      if (message != null) throw new ItemAlreadyExistsException(message);
+      if (!dft.getItemType().equals(itemType)) message = dft.getName() + " has invalid item type";  
+      if (dft.getMetadataReaderJarPath() == null)
+      {
+        dft.setMetadataReaderJarPath(jarPath);          
+      }
+      if (dft.getValidatorJarPath() == null)
+      {
+        dft.setValidatorJarPath(jarPath);   
+      }
+      if (!dft.getValidatorJarPath().equals(jarPath) || !dft.getMetadataReaderJarPath().equals(jarPath))
+      {
+        message = dft.getName() + " has invalid jar path.";
+      }
+      if (message != null) throw new ItemAlreadyExistsException(message + ". This can not be used.");
       
     }
@@ -449 +464 @@
       dft.setDescription(description);
       dft.setGenericType(genericType);
+      dft.setValidatorJarPath(jarPath);
+      dft.setMetadataReaderJarPath(jarPath);
     }
     return dft;