Changeset 562

Timestamp:

Jan 30, 2008, 2:46:49 PM (15 years ago)

Author:

Nicklas Nordborg

Message:

References #91: Import Illumina scandata files as raw data

The plug-in can now import data, but error handling is very limited.

Location:

trunk/net/sf/basedb/illumina/src/net/sf/basedb/illumina/plugins

Files:

• ScanDataImporter.java (modified) (11 diffs)
• ScandataSpot.java (added)

trunk/net/sf/basedb/illumina/src/net/sf/basedb/illumina/plugins/ScanDataImporter.java

@@ -25 +25 @@
 package net.sf.basedb.illumina.plugins;
 
+import java.io.IOException;
+import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
+import java.util.regex.Pattern;
 
 import net.sf.basedb.core.ArrayDesign;
@@ -35 +40 @@
 import net.sf.basedb.core.DataFileType;
 import net.sf.basedb.core.DbControl;
+import net.sf.basedb.core.FeatureIdentificationMethod;
 import net.sf.basedb.core.File;
 import net.sf.basedb.core.FileParameterType;
+import net.sf.basedb.core.FileStoreEnabled;
 import net.sf.basedb.core.FileStoreUtil;
 import net.sf.basedb.core.InvalidDataException;
@@ -52 +59 @@
 import net.sf.basedb.core.ProgressReporter;
 import net.sf.basedb.core.RawBioAssay;
+import net.sf.basedb.core.RawDataBatcher;
 import net.sf.basedb.core.RawDataType;
 import net.sf.basedb.core.RequestInformation;
 import net.sf.basedb.core.StringParameterType;
+import net.sf.basedb.core.data.RawData;
 import net.sf.basedb.core.plugin.About;
 import net.sf.basedb.core.plugin.AboutImpl;
@@ -65 +74 @@
 import net.sf.basedb.core.query.Hql;
 import net.sf.basedb.core.query.Restrictions;
+import net.sf.basedb.core.signal.SignalException;
+import net.sf.basedb.core.signal.SignalHandler;
+import net.sf.basedb.core.signal.SignalTarget;
+import net.sf.basedb.core.signal.ThreadSignalHandler;
 import net.sf.basedb.illumina.Illumina;
 import net.sf.basedb.plugins.util.Parameters;
+import net.sf.basedb.util.MD5;
 import net.sf.basedb.util.parser.FlatFileParser;
 
 /**
-  Plug-in that import raw scandata file.
+  Plug-in that import raw scandata files. This plug-in support one or more
+  files each one containing data from one stripe on the glass. If more
+  than one file is given the data for spots with the same Illumicode (=Array_Address_Id)
+  are merged into a single value. See 
+  http://www.blackwell-synergy.com/doi/pdf/10.1111/j.1467-9639.1994.tb00697.x
+  for the formula used to merg of the variance.
+  <p>
+  The format of the scandata files MUST be of this form:
+  <pre class="code">
+Illumicode,N,Mean GRN,Dev GRN
+10008,26,222,47
+10010,16,57,11
+10014,16,56,13
+....
+</pre>
+
+  The Illumicode value will be left-padded with zeroes to a total length of
+  10 characters to match the values for Array_Address_Id in the BGX files.
+  
   @author nicklas
 */
 public class ScanDataImporter 
   extends AbstractPlugin
-  implements InteractivePlugin
+  implements InteractivePlugin, SignalTarget
 {
 
@@ -105 +137 @@
     );
   
-  
-  /**
-    The parser we are using.
-  */
-  private FlatFileParser ffp;
-  
   /**
     Configuration parameters
@@ -125 +151 @@
   */
   private List<PluginParameter<File>> fileParameters;
+  
+  private ThreadSignalHandler signalHandler;
   
   public ScanDataImporter()
@@ -147 +175 @@
   public boolean supportsConfigurations() 
   {
-    return false;
-  }
-
+    return super.supportsConfigurations();
+    //return false;
+  }
   @Override
   public MainType getMainType() 
   {
     return MainType.IMPORT;
-  } 
+  }
   @Override
   public void run(Request request, Response response, ProgressReporter progress) 
   {
-    
+    if (signalHandler != null) signalHandler.setWorkerThread(null);
+    DbControl dc = sc.newDbControl();
+    try
+    {
+      if (progress != null) progress.display(0, "Loading items...");
+      RawBioAssay rba = (RawBioAssay)job.getValue("rawBioAssay");
+      rba = RawBioAssay.getById(dc, rba.getId());
+      
+      List<PlatformFileType> fileTypes = getPlatformFileTypes(dc, rba);
+      List<File> files = new ArrayList<File>(fileTypes.size());
+      for (PlatformFileType pft : fileTypes)
+      {
+        DataFileType dft = pft.getDataFileType();
+        File f = (File)job.getValue("file." + dft.getExternalId());
+        if (f != null) files.add(f);
+      }
+      
+      importScandata(rba, files, progress);
+      dc.commit();
+      
+      if (progress != null) progress.display(100, "");
+      response.setDone("ok");
+    }
+    catch (Throwable t)
+    {
+      response.setError(t.getMessage(), Arrays.asList(t));
+    }
+    finally
+    {
+      if (dc != null) dc.close();
+    }
   }
   // -------------------------------------------
@@ -278 +336 @@
   }
   // -------------------------------------------
+  /*
+    From the SignalTarget interface
+    -------------------------------------------
+  */
+  @Override
+  public SignalHandler getSignalHandler() 
+  {
+    signalHandler =  new ThreadSignalHandler();
+    return signalHandler;
+  }  
+  // -------------------------------------------
   
   
@@ -308 +377 @@
         
         fileParameters = new ArrayList<PluginParameter<File>>();
-        Platform p = rba.getPlatform();
-        PlatformVariant v = rba.getVariant();
-        ItemQuery<PlatformFileType> query = p.getFileTypes(v, true);
-        query.restrict(Restrictions.eq(Hql.property("dataFileType.itemType"), 
-          Expressions.integer(Item.RAWBIOASSAY.getValue())));
-        for (PlatformFileType pft : query.list(dc))
+        List<PlatformFileType> fileTypes = getPlatformFileTypes(dc, rba);
+        
+        for (PlatformFileType pft : fileTypes)
         {
           DataFileType dft = pft.getDataFileType();
@@ -337 +403 @@
       }
       
-      /*
-      // The raw data file to import from - if a file already hase
-      // been attached to the raw bioassay use it as a default choice
-      PluginParameter<File> fileParameter = new PluginParameter<File>(
-        "file",
-        "Scandata file",
-        "The file that contains the raw data that you want to import",
-        new FileParameterType(rawDataFiles == null || rawDataFiles.isEmpty() ? 
-          null : rawDataFiles.get(0), true, 1)
-      );
-      parameters.add(fileParameter);
-      */
-      
       parameters.add(Parameters.charsetParameter(null, null, null));
       parameters.add(Parameters.decimalSeparatorParameter(null, null, null));
@@ -371 +424 @@
     }
     return configureJob;
-  }   
+  }
+  
+  public FlatFileParser createFlatFileParser()
+  {
+    FlatFileParser ffp = new FlatFileParser();
+    ffp.setDataHeaderRegexp(Pattern.compile("Illumicode,N,Mean GRN,Dev GRN"));
+    ffp.setDataSplitterRegexp(Pattern.compile(","));
+    return ffp;
+  }
+  
+  public void importScandata(RawBioAssay rba, List<File> files, ProgressReporter progress)
+    throws IOException
+  {
+    // We assume that each file contains rougly the same number of lines
+    // Progress reporter should be divided into numFile + 1 steps
+    int offset = 0;
+    double deltaOffset = 100 / (files.size() + 1);
+    
+    // Parse data from the files and temporarily store everything
+    // in 'rawdata'
+    FlatFileParser ffp = createFlatFileParser();
+    Map<String, ScandataSpot> rawdata = new HashMap<String, ScandataSpot>();
+    for (File file : files)
+    {
+      double factor =  deltaOffset / file.getSize();
+      long current = 0;
+      System.out.println("factor = " +factor + "; offset=" + offset + "; deltaOffset=" + deltaOffset);
+      InputStream in = file.getDownloadStream(0);
+      ffp.setInputStream(in, "ISO-8859-1");
+      
+      FlatFileParser.LineType line = ffp.parseHeaders();
+      if (line != FlatFileParser.LineType.DATA_HEADER)
+      {
+        throw new InvalidDataException("Can't find start of data in file: " + file);
+      }
+      
+      while (ffp.hasMoreData())
+      {
+        // Check if user has aborted
+        if (Thread.interrupted()) throw new SignalException("Aborted by user.");
+        
+        // Progress reporting
+        int lines = ffp.getParsedLines();
+        if (progress != null && lines % 100 == 0)
+        {
+          String message = "Parsing file " + file.getName() + "; " + lines + " lines done.";
+          current = ffp.getParsedBytes();
+          int percent = (int)(offset + factor * current);
+          progress.display(percent, message);
+        }
+        
+        // Get next data line
+        FlatFileParser.Data data = ffp.nextData();
+        String featureId = MD5.leftPad(data.get(0), '0', 10);
+        int n = Integer.parseInt(data.get(1));
+        int mean = Integer.parseInt(data.get(2));
+        int dev = Integer.parseInt(data.get(3));
+        
+        ScandataSpot spot = rawdata.get(featureId);
+        if (spot == null)
+        {
+          spot = new ScandataSpot(featureId, n, mean, dev);
+          rawdata.put(featureId, spot);
+        }
+        else
+        {
+          spot.merge(n, mean, dev);
+        }
+      }
+      
+      offset += deltaOffset;
+    }
+
+    // Insert data into db
+    long current = 0;
+    RawDataBatcher batcher = rba.getRawDataBatcher(FeatureIdentificationMethod.FEATURE_ID);
+    double factor =  deltaOffset / rawdata.size();
+    for (ScandataSpot spot : rawdata.values())
+    {
+      // Check if user has aborted
+      if (Thread.interrupted()) throw new SignalException("Aborted by user.");
+      
+      // Progress reporting
+      if (progress != null && current % 100 == 0)
+      {
+        String message = "Saving to database; " + current + " spots done.";
+        current = ffp.getParsedBytes();
+        int percent = (int)(offset + factor * current);
+        progress.display(percent, message);
+      }
+
+      RawData raw = batcher.newRawData();
+      raw.setExtended("n", spot.getN());
+      raw.setExtended("mean", (float)spot.getMean());
+      raw.setExtended("dev", (float)spot.getDev());
+  
+      batcher.insert(raw, null, spot.getFeatureId());
+      current++;
+    }
+    batcher.flush();
+    batcher.close();
+  }
+  
+  public List<PlatformFileType> getPlatformFileTypes(DbControl dc, FileStoreEnabled item)
+  {
+    
+    Platform platform = item.getPlatform();
+    PlatformVariant variant = item.getVariant();
+    
+    ItemQuery<PlatformFileType> query = platform.getFileTypes(variant, false);
+    query.restrict(Restrictions.eq(Hql.property("dataFileType.itemType"), 
+      Expressions.integer(item.getType().getValue())));
+    return query.list(dc);
+  }
+
 }