Changeset 750


Ignore:
Timestamp:
Sep 1, 2008, 4:15:04 PM (13 years ago)
Author:
Martin Svensson
Message:

References # 119 The normalization algorithm is almost done. Some smaller adjustments and alot of testing is left to do.

Location:
plugins/base2/net.sf.basedb.normalizers/trunk/src/net/sf/basedb/plugins
Files:
1 added
5 edited

Legend:

Unmodified
Added
Removed
  • plugins/base2/net.sf.basedb.normalizers/trunk/src/net/sf/basedb/plugins/AbstractNormalizationPlugin.java

    r746 r750  
    8888    return permissions;
    8989  }
     90 
     91  /**
     92    Gets a 'greater than' restriction for the specified number of channels.
     93   **/
     94  protected Restriction getIntensityRestriction(int noOfChannels, float minIntensity)
     95  {
     96    Restriction intensityRestriction = null;   
     97    if (noOfChannels == 1)
     98    {
     99      intensityRestriction = Restrictions.gt(Dynamic.column(VirtualColumn.channel(1)), Expressions.aFloat(minIntensity));
     100    }
     101    else
     102    {
     103      Restriction[] restrictions = new Restriction[noOfChannels];
     104      for (int i=0; i<noOfChannels; i++)
     105      {
     106        restrictions[i] = Restrictions.gt(Dynamic.column(VirtualColumn.channel(i+1)), Expressions.aFloat(minIntensity));
     107      }
     108      intensityRestriction = Restrictions.and(restrictions);
     109    }
     110    return intensityRestriction;
     111  }
    90112}
  • plugins/base2/net.sf.basedb.normalizers/trunk/src/net/sf/basedb/plugins/AverageNormalization.java

    r748 r750  
    434434           
    435435          default:
    436             throw new BaseException(" channels raw data is not supported by this plug-in");
     436            throw new BaseException(noOfChannels + "channels raw data is not supported by this plug-in");
    437437          }
    438438          data.add(spot);
     
    511511    return meanValue;
    512512  }
    513  
    514   /*
    515     Gets a 'greater than' restriction for the specified number of channels.
    516    */
    517   private Restriction getIntensityRestriction(int noOfChannels, float minIntensity)
    518   {
    519     Restriction intensityRestriction = null;   
    520     if (noOfChannels == 1)
    521     {
    522       intensityRestriction = Restrictions.gt(Dynamic.column(VirtualColumn.channel(1)), Expressions.aFloat(minIntensity));
    523     }
    524     else
    525     {
    526       Restriction[] restrictions = new Restriction[noOfChannels];
    527       for (int i=0; i<noOfChannels; i++)
    528       {
    529         restrictions[i] = Restrictions.gt(Dynamic.column(VirtualColumn.channel(i+1)), Expressions.aFloat(minIntensity));
    530       }
    531       intensityRestriction = Restrictions.and(restrictions);
    532     }
    533     return intensityRestriction;
    534   }
    535513}
  • plugins/base2/net.sf.basedb.normalizers/trunk/src/net/sf/basedb/plugins/RankInvariantNormalization.java

    r749 r750  
    2626
    2727import net.sf.basedb.core.BaseException;
     28import net.sf.basedb.core.BioAssay;
    2829import net.sf.basedb.core.BioAssaySet;
    2930import net.sf.basedb.core.DbControl;
     31import net.sf.basedb.core.DynamicResultIterator;
     32import net.sf.basedb.core.DynamicSpotQuery;
    3033import net.sf.basedb.core.FloatParameterType;
    31 import net.sf.basedb.core.InvalidUseOfNullException;
    3234import net.sf.basedb.core.Job;
     35import net.sf.basedb.core.PluginParameter;
    3336import net.sf.basedb.core.ProgressReporter;
    3437import net.sf.basedb.core.RequestInformation;
    3538import net.sf.basedb.core.SpotBatcher;
    3639import net.sf.basedb.core.Transformation;
     40import net.sf.basedb.core.Type;
     41import net.sf.basedb.core.VirtualColumn;
    3742import net.sf.basedb.core.plugin.About;
    3843import net.sf.basedb.core.plugin.AboutImpl;
    3944import net.sf.basedb.core.plugin.GuiContext;
    4045import net.sf.basedb.core.plugin.InteractivePlugin;
    41 import net.sf.basedb.core.PluginParameter;
    4246import net.sf.basedb.core.plugin.Request;
    4347import net.sf.basedb.core.plugin.Response;
     48import net.sf.basedb.core.query.Dynamic;
     49import net.sf.basedb.core.query.Restriction;
     50import net.sf.basedb.core.query.Selects;
     51import net.sf.basedb.core.query.SqlResult;
    4452import net.sf.basedb.core.signal.SignalHandler;
    4553import net.sf.basedb.core.signal.SignalReceivedException;
     
    4957import net.sf.basedb.util.Values;
    5058
     59import java.io.BufferedReader;
     60import java.io.File;
     61import java.io.FileReader;
     62import java.io.FileWriter;
     63import java.io.IOException;
     64import java.sql.SQLException;
    5165import java.util.ArrayList;
    5266import java.util.Arrays;
     67import java.util.Collections;
     68import java.util.HashMap;
    5369import java.util.List;
     70import java.util.Map;
    5471
    5572/**
     
    121138      }
    122139      DbControl dc = null;
     140      if (progress != null) progress.display(0, "Preparing to normalize...");
    123141      try
    124142      {
     
    273291 
    274292  /*
    275     Normalize bio assay set data with the Rank invariant algorithm
    276    */
     293    Normalize bio assay-set data with the Rank invariant algorithm
     294   */
     295  @SuppressWarnings("unchecked")
    277296  private BioAssaySet normalize(DbControl dc, BioAssaySet source,
    278297      Job job, float minIntensity, ProgressReporter progress)
    279298  {
    280299    int noOfChannels = source.getRawDataType().getChannels();
    281        
    282     // Create Transformation
    283     Transformation t = source.newTransformation(job);
    284     t.setName(about.getName());
    285     dc.saveItem(t);
    286    
    287     // Create the normalized bioassay set
    288     BioAssaySet child = t.newProduct(null, "new", true);
    289     dc.saveItem(child);
    290    
    291     // Batcher for inserting normalized data
    292     SpotBatcher batcher = child.getSpotBatcher();
    293    
    294    
    295    
     300    long normalizedSpots = 0;
     301       
     302    // Query to receive spot data from data base
     303    DynamicSpotQuery query = source.getSpotData();
     304    // Restriction to get intensities greater than 'minIntensity' for each channel
     305    Restriction intensityRestriction = getIntensityRestriction(noOfChannels, minIntensity);
     306    query.restrictPermanent(intensityRestriction);
     307    // Get no spots in assay
     308    long numSpots = query.count(dc);
     309    query.reset();
     310   
     311    // Configure query
     312    query.select(Dynamic.select(VirtualColumn.POSITION));
     313    query.select(Selects.expression(ch1, "ch1"));
     314    if (noOfChannels > 1) query.select(Selects.expression(ch2, "ch2"));   
     315    query.restrict(bioAssayRestriction);
     316   
     317    if (progress != null) progress.display(
     318          (int)((100L * normalizedSpots)/numSpots),
     319          normalizedSpots + " spots normalized");
     320   
     321    double[] rowSummaries = null;
     322   
     323    List<BioAssay> assays = source.getBioAssays().list(dc);
     324    Map<Integer, File> fileMap = new HashMap<Integer, File>();
     325    int spotsPerAssay = -1;
     326   
     327    for (BioAssay assay : assays)
     328    {
     329      checkInterrupted();
     330     
     331      // Control that the number of spots per assay is the same in all assays
     332      if (spotsPerAssay > -1 && spotsPerAssay != assay.getNumSpots())
     333        throw new BaseException("The number of spots are not equal between the dispal\n " +
     334            "The normalization can not be done.");
     335      else if (spotsPerAssay == -1)
     336        spotsPerAssay = assay.getNumSpots(); 
     337     
     338      // Get spot data and sort it ascending
     339      List<AbstractSpotData> data = getSpots(dc, assay, query, noOfChannels);     
     340      Collections.sort(data);
     341     
     342      // Write spot data to file and calculate row summaries
     343      rowSummaries = new double[data.size()];     
     344      try
     345      {
     346        // Create temp-file to store the spot's original values and position.
     347        File tempFile = File.createTempFile(null, ".tmp");
     348        FileWriter fw = new FileWriter(tempFile);
     349        // Write spot information to a file and summarize the rows.
     350        for (int i=0; i<data.size(); i++)
     351        {
     352          AbstractSpotData spot = data.get(i);
     353          double newRowValue = assays.indexOf(assay) < 0 ?
     354              (rowSummaries[i] + spot.getNormalizableData()) :
     355              spot.getNormalizableData();
     356          rowSummaries[i] = newRowValue;
     357         
     358          if (i < 0)fw.write("\n");
     359          fw.write(spot.toString());
     360        }
     361        fw.flush();
     362        fw.close();
     363        fileMap.put((int)assay.getDataCubeColumnNo(), tempFile);
     364      }
     365      catch (IOException e)
     366      {
     367        throw new BaseException(e);
     368      }
     369    }
     370   
     371    // Calculate the row averages.
     372    float[] rowAverages = new float[rowSummaries.length];
     373    for (int i=0; i<rowSummaries.length; i++)
     374    {
     375      rowAverages[i] = new Float(rowSummaries[i]/assays.size());
     376    }
     377   
     378    try
     379    {
     380      // Create Transformation
     381      Transformation t = source.newTransformation(job);
     382      t.setName(about.getName());
     383      dc.saveItem(t);
     384     
     385      // Create the normalized bioassay set
     386      BioAssaySet child = t.newProduct(null, "new", true);
     387      dc.saveItem(child);
     388     
     389      // Batcher for inserting normalized data
     390      SpotBatcher batcher = child.getSpotBatcher();
     391           
     392      // Read each assay's file and take the position order and change to the normalized value.
     393      for (BioAssay assay : assays)
     394      {
     395        checkInterrupted();
     396       
     397        short columnNo = assay.getDataCubeColumnNo();
     398        File assayFile = fileMap.get((int)columnNo);
     399        BufferedReader input =  new BufferedReader(new FileReader(assayFile));
     400       
     401        String line = input.readLine();
     402        int rowIndex = 0;
     403        while ((line = input.readLine()) != null)
     404        {
     405          String[] lineInfo = line.split("\t");
     406          AbstractSpotData spot = getSpotDataFromString(lineInfo);
     407          spot.setNormalizableData(rowAverages[rowIndex]);
     408          batcher.insert(columnNo, spot.getPosition(), spot.getChannelData());
     409          normalizedSpots++;
     410        }
     411       
     412        if(progress != null) progress.display(
     413            (int)((100L * normalizedSpots)/numSpots),
     414          normalizedSpots + " spots normalized");
     415      }
     416    }
     417    catch (IOException e)
     418    {
     419      throw new BaseException(e);
     420    }
    296421    return null;
    297422  }
    298 
     423 
     424  @SuppressWarnings("unchecked")
     425  private List<AbstractSpotData> getSpots(DbControl dc, BioAssay assay, DynamicSpotQuery query, int channels)
     426  {
     427    List<AbstractSpotData> data = null;
     428    try
     429    {
     430      // Load spot data for this bioassay
     431      short bioassayColumn = assay.getDataCubeColumnNo();
     432      query.setParameter("bioAssayColumn", (int)bioassayColumn, Type.INT);
     433      DynamicResultIterator it = query.iterate(dc);
     434     
     435     
     436     
     437      // Handling spot data, writing to file and add to a sortable list     
     438      data = new ArrayList<AbstractSpotData>(assay.getNumSpots());
     439      int positionIndex = it.getIndex("position");
     440      int ch1Index = it.getIndex("ch1");
     441      while (it.hasNext())
     442      {
     443        SqlResult result = it.next();
     444        int position = result.getInt(positionIndex);
     445        float ch1Intensity = result.getFloat(ch1Index);
     446        AbstractSpotData spotData = null;
     447        switch (channels)
     448        {
     449        case 1:
     450          spotData = new SpotData1Ch(position, ch1Intensity);
     451          break;
     452         
     453        case 2:
     454          float ch2Intensity = result.getFloat(it.getIndex("ch2"));
     455          spotData = new SpotData2Ch(position, ch1Intensity, ch2Intensity);
     456          break;
     457         
     458          default:
     459            throw new BaseException(channels + " channel data is not supported by this plug-in");
     460        }         
     461        data.add(spotData);
     462      }
     463    }
     464    catch (SQLException e)
     465    {
     466      throw new BaseException(e);
     467    }
     468    return data;
     469  }
     470 
     471  private AbstractSpotData<?> getSpotDataFromString(String[] tabbedLine)
     472  {
     473    int position = new Integer(tabbedLine[0]);
     474    if (tabbedLine.length == 2) return new SpotData1Ch(position, new Float(tabbedLine[1]));
     475    else if (tabbedLine.length == 3) return new SpotData2Ch(position, new Float(tabbedLine[1]), new Float(tabbedLine[2]));
     476    else return null;
     477  }
    299478}
  • plugins/base2/net.sf.basedb.normalizers/trunk/src/net/sf/basedb/plugins/SpotData1Ch.java

    r746 r750  
    3333 */
    3434public class SpotData1Ch
    35   implements Normalizable, Comparable<SpotData1Ch>
     35  extends AbstractSpotData<SpotData1Ch>
    3636{
    3737  private final int position;
  • plugins/base2/net.sf.basedb.normalizers/trunk/src/net/sf/basedb/plugins/SpotData2Ch.java

    r746 r750  
    3333 */
    3434public class SpotData2Ch
    35   implements Normalizable, Comparable<SpotData2Ch>
     35  extends AbstractSpotData<SpotData2Ch>
    3636{
    3737
Note: See TracChangeset for help on using the changeset viewer.