Changeset 4212

Timestamp:

Apr 10, 2008, 12:53:41 PM (15 years ago)

Author:

Nicklas Nordborg

Message:

References #957: Base1PluginExecuter doesn't import plug-in generated BASEfiles as expected

The new code is in place. Also added better test cases for the the four cases:

1. No change in parent mapping
2. Merge assays
3. New positions
4. New positions + merge assays

Probably needs more testing with real plug-ins. A bit unsure about the performance when remapping raw data for large experiments.

Location:

branches/2.6-stable

Files:

• build.xml (modified) (1 diff)
• src/core/net/sf/basedb/core/data/RawData.java (modified) (1 diff)
• src/plugins/core/net/sf/basedb/plugins/Base1PluginExecuter.java (modified) (13 diffs)
• src/test/Base1TestPlugin.java (added)
• src/test/TestAll.java (modified) (1 diff)
• src/test/TestAnalyzePluginUtil.java (modified) (4 diffs)
• src/test/TestBase1PluginExecuter.java (modified) (5 diffs)
• src/test/TestExperiment.java (modified) (1 diff)
• src/test/TestFormula.java (modified) (3 diffs)
• src/test/TestJob.java (modified) (1 diff)
• src/test/TestLowessNormalization.java (modified) (1 diff)
• src/test/TestMedianRatioNormalization.java (modified) (1 diff)
• src/test/data/test.base1plugin.base (added)
• src/test/data/test.nullplugin.base (moved) (moved from branches/2.6-stable/src/test/data/test.plugin.base) (1 diff)
• src/test/nullplugin.sh (moved) (moved from branches/2.6-stable/src/test/runplugin.sh)
• src/test/testbase1plugin.sh (added)

branches/2.6-stable/build.xml

@@ -848 +848 @@
       jarfile="${test.build}/JarPlugin.jar" 
       basedir="${test.build}" 
-      includes="JarPlugin*,NullPlugin*,Base1NullPlugin*"
+      includes="JarPlugin*,NullPlugin*,Base1*"
       excludes="JarPlugin.jar,JarPluginAbout.*"
       manifest="${test.src}/data/JarPluginManifest.txt"

branches/2.6-stable/src/core/net/sf/basedb/core/data/RawData.java

@@ -46 +46 @@
 {
 
-  RawData()
+  public RawData()
   {}
 

branches/2.6-stable/src/plugins/core/net/sf/basedb/plugins/Base1PluginExecuter.java

@@ -35 +35 @@
 import net.sf.basedb.core.DbControl;
 import net.sf.basedb.core.Directory;
+import net.sf.basedb.core.DynamicQuery;
+import net.sf.basedb.core.DynamicResultIterator;
+import net.sf.basedb.core.DynamicSpotQuery;
 import net.sf.basedb.core.Experiment;
 import net.sf.basedb.core.ExtraValue;
@@ -48 +51 @@
 import net.sf.basedb.core.ItemQuery;
 import net.sf.basedb.core.Job;
+import net.sf.basedb.core.MappingBatcher;
 import net.sf.basedb.core.Path;
 import net.sf.basedb.core.PathParameterType;
@@ -55 +59 @@
 import net.sf.basedb.core.PositionBatcher;
 import net.sf.basedb.core.ProgressReporter;
-import net.sf.basedb.core.Reporter;
 import net.sf.basedb.core.RequestInformation;
 import net.sf.basedb.core.SpotBatcher;
@@ -63 +66 @@
 import net.sf.basedb.core.Type;
 import net.sf.basedb.core.User;
+import net.sf.basedb.core.VirtualColumn;
+import net.sf.basedb.core.VirtualTable;
+import net.sf.basedb.core.data.RawData;
 import net.sf.basedb.core.data.ReporterData;
 import net.sf.basedb.core.plugin.About;
@@ -72 +78 @@
 import net.sf.basedb.core.plugin.Request;
 import net.sf.basedb.core.plugin.Response;
+import net.sf.basedb.core.query.Dynamic;
 import net.sf.basedb.core.query.Hql;
+import net.sf.basedb.core.query.JoinType;
 import net.sf.basedb.core.query.Orders;
+import net.sf.basedb.core.query.SqlResult;
 import net.sf.basedb.core.signal.SignalException;
 import net.sf.basedb.core.signal.SignalHandler;
 import net.sf.basedb.core.signal.SignalTarget;
 import net.sf.basedb.core.signal.ThreadSignalHandler;
+import net.sf.basedb.util.Diff3;
 import net.sf.basedb.util.Enumeration;
 import net.sf.basedb.util.Values;
@@ -1331 +1341 @@
   /**
     Imports the data from the file stdout.txt in the execution directory.
-    
-    @return The id of the transformation the data is connected to.
     @throws IOException if there is any error reading from stdout.txt
   */
-  private int importData(DbControl dc, File stdout, Transformation t)
-    throws IOException
-  {
-    BioAssaySet bas = null;
-    SpotBatcher spotBatcher = null;
-    Boolean doPositionBatch = null;
-
-    HashMap<String, BioAssay> idMap = new HashMap<String, BioAssay>();
-    HashMap<Integer, ReporterData> posRepMap = new HashMap<Integer, ReporterData>();
-    HashMap<String, ExtraValueType> evtMap = new HashMap<String, ExtraValueType>();
-
+  private void importData(DbControl dc, File stdout, Transformation t)
+    throws SQLException, IOException
+  {
+    
+    // Position -> ReporterID mapping for the parent bioassay set
+    Map<Integer, Integer> parentReporterPositions = new HashMap<Integer, Integer>();
+    // Position -> ReporterID mapping for the child bioassay set
+    Map<Integer, Integer> childReporterPositions = new HashMap<Integer, Integer>();
+    // BioAssayID -> Column no mapping for the child bioassay set
+    Map<Integer, Short> bioAssayColumns = new HashMap<Integer, Short>();
+    // Column mapping from parent -> child bioassay
+    Map<Short, Short> columnMapping = new HashMap<Short, Short>();
+    // If the "assays" section contains a child->parent mapping for bioassays
+    boolean hasParentAssayMapping = false;
+    // If all position->reporter mappings are the same on the child and parent bioassay set
+    boolean hasSamePositionReporterMapping = true;
+    // If at least one spot section was found
+    boolean hasSpotSection = false;
+    // Information about child bioassays
+    Map<Integer, ChildBioAssay> childBioAssays = new HashMap<Integer, ChildBioAssay>();
+    
+    // Load the position -> reporter mapping from the parent bioassay set
+    BioAssaySet source = t.getSource();
+    DynamicQuery positionQuery = source.getPositionData();
+    positionQuery.select(Dynamic.select(VirtualColumn.POSITION));
+    positionQuery.select(Dynamic.select(VirtualColumn.REPORTER_ID));
+    DynamicResultIterator positionQueryIterator = positionQuery.iterate(dc);
+    while (positionQueryIterator.hasNext())
+    {
+      SqlResult result = positionQueryIterator.next();
+      parentReporterPositions.put(result.getInt(1), (Integer)result.getObject(2));
+    }
+    
+    // Parse the stdout file to verify child mappings
     FlatFileParser ffp = getInitializedFlatFileParser(stdout.getDownloadStream(0));
-    
     while (ffp.hasMoreSections())
     {
+      checkInterrupted();
       FlatFileParser.Line section = ffp.nextSection();
       ffp.parseHeaders();
       
-      if (section.name().equals("assays"))
-      {
-        bas = t.newProduct("new", "new", false);
-        doPositionBatch = true;
-        bas.setName(getSourceBioAssaySet(dc).getName());
+      if ("assays".equals(section.name()))
+      {
+        /*
+          Optional section
+          If it exists it must define 'id' and 'name' of child bioassays
+          The 'id' must be the same as the id of a parent bioassay,
+          UNLESS a 'parents' column is present
+        */
         List<String> columns = Arrays.asList(ffp.getHeader("columns").split("\\t"));
-        int idCol = columns.indexOf("id");
-        int nameCol = columns.indexOf("name");
-        int parentCol = columns.indexOf("parents");
-
+        int parentIndex = columns.indexOf("parents");
+        int idIndex = columns.indexOf("id");
+        int nameIndex = columns.indexOf("name");
+        hasParentAssayMapping =  parentIndex != -1;
+        if (idIndex == -1)
+        {
+          throw new BaseException("Missing column 'id' in section 'assays' at line " 
+              + ffp.getParsedLines() + " in file '" + stdout.getName() + "'");
+        }
+        if (nameIndex == -1)
+        {
+          throw new BaseException("Missing column 'name' in section 'assays' at line " 
+              + ffp.getParsedLines() + " in file '" + stdout.getName() + "'");  
+        }
+        
+        // Read data part of the 'assays' section
+        // Each line has ID, NAME and PARENTS (optional)
         ffp.setMinDataColumns(columns.size());
-
-        FlatFileParser.Data dataline = ffp.nextData();
-        while (dataline != null)
-        {
-          String id = dataline.get(idCol);
-          String name = dataline.get(nameCol);
-          Set<BioAssay> baParents;
-          if (parentCol > -1)
-          {
-            String[] parentIds;
-            parentIds = dataline.get(parentCol).split("/");
-            baParents = new HashSet<BioAssay>(parentIds.length);
-            for (String parent : parentIds)
-            {
-              baParents.add(BioAssay.getById(dc, new Integer(parent)));
-            }
-          }
-          else
-          {
+        FlatFileParser.Data data;
+        while ((data = ffp.nextData()) != null)
+        {
+          int assayId = Values.getInt(data.get(idIndex));
+          
+          // Don't allow duplicate assay ID:s
+          if (childBioAssays.containsKey(assayId))
+          {
+            throw new BaseException("Duplicate assay id (" + assayId + 
+                ") found in section 'assays' at line " + ffp.getParsedLines() + 
+                " in file '" + stdout.getName() + "'");
+          }
+          
+          // Extract name and parents and store as ChildBioAssay objects
+          String name = data.get(nameIndex);
+          Set<Integer> parents = null;
+          if (hasParentAssayMapping)
+          {
+            parents = new HashSet<Integer>(Arrays.asList(Values.getInt(data.get(parentIndex).split("/"))));
+            Integer parentId = null;            
             try
             {
-              baParents = Collections.singleton(BioAssay.getById(dc, Integer.parseInt(id)));
+              Iterator<Integer> it = parents.iterator();
+              while (it.hasNext())
+              {
+                parentId = it.next();
+                BioAssay.getById(dc, parentId);
+              }
             }
-            catch (ItemNotFoundException e)
+            catch (ItemNotFoundException ex)
             {
-              throw new BaseException("Couldn't find any parent for bioassay "+name, e);
+              throw new BaseException("Can't find parent bioassay '" + parentId +
+                "' in section 'assays' at line " + ffp.getParsedLines() + " in file '" +
+                stdout.getName() + "'"
+                );
             }
           }
-          BioAssay ba = bas.newBioAssay(baParents);
-          ba.setName(name);
-          idMap.put(id, ba);
-          dataline = ffp.nextData();
-        }
+          else
+          {
+            try
+            {
+              BioAssay.getById(dc, assayId);
+            }
+            catch (ItemNotFoundException ex)
+            {
+              throw new BaseException("Can't find bioassay '" + assayId +
+                "' in section 'assays' at line " + ffp.getParsedLines() + " in file '" +
+                stdout.getName() + "'"
+                );
+            }
+          }
+          childBioAssays.put(assayId, new ChildBioAssay(assayId, name, parents));
+        }
+      }
+      else if ("spots".equals(section.name()))
+      {
+        checkInterrupted();
+        /*
+          Required section. May appear more than once.
+          'position' and 'reporter' are required columns in the 'columns' header.
+          'assays' is a required column header, that has the ID of the assays
+          that have spot data in this section.
+        */
+        hasSpotSection = true;
+
+        // Get the "position" and "reporter" columns from the "column" header
+        List<String> columns = Arrays.asList(ffp.getHeader("columns").split("\\t"));
+        int positionIndex = columns.indexOf("position");
+        int reporterIndex = columns.indexOf("reporter");
+
+        // Error if the columns are missing
+        if (positionIndex == -1) 
+        {
+          throw new BaseException("Missing column 'position' in section 'spots' at line " +
+              ffp.getParsedLines() + " in file '" + stdout.getName() + "'");
+        }
+        if (reporterIndex == -1) 
+        {
+          throw new BaseException("Missing column 'reporter' in section 'spots' at line " +
+              ffp.getParsedLines() + " in file '" + stdout.getName() + "'");
+        }
+
+        // Get the "assays" header
+        List<Integer> assays = Arrays.asList(Values.getInt(ffp.getHeader("assays").split("\\t")));
+        if (assays == null || assays.size() == 0)
+        {
+          throw new BaseException("Missing header 'assays' in section 'spots' at line " +
+              ffp.getParsedLines() + " in file '" + stdout.getName() + "'");
+        }
+        
+        for (Integer assayId : assays)
+        {
+          if (!childBioAssays.containsKey(assayId))
+          {
+            try
+            {
+              BioAssay.getById(dc, assayId);
+            }
+            catch (ItemNotFoundException ex)
+            {
+              throw new BaseException("Can't find bioassay '" + assayId +
+                "' in section 'spots' at line " + ffp.getParsedLines() + " in file '" +
+                stdout.getName() + "'"
+                );
+            }
+            childBioAssays.put(assayId, new ChildBioAssay(assayId, null, null));
+          }
+        }
+        
+        // Parse data and check if each position has same reporter as before
+        FlatFileParser.Data data;
+        while ((data = ffp.nextData()) != null)
+        {
+          checkInterrupted();
+          Integer position = Values.getInteger(data.get(positionIndex), null);
+          Integer newReporterId = Values.getInteger(data.get(reporterIndex), null);
+          
+          // Error if position is missing
+          if (position == null)
+          {
+            throw new BaseException("Missing or invalid value for 'position' in section 'spots' at line " +
+                ffp.getParsedLines() + " in file '" + stdout.getName() + "'");
+          }
+          
+          // Is the new reporter the same as the existing?
+          // A 'null' reporter is considered the same as another 'null' reporter
+          if (hasSamePositionReporterMapping)
+          {     
+            Integer existingReporterId = parentReporterPositions.get(position);
+            if (!Diff3.isEqualOrNull(newReporterId, existingReporterId))
+            {
+              hasSamePositionReporterMapping = false;
+              parentReporterPositions.clear();
+            }
+          }
+          
+          // Is the new reporter already registered with the same position?
+          if (!childReporterPositions.containsKey(position))
+          {
+            childReporterPositions.put(position, newReporterId);
+          }
+          else
+          {
+            Integer registeredReporterId = childReporterPositions.get(position);
+            // Error if same position has different reporters in the stdout file
+            if (!Diff3.isEqualOrNull(newReporterId, registeredReporterId))
+            {
+              throw new BaseException("Invalid value for 'reporter' (" + newReporterId + 
+                ") for position '" + position + "' in section 'spots' at line " + 
+                ffp.getParsedLines() + " in file '" + stdout.getName() + 
+                "'. Expected '" + registeredReporterId + "'");
+            }
+          }
+        }
+      }
+    }
+    
+    if (!hasSpotSection) return;
+    
+    // Create the child bioassay set
+    BioAssaySet child = null;
+    boolean useNewDataCube = hasParentAssayMapping || !hasSamePositionReporterMapping;
+    if (useNewDataCube)
+    {
+      child = t.newProduct("new", "new", false);
+    }
+    else
+    {
+      child = t.newProduct(null, "new", false);
+    }
+    dc.saveItem(child);
+
+    // Create child bioassays
+    for (ChildBioAssay childData : childBioAssays.values())
+    {
+      BioAssay childBa = null;
+      if (useNewDataCube)
+      {
+        Collection<BioAssay> parents = null;
+        if (childData.parents == null)
+        {
+          parents = Collections.singletonList(BioAssay.getById(dc, childData.id));
+        }
+        else
+        {
+          parents = new ArrayList<BioAssay>(childData.parents.size());
+          for (Integer parentId : childData.parents)
+          {
+            parents.add(BioAssay.getById(dc, parentId));
+          }
+        }
+        childBa = child.newBioAssay(parents);
+        // Put column mappings
+        short childColumn = childBa.getDataCubeColumnNo();
+        for (BioAssay parentBa : parents)
+        {
+          columnMapping.put(parentBa.getDataCubeColumnNo(), childColumn);
+        }
+      }
+      else
+      {
+        BioAssay parentBa = BioAssay.getById(dc, childData.id);
+        childBa = child.newBioAssay(parentBa);
+        // Columns should be identical... put in the mapping just be be sure
+        columnMapping.put(parentBa.getDataCubeColumnNo(), childBa.getDataCubeColumnNo());
+      }
+      if (childData.name != null) childBa.setName(childData.name);
+      dc.saveItem(childBa);
+      bioAssayColumns.put(childData.id, childBa.getDataCubeColumnNo());
+    }
+    
+    // Create new position -> reporter mapping
+    if (useNewDataCube)
+    {
+      // Create position -> reporter mapping from the new data
+      PositionBatcher posBatcher = child.getPositionBatcher();
+      ReporterProxy proxy = new ReporterProxy();
+      for (Map.Entry<Integer, Integer> entry : childReporterPositions.entrySet())
+      {
+        checkInterrupted();
+        int position = entry.getKey();
+        Integer reporterId = entry.getValue();
+        if (reporterId == null)
+        {
+          posBatcher.insert(position, null);
+        }
+        else
+        {
+          proxy.setTheId(reporterId);
+          posBatcher.insert(position, proxy);
+        }
+      }
+      posBatcher.flush();
+      posBatcher.close();
+      
+      if (hasSamePositionReporterMapping)
+      {
+        // Create raw data mapping
+        MappingBatcher mapBatcher = child.getMappingBatcher();
+        DynamicSpotQuery spotQuery = source.getSpotData();
+        spotQuery.joinRawData(JoinType.INNER);
+        
+        spotQuery.select(Dynamic.select(VirtualColumn.COLUMN));
+        spotQuery.select(Dynamic.select(VirtualColumn.POSITION));
+        spotQuery.select(Dynamic.select(VirtualTable.RAWPARENTS, VirtualColumn.RAWDATA_ID));
+        
+        DynamicResultIterator spotIterator = spotQuery.iterate(dc);
+        RawDataProxy rawProxy = new RawDataProxy();
+        while (spotIterator.hasNext())
+        {
+          SqlResult result = spotIterator.next();
+          short parentColumn = result.getShort(1);
+          int position = result.getInt(2);
+          int rawDataId = result.getInt(3);
+          Short childColumn = columnMapping.get(parentColumn);
+          if (childColumn != null)
+          {
+            rawProxy.setTheId(rawDataId);
+            mapBatcher.insert(childColumn, position, rawProxy);
+          }
+        }
+        mapBatcher.flush();
+        mapBatcher.close();
+      }
+    }
+    
+    // Clean up things that are no longer needed
+    childBioAssays.clear();
+    childReporterPositions.clear();
+    parentReporterPositions.clear();
+    
+    
+    // Old code below
+
+    SpotBatcher spotBatcher = child.getSpotBatcher();
+    HashMap<String, ExtraValueType> evtMap = new HashMap<String, ExtraValueType>();
+    ffp = getInitializedFlatFileParser(stdout.getDownloadStream(0));
+    
+    while (ffp.hasMoreSections())
+    {
+      checkInterrupted();
+      FlatFileParser.Line section = ffp.nextSection();
+      ffp.parseHeaders();
+      
+      if (section.name().equals("assays"))
+      {
+        List<String> columns = Arrays.asList(ffp.getHeader("columns").split("\\t"));
+        ffp.setMinDataColumns(columns.size());
+        // Loop to the end of this section
+        while (ffp.nextData() != null)
+        {}
       }
       else if (section.name().equals("spots"))
       {
         List<String> columns = Arrays.asList(ffp.getHeader("columns").split("\\t"));
-        List<String> assays = Arrays.asList(ffp.getHeader("assays").split("\\t"));
+        List<Integer> assays = Arrays.asList(Values.getInt(ffp.getHeader("assays").split("\\t")));
         List<String> assayFields = Arrays.asList(ffp.getHeader("assayFields").split("\\t"));
         List<String> setExtraFloats = new ArrayList<String>(); 
@@ -1407 +1712 @@
           setExtraFloats = Arrays.asList(ffp.getHeader("setExtraFloats").split("\\t"));
         
-        if (bas == null)
-        {
-          bas = t.newProduct(null, "new", false);
-          doPositionBatch = false;
-        }
-        spotBatcher = bas.getSpotBatcher();
-
-        for (String assayId : assays)
-        {
-          if (idMap.get(assayId) == null)
-          {
-            try
-            {
-              BioAssay parent = BioAssay.getById(dc, Integer.parseInt(assayId));
-              BioAssay ba = bas.newBioAssay(parent);
-              idMap.put(assayId, ba);
-            }
-            catch (ItemNotFoundException e)
-            {
-              throw new BaseException("Couldn't find any parent for bioassay "+assayId);
-            }
-          }
-        }
-        
         ffp.setMinDataColumns(columns.size() - 1 + assays.size() * assayFields.size());
         
@@ -1436 +1717 @@
 
         int posCol = columns.indexOf("position");
-        int repCol = columns.indexOf("reporter");
         int dataCol = columns.indexOf("assayData");
-        int[] intCol = new int[bas.getRawDataType().getChannels()];
+        int[] intCol = new int[child.getRawDataType().getChannels()];
         for (int i = 0; i < intCol.length; ++i)
         {
@@ -1506 +1786 @@
             evtMap.put(name, evt);
           }
-          evBatcher.add(i, bas.getSpotExtraValueBatcher(Float.class, evt, Job.getById(dc, job.getId())));
+          evBatcher.add(i, child.getSpotExtraValueBatcher(Float.class, evt, Job.getById(dc, job.getId())));
         }
         
         while (ffp.hasMoreData())
         {
+          checkInterrupted();
           FlatFileParser.Data dataline = ffp.nextData();
           int index = 0;
-          for (String assayId : assays)
-          {
-            BioAssay ba = idMap.get(assayId);
+          for (Integer assayId : assays)
+          {
+            short dataCubeColumn = bioAssayColumns.get(assayId);
             try
             {
               float[] intensities = new float[intCol.length];
-              Integer reporter = dataline.get(repCol) == null ? null : new Integer(dataline.get(repCol));
               Integer position = dataline.get(posCol) == null ? null : new Integer(dataline.get(posCol));
 
@@ -1538 +1818 @@
               }
 
-              spotBatcher.insert(ba.getDataCubeColumnNo(), position, intensities);
-              
-              if (posRepMap.containsKey(position))
-              {
-                if (!reporter.equals(posRepMap.get(position).getId()))
-                  throw new BaseException("Invalid data. Position, "+position+", occurs twice with different reporter data.");
-              }
-              else
-              {
-                posRepMap.put(position, Reporter.getById(dc, reporter));
-              }
-              
+              spotBatcher.insert(dataCubeColumn, position, intensities);
+                    
               for (int i = 0; i < evBatcher.size(); i++)
               {
@@ -1555 +1825 @@
                 {
                   Float ev = new Float(dataline.get(extraFloatsCol[i] + index));
-                  evBatcher.get(i).insert(ba.getDataCubeColumnNo(), position, ev);
+                  evBatcher.get(i).insert(dataCubeColumn, position, ev);
                 }
                 catch (NumberFormatException e)
@@ -1569 +1839 @@
     }
 
-    //Batch reporters
-    if (bas != null && doPositionBatch)
-    {
-      PositionBatcher posBatcher = bas.getPositionBatcher();
-      for (Integer position : posRepMap.keySet())
-      {
-        posBatcher.insert(position, posRepMap.get(position));
-      }
-    }
-    
-    // save bas
-    if (bas != null)
-    {
-      dc.saveItem(bas);
-      for (BioAssay ba : idMap.values())
-      {
-        dc.saveItem(ba);
-      }
-    }
-    
-    int id = -1;
-    if (t != null) id = t.getId();
-    return id;
-  }
-  
-
+  }
+  
   private void importFiles(DbControl dc, Transformation trans)
   {
@@ -2079 +2325 @@
     }
   }
+  
+  private static class ReporterProxy
+    extends ReporterData
+  {
+    private int id;
+    ReporterProxy()
+    {}
+    public void setTheId(int id)
+    {
+      this.id = id;
+    }
+    public int getId()
+    {
+      return id;
+    }
+  }
+  
+  private static class RawDataProxy
+    extends RawData
+  {
+    private int id;
+    RawDataProxy()
+    {}
+    public void setTheId(int id)
+    {
+      this.id = id;
+    }
+    public int getId()
+    {
+      return id;
+    }
+  } 
+  
+  private static class ChildBioAssay
+  {
+    final int id;
+    final String name;
+    final Set<Integer> parents;
+    
+    ChildBioAssay(int id, String name, Set<Integer> parents)
+    {
+      this.id = id;
+      this.name = name;
+      this.parents = parents;
+    }
+  }
 }

branches/2.6-stable/src/test/TestAll.java

@@ -150 +150 @@
     results.put("TestFormula", TestFormula.test_all());
     results.put("TestExperiment", TestExperiment.test_all());
+    
+    // Complex tests
+    results.put("TestBase1PluginExecuter", TestBase1PluginExecuter.test_all());
 
     int failures = 0;

branches/2.6-stable/src/test/TestAnalyzePluginUtil.java

@@ -35 +35 @@
 import net.sf.basedb.core.RawDataBatcher;
 import net.sf.basedb.core.RawDataTypes;
+import net.sf.basedb.core.ReporterBatcher;
 import net.sf.basedb.core.data.RawData;
 import net.sf.basedb.util.FileUtil;
+import net.sf.basedb.util.Values;
 import net.sf.basedb.util.parser.FlatFileParser;
 
@@ -43 +45 @@
   /**
     Creates a randomly generated experiment.
-    @param rawBioAssays number of {@link net.sf.basedb.core.RawBioAssay RawBioAssays} in experiment
-    @param spots number of spots in each RawBioAssay 
+   * @param file TODO
+   * @param rawBioAssays number of {@link net.sf.basedb.core.RawBioAssay RawBioAssays} in experiment
+   * @param spots number of spots in each RawBioAssay 
   */
-  static int test_createExperiment(int rawBioAssays, int spots)
+  static int test_createExperiment(String file, int rawBioAssays, int spots)
   {
     if (!TestUtil.hasPermission(Permission.CREATE, Item.EXPERIMENT)) return 0;
@@ -66 +69 @@
         parser.setDataHeaderRegexp(Pattern.compile("\"Block\"\\t\"Column\"\\t\"Row\"\\t\"Name\"\\t\"ID\".*"));
         parser.setDataSplitterRegexp(Pattern.compile("\\t"));
-        parser.setInputStream(FileUtil.getInputStream(new java.io.File("data/test.rawdata.import.txt")), "ISO-8859-1");
+        parser.setInputStream(FileUtil.getInputStream(new java.io.File(file)), "ISO-8859-1");
         parser.parseHeaders();
         
@@ -73 +76 @@
         rba.setDescription("RawBioAssay created by TestAnalyzePluginUtil. Number of spots should be "+spots);
         RawDataBatcher rawDataBatcher = rba.getRawDataBatcher(null);
+        ReporterBatcher reporterBatcher = ReporterBatcher.getNew(dc);
         for (int j = 0; parser.hasMoreData() && j < spots; j++)
         {
           FlatFileParser.Data parsedData = parser.nextData();
           RawData rawData = rawDataBatcher.newRawData();
-          rawData.setBlock(Integer.valueOf(parsedData.get(0)));
-          rawData.setRow(Integer.valueOf(parsedData.get(2)));
-          rawData.setColumn(Integer.valueOf(parsedData.get(1)));
-          rawData.setX(Float.valueOf(parsedData.get(5)));
-          rawData.setY(Float.valueOf(parsedData.get(6)));
-//          String reporterId = parsedData.get(4);
-          rawData.setReporter(null);
+          rawData.setBlock(Values.getInteger(parsedData.get(0), null));
+          rawData.setRow(Values.getInteger(parsedData.get(2), null));
+          rawData.setColumn(Values.getInteger(parsedData.get(1), null));
+          rawData.setX(Values.getFloat(parsedData.get(5), null));
+          rawData.setY(Values.getFloat(parsedData.get(6), null));
+          String reporterId = parsedData.get(4);
+          rawData.setReporter(reporterId == null ? null : reporterBatcher.getByExternalId(reporterId));
           
           Map<String, Object> extraData = rawData.getAllExtended();
-          extraData.put("diameter", Float.valueOf(parsedData.get(7)));
-          extraData.put("ch1FgMedian", Float.valueOf(parsedData.get(8)));
-          extraData.put("ch1FgMean", Float.valueOf(parsedData.get(9)));
-          extraData.put("ch1FgSd", Float.valueOf(parsedData.get(10)));
-          extraData.put("ch1BgMedian", Float.valueOf(parsedData.get(11)));
-          extraData.put("ch1BgMean", Float.valueOf(parsedData.get(12)));
-          extraData.put("ch1BgSd", Float.valueOf(parsedData.get(13)));
-          extraData.put("ch1PercSd1", Integer.valueOf(parsedData.get(14)));
-          extraData.put("ch1PercSd2", Integer.valueOf(parsedData.get(15)));
-          extraData.put("ch1PercSat", Integer.valueOf(parsedData.get(16)));
-          extraData.put("ch2FgMedian", Float.valueOf(parsedData.get(17)));
-          extraData.put("ch2FgMean", Float.valueOf(parsedData.get(18)));
-          extraData.put("ch2FgSd", Float.valueOf(parsedData.get(19)));
-          extraData.put("ch2BgMedian", Float.valueOf(parsedData.get(20)));
-          extraData.put("ch2BgMean", Float.valueOf(parsedData.get(21)));
-          extraData.put("ch2BgSd", Float.valueOf(parsedData.get(22)));
-          extraData.put("ch2PercSd1", Integer.valueOf(parsedData.get(23)));
-          extraData.put("ch2PercSd2", Integer.valueOf(parsedData.get(24)));
-          extraData.put("ch2PercSat", Integer.valueOf(parsedData.get(25)));
+          extraData.put("diameter", Values.getFloat(parsedData.get(7), null));
+          extraData.put("ch1FgMedian", Values.getFloat(parsedData.get(8), null));
+          extraData.put("ch1FgMean", Values.getFloat(parsedData.get(9), null));
+          extraData.put("ch1FgSd", Values.getFloat(parsedData.get(10), null));
+          extraData.put("ch1BgMedian", Values.getFloat(parsedData.get(11), null));
+          extraData.put("ch1BgMean", Values.getFloat(parsedData.get(12), null));
+          extraData.put("ch1BgSd", Values.getFloat(parsedData.get(13), null));
+          extraData.put("ch1PercSd1", Values.getInteger(parsedData.get(14), null));
+          extraData.put("ch1PercSd2", Values.getInteger(parsedData.get(15), null));
+          extraData.put("ch1PercSat", Values.getInteger(parsedData.get(16), null));
+          extraData.put("ch2FgMedian", Values.getFloat(parsedData.get(17), null));
+          extraData.put("ch2FgMean", Values.getFloat(parsedData.get(18), null));
+          extraData.put("ch2FgSd", Values.getFloat(parsedData.get(19), null));
+          extraData.put("ch2BgMedian", Values.getFloat(parsedData.get(20), null));
+          extraData.put("ch2BgMean", Values.getFloat(parsedData.get(21), null));
+          extraData.put("ch2BgSd", Values.getFloat(parsedData.get(22), null));
+          extraData.put("ch2PercSd1", Values.getInteger(parsedData.get(23), null));
+          extraData.put("ch2PercSd2", Values.getInteger(parsedData.get(24), null));
+          extraData.put("ch2PercSat", Values.getInteger(parsedData.get(25), null));
           
           rawDataBatcher.insert(rawData);

branches/2.6-stable/src/test/TestBase1PluginExecuter.java

@@ -57 +57 @@
   {
     write("++Testing TestBase1PluginExecuter");
-    int experimentId = TestAnalyzePluginUtil.test_createExperiment(2, 100);
-
-    int formulaId = TestFormula.test_create(Formula.Type.INTENSITY_EXPRESSION, "genepix",
-        new String[] { "raw('ch1FgMedian')", "raw('ch2FgMedian')" }, false);
-    
-    int fileId = TestFile.test_create("data/test.plugin.base", false, false);
+    
+    // Import reporters
+    TestReporter.test_import_from_file("data/test.rawdata.import.txt", 
+      "\"Block\"\\t\"Column\"\\t\"Row\"\\t\"Name\"\\t\"ID\".*", "\\t", 4, 3);
+    
+    // Create experiment
+    int experimentId = TestAnalyzePluginUtil.test_createExperiment("data/test.rawdata.import.txt", 2, 100);
+
+    int formulaId = TestFormula.test_create("raw fg mean", Formula.Type.INTENSITY_EXPRESSION,
+        "genepix", new String[] { "raw('ch1FgMedian')", "raw('ch2FgMedian')" }, false);
+    
+    int formulaChromosomeId = TestFormula.test_create("chromosome", 
+        Formula.Type.COLUMN_EXPRESSION, "genepix", new String[] { "rep('chromosome')" }, false);
+    
+    int fileId = TestFile.test_create("data/test.nullplugin.base", false, false);
+    int fileId2 = TestFile.test_create("data/test.base1plugin.base", false, false);
     int directoryId = TestDirectory.test_create(true, "Base1PluginExecuter");
     
@@ -68 +78 @@
     int pluginDefinitionId = TestPluginDefinition.test_get("net.sf.basedb.plugins.Base1PluginExecuter");
     int pluginConfigurationId = test_create_configuration(pluginDefinitionId, fileId);
+    int pluginConfigurationId2 = test_create_configuration(pluginDefinitionId, fileId2);
+    
+    // Copy input to output
     int jobId = test_create_job(pluginConfigurationId, experimentId, directoryId);
     TestJob.test_execute(jobId);
     
+    // One-to-one mapping on assays and reporters
+    int jobId2 = test_create_job(pluginConfigurationId2, experimentId, directoryId);
+    TestJob.test_execute(jobId2);
+
+    // Merge assays
+    int jobId3 = test_create_job(pluginConfigurationId2, experimentId, directoryId, 
+        new Parameter("mergeAssays", "1"));
+    TestJob.test_execute(jobId3);
+
+    // Offset positions
+    int jobId4 = test_create_job(pluginConfigurationId2, experimentId, directoryId, 
+        new Parameter("offsetPositions", "1"));
+    TestJob.test_execute(jobId4);
+
+    // Offset positions and merge assays
+    int jobId5 = test_create_job(pluginConfigurationId2, experimentId, directoryId, 
+        new Parameter("mergeAssays", "1"), new Parameter("offsetPositions", "1"));
+    TestJob.test_execute(jobId5);
+
     if (TestUtil.waitBeforeDelete()) TestUtil.waitForEnter();
     
     // Cleanup
     TestExperiment.test_delete(experimentId);
-    TestJob.test_delete(jobId);
     TestPluginConfiguration.test_delete(pluginConfigurationId);
+    TestPluginConfiguration.test_delete(pluginConfigurationId2);
     TestFormula.test_delete(formulaId);
+    TestFormula.test_delete(formulaChromosomeId);
     TestFile.test_delete(fileId);
+    TestFile.test_delete(fileId2);
     TestDirectory.test_delete(directoryId, true);
+    TestReporter.test_delete();
     return ok;
   }
@@ -129 +164 @@
   }
   
-  static int test_create_job(int pluginConfigurationId, int experimentId, int directoryId)
+  static int test_create_job(int pluginConfigurationId, int experimentId, int directoryId, Parameter... parameters)
   {
     if (pluginConfigurationId == 0 || !TestUtil.hasPermission(Permission.CREATE, Item.JOB)) return 0;
@@ -146 +181 @@
       
       PluginConfigurationRequest request = j.configure(null);
+      
+      // First, set default values
       for (PluginParameter<?> p : request.getRequestInformation().getParameters())
       {
-        Object value = p.getParameterType().getDefaultValue();
-
-        request.setParameterValue(p.getName(), value);
-      }
+        request.setParameterValue(p.getName(), p.getParameterType().getDefaultValue());
+      }
+      
+      // Then, set values from method call
+      if (parameters != null)
+      {
+        for (Parameter p : parameters)
+        {
+          request.setParameterValue(p.name, p.value);
+        }
+      }
+      
       request.setParameterValue("source", bioAssaySet);
       request.setParameterValue("pluginDirectory", Directory.getById(dc, directoryId).getPath().toString());
@@ -202 +247 @@
     }
   }
+  
+  static class Parameter
+  {
+    final String name;
+    final Object value;
+    Parameter(String name, Object value)
+    {
+      this.name = name;
+      this.value = value;
+    }
+  }
+  
 }

branches/2.6-stable/src/test/TestExperiment.java

@@ -102 +102 @@
     
     
-    int formulaId = TestFormula.test_create(Formula.Type.INTENSITY_EXPRESSION, "genepix",
-        new String[] { "raw('ch1FgMean')", "raw('ch2FgMean')" }, false);
+    int formulaId = TestFormula.test_create(null, Formula.Type.INTENSITY_EXPRESSION,
+        "genepix", new String[] { "raw('ch1FgMean')", "raw('ch2FgMean')" }, false);
     int jobId = test_create_root_bioassayset_using_plugin(id, formulaId, rbaId1, rbaId2);
     TestJob.test_execute(jobId);

branches/2.6-stable/src/test/TestFormula.java

@@ -44 +44 @@
     write_header();
     // Standard tests: create, load, list
-    int id = test_create(Formula.Type.COLUMN_EXPRESSION, "genepix", 
-      new String[] { "ch(1) + ch(2)" },  true);
-    int id2 = test_create(Formula.Type.INTENSITY_EXPRESSION, "genepix",
-      new String[] { "raw('ch1FgMean')", "raw('ch2FgMean')" }, false);
+    int id = test_create("add", Formula.Type.COLUMN_EXPRESSION, 
+      "genepix",  new String[] { "ch(1) + ch(2)" }, true);
+    int id2 = test_create("raw fg mean", Formula.Type.INTENSITY_EXPRESSION,
+      "genepix", new String[] { "raw('ch1FgMean')", "raw('ch2FgMean')" }, false);
     test_load(id);
     test_list(-1);
@@ -59 +59 @@
   }
 
-  static int test_create(Formula.Type formulaType, String rawDataType, String[] formulas, boolean setAll)
+  static int test_create(String name, Formula.Type formulaType, String rawDataType, String[] formulas, boolean setAll)
   {
     if (!TestUtil.hasPermission(Permission.CREATE, Item.FORMULA)) return 0;
@@ -75 +75 @@
         f.setParser(Formula.Parser.JEP);
       }
+      if (name != null) f.setName(name);
       f.setFormulaType(formulaType);
       f.setRawDataType(RawDataTypes.getRawDataType(rawDataType));

branches/2.6-stable/src/test/TestJob.java

@@ -95 +95 @@
       if (response.getStatus() == Response.Status.ERROR)
       {
-        throw new BaseException(response.getMessage(), response.getErrorList().get(0));
+        Throwable cause = response.getErrorList() != null &&
+          response.getErrorList().size() > 0 ? response.getErrorList().get(0) : null;
+        throw new BaseException(response.getMessage(), cause);
       }
       

branches/2.6-stable/src/test/TestLowessNormalization.java

@@ -54 +54 @@
   {
     write("++Testing TestLowessNormalization");
-    int experimentId = TestAnalyzePluginUtil.test_createExperiment(2, 100);
+    int experimentId = TestAnalyzePluginUtil.test_createExperiment("data/test.rawdata.import.txt", 2, 100);
 
-    int formulaId = TestFormula.test_create(Formula.Type.INTENSITY_EXPRESSION, "genepix",
-        new String[] { "raw('ch1FgMedian')", "raw('ch2FgMedian')" }, false);
+    int formulaId = TestFormula.test_create(null, Formula.Type.INTENSITY_EXPRESSION,
+        "genepix", new String[] { "raw('ch1FgMedian')", "raw('ch2FgMedian')" }, false);
     
     TestAnalyzePluginUtil.test_createRootBioAssaySet(experimentId, formulaId);

branches/2.6-stable/src/test/TestMedianRatioNormalization.java

@@ -49 +49 @@
   {
     write("++Testing MedianRatioNormalization");
-    int experimentId = TestAnalyzePluginUtil.test_createExperiment(2, 100);
-    int formulaId = TestFormula.test_create(Formula.Type.INTENSITY_EXPRESSION, "genepix",
-        new String[] { "raw('ch1FgMedian')", "raw('ch2FgMedian')" }, false);
+    int experimentId = TestAnalyzePluginUtil.test_createExperiment("data/test.rawdata.import.txt", 2, 100);
+    int formulaId = TestFormula.test_create(null, Formula.Type.INTENSITY_EXPRESSION,
+        "genepix", new String[] { "raw('ch1FgMedian')", "raw('ch2FgMedian')" }, false);
     TestAnalyzePluginUtil.test_createRootBioAssaySet(experimentId, formulaId);
     int pluginDefinitionId = TestPluginDefinition.test_get("net.sf.basedb.plugins.MedianRatioNormalization");

branches/2.6-stable/src/test/data/test.nullplugin.base

@@ -5 +5 @@
 name  Base1NullPlugin
 descr Test base1pluginexecuter
-execName  runplugin.sh
+execName  nullplugin.sh
 geneAverages  0
 serialFormat  1