Changeset 3622


Ignore:
Timestamp:
Aug 2, 2007, 12:36:07 PM (16 years ago)
Author:
Johan Enell
Message:

Fixes #700, #590

Location:
trunk/src
Files:
3 edited

Legend:

Unmodified
Added
Removed

  • trunk/src/core/net/sf/basedb/core/Formula.java

    r3616 r3622  
    378378    NONE(0, "None")
    379379    {
     380      @Override
    380381      public Expression getAverageExpression(Expression e)
    381382      {
    … …  
    385386    GEOMETRIC_MEAN(1, "Geometric mean")
    386387    {
     388      @Override
    387389      public Expression getAverageExpression(Expression e)
    388390      {
    … …  
    392394    ARITHMETIC_MEAN(2, "Arithmetic mean")
    393395    {
     396      @Override
    394397      public Expression getAverageExpression(Expression e)
    395398      {
    … …  
    400403    MIN(3, "Min")
    401404    {
     405      @Override
    402406      public Expression getAverageExpression(Expression e)
    403407      {
    … …  
    408412    MAX(4, "Max")
    409413    {
     414      @Override
    410415      public Expression getAverageExpression(Expression e)
    411416      {

  • trunk/src/plugins/core/net/sf/basedb/plugins/BioAssaySetExporter.java

    r3616 r3622  
    5353import net.sf.basedb.core.StringParameterType;
    5454import net.sf.basedb.core.VirtualColumn;
     55import net.sf.basedb.core.VirtualTable;
    5556import net.sf.basedb.core.data.ReporterData;
    5657import net.sf.basedb.core.plugin.About;
    … …  
    6566import net.sf.basedb.core.plugin.Response;
    6667import net.sf.basedb.core.query.Dynamic;
     68import net.sf.basedb.core.query.Expression;
     69import net.sf.basedb.core.query.Expressions;
    6770import net.sf.basedb.core.query.Hql;
    6871import net.sf.basedb.core.query.JoinType;
    6972import net.sf.basedb.core.query.Orders;
     73import net.sf.basedb.core.query.Select;
    7074import net.sf.basedb.core.query.Selects;
    7175import net.sf.basedb.core.query.SqlResult;
    … …  
    283287        throw new BaseException("Unknown field in reporter fields: "+field);
    284288      }
    285       query.select(Selects.expression(BioAssaySetUtil.createJepExpression(dc, bas, qi.getJep(), true), qi.getQueryAlias()));
     289      query.select(Selects.expression(qi.getExpression(dc, bas, mergeReporters), qi.getQueryAlias()));
    286290    }
    287291    for (String field : spotFields)
    … …  
    292296        throw new BaseException("Unknown field in spot fields: "+field);
    293297      }
    294       query.select(Selects.expression(BioAssaySetUtil.createJepExpression(dc, bas, qi.getJep(), true), qi.getQueryAlias()));
    295     }
    296     DynamicSpotQuery count = bas.getSpotData();
    297     count.setDistinct(true);
     298      query.select(Selects.expression(qi.getExpression(dc, bas, mergeReporters), qi.getQueryAlias()));
     299    }
     300
    298301    if (mergeReporters)
    299302    {
    300       count.select(Selects.expression(Dynamic.column(VirtualColumn.REPORTER_ID), "rep", true));
    301     }
    302     else
    303     {
    304       count.select(Selects.expression(Dynamic.column(VirtualColumn.POSITION), "pos", true));
    305     }
    306    
    307     if (mergeReporters)
    308     {
    309       throw new BaseException("'Merge on reporter' not implemented");
    310     }
    311    
    312     exportBaseFileSectionSpots(out, dc, bas, query, reporterFields, spotFields, bas.getBioAssays().list(dc), qis, count.count(dc));
     303      QueryItem qi = qis.get("rep.id");
     304      Select rep = Selects.expression(qi.getExpression(dc, bas, mergeReporters), qi.getQueryAlias());
     305      if (!reporterFields.contains(qi.getExportName()))
     306      {
     307        query.select(rep);
     308      }
     309      query.group(Dynamic.column(VirtualColumn.COLUMN));
     310      query.group(Expressions.selected(rep));
     311    }
     312   
     313    long count = getBaseFileSpotSectionCount(dc, bas.getSpotData(), mergeReporters);
     314    exportBaseFileSectionSpots(out, dc, bas, query, reporterFields, spotFields, bas.getBioAssays().list(dc), qis, count);
    313315    out.flush();
    314316  }
    … …  
    416418          throw new BaseException("Unknown column: "+field+" (you can add this field using a formula)");
    417419        }
    418         query.select(Selects.expression(BioAssaySetUtil.createJepExpression(dc, bas, qi.getJep(), true), qi.getQueryAlias()));
     420        query.select(Selects.expression(qi.getExpression(dc, bas, mergeReporters), qi.getQueryAlias()));
    419421      }
    420422      for (String field : spotFields)
    … …  
    425427          throw new BaseException("Unknwon field: "+field+" (you can add this field using a formula)");
    426428        }
    427         query.select(Selects.expression(BioAssaySetUtil.createJepExpression(dc, bas, qi.getJep(), true), qi.getQueryAlias()));
    428       }
    429       DynamicSpotQuery count = ba.getSpotData();
    430       count.setDistinct(true);
     429        query.select(Selects.expression(qi.getExpression(dc, bas, mergeReporters), qi.getQueryAlias()));
     430      }
     431
    431432      if (mergeReporters)
    432433      {
    433         count.select(Selects.expression(Dynamic.column(VirtualColumn.REPORTER_ID), "rep", true));
    434       }
    435       else
    436       {
    437         count.select(Selects.expression(Dynamic.column(VirtualColumn.POSITION), "pos", true));
    438       }
    439      
    440       if (mergeReporters)
    441       {
    442         throw new BaseException("'Merge on reporter' not implemented");
    443       }
    444 
    445       exportBaseFileSectionSpots(out, dc, bas, query, reporterFields, spotFields, Collections.singletonList(ba), qis, count.count(dc));
     434        QueryItem qi = qis.get("rep.id");
     435        Select rep = Selects.expression(qi.getExpression(dc, bas, mergeReporters), qi.getQueryAlias());
     436        if (!reporterFields.contains(qi.getExportName()))
     437        {
     438          query.select(rep);
     439        }
     440        query.group(rep.getExpression());
     441      }
     442
     443      long count = getBaseFileSpotSectionCount(dc, ba.getSpotData(), mergeReporters);
     444      exportBaseFileSectionSpots(out, dc, bas, query, reporterFields, spotFields, Collections.singletonList(ba), qis, count);
    446445     
    447446      out.println();
    … …  
    817816 
    818817 
     818  private long getBaseFileSpotSectionCount(DbControl dc, DynamicSpotQuery count, boolean mergeReporters)
     819  {
     820    count.setDistinct(true);
     821    if (mergeReporters)
     822    {
     823      count.joinReporters(JoinType.INNER);
     824      count.select(Selects.expression(Dynamic.column(VirtualTable.POSITION, VirtualColumn.REPORTER_ID), "rep", true));
     825    }
     826    else
     827    {
     828      count.select(Selects.expression(Dynamic.column(VirtualColumn.POSITION), "pos", true));
     829    }
     830    return count.count(dc);
     831  }
     832 
     833 
    819834  private List<QueryItem> getBaseFileExtraQI(DbControl dc, BioAssaySet bas)
    820835  {
    … …  
    826841      ExtraValueType evt = ev.getExtraValueType();
    827842      items.add(new QueryItem("xtra."+evt.getExternalId(), "[Extra] "+evt.getName(),
    828         evt.getName(), "xtra('" + evt.getExternalId() + "')"));
     843        evt.getName(), "xtra('" + evt.getExternalId() + "')", evt.getAverageMethod()));
    829844    }
    830845    Collections.sort(items, new QueryItemTitleComparator());
    … …  
    841856    for (Formula f : queryFormula.list(dc))
    842857    {
    843       items.add(new QueryItem("frm." + f.getId(), "[Formula] "+f.getName(), f.getName(), f.getFormula(0)));
     858      items.add(new QueryItem("frm." + f.getId(), "[Formula] "+f.getName(), f.getName(), f.getFormula(0), f.getAverageMethod()));
    844859    }
    845860    Collections.sort(items, new QueryItemTitleComparator());
    … …  
    863878  {
    864879    List<QueryItem> items = new ArrayList<QueryItem>();
    865     items.add(new QueryItem("raw.position", "[Raw] Position", "position", "raw('position')"));
    866     items.add(new QueryItem("raw.block", "[Raw] Array block", "block", "raw('block')"));
    867     items.add(new QueryItem("raw.metaGridX", "[Raw] Meta grid X", "metaGridX", "raw('metaGridX')"));
    868     items.add(new QueryItem("raw.metaGridY", "[Raw] Meta grid Y", "metaGridY", "raw('metaGridY')"));
    869     items.add(new QueryItem("raw.row", "[Raw] Row in block", "row", "raw('row')"));
    870     items.add(new QueryItem("raw.col", "[Raw] Column in block", "col", "raw('column')"));
    871     items.add(new QueryItem("raw.x", "[Raw] X coordinate", "x", "raw('x')"));
    872     items.add(new QueryItem("raw.y", "[Raw] Y coordinate", "y", "raw('y')"));
     880    items.add(new QueryItem("raw.position", "[Raw] Position", "position", "raw('position')", Formula.AverageMethod.NONE));
     881    items.add(new QueryItem("raw.block", "[Raw] Array block", "block", "raw('block')", Formula.AverageMethod.NONE));
     882    items.add(new QueryItem("raw.metaGridX", "[Raw] Meta grid X", "metaGridX", "raw('metaGridX')", Formula.AverageMethod.NONE));
     883    items.add(new QueryItem("raw.metaGridY", "[Raw] Meta grid Y", "metaGridY", "raw('metaGridY')", Formula.AverageMethod.NONE));
     884    items.add(new QueryItem("raw.row", "[Raw] Row in block", "row", "raw('row')", Formula.AverageMethod.NONE));
     885    items.add(new QueryItem("raw.col", "[Raw] Column in block", "col", "raw('column')", Formula.AverageMethod.NONE));
     886    items.add(new QueryItem("raw.x", "[Raw] X coordinate", "x", "raw('x')", Formula.AverageMethod.NONE));
     887    items.add(new QueryItem("raw.y", "[Raw] Y coordinate", "y", "raw('y')", Formula.AverageMethod.NONE));
    873888    for (RawDataProperty rdp : bas.getRawDataType().getProperties())
    874889    {
    875       items.add(new QueryItem("raw." + rdp.getName(), "[Raw] "+rdp.getTitle(), rdp.getName(), "raw('"+rdp.getName()+"')"));
     890      items.add(new QueryItem("raw." + rdp.getName(), "[Raw] "+rdp.getTitle(), rdp.getName(), "raw('"+rdp.getName()+"')", rdp.getAverageMethod()));
    876891    }
    877892    Collections.sort(items, new QueryItemTitleComparator());
    … …  
    883898  {
    884899    List<QueryItem> items = new ArrayList<QueryItem>();
    885     items.add(new QueryItem("rep.id", "Reporter (internal id)", "reporter", "rep('id')"));
    886     items.add(new QueryItem("rep.position", "Position", "position", "pos()"));
    887     items.add(new QueryItem("rep.name", "Reporter Name", "name", "rep('name')"));
    888     items.add(new QueryItem("rep.externalId", "Reporter ID", "externalId", "rep('externalId')"));
    889     items.add(new QueryItem("rep.symbol", "Gene symbol", "symbol", "rep('symbol')"));
    890     items.add(new QueryItem("rep.description", "Description", "description", "rep('description')"));
    891     items.add(new QueryItem("rep.lastUpdate", "Last updated", "lastUpdate", "rep('lastUpdate')"));
     900    items.add(new QueryItem("rep.id", "Reporter (internal id)", "reporter", "rep('id')", null));
     901    items.add(new QueryItem("rep.position", "Position", "position", "pos()", Formula.AverageMethod.NONE));
     902    items.add(new QueryItem("rep.name", "Reporter Name", "name", "rep('name')", Formula.AverageMethod.MAX));
     903    items.add(new QueryItem("rep.externalId", "Reporter ID", "externalId", "rep('externalId')", Formula.AverageMethod.MAX));
     904    items.add(new QueryItem("rep.symbol", "Gene symbol", "symbol", "rep('symbol')", Formula.AverageMethod.MAX));
     905    items.add(new QueryItem("rep.description", "Description", "description", "rep('description')", Formula.AverageMethod.MAX));
     906    items.add(new QueryItem("rep.lastUpdate", "Last updated", "lastUpdate", "rep('lastUpdate')", Formula.AverageMethod.MAX));
    892907    for (ExtendedProperty ep : ExtendedProperties.getProperties(ReporterData.class.toString()))
    893908    {
    894       items.add(new QueryItem("rep." + ep.getName(), ep.getTitle(), ep.getName(), "rep('"+ep.getName()+"')"));
     909      items.add(new QueryItem("rep." + ep.getName(), ep.getTitle(), ep.getName(), "rep('"+ep.getName()+"')", Formula.AverageMethod.MAX));
    895910    }
    896911    Collections.sort(items, new QueryItemTitleComparator());
    … …  
    902917  {
    903918    List<QueryItem> items = new ArrayList<QueryItem>();
    904     int channels = bas.getRawDataType().getChannels(); 
     919    int channels = bas.getRawDataType().getChannels();
    905920    for (int channel = 1; channel <= channels; ++channel)
    906921    {
    907       items.add(new QueryItem("intensity"+channel, "Int ch"+channel, "intensity" + channel, "ch("+channel+")"));
     922      items.add(new QueryItem("intensity"+channel, "Int ch"+channel, "intensity" + channel, "ch("+channel+")", Formula.AverageMethod.ARITHMETIC_MEAN));
    908923    }
    909924    Collections.sort(items, new QueryItemTitleComparator());
    … …  
    12661281    private String queryAlias;
    12671282   
    1268     QueryItem(String name, String title, String exportName, String jep)
     1283    private Formula.AverageMethod avg;
     1284   
     1285    QueryItem(String name, String title, String exportName, String jep, Formula.AverageMethod avg)
    12691286    {
    12701287      this.name = name;
    … …  
    12731290      this.jep = jep;
    12741291      this.queryAlias = "a" + Math.abs(System.identityHashCode(name));
    1275   }
    1276 
    1277     public final String getJep()
    1278     {
    1279       return jep;
     1292      this.avg = avg;
     1293    }
     1294
     1295    public final Expression getExpression(DbControl dc, BioAssaySet bas, boolean average)
     1296    {
     1297      Expression e = BioAssaySetUtil.createJepExpression(dc, bas, jep, true);
     1298      if (average && avg != null)
     1299      {
     1300        e = avg.getAverageExpression(e);
     1301        if (e == null)
     1302        {
     1303          e = Expressions.string("");
     1304        }
     1305      }
     1306      return e;
    12801307    }
    12811308

  • trunk/src/test/TestBioAsssaySetExporter.java

    r3554 r3622  
    3434import java.util.Set;
    3535import java.util.TreeMap;
     36import java.util.regex.Matcher;
    3637import java.util.regex.Pattern;
    3738
    … …  
    7071  static boolean ok = true;
    7172 
     73  static ArrayList<String> readlines = new ArrayList<String>();
     74 
    7275  public static void main(String[] args)
    7376  {
    … …  
    110113     
    111114      test_exportBaseFile(bas, file, false, false);
    112 //      test_exportBaseFile(bas, file, true, false);
    113 //      test_exportBaseFile(bas, file, false, true);
    114 //      test_exportBaseFile(bas, file, true, true);
     115      test_exportBaseFile(bas, file, true, false);
     116      test_exportBaseFile(bas, file, false, true);
     117      test_exportBaseFile(bas, file, true, true);
    115118    }
    116119   
    … …  
    178181        if (i < 2) rd.setReporter(r1);
    179182        else rd.setReporter(r2);
    180         rd.setExtended("Flags", Integer.valueOf(0));
     183        rd.setExtended("flags", Integer.valueOf(0));
    181184        rd.setExtended("ch1FgMedian", Float.valueOf(1));
    182185        rd.setExtended("ch2FgMedian", Float.valueOf(2));
    … …  
    192195      rd.setRow(2);
    193196      rd.setReporter(r3);
    194       rd.setExtended("Flags", Integer.valueOf(-50));
     197      rd.setExtended("flags", Integer.valueOf(-50));
    195198      rd.setExtended("ch1FgMedian", Float.valueOf(10));
    196199      rd.setExtended("ch2FgMedian", Float.valueOf(20));
    … …  
    446449      ffp.setUseNullIfEmpty(false);
    447450
     451      ArrayList<String[]> rows = new ArrayList<String[]>();
     452     
     453      readlines.clear();
    448454      // Test settings section
    449455      ffp.parseHeaders();
     456      for (Line l : ffp.getLines()) readlines.add(l.toString());
    450457      TestFlatFileParser.testLine(ffp.getLine(0), "BASEfile");
    451458      TestFlatFileParser.testSection(ffp.nextSection(), "settings");
    452459      ffp.parseHeaders();
     460      for (Line l : ffp.getLines()) readlines.add(l.toString());
    453461      TestFlatFileParser.testHeaderLine(ffp.getLine(0), "a", "some other parameter");
    454462      TestFlatFileParser.testHeaderLine(ffp.getLine(1), "b", "some parameter");
    455463     
     464     
     465      readlines.clear();
    456466      // Test assays section
    457467      TestFlatFileParser.testSection(ffp.nextSection(), "assays");
    458468      ffp.parseHeaders();
     469      for (Line l : ffp.getLines()) readlines.add(l.toString());
    459470     
    460471      test_baseFileSectionHeader("annotationColumns", ffp.getHeader("annotationColumns"), assaysAnnotationColumns);
    461472      test_baseFileSectionHeader("columns", ffp.getHeader("columns"), assaysColumns);
    462473      test_baseFileSectionHeader("count", ffp.getHeader("count"), assaysCount);
    463       test_baseFileSectionData(ffp, 2, 3);
    464    
    465      
     474      rows.add(null);
     475      rows.add(null);
     476      test_baseFileSectionData(ffp, rows);
     477     
     478      readlines.clear();
    466479      // Test spot section(s)
    467480      int numOfSections = matrix ? 1 : (int) bas.getBioAssays().count(dc);
    … …  
    471484        TestFlatFileParser.testSection(section, "spots");
    472485        ffp.parseHeaders();
     486        for (Line l : ffp.getLines()) readlines.add(l.toString());
    473487        test_baseFileSectionHeader("channels", ffp.getHeader("channels"), spotsChannels);
    474488        test_baseFileSectionHeader("assayFields", ffp.getHeader("assayFields"), spotsAssayFields);
    … …  
    477491        test_baseFileSectionHeader("count", ffp.getHeader("count"), spotsCount);
    478492
     493        rows.clear();
    479494        if (matrix)
    480495        {
    481           int columns = (spotsColumns.size()-1)+spotsAssayFields.size()*spotsAssays.size();
    482           if (mergeReporters) test_baseFileSectionData(ffp, 2, columns);
    483           else test_baseFileSectionData(ffp, 6, columns);
     496          if (mergeReporters)
     497          {
     498            String[] cols = {"\\d+","New reporter","","[14]","","","","0\\.5","[14]","","","","0\\.5"};
     499            rows.add(cols);
     500            rows.add(cols);
     501          }
     502          else
     503          {
     504            String[] cols1 = {"\\d+","New reporter","[1234]","1","[01]","[01]","0","0\\.5","1","[01]","[01]","0","0\\.5"};
     505            String[] cols2 = {"\\d+","New reporter","[56]","(10)|","2|","0|","-50|","0\\.5|","(10)|","2|","0|","-50|","0\\.5|"};
     506            rows.add(cols1);
     507            rows.add(cols1);
     508            rows.add(cols1);
     509            rows.add(cols1);
     510            rows.add(cols2);
     511            rows.add(cols2);
     512          }
    484513        }
    485514        else
    486515        {
    487           int columns = (spotsColumns.size()-1)+spotsAssayFields.size();
    488           if (mergeReporters) test_baseFileSectionData(ffp, 2, columns);
    489           else test_baseFileSectionData(ffp, 5, columns);
     516          if (mergeReporters)
     517          {
     518            String[] cols = {"\\d+","New reporter","","[14]","","","","0\\.5"};
     519            rows.add(cols);
     520            rows.add(cols);
     521          }
     522          else
     523          {
     524            String[] cols = {"\\d+","New reporter","[123456]","10?","[012]","[01]","(0)|(-50)","0\\.5"};
     525            rows.add(cols);
     526            rows.add(cols);
     527            rows.add(cols);
     528            rows.add(cols);
     529            rows.add(cols);
     530          }
    490531        }
     532        test_baseFileSectionData(ffp, rows);
    491533       
    492534        section = ffp.nextSection();
    … …  
    499541    {
    500542      write("--Exporting basefile [matrix:"+matrix+", mergeReporters:"+mergeReporters+"] FAILED");
    501       write(ffp);
     543      for (String line : readlines)
     544      {
     545        write("["+line+"]");
     546      }
    502547      ex.printStackTrace();
    503548      ok = false;
    … …  
    509554  }
    510555 
    511   private static ArrayList<FlatFileParser.Data> test_baseFileSectionData(FlatFileParser ffp, int rows, int columns)
     556  private static ArrayList<FlatFileParser.Data> test_baseFileSectionData(FlatFileParser ffp, ArrayList<String[]> rows)
    512557    throws Exception
    513558  {
    514     ArrayList<FlatFileParser.Data> datalines = new ArrayList<FlatFileParser.Data>(rows);
     559    ArrayList<FlatFileParser.Data> datalines = new ArrayList<FlatFileParser.Data>(rows.size());
    515560    while (ffp.hasMoreData())
    516561    {
    517562      FlatFileParser.Data dataline = ffp.nextData();
    518       if (dataline.columns() != columns)
    519         throw new Exception("Wrong number of columns. Found "+dataline.columns()+" expected "+columns+". ["+dataline+"]");
     563      readlines.add(dataline.toString());
    520564      datalines.add(dataline);
    521565    }
    522     if (datalines.size() != rows)
    523       throw new Exception("Wrong number of data lines. Found "+datalines.size()+" expected "+rows);
     566   
     567    if (datalines.size() != rows.size())
     568      throw new Exception("Wrong number of data lines. Found "+datalines.size()+" expected "+rows.size());
     569   
     570    for (int i = 0; i < rows.size(); ++i)
     571    {
     572      FlatFileParser.Data dataline = datalines.get(i);
     573     
     574      String[] row = rows.get(i);
     575      if (row != null)
     576      {
     577        if (dataline.columns() != row.length)
     578          throw new Exception("Wrong number of columns. Found "+dataline.columns()+" expected "+row.length+". ["+dataline+"]");
     579       
     580        for (int j = 0; j < row.length; ++j)
     581        {
     582          String col = row[j];
     583          Pattern p = Pattern.compile(col);
     584          Matcher m = p.matcher(dataline.get(j));
     585          if (!m.matches())
     586          {
     587            throw new Exception("The value["+dataline.get(j)+"] at column number "+(j+1)+" dosent match the pattern '"+p+"'. ["+dataline+"]");
     588          }
     589        }
     590      }
     591    }
     592   
    524593    return datalines;
    525594  }
    … …  
    533602   
    534603  }
    535  
    536   private static void write(FlatFileParser ffp)
    537   {
    538     if (!TestUtil.getSilent() && ffp != null)
    539     {
    540       for (Line line : ffp.getLines())
    541       {
    542         write(line.toString());
    543       }
    544     }
    545   }
    546604
    547605  static void write(String message)
Note: See TracChangeset for help on using the changeset viewer.