source: trunk/test/normalization_test.cc @ 2156

Last change on this file since 2156 was 2156, checked in by Peter, 13 years ago

refs #498. testing that Gauss works with mixed iterators

  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 15.7 KB
Line 
1// $Id: normalization_test.cc 2156 2010-01-17 23:37:11Z peter $
2
3/*
4  Copyright (C) 2008, 2009 Jari Häkkinen, Peter Johansson
5
6  This file is part of the yat library, http://dev.thep.lu.se/yat
7
8  The yat library is free software; you can redistribute it and/or
9  modify it under the terms of the GNU General Public License as
10  published by the Free Software Foundation; either version 3 of the
11  License, or (at your option) any later version.
12
13  The yat library is distributed in the hope that it will be useful,
14  but WITHOUT ANY WARRANTY; without even the implied warranty of
15  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16  General Public License for more details.
17
18  You should have received a copy of the GNU General Public License
19  along with yat. If not, see <http://www.gnu.org/licenses/>.
20*/
21
22#include "Suite.h"
23
24#include "yat/normalizer/Centralizer.h"
25#include "yat/normalizer/ColumnNormalizer.h"
26#include "yat/normalizer/Gauss.h"
27#include "yat/normalizer/qQuantileNormalizer.h"
28#include "yat/normalizer/QuantileNormalizer.h"
29#include "yat/normalizer/RowNormalizer.h"
30#include "yat/normalizer/Spearman.h"
31#include "yat/normalizer/Zscore.h"
32
33#include "yat/utility/DataIterator.h"
34#include "yat/utility/FileUtil.h"
35#include "yat/utility/Matrix.h"
36#include "yat/utility/MatrixWeighted.h"
37#include "yat/utility/WeightIterator.h"
38
39#include <boost/concept_archetype.hpp>
40
41#include <climits>
42#include <fstream>
43#include <limits>
44#include <vector>
45
46using namespace theplu::yat;
47void test_centralizer(test::Suite&);
48void test_column_normalize(test::Suite&);
49void test_gauss_normalize(test::Suite&);
50void test_qquantile_normalize(test::Suite&);
51void test_qquantile_normalize_weighted(test::Suite&);
52void test_quantile_normalize(test::Suite&);
53void test_row_normalize(test::Suite&);
54void test_spearman(test::Suite&);
55void test_spearman_weighted(test::Suite&);
56void test_z_score(test::Suite&);
57
58int main(int argc, char* argv[])
59{ 
60  test::Suite suite(argc, argv);
61  suite.err() << "testing normalizations ... " << std::endl;
62
63  test_centralizer(suite);
64  test_column_normalize(suite);
65  test_qquantile_normalize(suite);
66  test_qquantile_normalize_weighted(suite);
67  test_quantile_normalize(suite);
68  test_gauss_normalize(suite);
69  test_row_normalize(suite);
70  test_spearman(suite);
71  test_z_score(suite);
72
73  return suite.return_value();
74}
75
76
77void test_centralizer(test::Suite& suite)
78{
79  suite.err() << "Testing Centralizer\n";
80  std::vector<double> vec;
81  vec.push_back(1);
82  vec.push_back(2);
83  vec.push_back(3);
84  normalizer::Centralizer<> c;
85  c(vec.begin(), vec.end(), vec.begin());
86  for (size_t i=0; i<vec.size(); ++i)
87    suite.add(suite.equal(vec[i], static_cast<double>(i)-1.0));
88
89  std::vector<utility::DataWeight> vec2;
90  vec2.push_back(utility::DataWeight(1,1));
91  vec2.push_back(utility::DataWeight(2,0.5));
92  vec2.push_back(utility::DataWeight(2,0.5));
93  std::vector<utility::DataWeight> vec3(vec2.size());
94  c(vec2.begin(), vec2.end(), vec3.begin());
95  for (size_t i=0; i<vec2.size(); ++i)
96    suite.add(suite.equal(vec3[i].weight(), vec2[i].weight()));
97  suite.add(suite.equal(vec3[0].data(), -0.5));
98  suite.add(suite.equal(vec3[1].data(), 0.5));
99  suite.add(suite.equal(vec3[2].data(), 0.5));
100
101  // compile test should not be run
102  if (false) {
103    boost::detail::dummy_constructor dummy_cons;
104    c(boost::input_iterator_archetype<double>(), 
105      boost::input_iterator_archetype<double>(),
106      boost::output_iterator_archetype<double>(dummy_cons));
107  }
108}
109
110
111void test_column_normalize(test::Suite& suite)
112{
113  using namespace normalizer;
114  suite.err() << "Testing ColumnNormalizer\n";
115 
116  utility::Matrix m(2,2);
117  m(0,0) = 0;
118  m(0,1) = 10;
119  m(1,0) = 2;
120  m(1,1) = 4;
121  ColumnNormalizer<Centralizer<> > qn;
122  qn(m, m);
123  suite.err() << "Testing m(0,0)\n";
124  suite.add(suite.equal(m(0,0), -1));
125  suite.err() << "Testing m(0,1)\n";
126  suite.add(suite.equal(m(0,1), 3));
127  suite.err() << "Testing m(1,0)\n";
128  suite.add(suite.equal(m(1,0), 1));
129  suite.err() << "Testing m(1,1)\n";
130  suite.add(suite.equal(m(1,1), -3));
131
132  if (false) { // do not run compile tests
133    test::container2d_archetype<double> container2d; 
134    test::mutable_container2d_archetype<double> mutable_container2d; 
135    qn(container2d, mutable_container2d);
136  }
137}
138
139
140void test_qquantile_normalize(test::Suite& suite)
141{
142  using namespace normalizer;
143
144  suite.err() << "Testing qQuantileNormalizer\n";
145  std::string data(test::filename("data/normalization_test.data"));
146  if (utility::FileUtil(data.c_str()).permissions("r")) {
147    suite.add(false);
148    suite.err() << "Cannot access file " << data << '\n';
149    return;
150  }
151  std::ifstream data_stream(data.c_str());
152
153  utility::Matrix m(data_stream);
154
155  suite.err() << "testing number of parts (Q) boundary conditions\n";
156  qQuantileNormalizer(m.begin_column(0), m.end_column(0), m.rows());
157  qQuantileNormalizer(m.begin_column(0), m.end_column(0), 3);
158
159  // first column as target
160  qQuantileNormalizer qqn(m.begin_column(0), m.end_column(0) ,9); 
161  ColumnNormalizer<qQuantileNormalizer> cn(qqn);
162  utility::Matrix result(m.rows(),m.columns());
163  cn(m, result);
164
165  suite.err() << "test that result can be stored in the source matrix...";
166  cn(m,m);
167  if (suite.add(result==m))
168    suite.err() << " ok.\n";
169  else 
170    suite.err() << " failed.\n";
171
172  // Enough iteration will make all columns to have the same values as
173  // the target.
174  suite.err() << "Testing that q=matrix rows gives QuantileNormalization\n";
175  utility::Matrix m2(4,2);
176  m2(0,0) = 0; m2(0,1) = 10;
177  m2(1,0) = 2; m2(1,1) = 4;
178  m2(2,0) = 1; m2(2,1) = 0;
179  m2(3,0) = 3; m2(3,1) = 7;
180  qQuantileNormalizer qqn2(m2.begin_column(0), m2.end_column(0), m2.rows());
181  ColumnNormalizer<qQuantileNormalizer> cn2(qqn2);
182  utility::Matrix result2(m2.rows(),m2.columns());
183  cn2(m2,result2);
184  suite.add( suite.equal_fix(m2(0,0),result2(2,1),1.0e-12) &&
185             suite.equal_fix(m2(1,0),result2(3,1),1.0e-12) &&
186             suite.equal_fix(m2(2,0),result2(1,1),1.0e-12) &&
187             suite.equal_fix(m2(3,0),result2(0,1),1.0e-12) );
188  // compile test should not be run
189  if (false) {
190    qQuantileNormalizer qqn3(boost::forward_iterator_archetype<double>(),
191                             boost::forward_iterator_archetype<double>(),
192                             100);
193    qqn3(boost::random_access_iterator_archetype<double>(), 
194         boost::random_access_iterator_archetype<double>(),
195         boost::mutable_random_access_iterator_archetype<double>());
196    using utility::DataWeight;
197    qQuantileNormalizer qqn4(boost::forward_iterator_archetype<DataWeight>(),
198                             boost::forward_iterator_archetype<DataWeight>(),
199                             100);
200    qqn4(boost::random_access_iterator_archetype<DataWeight>(), 
201         boost::random_access_iterator_archetype<DataWeight>(),
202         boost::mutable_random_access_iterator_archetype<DataWeight>());
203  }
204}
205
206
207void test_qquantile_normalize_weighted(test::Suite& suite)
208{
209  using namespace normalizer;
210
211  suite.err() << "Testing qQuantileNormalizer weighted\n";
212
213  // test with unweighted target and source
214  std::vector<double> target;
215  target.reserve(1000);
216  while (target.size()<1000)
217    target.push_back(target.size());
218  qQuantileNormalizer qQN(target.begin(), target.end(), 4);
219  std::vector<double> source;
220  while (source.size()<10)
221    source.push_back(source.size()*10);
222  std::vector<double> result(source.size());
223 
224  qQN(source.begin(), source.end(), result.begin());
225 
226  using utility::DataWeight;
227  suite.err() << "Testing with unweighted target and weighted source\n";
228  std::vector<utility::DataWeight> src_w(source.size(), DataWeight(0, 1));
229  std::copy(source.begin(), source.end(),
230            utility::data_iterator(src_w.begin()));
231
232  std::vector<utility::DataWeight> result_w(src_w.size());
233  qQN(src_w.begin(), src_w.end(), result_w.begin());
234  suite.add(suite.equal_range(result.begin(), result.end(),
235                              utility::data_iterator(result_w.begin())));
236
237  suite.err() << "Testing with missing value in source\n";
238  // adding a missing value
239  std::vector<utility::DataWeight>::iterator MWi=src_w.begin();
240  MWi+=5;
241  src_w.insert(MWi, DataWeight(std::numeric_limits<double>::quiet_NaN(), 0.0));
242  std::vector<utility::DataWeight> result_w2(src_w.size());
243  qQN(src_w.begin(), src_w.end(), result_w2.begin());
244  // excluding missing value from comparison in suite.equal_range
245  MWi=result_w2.begin();
246  MWi+=5;
247  result_w2.erase(MWi);
248  suite.add(suite.equal_range(utility::data_iterator(result_w.begin()), 
249                              utility::data_iterator(result_w.end()),
250                              utility::data_iterator(result_w2.begin())));
251
252  suite.err() << "testing with weighted target" << std::endl;
253  std::vector<utility::DataWeight> target_w(target.size()+1, DataWeight(0, 1));
254  target_w[0] = DataWeight(5.3, 0);
255  std::copy(target.begin(), target.end(),
256            utility::data_iterator(target_w.begin()+1));
257  qQuantileNormalizer qQNw(target_w.begin(), target_w.end(), 4);
258  std::vector<utility::DataWeight> result_w3(src_w.size());
259  qQNw(src_w.begin(), src_w.end(), result_w3.begin());
260  // excluding missing value from comparison in suite.equal_range
261  MWi=result_w3.begin();
262  MWi+=5;
263  result_w3.erase(MWi);
264  suite.add(suite.equal_range(utility::data_iterator(result_w3.begin()), 
265                              utility::data_iterator(result_w3.end()),
266                              utility::data_iterator(result_w2.begin())));
267 
268}
269
270
271void test_quantile_normalize(test::Suite& suite)
272{
273  suite.err() << "Testing QuantileNormalizer\n";
274 
275  utility::Matrix m(2,2);
276  m(0,0) = 0;
277  m(0,1) = 10;
278  m(1,0) = 2;
279  m(1,1) = 4;
280  normalizer::QuantileNormalizer qn;
281  qn(m, m);
282  suite.err() << "Testing m(0,0)\n";
283  suite.add(suite.equal(m(0,0), 2));
284  suite.err() << "Testing m(0,1)\n";
285  suite.add(suite.equal(m(0,1), 6));
286  suite.err() << "Testing m(1,0)\n";
287  suite.add(suite.equal(m(1,0), 6));
288  suite.err() << "Testing m(1,1)\n";
289  suite.add(suite.equal(m(1,1), 2));
290}
291
292void test_row_normalize(test::Suite& suite)
293{
294  using namespace normalizer;
295  suite.err() << "Testing RowNormalizer\n";
296 
297  utility::Matrix m(2,3);
298  m(0,0) = 0;
299  m(0,1) = 10;
300  m(1,0) = 2;
301  m(1,1) = 4;
302  utility::Matrix m2(m);
303  m2.transpose();
304  ColumnNormalizer<Centralizer<> > cn;
305  RowNormalizer<Centralizer<> > rn;
306  cn(m, m);
307  rn(m2, m2);
308  m2.transpose();
309  suite.equal_range(m.begin(), m.end(), m2.begin());
310  if (false) { // do not run compile tests
311    test::container2d_archetype<double> container2d; 
312    test::mutable_container2d_archetype<double> mutable_container2d; 
313    rn(container2d, mutable_container2d);
314  }
315}
316
317void test_spearman(test::Suite& suite)
318{
319  suite.err() << "Testing Spearman\n";
320  normalizer::Spearman spearman;
321  std::vector<double> vec;
322  vec.push_back(0);
323  vec.push_back(2);
324  vec.push_back(3);
325  vec.push_back(1);
326  spearman(vec.begin(), vec.end(), vec.begin());
327  std::vector<double> correct;
328  correct.push_back(1.0/8);
329  correct.push_back(5.0/8);
330  correct.push_back(7.0/8);
331  correct.push_back(3.0/8);
332  suite.add(suite.equal_range(vec.begin(), vec.end(), correct.begin()));
333  suite.err() << "Testing Spearman with ties\n";
334  vec[1]=vec[2];
335  correct[1] = correct[2] = (correct[1]+correct[2])/2;
336  spearman(vec.begin(), vec.end(), vec.begin());
337  suite.add(suite.equal_range(vec.begin(), vec.end(), correct.begin()));
338  test_spearman_weighted(suite);
339}
340
341
342void test_gauss_normalize(test::Suite& suite)
343{
344  suite.err() << "Testing Gauss\n";
345  normalizer::Gauss gauss;
346  std::vector<double> vec;
347  vec.push_back(1);
348  gauss(vec.begin(), vec.end(), vec.begin());
349  suite.add(suite.equal(vec.front(), 0));
350  vec.push_back(1);
351  gauss(vec.begin(), vec.end(), vec.begin());
352  suite.add(suite.equal(vec.front(), -vec.back()));
353  // compile test should not be run
354  if (false) {
355    gauss(boost::random_access_iterator_archetype<double>(), 
356          boost::random_access_iterator_archetype<double>(),
357          boost::mutable_random_access_iterator_archetype<double>());
358    using utility::DataWeight;
359    gauss(boost::random_access_iterator_archetype<DataWeight>(), 
360          boost::random_access_iterator_archetype<DataWeight>(),
361          boost::mutable_random_access_iterator_archetype<DataWeight>());
362    gauss(boost::random_access_iterator_archetype<DataWeight>(), 
363          boost::random_access_iterator_archetype<DataWeight>(),
364          boost::mutable_random_access_iterator_archetype<double>());
365    gauss(boost::random_access_iterator_archetype<double>(), 
366          boost::random_access_iterator_archetype<double>(),
367          boost::mutable_random_access_iterator_archetype<DataWeight>());
368  }
369}
370
371void test_spearman_weighted(test::Suite& suite)
372{
373  suite.err() << "Testing Weighted Spearman\n";
374  normalizer::Spearman spearman;
375
376  suite.err() << "Testing that unity weights reproduces unweighted case\n";
377  utility::MatrixWeighted m(1,4,0,1);
378  utility::MatrixWeighted res(m.rows(), m.columns(),3.14,0);
379  m(0,0).data()=0;
380  m(0,1).data()=2;
381  m(0,2).data()=3;
382  m(0,3).data()=1;
383  std::vector<double> correct(m.columns());
384  std::vector<double> correct_w(m.columns(), 1.0);
385  std::copy(utility::data_iterator(m.begin_row(0)),
386            utility::data_iterator(m.end_row(0)),
387            correct.begin());
388  spearman(correct.begin(), correct.end(), correct.begin());
389  spearman(m.begin_row(0), m.end_row(0), res.begin_row(0));
390
391  using utility::data_iterator;
392  suite.add(suite.equal_range(data_iterator(res.begin_row(0)),
393                               data_iterator(res.end_row(0)),
394                               correct.begin()));
395  using utility::weight_iterator;
396  suite.add(suite.equal_range(weight_iterator(res.begin_row(0)),
397                               weight_iterator(res.end_row(0)),
398                               correct_w.begin()));
399
400  suite.err() << "Testing rescaling of weights\n";
401  for (size_t i=0; i<m.columns(); ++i) {
402    m(0,i).weight() *= 2;
403    correct_w[i] *= 2;
404  }   
405  spearman(m.begin_row(0), m.end_row(0), res.begin_row(0));
406  suite.add(suite.equal_range(data_iterator(res.begin_row(0)),
407                               data_iterator(res.end_row(0)),
408                               correct.begin()));
409  suite.add(suite.equal_range(weight_iterator(res.begin_row(0)),
410                               weight_iterator(res.end_row(0)),
411                               correct_w.begin()));
412
413 
414  suite.err() << "Testing case with a zero weight\n";
415  m(0,1).data() = std::numeric_limits<double>::quiet_NaN();
416  m(0,1).weight() = 0.0;
417  spearman(m.begin_row(0), m.end_row(0), res.begin_row(0));
418  suite.add(suite.equal(res(0,0).data(), 0.5/3)); 
419  suite.add(suite.equal(res(0,2).data(), 2.5/3)); 
420  suite.add(suite.equal(res(0,3).data(), 1.5/3)); 
421
422  suite.err() << "Testing case with ties\n";
423  m(0,0).data() = m(0,2).data();
424  spearman(m.begin_row(0), m.end_row(0), res.begin_row(0));
425  suite.add(suite.equal(res(0,0).data(), 2.0/3)); 
426  suite.add(suite.equal(res(0,2).data(), 2.0/3)); 
427  suite.add(suite.equal(res(0,3).data(), 0.5/3)); 
428  // compile test should not be run
429  if (false) {
430    spearman(boost::random_access_iterator_archetype<double>(), 
431             boost::random_access_iterator_archetype<double>(),
432             boost::mutable_random_access_iterator_archetype<double>());
433    using utility::DataWeight;
434    spearman(boost::random_access_iterator_archetype<DataWeight>(), 
435             boost::random_access_iterator_archetype<DataWeight>(),
436             boost::mutable_random_access_iterator_archetype<DataWeight>());
437    spearman(boost::random_access_iterator_archetype<double>(), 
438             boost::random_access_iterator_archetype<double>(),
439             boost::mutable_random_access_iterator_archetype<DataWeight>());
440    spearman(boost::random_access_iterator_archetype<DataWeight>(), 
441             boost::random_access_iterator_archetype<DataWeight>(),
442             boost::mutable_random_access_iterator_archetype<double>());
443  }
444
445}
446
447void test_z_score(test::Suite& suite)
448{
449  suite.err() << "Testing Zscore\n";
450  std::vector<double> vec;
451  vec.push_back(0);
452  vec.push_back(3.14);
453  normalizer::Zscore zscore;
454  zscore(vec.begin(), vec.end(), vec.begin());
455  for (size_t i=0; i<vec.size(); ++i)
456    suite.add(suite.equal(vec[i], 2.0*i-1.0));
457
458  std::vector<utility::DataWeight> vec2;
459  vec2.push_back(utility::DataWeight(1,1));
460  vec2.push_back(utility::DataWeight(2.13,0.5));
461  vec2.push_back(utility::DataWeight(2.13,0.5));
462  std::vector<utility::DataWeight> vec3(vec2.size());
463  zscore(vec2.begin(), vec2.end(), vec3.begin());
464  for (size_t i=0; i<vec2.size(); ++i)
465    suite.add(suite.equal(vec3[i].weight(), vec2[i].weight()));
466  suite.add(suite.equal(vec3[0].data(), -1.0));
467  suite.add(suite.equal(vec3[1].data(), 1.0));
468  suite.add(suite.equal(vec3[2].data(), 1.0));
469}
470
471
Note: See TracBrowser for help on using the repository browser.