source: trunk/test/roc.cc @ 2601

Last change on this file since 2601 was 2601, checked in by Peter, 11 years ago

merge release 0.7.3 into trunk. A couple of conflict and needed fix test related to #669

  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 5.9 KB
Line 
1// $Id: roc.cc 2601 2011-10-30 05:08:14Z peter $
2
3/*
4  Copyright (C) 2007, 2008 Jari Häkkinen, Peter Johansson
5  Copyright (C) 2011 Peter Johansson
6
7  This file is part of the yat library, http://dev.thep.lu.se/yat
8
9  The yat library is free software; you can redistribute it and/or
10  modify it under the terms of the GNU General Public License as
11  published by the Free Software Foundation; either version 3 of the
12  License, or (at your option) any later version.
13
14  The yat library is distributed in the hope that it will be useful,
15  but WITHOUT ANY WARRANTY; without even the implied warranty of
16  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17  General Public License for more details.
18
19  You should have received a copy of the GNU General Public License
20  along with yat. If not, see <http://www.gnu.org/licenses/>.
21*/
22
23#include "Suite.h"
24
25#include "yat/classifier/DataLookupWeighted1D.h"
26#include "yat/classifier/Target.h"
27#include "yat/statistics/Fisher.h"
28#include "yat/statistics/ROC.h"
29#include "yat/statistics/utility.h"
30#include "yat/utility/Vector.h"
31
32#include <cassert>
33#include <cmath>
34#include <fstream>
35#include <iostream>
36
37
38using namespace theplu::yat;
39
40void test_empty(test::Suite&);
41void test_ties(test::Suite& suite);
42void test_p_exact(test::Suite& suite);
43void test_p_approx(test::Suite& suite);
44void test_p_exact_with_ties(test::Suite& suite);
45void test_p_approx_with_ties(test::Suite& suite);
46
47int main(int argc, char* argv[])
48{ 
49  test::Suite suite(argc, argv);
50
51  suite.err() << "testing ROC" << std::endl;
52  utility::Vector value(31);
53  std::vector<std::string> label(31,"negative");
54  for (size_t i=0; i<16; i++) 
55    label[i] = "positive";
56  classifier::Target target(label);
57  for (size_t i=0; i<value.size(); i++) 
58    value(i)=i;
59  statistics::ROC roc;
60  add(roc, value.begin(), value.end(), target);
61  double area = roc.area();
62  if (!suite.equal(area,0.0)){
63    suite.err() << "test_roc: area is " << area << " should be 0.0" 
64           << std::endl;
65    suite.add(false);
66  }
67  target.set_binary(0,false);
68  target.set_binary(1,true);
69  roc.reset();
70  add(roc, value.begin(), value.end(), target);
71  area = roc.area();
72  if (!suite.equal(area,1.0)){
73    suite.err() << "test_roc: area is " << area << " should be 1.0" 
74           << std::endl;
75    suite.add(false);
76  }
77 
78  double p = roc.p_value_one_sided();
79  double p2 = roc.p_value();
80  double p_matlab = 0.00000115;
81  if (!(p/p_matlab < 1.01 && p/p_matlab > 0.99)){
82    suite.err() << "get_p_approx: p-value not correct" << std::endl;
83    suite.err() << p << " expected " << p_matlab << std::endl;
84    suite.add(false);
85  }
86  if (!(p2==2*p)) {
87    suite.add(false);
88    suite.err() << "Two-sided P-value should equal 2 * one-sided P-value.\n";
89  }
90  roc.minimum_size() = 20;
91  p = roc.p_value_one_sided();
92  p2 = roc.p_value();
93  if (!( p < 1e-8 && p > 1e-9) ){
94    suite.err() << "get_p_exact: p-value not correct" << std::endl;
95    suite.add(false);
96  }
97  if (!( p2==2*p)) {
98    suite.add(false);
99    suite.err() << "Two-sided P-value should equal 2 * one-sided P-value.\n";
100  }
101 
102  classifier::DataLookupWeighted1D dlw(target.size(),1.3);
103  add(roc, dlw.begin(), dlw.end(), target);
104  test_ties(suite);
105  test_p_approx_with_ties(suite);
106  test_p_exact_with_ties(suite);
107  test_p_approx(suite);
108  test_p_exact(suite);
109  test_empty(suite);
110  return suite.return_value();
111}
112
113
114void test_p_exact_with_ties(test::Suite& suite)
115{
116  suite.out() << "test p exact with ties\n";
117  statistics::ROC roc;
118  /*
119    +++-- 6
120    ++-+- 5 4.5 *** our case ***
121    +-++- 4 4.5
122    ++--+ 4 3.5
123    +-+-+ 3 3.5
124    +--++ 2 2
125    -+++- 3 3
126    -++-+ 2 2
127    -+-++ 1 0.5 *** our second case ***
128    --+++ 0 0.5
129   */
130  roc.add(2, true);
131  roc.add(1, true);
132  roc.add(1, false);
133  roc.add(0, true);
134  roc.add(-1, false);
135  roc.area();
136  if (!suite.equal(roc.p_value_one_sided(), 3.0/10.0)) {
137    suite.add(false);
138    suite.out() << "  p_value_one_sided: expected 0.3\n";
139  }
140  else
141    suite.add(true);
142  if (!suite.equal(roc.p_value(), 5.0/10.0)) {
143    suite.add(false);
144    suite.out() << "  (two-sided) p_value: expected 0.5\n";
145  }
146  else
147    suite.add(true);
148
149  suite.out() << "test p exact with ties II\n";
150  roc.reset();
151  roc.add(2, false);
152  roc.add(1, true);
153  roc.add(1, false);
154  roc.add(0, true);
155  roc.add(-1, true);
156  suite.add(suite.equal(roc.area(), 0.5/6));
157  if (!suite.add(suite.equal(roc.p_value_one_sided(), 10.0/10.0)))
158    suite.out() << "  p_value_one_sided: expected 0.3\n";
159  if (!suite.add(suite.equal(roc.p_value(), 3.0/10.0)))
160    suite.out() << "  (two-sided) p_value: expected 0.5\n";
161}
162
163
164void test_p_approx_with_ties(test::Suite& suite)
165{
166  suite.out() << "test p approx with ties\n";
167  statistics::ROC roc;
168  for (size_t i=0; i<100; ++i) {
169    roc.add(1, i<60);
170    roc.add(0, i<40);
171  }
172  suite.add(suite.equal(roc.area(), 0.6));
173  // Having only two data values, 0 and 1, data can be represented as
174  // a 2x2 contigency table, and ROC test is same as Fisher's exact
175  // test.
176  statistics::Fisher fisher;
177  fisher.oddsratio(60, 40, 40, 60);
178  suite.add(suite.equal_fix(roc.p_value(), fisher.p_value(), 0.0002));
179}
180
181void test_ties(test::Suite& suite)
182{
183  suite.out() << "test ties\n";
184  statistics::ROC roc;
185  for (size_t i=0; i<20; ++i)
186    roc.add(10.0, i<10);
187  if (!suite.add(suite.equal(roc.area(), 0.5))) {
188    suite.err() << "error: roc with ties: area: " << roc.area() << "\n";
189  }
190}
191
192void test_p_exact(test::Suite& suite)
193{
194  suite.out() << "test_p_exact\n";
195  statistics::ROC roc;
196  for (size_t i=0; i<9; ++i)
197    roc.add(i, i<5);
198  if (roc.p_value_one_sided()<0.5) {
199    suite.add(false);
200    suite.err() << "error: expected p-value>0.5\n  found: " 
201                << roc.p_value_one_sided() << "\n";
202  }
203}
204
205
206void test_p_approx(test::Suite& suite)
207{
208  suite.out() << "test_p_approx\n";
209  statistics::ROC roc;
210  for (size_t i=0; i<100; ++i)
211    roc.add(i, i<50);
212  if (roc.p_value_one_sided()<0.5) {
213    suite.add(false);
214    suite.err() << "error: expected p-value>0.5\n  found: " 
215                << roc.p_value_one_sided() << "\n";
216  }
217}
218
219
220void test_empty(test::Suite& suite)
221{
222  suite.err() << "test empty\n";
223  // testing buf #669
224  statistics::ROC roc;
225  roc.p_value();
226  roc.area();
227  suite.err() << "test empty done\n";
228}
Note: See TracBrowser for help on using the repository browser.