[1003] | 1 | // $Id: KolmogorovSmirnov.cc 1612 2008-11-04 23:12:43Z peter $ |
---|
| 2 | |
---|
[1004] | 3 | /* |
---|
| 4 | Copyright (C) 2008 Peter Johansson |
---|
| 5 | |
---|
[1437] | 6 | This file is part of the yat library, http://dev.thep.lu.se/yat |
---|
[1004] | 7 | |
---|
| 8 | The yat library is free software; you can redistribute it and/or |
---|
| 9 | modify it under the terms of the GNU General Public License as |
---|
[1486] | 10 | published by the Free Software Foundation; either version 3 of the |
---|
[1004] | 11 | License, or (at your option) any later version. |
---|
| 12 | |
---|
| 13 | The yat library is distributed in the hope that it will be useful, |
---|
| 14 | but WITHOUT ANY WARRANTY; without even the implied warranty of |
---|
| 15 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
---|
| 16 | General Public License for more details. |
---|
| 17 | |
---|
| 18 | You should have received a copy of the GNU General Public License |
---|
[1487] | 19 | along with yat. If not, see <http://www.gnu.org/licenses/>. |
---|
[1004] | 20 | */ |
---|
| 21 | |
---|
[1003] | 22 | #include "KolmogorovSmirnov.h" |
---|
| 23 | #include "yat/random/random.h" |
---|
| 24 | #include "yat/utility/stl_utility.h" |
---|
| 25 | |
---|
| 26 | #include <algorithm> |
---|
| 27 | #include <cassert> |
---|
| 28 | #include <cmath> |
---|
| 29 | #include <deque> |
---|
| 30 | #include <functional> |
---|
| 31 | |
---|
| 32 | namespace theplu { |
---|
| 33 | namespace yat { |
---|
| 34 | namespace statistics { |
---|
| 35 | |
---|
| 36 | |
---|
| 37 | KolmogorovSmirnov::KolmogorovSmirnov(void) |
---|
| 38 | : cached_(true), score_(0.0), sum_w1_(0.0), sum_w2_(0.0) |
---|
| 39 | { |
---|
| 40 | } |
---|
| 41 | |
---|
| 42 | |
---|
| 43 | void KolmogorovSmirnov::add(double x, bool target, double w) |
---|
| 44 | { |
---|
| 45 | if (w==0) // ignoring zero weight data |
---|
| 46 | return; |
---|
| 47 | |
---|
| 48 | data_.insert(std::make_pair(std::make_pair(x,w),target)); |
---|
| 49 | if (target){ |
---|
| 50 | sum_w1_+=w; |
---|
| 51 | } |
---|
| 52 | else { |
---|
| 53 | sum_w2_+=w; |
---|
| 54 | } |
---|
| 55 | cached_=false; |
---|
| 56 | } |
---|
| 57 | |
---|
| 58 | |
---|
[1593] | 59 | double KolmogorovSmirnov::p_value(void) const |
---|
| 60 | { |
---|
| 61 | double res=0; |
---|
| 62 | double res2=0; |
---|
[1600] | 63 | double s2 = score() * score() * sum_w1_*sum_w2_/(sum_w1_+sum_w2_); |
---|
[1593] | 64 | int sign = 1; |
---|
| 65 | for (size_t k = 1; k<100; ++k) { |
---|
[1600] | 66 | res += sign * 2 * exp(-2.0 * k * k * s2); |
---|
[1593] | 67 | sign *= -1; |
---|
[1600] | 68 | // ignore remaining terms as they are small |
---|
[1593] | 69 | if (res==res2) |
---|
| 70 | return res; |
---|
| 71 | res2 = res; |
---|
| 72 | } |
---|
| 73 | |
---|
| 74 | return res; |
---|
| 75 | } |
---|
| 76 | |
---|
| 77 | |
---|
[1003] | 78 | double KolmogorovSmirnov::p_value(size_t perm) const |
---|
| 79 | { |
---|
| 80 | size_t count=0; |
---|
| 81 | std::deque<bool> targets; |
---|
| 82 | typedef data_w::const_iterator const_iter; |
---|
| 83 | for (const_iter i(data_.begin()); i!=data_.end(); ++i) |
---|
| 84 | targets.push_back(i->second); |
---|
| 85 | |
---|
| 86 | for (size_t i=0; i<perm; ++i){ |
---|
[1199] | 87 | random::random_shuffle(targets.begin(), targets.end()); |
---|
[1003] | 88 | KolmogorovSmirnov ks; |
---|
| 89 | std::deque<bool>::const_iterator target_i(targets.begin()); |
---|
| 90 | const_iter end(data_.end()); |
---|
| 91 | for (const_iter i(data_.begin()); i!=end; ++i, ++target_i) |
---|
| 92 | ks.add(i->first.first, *target_i, i->first.second); |
---|
| 93 | |
---|
| 94 | if (ks.score()>=score()) |
---|
| 95 | ++count; |
---|
| 96 | } |
---|
| 97 | return static_cast<double>(count)/perm; |
---|
| 98 | } |
---|
| 99 | |
---|
| 100 | |
---|
[1612] | 101 | void KolmogorovSmirnov::reset(void) |
---|
| 102 | { |
---|
| 103 | cached_=false; |
---|
| 104 | data_.clear(); |
---|
| 105 | sum_w1_=0; |
---|
| 106 | sum_w2_=0; |
---|
| 107 | } |
---|
| 108 | |
---|
| 109 | |
---|
[1003] | 110 | double KolmogorovSmirnov::score(void) const |
---|
| 111 | { |
---|
[1595] | 112 | return std::abs(signed_score()); |
---|
| 113 | } |
---|
| 114 | |
---|
| 115 | |
---|
| 116 | double KolmogorovSmirnov::signed_score(void) const |
---|
| 117 | { |
---|
[1003] | 118 | if (cached_) |
---|
| 119 | return score_; |
---|
| 120 | if (!sum_w1_ || !sum_w2_) |
---|
| 121 | return 0.0; |
---|
| 122 | score_=0; |
---|
| 123 | std::vector<double> v; |
---|
| 124 | scores(v); |
---|
| 125 | std::less<double> l; |
---|
| 126 | utility::abs<double> a; |
---|
| 127 | using utility::make_compose_f_gx_hy; |
---|
[1595] | 128 | score_ = *std::max_element(v.begin(), v.end(), |
---|
| 129 | make_compose_f_gx_hy(l, a, a)); |
---|
| 130 | cached_=true; |
---|
[1003] | 131 | return score_; |
---|
| 132 | } |
---|
| 133 | |
---|
| 134 | |
---|
| 135 | void KolmogorovSmirnov::scores(std::vector<double>& res) const |
---|
| 136 | { |
---|
| 137 | res.clear(); |
---|
| 138 | res.reserve(data_.size()); |
---|
| 139 | data_w::const_iterator iter(data_.begin()); |
---|
| 140 | double f1=0; |
---|
| 141 | double f2=0; |
---|
| 142 | // Peter, we should take care of ties! |
---|
| 143 | while(iter!=data_.end()){ |
---|
| 144 | if (iter->second) |
---|
| 145 | f1 += iter->first.second; |
---|
| 146 | else |
---|
| 147 | f2 += iter->first.second; |
---|
| 148 | res.push_back(f1/sum_w1_-f2/sum_w2_); |
---|
[1198] | 149 | ++iter; |
---|
[1003] | 150 | } |
---|
| 151 | } |
---|
| 152 | |
---|
| 153 | |
---|
| 154 | std::ostream& operator<<(std::ostream& os, const KolmogorovSmirnov& ks) |
---|
| 155 | { |
---|
| 156 | std::vector<double> s; |
---|
| 157 | ks.scores(s); |
---|
| 158 | for (size_t i=0; i<s.size(); ++i) |
---|
| 159 | os << i << "\t" << s[i] << "\n"; |
---|
| 160 | return os; |
---|
| 161 | } |
---|
| 162 | |
---|
| 163 | |
---|
| 164 | }}} // of namespace theplu yat statistics |
---|