source: trunk/test/crossvalidation_test.cc @ 704

// $Id: crossvalidation_test.cc 704 2006-12-18 16:01:41Z markus $

/*
  Copyright (C) The authors contributing to this file.

  This file is part of the yat library, http://lev.thep.lu.se/trac/yat

  The yat library is free software; you can redistribute it and/or
  modify it under the terms of the GNU General Public License as
  published by the Free Software Foundation; either version 2 of the
  License, or (at your option) any later version.

  The yat library is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  General Public License for more details.

  You should have received a copy of the GNU General Public License
  along with this program; if not, write to the Free Software
  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
  02111-1307, USA.
*/

#include "yat/classifier/CrossValidationSampler.h"
#include "yat/classifier/SubsetGenerator.h"
#include "yat/classifier/MatrixLookup.h"
#include "yat/classifier/Target.h"
#include "yat/utility/matrix.h"

#include <cstdlib>
#include <fstream>
#include <iostream>
#include <string>
#include <vector>

// forward declaration
void class_count_test(const std::vector<size_t>&, std::ostream*, bool&);
void sample_count_test(const std::vector<size_t>&, std::ostream*, bool&);


int main(const int argc,const char* argv[])
{  
  using namespace theplu::yat;
  
  std::ostream* error;
  if (argc>1 && argv[1]==std::string("-v"))
    error = &std::cerr;
  else {
    error = new std::ofstream("/dev/null");
    if (argc>1)
      std::cout << "crossvalidation_test -v : for printing extra information\n";
  }
  *error << "testing crosssplitter" << std::endl;
  bool ok = true;

  std::vector<std::string> label(10,"default");
  label[2]=label[7]="white";
  label[4]=label[5]="black";
  label[6]=label[3]="green";
  label[8]=label[9]="red";
                 
  classifier::Target target(label);
  utility::matrix raw_data(10,10);
  classifier::MatrixLookup data(raw_data);
  classifier::CrossValidationSampler cv(target,3,3);
  
  std::vector<size_t> sample_count(10,0);
  for (size_t j=0; j<cv.size(); ++j){
    std::vector<size_t> class_count(5,0);
    assert(j<cv.size());
    if (cv.training_index(j).size()+cv.validation_index(j).size()!=
        target.size()){
      ok = false;
      *error << "ERROR: size of training samples plus " 
             << "size of validation samples is invalid." << std::endl;
    }
    if (cv.validation_index(j).size()!=3 && cv.validation_index(j).size()!=4){
      ok = false;
      *error << "ERROR: size of validation samples is invalid." 
             << "expected size to be 3 or 4" << std::endl;
    }
    for (size_t i=0; i<cv.validation_index(j).size(); i++) {
      assert(cv.validation_index(j)[i]<sample_count.size());
      sample_count[cv.validation_index(j)[i]]++;
    }
    for (size_t i=0; i<cv.training_index(j).size(); i++) {
      class_count[target(cv.training_index(j)[i])]++;
    }
    class_count_test(class_count,error,ok);
  }
  sample_count_test(sample_count,error,ok);
  
  //
  // Test two nested CrossSplitters 
  //

  *error << "\ntesting two nested crossplitters" << std::endl;
  label.resize(9);
  label[0]=label[1]=label[2]="0";
  label[3]=label[4]=label[5]="1";
  label[6]=label[7]=label[8]="2";
                 
  target=classifier::Target(label);
  utility::matrix raw_data2(2,9);
  for(size_t i=0;i<raw_data2.rows();i++)
    for(size_t j=0;j<raw_data2.columns();j++)
      raw_data2(i,j)=i*10+10+j+1;
    
  classifier::MatrixLookup data2(raw_data2);
  classifier::CrossValidationSampler cv2(target,3,3);
  classifier::SubsetGenerator cv_test(cv2,data2);

  std::vector<size_t> test_sample_count(9,0);
  std::vector<size_t> test_class_count(3,0);
  std::vector<double> test_value1(4,0);
  std::vector<double> test_value2(4,0);
  std::vector<double> t_value(4,0);
  std::vector<double> v_value(4,0); 
  for(u_long k=0;k<cv_test.size();k++) {
    
    const classifier::DataLookup2D& tv_view=cv_test.training_data(k);
    const classifier::Target& tv_target=cv_test.training_target(k);
    const std::vector<size_t>& tv_index=cv_test.training_index(k);
    const classifier::DataLookup2D& test_view=cv_test.validation_data(k);
    const classifier::Target& test_target=cv_test.validation_target(k);
    const std::vector<size_t>& test_index=cv_test.validation_index(k);

    for (size_t i=0; i<test_index.size(); i++) {
      assert(test_index[i]<sample_count.size());
      test_sample_count[test_index[i]]++;
      test_class_count[target(test_index[i])]++;
      test_value1[0]+=test_view(0,i);
      test_value2[0]+=test_view(1,i);
      test_value1[test_target(i)+1]+=test_view(0,i);
      test_value2[test_target(i)+1]+=test_view(1,i);
      if(test_target(i)!=target(test_index[i])) {
        ok=false;
        *error << "ERROR: incorrect mapping of test indices" << std:: endl;
      }        
    }
    
    classifier::CrossValidationSampler sampler_training(tv_target,2,2);
    classifier::SubsetGenerator cv_training(sampler_training,tv_view);
    std::vector<size_t> v_sample_count(6,0);
    std::vector<size_t> t_sample_count(6,0);
    std::vector<size_t> v_class_count(3,0);
    std::vector<size_t> t_class_count(3,0);
    std::vector<size_t> t_class_count2(3,0);
    for(u_long l=0;l<cv_training.size();l++) {
      const classifier::DataLookup2D& t_view=cv_training.training_data(l);
      const classifier::Target& t_target=cv_training.training_target(l);
      const std::vector<size_t>& t_index=cv_training.training_index(l);
      const classifier::DataLookup2D& v_view=cv_training.validation_data(l);
      const classifier::Target& v_target=cv_training.validation_target(l);
      const std::vector<size_t>& v_index=cv_training.validation_index(l);
      
      if (test_index.size()+tv_index.size()!=target.size() 
          || t_index.size()+v_index.size() != tv_target.size() 
          || test_index.size()+v_index.size()+t_index.size() !=  target.size()){
        ok = false;
        *error << "ERROR: size of training samples, validation samples " 
               << "and test samples in is invalid." 
               << std::endl;
      }
      if (test_index.size()!=3 || tv_index.size()!=6 || t_index.size()!=3 ||
          v_index.size()!=3){
        ok = false;
        *error << "ERROR: size of training, validation, and test samples"
               << " is invalid." 
               << " Expected sizes to be 3" << std::endl;
      }     

      std::vector<size_t> tv_sample_count(6,0);
      for (size_t i=0; i<t_index.size(); i++) {
        assert(t_index[i]<t_sample_count.size());
        tv_sample_count[t_index[i]]++;
        t_sample_count[t_index[i]]++;
        t_class_count[t_target(i)]++;
        t_class_count2[tv_target(t_index[i])]++;
        t_value[0]+=t_view(0,i);
        t_value[t_target(i)+1]+=t_view(0,i);        
      }
      for (size_t i=0; i<v_index.size(); i++) {
        assert(v_index[i]<v_sample_count.size());
        tv_sample_count[v_index[i]]++;
        v_sample_count[v_index[i]]++;
        v_class_count[v_target(i)]++;
        v_value[0]+=v_view(0,i);
        v_value[v_target(i)+1]+=v_view(0,i);
      }
  
      sample_count_test(tv_sample_count,error,ok);      

    }
    sample_count_test(v_sample_count,error,ok);
    sample_count_test(t_sample_count,error,ok);
    
    class_count_test(t_class_count,error,ok);
    class_count_test(t_class_count2,error,ok);
    class_count_test(v_class_count,error,ok);


  }
  sample_count_test(test_sample_count,error,ok);
  class_count_test(test_class_count,error,ok);
  
  if(test_value1[0]!=135 || test_value1[1]!=36 || test_value1[2]!=45 ||
     test_value1[3]!=54) {
    ok=false;
    *error << "ERROR: incorrect sums of test values in row 1" 
           << " found: " << test_value1[0] << ", "  << test_value1[1] 
           << ", "  << test_value1[2] << " and "  << test_value1[3] 
           << std::endl;
  }

  
  if(test_value2[0]!=225 || test_value2[1]!=66 || test_value2[2]!=75 ||
     test_value2[3]!=84) {
    ok=false;
    *error << "ERROR: incorrect sums of test values in row 2" 
           << " found: " << test_value2[0] << ", "  << test_value2[1] 
           << ", "  << test_value2[2] << " and "  << test_value2[3] 
           << std::endl;
  }

  if(t_value[0]!=270 || t_value[1]!=72 || t_value[2]!=90 || t_value[3]!=108)  {
    ok=false;
    *error << "ERROR: incorrect sums of training values in row 1" 
           << " found: " << t_value[0] << ", "  << t_value[1] 
           << ", "  << t_value[2] << " and "  << t_value[3] 
           << std::endl;    
  }

  if(v_value[0]!=270 || v_value[1]!=72 || v_value[2]!=90 || v_value[3]!=108)  {
    ok=false;
    *error << "ERROR: incorrect sums of validation values in row 1" 
           << " found: " << v_value[0] << ", "  << v_value[1] 
           << ", "  << v_value[2] << " and "  << v_value[3] 
           << std::endl;    
  }



  if (error!=&std::cerr)
    delete error;
  
  if (ok)
    return 0;
  return -1;
}


void class_count_test(const std::vector<size_t>& class_count, 
                      std::ostream* error, bool& ok)  
{
  for (size_t i=0; i<class_count.size(); i++)
    if (class_count[i]==0){
      ok = false;
      *error << "ERROR: class " << i << " was not in set." 
             << " Expected at least one sample from each class." 
             << std::endl;
    }
}

void sample_count_test(const std::vector<size_t>& sample_count, 
                       std::ostream* error, bool& ok)  
{
  for (size_t i=0; i<sample_count.size(); i++){
    if (sample_count[i]!=1){
      ok = false;
      *error << "ERROR: sample " << i << " was in a group " << sample_count[i] 
             << " times." << " Expected to be 1 time" << std::endl;
    }
  }
}