Changeset 824 for trunk/test

Timestamp:

Mar 19, 2007, 1:12:04 PM (16 years ago)

Author:

Peter

Message:

Moved Ensemble stuff in subset_egnerator_test to ensemble_test. Also merged crossvalidation and subsetgenerator to one test, and added boostrap_test into this test. Refs #175

Location:

trunk/test

Files:

• Makefile.am (modified) (2 diffs)
• crossvalidation_test.cc (modified) (2 diffs)
• ensemble_test.cc (modified) (2 diffs)
• subset_generator_test.cc (modified) (4 diffs)

trunk/test/Makefile.am

@@ -27 +27 @@
 
 TESTS = alignment_test averager_test                  \
-  consensus_inputranker_test crossvalidation_test data_lookup_1d_test \
+  consensus_inputranker_test data_lookup_1d_test  \
   distance_test \
   ensemble_test feature_selection_test fileutil_test inputranker_test \
@@ -49 +49 @@
 averager_test_SOURCES = averager_test.cc
 consensus_inputranker_test_SOURCES = consensus_inputranker_test.cc
-crossvalidation_test_SOURCES = crossvalidation_test.cc
 data_lookup_1d_test_SOURCES = data_lookup_1d_test.cc
 distance_test_SOURCES = distance_test.cc

trunk/test/crossvalidation_test.cc

@@ -23 +23 @@
 
 #include "yat/classifier/CrossValidationSampler.h"
-#include "yat/classifier/SubsetGenerator.h"
 #include "yat/classifier/MatrixLookup.h"
 #include "yat/classifier/Target.h"
@@ -54 +53 @@
   *error << "testing crosssplitter" << std::endl;
   bool ok = true;
-
-  std::vector<std::string> label(10,"default");
-  label[2]=label[7]="white";
-  label[4]=label[5]="black";
-  label[6]=label[3]="green";
-  label[8]=label[9]="red";
-                 
-  classifier::Target target(label);
-  utility::matrix raw_data(10,10);
-  classifier::MatrixLookup data(raw_data);
-  classifier::CrossValidationSampler cv(target,3,3);
-  
-  std::vector<size_t> sample_count(10,0);
-  for (size_t j=0; j<cv.size(); ++j){
-    std::vector<size_t> class_count(5,0);
-    assert(j<cv.size());
-    if (cv.training_index(j).size()+cv.validation_index(j).size()!=
-        target.size()){
-      ok = false;
-      *error << "ERROR: size of training samples plus " 
-             << "size of validation samples is invalid." << std::endl;
-    }
-    if (cv.validation_index(j).size()!=3 && cv.validation_index(j).size()!=4){
-      ok = false;
-      *error << "ERROR: size of validation samples is invalid." 
-             << "expected size to be 3 or 4" << std::endl;
-    }
-    for (size_t i=0; i<cv.validation_index(j).size(); i++) {
-      assert(cv.validation_index(j)[i]<sample_count.size());
-      sample_count[cv.validation_index(j)[i]]++;
-    }
-    for (size_t i=0; i<cv.training_index(j).size(); i++) {
-      class_count[target(cv.training_index(j)[i])]++;
-    }
-    class_count_test(class_count,error,ok);
-  }
-  sample_count_test(sample_count,error,ok);
-  
-  //
-  // Test two nested CrossSplitters 
-  //
-
-  *error << "\ntesting two nested crossplitters" << std::endl;
-  label.resize(9);
-  label[0]=label[1]=label[2]="0";
-  label[3]=label[4]=label[5]="1";
-  label[6]=label[7]=label[8]="2";
-                 
-  target=classifier::Target(label);
-  utility::matrix raw_data2(2,9);
-  for(size_t i=0;i<raw_data2.rows();i++)
-    for(size_t j=0;j<raw_data2.columns();j++)
-      raw_data2(i,j)=i*10+10+j+1;
-    
-  classifier::MatrixLookup data2(raw_data2);
-  classifier::CrossValidationSampler cv2(target,3,3);
-  classifier::SubsetGenerator cv_test(cv2,data2);
-
-  std::vector<size_t> test_sample_count(9,0);
-  std::vector<size_t> test_class_count(3,0);
-  std::vector<double> test_value1(4,0);
-  std::vector<double> test_value2(4,0);
-  std::vector<double> t_value(4,0);
-  std::vector<double> v_value(4,0); 
-  for(u_long k=0;k<cv_test.size();k++) {
-    
-    const classifier::DataLookup2D& tv_view=cv_test.training_data(k);
-    const classifier::Target& tv_target=cv_test.training_target(k);
-    const std::vector<size_t>& tv_index=cv_test.training_index(k);
-    const classifier::DataLookup2D& test_view=cv_test.validation_data(k);
-    const classifier::Target& test_target=cv_test.validation_target(k);
-    const std::vector<size_t>& test_index=cv_test.validation_index(k);
-
-    for (size_t i=0; i<test_index.size(); i++) {
-      assert(test_index[i]<sample_count.size());
-      test_sample_count[test_index[i]]++;
-      test_class_count[target(test_index[i])]++;
-      test_value1[0]+=test_view(0,i);
-      test_value2[0]+=test_view(1,i);
-      test_value1[test_target(i)+1]+=test_view(0,i);
-      test_value2[test_target(i)+1]+=test_view(1,i);
-      if(test_target(i)!=target(test_index[i])) {
-        ok=false;
-        *error << "ERROR: incorrect mapping of test indices" << std:: endl;
-      }        
-    }
-    
-    classifier::CrossValidationSampler sampler_training(tv_target,2,2);
-    classifier::SubsetGenerator cv_training(sampler_training,tv_view);
-    std::vector<size_t> v_sample_count(6,0);
-    std::vector<size_t> t_sample_count(6,0);
-    std::vector<size_t> v_class_count(3,0);
-    std::vector<size_t> t_class_count(3,0);
-    std::vector<size_t> t_class_count2(3,0);
-    for(u_long l=0;l<cv_training.size();l++) {
-      const classifier::DataLookup2D& t_view=cv_training.training_data(l);
-      const classifier::Target& t_target=cv_training.training_target(l);
-      const std::vector<size_t>& t_index=cv_training.training_index(l);
-      const classifier::DataLookup2D& v_view=cv_training.validation_data(l);
-      const classifier::Target& v_target=cv_training.validation_target(l);
-      const std::vector<size_t>& v_index=cv_training.validation_index(l);
-      
-      if (test_index.size()+tv_index.size()!=target.size() 
-          || t_index.size()+v_index.size() != tv_target.size() 
-          || test_index.size()+v_index.size()+t_index.size() !=  target.size()){
-        ok = false;
-        *error << "ERROR: size of training samples, validation samples " 
-               << "and test samples in is invalid." 
-               << std::endl;
-      }
-      if (test_index.size()!=3 || tv_index.size()!=6 || t_index.size()!=3 ||
-          v_index.size()!=3){
-        ok = false;
-        *error << "ERROR: size of training, validation, and test samples"
-               << " is invalid." 
-               << " Expected sizes to be 3" << std::endl;
-      }     
-
-      std::vector<size_t> tv_sample_count(6,0);
-      for (size_t i=0; i<t_index.size(); i++) {
-        assert(t_index[i]<t_sample_count.size());
-        tv_sample_count[t_index[i]]++;
-        t_sample_count[t_index[i]]++;
-        t_class_count[t_target(i)]++;
-        t_class_count2[tv_target(t_index[i])]++;
-        t_value[0]+=t_view(0,i);
-        t_value[t_target(i)+1]+=t_view(0,i);        
-      }
-      for (size_t i=0; i<v_index.size(); i++) {
-        assert(v_index[i]<v_sample_count.size());
-        tv_sample_count[v_index[i]]++;
-        v_sample_count[v_index[i]]++;
-        v_class_count[v_target(i)]++;
-        v_value[0]+=v_view(0,i);
-        v_value[v_target(i)+1]+=v_view(0,i);
-      }
-  
-      sample_count_test(tv_sample_count,error,ok);      
-
-    }
-    sample_count_test(v_sample_count,error,ok);
-    sample_count_test(t_sample_count,error,ok);
-    
-    class_count_test(t_class_count,error,ok);
-    class_count_test(t_class_count2,error,ok);
-    class_count_test(v_class_count,error,ok);
-
-
-  }
-  sample_count_test(test_sample_count,error,ok);
-  class_count_test(test_class_count,error,ok);
-  
-  if(test_value1[0]!=135 || test_value1[1]!=36 || test_value1[2]!=45 ||
-     test_value1[3]!=54) {
-    ok=false;
-    *error << "ERROR: incorrect sums of test values in row 1" 
-           << " found: " << test_value1[0] << ", "  << test_value1[1] 
-           << ", "  << test_value1[2] << " and "  << test_value1[3] 
-           << std::endl;
-  }
-
-  
-  if(test_value2[0]!=225 || test_value2[1]!=66 || test_value2[2]!=75 ||
-     test_value2[3]!=84) {
-    ok=false;
-    *error << "ERROR: incorrect sums of test values in row 2" 
-           << " found: " << test_value2[0] << ", "  << test_value2[1] 
-           << ", "  << test_value2[2] << " and "  << test_value2[3] 
-           << std::endl;
-  }
-
-  if(t_value[0]!=270 || t_value[1]!=72 || t_value[2]!=90 || t_value[3]!=108)  {
-    ok=false;
-    *error << "ERROR: incorrect sums of training values in row 1" 
-           << " found: " << t_value[0] << ", "  << t_value[1] 
-           << ", "  << t_value[2] << " and "  << t_value[3] 
-           << std::endl;    
-  }
-
-  if(v_value[0]!=270 || v_value[1]!=72 || v_value[2]!=90 || v_value[3]!=108)  {
-    ok=false;
-    *error << "ERROR: incorrect sums of validation values in row 1" 
-           << " found: " << v_value[0] << ", "  << v_value[1] 
-           << ", "  << v_value[2] << " and "  << v_value[3] 
-           << std::endl;    
-  }
-
-
 
   if (error!=&std::cerr)

trunk/test/ensemble_test.cc

@@ -35 +35 @@
 #include "yat/classifier/PolynomialKernelFunction.h"
 #include "yat/classifier/SVM.h"
+#include "yat/statistics/AUC.h"
 
 #include <cassert>
@@ -88 +89 @@
   ensemble.build();
   
+  utility::vector out(target.size(),0);
+  for (size_t i = 0; i<out.size(); ++i)
+    out(i)=ensemble.validate()[0][i].mean(); 
+  statistics::AUC roc;
+  *error << roc.score(target,out) << std::endl;
+
   delete kf;
 

trunk/test/subset_generator_test.cc

@@ -22 +22 @@
 */
 
+#include "yat/classifier/BootstrapSampler.h"
 #include "yat/classifier/CrossValidationSampler.h"
-#include "yat/classifier/EnsembleBuilder.h"
 #include "yat/classifier/FeatureSelectorIR.h"
 #include "yat/classifier/Kernel_SEV.h"
@@ -43 +43 @@
 using namespace theplu::yat;
 
+bool class_count_test(const std::vector<size_t>&, std::ostream*);
+bool sample_count_test(const std::vector<size_t>&, std::ostream*);
+bool test_nested(std::ostream* error);
+bool test_cv(std::ostream*);
+bool test_creation(std::ostream* error);
+bool test_bootstrap(std::ostream* error);
+
+
 int main(const int argc,const char* argv[])
 {  
@@ -51 +59 @@
     error = new std::ofstream("/dev/null");
     if (argc>1)
-      std::cout << "feature_selection -v : for printing extra information\n";
-  }
-  *error << "testing ferature_selection" << std::endl;
+      std::cout << "subset_generator -v : for printing extra information\n";
+  }
+  *error << "testing subset_generator" << std::endl;
   bool ok = true;
 
-
+  ok = ok && test_creation(error);
+  ok = ok && test_nested(error);
+  ok = ok && test_cv(error);
+
+  if (ok)
+    return 0;
+  return -1;
+}
+
+
+bool test_creation(std::ostream* error)
+{
+  bool ok=true;
   std::ifstream is("data/nm_target_bin.txt");
   *error << "loading target " << std::endl;
@@ -84 +104 @@
   classifier::SubsetGenerator subset_data(sampler, data, fs);
   classifier::SubsetGenerator subset_kernel(sampler, kernel, fs);
-
-  classifier::SVM svm(kernel,target);
-  statistics::PearsonDistance distance;
-  classifier::NCC ncc(data,target,distance);
-  *error << "building Ensemble" << std::endl;
-  //  classifier::EnsembleBuilder ensemble_ncc(ncc,subset_data);
-  //ensemble_ncc.build();
-  classifier::EnsembleBuilder ensemble_svm(svm,sampler);
-  ensemble_svm.build();
-  
-  utility::vector out(target.size(),0);
-  for (size_t i = 0; i<out.size(); ++i)
-    out(i)=ensemble_svm.validate()[0][i].mean(); 
-  statistics::AUC roc;
-  *error << roc.score(target,out) << std::endl;
-
-  if (ok)
-    return 0;
-  return -1;
-}
+  return ok;
+}
+
+bool test_nested(std::ostream* error)
+{
+  bool ok=true;
+  //
+  // Test two nested CrossSplitters 
+  //
+
+  *error << "\ntesting two nested crossplitters" << std::endl;
+  std::vector<std::string> label(9);
+  label[0]=label[1]=label[2]="0";
+  label[3]=label[4]=label[5]="1";
+  label[6]=label[7]=label[8]="2";
+                 
+  classifier::Target target(label);
+  utility::matrix raw_data2(2,9);
+  for(size_t i=0;i<raw_data2.rows();i++)
+    for(size_t j=0;j<raw_data2.columns();j++)
+      raw_data2(i,j)=i*10+10+j+1;
+    
+  classifier::MatrixLookup data2(raw_data2);
+  classifier::CrossValidationSampler cv2(target,3,3);
+  classifier::SubsetGenerator cv_test(cv2,data2);
+
+  std::vector<size_t> sample_count(10,0);
+  std::vector<size_t> test_sample_count(9,0);
+  std::vector<size_t> test_class_count(3,0);
+  std::vector<double> test_value1(4,0);
+  std::vector<double> test_value2(4,0);
+  std::vector<double> t_value(4,0);
+  std::vector<double> v_value(4,0); 
+  for(u_long k=0;k<cv_test.size();k++) {
+    
+    const classifier::DataLookup2D& tv_view=cv_test.training_data(k);
+    const classifier::Target& tv_target=cv_test.training_target(k);
+    const std::vector<size_t>& tv_index=cv_test.training_index(k);
+    const classifier::DataLookup2D& test_view=cv_test.validation_data(k);
+    const classifier::Target& test_target=cv_test.validation_target(k);
+    const std::vector<size_t>& test_index=cv_test.validation_index(k);
+
+    for (size_t i=0; i<test_index.size(); i++) {
+      assert(test_index[i]<sample_count.size());
+      test_sample_count[test_index[i]]++;
+      test_class_count[target(test_index[i])]++;
+      test_value1[0]+=test_view(0,i);
+      test_value2[0]+=test_view(1,i);
+      test_value1[test_target(i)+1]+=test_view(0,i);
+      test_value2[test_target(i)+1]+=test_view(1,i);
+      if(test_target(i)!=target(test_index[i])) {
+        ok=false;
+        *error << "ERROR: incorrect mapping of test indices" << std:: endl;
+      }        
+    }
+    
+    classifier::CrossValidationSampler sampler_training(tv_target,2,2);
+    classifier::SubsetGenerator cv_training(sampler_training,tv_view);
+    std::vector<size_t> v_sample_count(6,0);
+    std::vector<size_t> t_sample_count(6,0);
+    std::vector<size_t> v_class_count(3,0);
+    std::vector<size_t> t_class_count(3,0);
+    std::vector<size_t> t_class_count2(3,0);
+    for(u_long l=0;l<cv_training.size();l++) {
+      const classifier::DataLookup2D& t_view=cv_training.training_data(l);
+      const classifier::Target& t_target=cv_training.training_target(l);
+      const std::vector<size_t>& t_index=cv_training.training_index(l);
+      const classifier::DataLookup2D& v_view=cv_training.validation_data(l);
+      const classifier::Target& v_target=cv_training.validation_target(l);
+      const std::vector<size_t>& v_index=cv_training.validation_index(l);
+      
+      if (test_index.size()+tv_index.size()!=target.size() 
+          || t_index.size()+v_index.size() != tv_target.size() 
+          || test_index.size()+v_index.size()+t_index.size() !=  target.size()){
+        ok = false;
+        *error << "ERROR: size of training samples, validation samples " 
+               << "and test samples in is invalid." 
+               << std::endl;
+      }
+      if (test_index.size()!=3 || tv_index.size()!=6 || t_index.size()!=3 ||
+          v_index.size()!=3){
+        ok = false;
+        *error << "ERROR: size of training, validation, and test samples"
+               << " is invalid." 
+               << " Expected sizes to be 3" << std::endl;
+      }     
+
+      std::vector<size_t> tv_sample_count(6,0);
+      for (size_t i=0; i<t_index.size(); i++) {
+        assert(t_index[i]<t_sample_count.size());
+        tv_sample_count[t_index[i]]++;
+        t_sample_count[t_index[i]]++;
+        t_class_count[t_target(i)]++;
+        t_class_count2[tv_target(t_index[i])]++;
+        t_value[0]+=t_view(0,i);
+        t_value[t_target(i)+1]+=t_view(0,i);        
+      }
+      for (size_t i=0; i<v_index.size(); i++) {
+        assert(v_index[i]<v_sample_count.size());
+        tv_sample_count[v_index[i]]++;
+        v_sample_count[v_index[i]]++;
+        v_class_count[v_target(i)]++;
+        v_value[0]+=v_view(0,i);
+        v_value[v_target(i)+1]+=v_view(0,i);
+      }
+  
+      ok = ok && sample_count_test(tv_sample_count,error);      
+
+    }
+    ok = ok && sample_count_test(v_sample_count,error);
+    ok = ok && sample_count_test(t_sample_count,error);
+    
+    ok = ok && class_count_test(t_class_count,error);
+    ok = ok && class_count_test(t_class_count2,error);
+    ok = ok && class_count_test(v_class_count,error);
+
+
+  }
+  ok = ok && sample_count_test(test_sample_count,error);
+  ok = ok && class_count_test(test_class_count,error);
+  
+  if(test_value1[0]!=135 || test_value1[1]!=36 || test_value1[2]!=45 ||
+     test_value1[3]!=54) {
+    ok=false;
+    *error << "ERROR: incorrect sums of test values in row 1" 
+           << " found: " << test_value1[0] << ", "  << test_value1[1] 
+           << ", "  << test_value1[2] << " and "  << test_value1[3] 
+           << std::endl;
+  }
+
+  
+  if(test_value2[0]!=225 || test_value2[1]!=66 || test_value2[2]!=75 ||
+     test_value2[3]!=84) {
+    ok=false;
+    *error << "ERROR: incorrect sums of test values in row 2" 
+           << " found: " << test_value2[0] << ", "  << test_value2[1] 
+           << ", "  << test_value2[2] << " and "  << test_value2[3] 
+           << std::endl;
+  }
+
+  if(t_value[0]!=270 || t_value[1]!=72 || t_value[2]!=90 || t_value[3]!=108)  {
+    ok=false;
+    *error << "ERROR: incorrect sums of training values in row 1" 
+           << " found: " << t_value[0] << ", "  << t_value[1] 
+           << ", "  << t_value[2] << " and "  << t_value[3] 
+           << std::endl;    
+  }
+
+  if(v_value[0]!=270 || v_value[1]!=72 || v_value[2]!=90 || v_value[3]!=108)  {
+    ok=false;
+    *error << "ERROR: incorrect sums of validation values in row 1" 
+           << " found: " << v_value[0] << ", "  << v_value[1] 
+           << ", "  << v_value[2] << " and "  << v_value[3] 
+           << std::endl;    
+  }
+  return ok;
+}
+
+bool class_count_test(const std::vector<size_t>& class_count, 
+                      std::ostream* error)  
+{
+  bool ok=true;
+  for (size_t i=0; i<class_count.size(); i++)
+    if (class_count[i]==0){
+      ok = false;
+      *error << "ERROR: class " << i << " was not in set." 
+             << " Expected at least one sample from each class." 
+             << std::endl;
+    }
+  return ok;
+}
+
+bool sample_count_test(const std::vector<size_t>& sample_count, 
+                       std::ostream* error)  
+{
+  bool ok=true;
+  for (size_t i=0; i<sample_count.size(); i++){
+    if (sample_count[i]!=1){
+      ok = false;
+      *error << "ERROR: sample " << i << " was in a group " << sample_count[i] 
+             << " times." << " Expected to be 1 time" << std::endl;
+    }
+  }
+  return ok;
+}
+
+
+bool test_bootstrap(std::ostream* error)
+{
+  bool ok=true;
+  std::vector<std::string> label(10,"default");
+  label[2]=label[7]="white";
+  label[4]=label[5]="black";
+  label[6]=label[3]="green";
+  label[8]=label[9]="red";
+                 
+  classifier::Target target(label);
+  utility::matrix raw_data(10,10);
+  classifier::MatrixLookup data(raw_data);
+  classifier::BootstrapSampler cv(target,3);
+  return ok;
+}
+
+
+bool test_cv(std::ostream* error)
+{
+  bool ok=true;
+  std::vector<std::string> label(10,"default");
+  label[2]=label[7]="white";
+  label[4]=label[5]="black";
+  label[6]=label[3]="green";
+  label[8]=label[9]="red";
+                 
+  classifier::Target target(label);
+  utility::matrix raw_data(10,10);
+  classifier::MatrixLookup data(raw_data);
+  classifier::CrossValidationSampler cv(target,3,3);
+  
+  std::vector<size_t> sample_count(10,0);
+  for (size_t j=0; j<cv.size(); ++j){
+    std::vector<size_t> class_count(5,0);
+    assert(j<cv.size());
+    if (cv.training_index(j).size()+cv.validation_index(j).size()!=
+        target.size()){
+      ok = false;
+      *error << "ERROR: size of training samples plus " 
+             << "size of validation samples is invalid." << std::endl;
+    }
+    if (cv.validation_index(j).size()!=3 && cv.validation_index(j).size()!=4){
+      ok = false;
+      *error << "ERROR: size of validation samples is invalid." 
+             << "expected size to be 3 or 4" << std::endl;
+    }
+    for (size_t i=0; i<cv.validation_index(j).size(); i++) {
+      assert(cv.validation_index(j)[i]<sample_count.size());
+      sample_count[cv.validation_index(j)[i]]++;
+    }
+    for (size_t i=0; i<cv.training_index(j).size(); i++) {
+      class_count[target(cv.training_index(j)[i])]++;
+    }
+    ok = ok && class_count_test(class_count,error);
+  }
+  ok = ok && sample_count_test(sample_count,error);
+  
+  return ok;
+}