source: trunk/lib/Stats.cc @ 1280

// $Id: Stats.cc 1280 2010-11-06 14:45:23Z peter $

/*
  Copyright (C) 2005 Peter Johansson
  Copyright (C) 2006, 2007, 2008, 2009 Jari Häkkinen, Peter Johansson
  Copyright (C) 2010 Peter Johansson

  This file is part of svndigest, http://dev.thep.lu.se/svndigest

  svndigest is free software; you can redistribute it and/or modify it
  under the terms of the GNU General Public License as published by
  the Free Software Foundation; either version 3 of the License, or
  (at your option) any later version.

  svndigest is distributed in the hope that it will be useful, but
  WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  General Public License for more details.

  You should have received a copy of the GNU General Public License
  along with svndigest. If not, see <http://www.gnu.org/licenses/>.
*/

#include "Stats.h"

#include "Colors.h"
#include "Configuration.h"
#include "Functor.h"
#include "Graph.h"
#include "SVNblame.h"
#include "SVNinfo.h"
#include "utility.h"

#include <algorithm>
#include <cassert>
#include <cstdlib>
#include <fstream>
#include <iostream>
#include <iterator>
#include <map>
#include <numeric>
#include <string>
#include <sstream>
#include <unistd.h>
#include <utility>
#include <vector>

namespace theplu{
namespace svndigest{


  Stats::Stats(const std::string& path)
    : stats_(std::vector<Author2Vector>(LineTypeParser::total+1))
  {
    // Make sure latest revision is set properly
    SVNinfo svn_info(path);
    revision_=svn_info.rev();
    last_changed_rev_=svn_info.last_changed_rev();
    reset();

    std::ostringstream ss;
    if (const std::vector<std::pair<std::string, std::string> >* 
        codons = Configuration::instance().codon(path)) {
      using std::string;
      typedef std::vector<std::pair<string, string> >::const_iterator citerator;
      for ( citerator codon=codons->begin(); codon!=codons->end(); ++codon) 
        ss << codon->first << codon->second;
    }
    config_code_ = ss.str();
    // we don't allow newline in codon because we parse the string
    // with newline and thus want the string to be a one-liner.
    std::replace(config_code_.begin(), config_code_.end(), '\n', ' ');
  }


  Stats::~Stats(void)
  {
  }

  
  void Stats::add(const std::vector<std::map<std::string, SparseVector> >& data)
  {
    // loop over line types
    for (size_t lt = 0; lt<data.size(); ++lt) {
      std::map<std::string, SparseVector>::const_iterator iter=data[lt].begin();
      std::map<std::string, SparseVector>::const_iterator last=data[lt].end();
      // loop over users
      for ( ; iter!=last; ++iter) {
        add_author(iter->first);
        SumVector tmpvec;
        accumulate(iter->second, tmpvec);
        stats_[lt][iter->first] += tmpvec;
      }
    }
  }


  /*
  void Stats::accumulate(std::vector<unsigned int>& vec,
                         svn_revnum_t rev) const
  {
    assert(rev>0);
    if (vec.empty()){
      // just to allow call to vec.back() below
      vec.resize(1,0);
    }
    else if (vec.begin()+rev-1 < vec.end())
      std::partial_sum(vec.begin()+rev-1,vec.end(),vec.begin()+rev-1);
    // static_cast to remove annoying compiler warning
    if (vec.size() < static_cast<size_t>(revision()+1))
      vec.resize(revision()+1, vec.back());
  }
  */
  /*
  void Stats::accumulate_stats(svn_revnum_t rev)
  {
    if (!rev)
      rev = 1;
    for (std::set<std::string>::const_iterator iter(authors().begin());
         iter!=authors().end(); ++iter) {
      std::vector<unsigned int>& code = code_stats()[*iter];
      accumulate(code, rev);
      std::vector<unsigned int>& comments = comment_stats()[*iter];
      accumulate(comments, rev);
      std::vector<unsigned int>& other = other_stats()[*iter];
      accumulate(other, rev);
      std::vector<unsigned int>& copyright = copyright_stats()[*iter];
      accumulate(copyright, rev);
    }
  }
  */
  /*
  void Stats::add(const std::string& user, svn_revnum_t rev, 
                  LineTypeParser::line_type lt, unsigned int n)
  {
    assert(user.size());
    add_author(user);

    if (lt==LineTypeParser::code)
      add(code_stats()[user], rev, n);
    else if (lt==LineTypeParser::comment)
      add(comment_stats()[user], rev, n);
    else if (lt==LineTypeParser::other)
      add(other_stats()[user], rev, n);
    else if (lt==LineTypeParser::copyright)
      add(copyright_stats()[user], rev, n);
  }

  void Stats::add(SumVector& vec, svn_revnum_t rev, unsigned int n)
  { 
    vec.set(rev, vec[rev] + n);
  }
  */


  void Stats::add_author(std::string name)
  {
    authors_.insert(name);
  }


  void Stats::add_authors(std::set<std::string>::const_iterator first, 
                          std::set<std::string>::const_iterator last)
  {
    authors_.insert(first, last);
  }


  const std::set<std::string>& Stats::authors(void) const
  {
    return authors_;
  }


  void Stats::calc_all(void)
  {
    for (int lt=0; lt <= 4; ++lt) {
      stats_[lt]["all"].clear();
      for (std::map<std::string, SumVector>::iterator i = stats_[lt].begin();
           i!=stats_[lt].end(); ++i) {
        stats_[lt]["all"] += i->second;
      }
    }

    comment_or_copy_stats()["all"] = comment_stats()["all"];
    comment_or_copy_stats()["all"] += copyright_stats()["all"];

    total_stats()["all"] = comment_or_copy_stats()["all"];
    total_stats()["all"] += code_stats()["all"];
    total_stats()["all"] += other_stats()["all"];
  }


  void Stats::calc_total(void)
  {
    for (std::set<std::string>::const_iterator iter(authors().begin());
         iter!=authors().end(); ++iter) {
      SumVector& total = total_stats()[*iter];
      total = code_stats()[*iter];
      total += comment_stats()[*iter];
      total += other_stats()[*iter];
      total += copyright_stats()[*iter];
    }
  }


  void Stats::calc_comment_or_copy(void)
  {
    for (std::set<std::string>::const_iterator iter(authors().begin());
         iter!=authors().end(); ++iter) {
      const SumVector& comments = comment_stats()[*iter];
      const SumVector& copy = copyright_stats()[*iter];

      comment_or_copy_stats()[*iter] = comments;
      comment_or_copy_stats()[*iter] += copy;
    }

  }


  unsigned int Stats::code(const std::string& user) const
  {
    return get_back(code_stats(), user);
  }


  unsigned int Stats::comments(const std::string& user) const
  {
    return get_back(comment_or_copy_stats(), user);
  }


  unsigned int Stats::empty(const std::string& user) const
  {
    return get_back(other_stats(), user);
  }


  unsigned int Stats::get_back(const Author2Vector& m, std::string user) const
  {
    A2VConstIter iter(m.find(std::string(user)));
    if (iter==m.end() || iter->second.empty()) 
      return 0;
    return iter->second.back();
  }


  const SumVector& Stats::get_vector(const Author2Vector& m, 
                                     const std::string& user) const
  {
    A2VConstIter iter(m.find(std::string(user)));
    if (iter==m.end()) 
      throw std::runtime_error(user+std::string(" not found i Stats"));
    return iter->second;
  }


  svn_revnum_t Stats::last_changed_rev(void) const
  {
    return last_changed_rev_;
  }


  unsigned int Stats::lines(const std::string& user) const
  {
    return get_back(total_stats(), user);
  }


  void Stats::load(std::istream& is, Author2Vector& m)
  {
    while (m.size() < authors().size()+1 && is.good()) {
      std::string name;
      std::getline(is, name);
      SparseVector vec;
      std::string line;
      std::getline(is, line);
      std::stringstream ss(line);
      while (ss.good()) {
        svn_revnum_t rev=0;
        unsigned int count=0;
        ss >> rev;
        ss >> count;
        assert(rev<=revision_);
        if (!count)
          break;
        vec.set(rev, count);
      }
      SumVector& sumvec = m[name];
      accumulate(vec, sumvec);
    }
  }


  svn_revnum_t Stats::load_cache(std::istream& is, bool& latest_ver)
  {
    std::string str;
    getline(is, str);
    if (str == "CACHE FILE VERSION 8") {
      latest_ver = true;
      return load_cache8(is);
    }
    latest_ver = false;
    if (str == "CACHE FILE VERSION 7")
      return load_cache7(is);

    if (str == "CACHE FILE VERSION 6")
      std::cout << "cache file is obsolete; "
                << "retrieving statistics from repository.\n";
    return 0;
  }


  svn_revnum_t Stats::load_cache8(std::istream& is)
  {
    std::string line;
    getline(is, line);
    if (line!=config_code_) {
      std::cout << "cache file is for different configuration.\n"
                << "config code: '" << config_code_ << "'\n"
                << "config code in cache file: '" << line << "'\n"
                << "retrieving statistics from repository.\n";
      return 0;
    }
    return load_cache7(is);
  }


  svn_revnum_t Stats::load_cache7(std::istream& is)
  {
    svn_revnum_t rev;
    is >> rev;
    reset();
    size_t a_size=0;
    is >> a_size;
    std::string str;
    while (authors().size()<a_size && is.good()){
      getline(is, str);
      if (str.size())
        add_author(str);
    }
    getline(is, str);
    if (str!=cache_check_str()) {
      return 0;
    }
    for (size_t i=0; i<stats_.size(); ++i){
      load(is, stats_[i]);
      getline(is, str);
      if (str!=cache_check_str()) {
        return 0;
      }
    }
    return rev;
  }


  void Stats::map_add(A2VConstIter first1, A2VConstIter last1, 
                      Author2Vector& map)
  {
    A2VIter first2(map.begin());
    Author2Vector::key_compare compare;
    while ( first1 != last1) { 
      // key of first1 less than key of first2
      if (first2==map.end() || compare(first1->first,first2->first)) {
        first2 = map.insert(first2, *first1);
        ++first1;
      }
      // key of first2 less than key of first1
      else if ( compare(first2->first, first1->first)) {
        ++first2;
      }
      // keys are equivalent
      else {
        first2->second += first1->second;
        ++first1;
        ++first2;
      }
    }
  }


  unsigned int Stats::max_element(const SumVector& vec) const
  {
    return std::max_element(vec.begin(), vec.end(), 
       pair_value_compare<const svn_revnum_t, unsigned int>())->second;
  }


  void Stats::parse(const std::string& path, svn_revnum_t rev)
  {
    // reset stats to zero for [rev, inf)
    for (size_t i=0; i<stats_.size(); ++i)
      for (A2VIter iter=stats_[i].begin(); iter!=stats_[i].end(); ++iter) {
        //iter->second.resize(rev,0);
        //iter->second.resize(revision(),0);
      }
    do_parse(path, rev);
    calc_comment_or_copy();
    calc_total();
    calc_all();
    
    assert(total_stats().size());
    assert(code_stats().size());
    assert(comment_or_copy_stats().size());
    assert(other_stats().size());
  }


  std::string Stats::plot(const std::string& filename,
                          const std::string& linetype) const
  {
    const std::string& format = Configuration::instance().image_format();
    if (format=="none")
      return filename;
    plot(filename, linetype, format);
    const std::string& anchor_format = 
      Configuration::instance().image_anchor_format();

    if (format!=anchor_format)
      plot(filename, linetype, anchor_format);
    return filename;
  }


  void Stats::plot(const std::string& filename,
                   const std::string& linetype,
                   const std::string& format) const
  {
    Graph gp(filename+"."+format, format);
    const Author2Vector* stat=NULL;
    if (linetype=="total")
      stat = &total_stats();
    else if (linetype=="code")
      stat = &code_stats();
    else if (linetype=="comments")
      stat = &comment_or_copy_stats();
    else if (linetype=="empty")
      stat = &other_stats();
    assert(stat);
    assert(stat->size());
    assert(stat->find("all")!=stat->end());
    const SumVector& total=get_vector(*stat, "all");    
    double yrange_max=1.03 * max_element(total) +1.0;
    gp.ymax(yrange_max);

    typedef std::vector<std::pair<std::string, SumVector> > vec_type;
    vec_type author_cont;
    author_cont.reserve(stat->size());
    for (std::set<std::string>::const_iterator i=authors_.begin(); 
         i != authors_.end(); ++i) {
      assert(stat->find(*i)!=stat->end());
      const SumVector& vec = get_vector(*stat,*i);
      if (max_element(vec)) {
        author_cont.push_back(std::make_pair(*i,vec));
      }
    }

    LessReversed<SumVector> lr;
    PairSecondCompare<std::string, SumVector, LessReversed<SumVector> > 
      compare(lr);
    std::sort(author_cont.begin(), author_cont.end(), compare);

    vec_type::iterator end(author_cont.end());
    vec_type::iterator i(author_cont.begin());
    const vec_type::size_type maxauthors=8;
    int authskip=author_cont.size()-maxauthors;
    if (authskip>1) {
      // only use others if there is more than 1 author to be skipped,
      // there is no reason to add only 1 author to others.
      vec_type::iterator j(i);
      i+=authskip;
      SumVector others;
      sum(j, i, others, PairValuePlusAssign<std::string, SumVector>());
      unsigned char r, g, b;
      std::string label("others");
      Colors::instance().get_color(label, r,g,b);
      gp.current_color(r,g,b);
      gp.plot(others, label, others.back());
    }
    for ( ; i!=end; ++i) {
      unsigned char r, g, b;
      Colors::instance().get_color(i->first,r,g,b);
      gp.current_color(r,g,b);
      gp.plot(i->second, i->first, get_back(*stat, i->first));
    }
    gp.current_color(255,0,0);
    gp.plot(total, "total", get_back(*stat, "all"));
  }


  void Stats::plot_summary(const std::string& filename) const
  {
    const std::string& format = Configuration::instance().image_format();
    if (format=="none")
      return;
    plot_summary(filename, format);
    const std::string& anchor_format = 
      Configuration::instance().image_anchor_format();

    if (format!=anchor_format)
      plot_summary(filename, anchor_format);
  }


  void Stats::plot_summary(const std::string& filename, 
                           const std::string& format) const
  {
    Graph gp(filename+"."+format, format);
    const SumVector& total = get_vector(total_stats(), "all");
    double yrange_max=1.03*max_element(total)+1;
    gp.ymax(yrange_max);
    
    const SumVector& x(get_vector(code_stats(), "all"));
    gp.current_color(255,255,0);
    gp.plot(x, "code", x.back());

    const SumVector& y = get_vector(comment_or_copy_stats(), "all");
    gp.current_color(0,0,255);
    gp.plot(y, "comment", y.back());

    const SumVector& z = get_vector(other_stats(), "all");
    gp.current_color(0,255,0);
    gp.plot(z, "other", z.back());

    gp.current_color(255,0,0);
    gp.plot(total, "total", total.back());
  }


  void Stats::print(std::ostream& os) const
  {
    // indicate that cache file is version 8 here , but keep remaning
    // 'CACHE FILE VERSION' as 'VERSION 7' to allow load_cache7() to
    // be used from load_cache8().
    os << "CACHE FILE VERSION 8\n";
    os << config_code_ << "\n";
    os << last_changed_rev() << " ";
    os << authors().size() << "\n";

    std::copy(authors().begin(), authors().end(), 
              std::ostream_iterator<std::string>(os, "\n"));
    os << cache_check_str() << "\n";
    for (size_t i=0; i<stats_.size(); ++i){
      print(os, stats_[i]);
      os << cache_check_str() << "\n";
    }
  }


  void Stats::print(std::ostream& os, const Author2Vector& m) const
  {
    for (A2VConstIter i(m.begin()); i!=m.end(); ++i){
      os << i->first << "\n";
      const SumVector& vec = i->second;
      if (vec.size()) {
        SumVector::const_iterator v = vec.begin();
        if (v->second)
          os << v->first << " " << v->second << " ";
        ++v;
        SumVector::const_iterator prev = vec.begin();
        while (v != vec.end()) {
          assert(vec[v->first - 1] == prev->second);
          // FIXME: this if should not be needed if SumVector was
          // truly sparse and collapsed when possible.
          if (v->second != prev->second)
            os << v->first << " " << v->second - prev->second << " ";
          ++v;
          ++prev;
        }
      }
      os << "\n";
    }
  }

  void Stats::reset(void)
  {
    for (size_t i=0; i<stats_.size(); ++i){
      stats_[i].clear();
      stats_[i]["all"].resize(revision()+1);
    }
    authors_.clear();
  }


  void Stats::resize(svn_revnum_t rev)
  {
    // set size on vectors
    for (size_t i=0; i<stats_.size(); ++i) {
      for (A2VIter iter=stats_[i].begin(); iter!=stats_[i].end(); ++iter) {
        iter->second.resize(rev);
      }
    }
  }


  svn_revnum_t Stats::revision(void) const 
  { 
    return revision_; 
  }


  Stats& Stats::operator+=(const Stats& rhs)
  {
    revision_ = std::max(revision_, rhs.revision_);
    last_changed_rev_ = std::max(last_changed_rev_, rhs.last_changed_rev_);
    add_authors(rhs.authors().begin(), rhs.authors().end());
    assert(stats_.size()==rhs.stats_.size());
    for (size_t i=0; i<stats_.size(); ++i)
      map_add(rhs.stats_[i].begin(), rhs.stats_[i].end(), stats_[i]);
    
    return *this;
  }

  
  size_t Stats::operator()(int linetype, std::string author, 
                           svn_revnum_t rev) const
  {
    assert(linetype<=LineTypeParser::total);
    assert(static_cast<size_t>(linetype) < stats_.size());
    assert(rev>=0);
    A2VConstIter i = stats_[linetype].find(author);
    if (i==stats_[linetype].end()){
      std::stringstream msg;
      msg << __FILE__ << ": author: " << author << " does not exist";  
      throw std::runtime_error(msg.str());
    }
    assert(rev <= revision());
    //    assert(rev < static_cast<svn_revnum_t>(i->second.size()));
    return i->second[rev];
  }

}} // end of namespace svndigest and namespace theplu