STK_ReadWriteCsv.h

Go to the documentation of this file.

/*--------------------------------------------------------------------*/
/*     Copyright (C) 2004-2016  Serge Iovleff, Université Lille 1, Inria
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU Lesser General Public License as
    published by the Free Software Foundation; either version 2 of the
    License, or (at your option) any later version.
 
    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Lesser General Public License for more details.
 
    You should have received a copy of the GNU Lesser General Public
    License along with this program; if not, write to the
    Free Software Foundation, Inc.,
    59 Temple Place,
    Suite 330,
    Boston, MA 02111-1307
    USA
 
    Contact : S..._Dot_I..._At_stkpp_Dot_org (see copyright for ...)
*/
 
/*
 * Project:  stkpp::DManager
 * Purpose:  Declaration of the class ReaWriteCsv.
 * Author:   Serge Iovleff, S..._Dot_I..._At_stkpp_Dot_org (see copyright for ...)
 *
 **/
 
#ifndef STK_READWRITECSV_H
#define STK_READWRITECSV_H
 
#include <iomanip>
#include <Arrays/include/STK_Array1D.h>
#include <Arrays/include/STK_ArrayBase.h>
#include "STK_Variable.h"
 
namespace STK
{
 
template<typename Type> class TReadWriteCsv;
 
typedef class TReadWriteCsv<String> ReadWriteCsv;
 
namespace Csv
{
  static const bool   DEFAULT_READNAMES = true;
  static const bool   DEFAULT_MAPPING = false;
  static const bool   DEFAULT_LAST_NL = true;
  static const int DEFAULT_RESERVE = int(0x0FFFF);
  static const String DEFAULT_DELIMITER  = _T(",");
  static const String ERRORCODES[] =
  {
    _T(""), // no message
    _T("An unknown error occurred!"),
    _T("Variable name not found!"),
    _T("Filename not set!"),
    _T("File not found!"),
    _T("The Number of names is different from the Number of Data Columns!")
   };
  enum readflags
  {
    RW_APPEND_DATA    = 0x00000004,
    RW_REPLACE_DATA   = 0x00000008
  };
}
 
template<typename Type>
class TReadWriteCsv
{
  public:
    typedef Variable<Type> Var;
    inline TReadWriteCsv() : file_name_()
                           , with_names_(Csv::DEFAULT_READNAMES)
                           , with_mapping_(Csv::DEFAULT_MAPPING)
                           , with_last_nl_(Csv::DEFAULT_LAST_NL)
                           , delimiters_(Csv::DEFAULT_DELIMITER)
                           , reserve_(Csv::DEFAULT_RESERVE)
                           , msg_error_()
                           , nbVars_(0)
                           , nbRows_(0)
 
    {}
    inline TReadWriteCsv( std::string const& file_name
                        , bool read_names = Csv::DEFAULT_READNAMES
                        , String const& delimiters = Csv::DEFAULT_DELIMITER
                        )
                        : file_name_(file_name)
                        , with_names_(read_names)
                        , with_mapping_(Csv::DEFAULT_MAPPING)
                        , with_last_nl_(Csv::DEFAULT_LAST_NL)
                        , delimiters_(delimiters)
                        , reserve_(Csv::DEFAULT_RESERVE)
                        , msg_error_()
                        , nbVars_(0)
                        , nbRows_(0)
    {}
    inline TReadWriteCsv(TReadWriteCsv const& rw) { *this = rw;}
    inline ~TReadWriteCsv() {}
    void clear()
    {
      msg_error_.clear();
      str_data_.clear();
      with_names_   = Csv::DEFAULT_READNAMES;
      with_mapping_ = Csv::DEFAULT_MAPPING;
      with_last_nl_ = Csv::DEFAULT_LAST_NL;
      delimiters_   = Csv::DEFAULT_DELIMITER;
      reserve_      = Csv::DEFAULT_RESERVE;
    }
    inline int begin() const { return str_data_.begin(); }
    inline int end() const { return str_data_.end(); }
    inline int size() const { return str_data_.size(); }
 
    inline Range cols() const { return str_data_.range(); }
    inline int beginCols() const { return str_data_.begin(); }
    inline int endCols() const { return str_data_.end(); }
    inline int sizeCols() const { return str_data_.size(); }
 
    inline int firstRow( int icol) const { return str_data_.at(icol).begin();}
    inline int endRow( int icol) const { return str_data_.at(icol).end();}
    inline int lastRow( int icol) const { return str_data_.at(icol).lastIdx();}
    inline int sizeRow( int icol) const { return str_data_.at(icol).size();}
    inline int beginRows() const
    {
      if (size()<= 0) return baseIdx;
      int retVal = firstRow(begin());
      for (int i=begin()+1; i<=lastIdx(); i++)
      { retVal = std::min(retVal, firstRow(i));}
      return retVal;
    }
    inline int endRows() const
    {
      if (size()<= 0) return baseIdx;
      int retVal = lastRow(begin());
      for (int i=begin()+1; i<end(); i++)
      { retVal = std::max(retVal, endRow(i));}
      return retVal;
    }
    inline int sizeRows() const
    {
      if (size()<= 0) return 0;
      int retVal = sizeRow(begin());
      for (int i=begin()+1; i<end(); i++)
      { retVal = std::max(retVal, sizeRow(i));}
      return retVal;
    }
    inline int lastIdxRows() const
    {
      if (size()<= 0) return baseIdx-1;
      int retVal = lastRow(begin());
      for (int i=begin()+1; i<end(); i++)
      { retVal = std::max(retVal, lastRow(i));}
      return retVal;
    }
    inline Range rows() const { return Range(beginRows(), lastIdxRows(), 0); }
 
    inline int lastIdx() const { return str_data_.lastIdx(); }
    inline int lastIdxCols() const { return str_data_.lastIdx(); }
 
    inline String const& error() const { return msg_error_; }
    inline String const& delimiters() const { return delimiters_; }
    inline bool withNames() const { return with_names_; }
    inline void setDelimiters( String const& delimiters) const { delimiters_ = delimiters; }
    inline void setWithNames( bool with_names) const { with_names_ = with_names; }
    inline void setWithMapping( bool with_mapping) const { with_mapping_ = with_mapping; }
    inline void setLast_nl( bool with_last_nl) const { with_last_nl_ = with_last_nl; }
    inline void setReserve( int const& reserve) const { reserve_ = reserve; }
    inline void setInMapping( std::map<String, Type> const& imapping)
    { imapping_ = imapping;}
    inline void setOutMapping( std::map<String, Type> const& omapping)
    { omapping_ = omapping;}
    bool setName( int const& icol, String const& name)
    {
      try
      {
        str_data_.at(icol).setName(name);
        return true;
      }
      catch( Exception const& e) { msg_error_ = e.error(); }
      catch(...)  { msg_error_ = Csv::ERRORCODES[1]; }
      return false;
    }
 
    inline Type& operator()( int const& iRow, int const& iCol)
    { return str_data_.elt(iCol).elt(iRow);}
    inline Type const operator()( int const& iRow, int const& iCol) const
    { return str_data_.elt(iCol).elt(iRow);}
    inline Var* clone(int icol) const { return str_data_.elt(icol).clone(); }
    inline Var& var(int j) { return str_data_.elt(j); }
    inline Var const& var(int icol) const { return str_data_.elt(icol); }
    inline Var& at(int icol) { return str_data_.at(icol); }
    inline Var const& at(int icol) const { return str_data_.at(icol); }
    inline Var& operator[](int const& icol) { return str_data_[icol]; }
    inline Var const operator[](int const& icol) const { return str_data_[icol]; }
    inline Var& front() { return str_data_.at(begin());}
    inline Var const& front() const { return str_data_.at(begin());}
    inline Var& back() { return str_data_.at(lastIdx());}
    inline Var const& back() const { return str_data_.at(lastIdx());}
    bool push_back( Var const& data = Var())
    {
      try
      {
        str_data_.push_back(data);
        //str_data_.back().reserve(reserve_);
        return true;
      }
      catch(const Exception& e) { msg_error_ = e.error(); }
      catch(...) { msg_error_ = Csv::ERRORCODES[1]; }
      return false;
    }
    bool push_front( Var const& data = Var())
    {
      try
      {
        str_data_.push_front(data);
        return true;
      }
      catch(const Exception& e) { msg_error_ = e.error(); }
      catch(...) { msg_error_ = Csv::ERRORCODES[1]; }
      return false;
    }
    inline void resize( int sizeRows, int sizeCols)
    {
      resizeCols(sizeCols);
      resizeRows(sizeRows);
    }
    bool appendData( int const& icol, Type const& value)
    {
      try
      {
        if (Arithmetic<Type>::isNA(value))
          str_data_[icol].push_back(Arithmetic<Type>::NA());
        else
          str_data_[icol].push_back(value);
        return true;
      }
      catch(const Exception& e) { msg_error_ = e.error(); }
      catch(...) { msg_error_ = Csv::ERRORCODES[1]; }
      return false;
    }
    bool eraseColumn(int const& icol)
    {
      try
      {
        str_data_.erase(icol);
        return true;
      }
      catch( Exception const& e) { msg_error_ = e.error(); }
      catch(...) { msg_error_ = Csv::ERRORCODES[1]; }
      return false;
    }
    TReadWriteCsv& operator=( TReadWriteCsv const& rw)
    {
      file_name_     = rw.file_name_;
      with_names_    = rw.with_names_;
      delimiters_    = rw.delimiters_;
      reserve_       = rw.reserve_;
      msg_error_     = rw.msg_error_;
      str_data_      = rw.str_data_;
      nbVars_        = rw.nbVars_;
      nbRows_        = rw.nbRows_;
      return *this;
    }
    TReadWriteCsv& operator+=( TReadWriteCsv const& rw)
    {
      for ( int i=rw.begin(); i<=rw.lastIdx(); i++)
      { str_data_.push_back(rw.str_data_[i]);}
      return *this;
    }
    TReadWriteCsv operator+( TReadWriteCsv const& rw) const
    { return TReadWriteCsv((*this)) += rw;}
    template<class Array>
    void exporter(ArrayBase<Array>& array) const
    {
      array.asDerived().resize(sizeRows(), sizeCols());
      for (int j= beginCols(); j < endCols(); ++j)
      {
        for (int i= beginRows(); i < endRows(); ++i)
        {
          array.elt(i,j) = var(j)[i];
        }
 
      }
    }
    inline bool read() { return read(file_name_);}
    inline bool read(std::map<String, Type> const& mapping)
    { return read(file_name_);}
    bool read(std::string const& file_name)
    {
      try
      {
        // update current file_name
        file_name_ = file_name;
        // input file stream
        ifstream inFile;
        // open file
        inFile.open(file_name.c_str(),std::ios::in);
        // check error
        if (inFile.rdstate() & std::ios::failbit)
        {
          inFile.close();
          msg_error_ = Csv::ERRORCODES[4];
          msg_error_ += "\nFile: " + file_name;
          return false;
        }
        read(inFile);                  // and read it
        inFile.close();
        // return
        return true;
      }
      catch(const Exception& e)
      { msg_error_ = e.error();}
      catch(...)
      { msg_error_ = Csv::ERRORCODES[1];}
      return false;
    }
    bool read( istream& inBuffer);
    inline bool write() { return write(file_name_);}
    bool write( std::string const& file_name) const
    {
      file_name_  = file_name;
      try
      {
        ofstream os(file_name.c_str());
        writeSelection( os, beginRows(), lastIdxRows(), begin(), lastIdx());
        os.close();
        return true;
      }
      catch(const Exception& e) { msg_error_ = e.error(); }
      catch(...) { msg_error_ = Csv::ERRORCODES[1]; }
      return false;
    }
    void write( ostream& os) const
    { writeSelection(os, beginRows(), lastIdxRows(), begin(), lastIdx());}
    void writeSelection( ostream& os, int top, int bottom, int left, int right) const;
 
    friend istream& operator>>( istream& in, TReadWriteCsv& rw)
    {
      if (!rw.read(in)) throw Exception(rw.msg_error_);
      return in;
    }
    friend ostream& operator<<( ostream& os, TReadWriteCsv const& rw)
    {
      try
      { rw.writeSelection( os, rw.beginRows(), rw.lastIdxRows(), rw.begin(), rw.lastIdx());}
      // catch and re-throw any Exceptions
      catch(const Exception& e) { throw e; }
      catch(...) { throw Exception(Csv::ERRORCODES[1]); }
      return os;
    }
 
  protected:
    mutable std::string file_name_;
    mutable bool with_names_;
    mutable bool with_mapping_;
    mutable bool with_last_nl_;
    mutable String  delimiters_;
    mutable int reserve_;
    mutable String msg_error_;
    Array1D< Var > str_data_;
    std::map<String, Type> imapping_;
    std::map<Type, String> omapping_;
    int nbVars_;
    int nbRows_;
    int countFields( String const& line, Array1D<Char>& listDelimiters)
    {
      listDelimiters.clear();
      listDelimiters.reserve(nbVars_);
      // position of the delimiters in the file
      String::size_type idx =0;
      do
      {
        // Find delimiter
        idx = line.find_first_of(delimiters_, idx);
        // if the position of the delimiter char is not found
        if (idx == line.npos) break;
        listDelimiters.push_back(line[idx]);
        ++idx;
      } while(1);
      // last delimiter is the end of line
      listDelimiters.push_back(CHAR_NL);
      listDelimiters.shift(1);
      // return the number of fields
      return listDelimiters.size();
    }
    int readCurrentLine(istream& inBuffer, String& currentLine, Array1D<Char>& listDelimiters)
    {
      if (inBuffer.eof()) return 0;
      int nbField =0;
      do
      {
        currentLine.clear();
        std::getline(inBuffer, currentLine);
        if (currentLine.size() == 0) {nbField = 0;}
        else
        {
          DManager::removeCharBeforeAndAfter(currentLine, CHAR_BLANK);
          nbField = (currentLine.size() == 0)
                  ? 0 : countFields( currentLine, listDelimiters);
        }
      }
      while ((nbField == 0)&&(!inBuffer.eof()));
      return nbField;
    }
    void readNames(istream& inBuffer)
    {
      Array1D<Char> listDelimiters;
      String currentLine;
      // Count the number of names of the first line
      int nbField = readCurrentLine(inBuffer, currentLine, listDelimiters);
      if (nbField == 0) return;
      // Declare an input string stream
      istringstream instream(currentLine);
      for(int icol=1; icol<=nbField; icol++)
      {
        // Append a Col
        String name;
        DManager::getField( instream, name, listDelimiters.elt(icol));
        push_back(Var(nbRows_, name));
      }
      // Update the number of var
      nbVars_ = std::max(nbVars_, nbField);
    }
    void initRead(istream& inBuffer)
    {
      Array1D<Char>  listDelimiters;
      String currentLine;
      // clear previous ReadCvs if any
      str_data_.clear();
      // get current position in the stream
      std::streampos pos = inBuffer.tellg();
      // Count the number of rows of the input stream
      nbRows_ = DManager::nbEndOfLine(inBuffer);
      setReserve(nbRows_);
      // revert to the current position in the stream
      inBuffer.seekg(pos);
      // initialize the initial number of variables to 0
      nbVars_ = 0;
      // If the names are at the top line
      if (with_names_) { readNames(inBuffer);}
      // get current position
      pos = inBuffer.tellg();
      // count number of fields in the first line
      int nbField = readCurrentLine(inBuffer, currentLine, listDelimiters);
      nbVars_ = std::max(nbVars_, nbField);
      inBuffer.seekg(pos);
    }
    inline void fastSetData( int const& icol, int const& irow, Type const& value)
    { str_data_.elt(icol).elt(irow) = value;}
    inline void resizeRows( Range const& rows)
    {
      for (int iVar=begin(); iVar<=lastIdx(); iVar++ )
      { str_data_.elt(iVar).resize(rows);}
    }
    inline void resizeCols( Range const& cols)
    { str_data_.resize(cols);}
};
 
/* Reads the specified file with the specified read flags.
 *  @param file_name name of the file to read
 *  @return  @c true if successful, @c false if an error is encountered.
 **/
template <typename Type>
bool TReadWriteCsv<Type>::read( istream& inBuffer)
{
  Array1D<Char> listDelimiters;
  String currentLine;
  try
  {
    initRead(inBuffer);
    resizeCols(nbVars_);
    resizeRows(nbRows_);
    // Read data : loop for all rows
    int currentRow=beginRows()-1, countRows = 0;
    for (; !inBuffer.eof();)
    {
      int nbField = readCurrentLine(inBuffer, currentLine, listDelimiters);
      if (nbField == 0) { break;}
      //  add on the non-existing columns: will be done if (nbField >= nbVars_)
      for (int iField=nbVars_+1; iField<=nbField; iField++)
      { push_back(Var( nbRows_, Arithmetic<Type>::NA(), stringNa));
        nbVars_++;
      }
      // first loop on the existing columns with data
      currentRow++; countRows++;
      istringstream instream(currentLine);
      int icol=begin();
      for (int iField=1 ; iField<=nbField; iField++, icol++)
      {
        String field;
        DManager::getField( instream, field, listDelimiters.elt(iField));
        // append Data to the column
        with_mapping_ ?
            fastSetData(icol, currentRow, imapping_.count(field) ? imapping_.find(field)->second : Arithmetic<Type>::NA())
         :  fastSetData(icol, currentRow, stringToType<Type>(field));
      }
      // Append NA values if the row is not complete
      for (; icol<end(); icol++) { fastSetData(icol, currentRow, Arithmetic<Type>::NA());}
    }
    // resize in case there exists too much rows
    resizeRows(Range(countRows));
    return true;
  }
  catch( Exception const& e) { msg_error_ = e.error(); }
  catch(...) { msg_error_ = Csv::ERRORCODES[1]; }
  return false;
}
 
template <typename Type>
void TReadWriteCsv<Type>::writeSelection( ostream& os, int top, int bottom, int left, int right) const
{
  if ((right < left)|| (bottom < top)) return;
  // create a vector for the format of the output
  Array1D<String::size_type>  format(Range(left, right, 0), 0);
  // for each variable, find the largest size
  for(int iVar=left; iVar<=right; iVar++)
  { format[iVar] = str_data_[iVar].maxLength(with_names_);}
  // write if needed names variables
  if (with_names_)
  {
    for(int iVar=left; iVar<right; iVar++)
    {
      os << std::setw(int(format[iVar])) << std::right
         << Proxy<String>(str_data_.at(iVar).name())
         << delimiters_.at(0);
    }
    os << std::setw(int(format[right])) << std::right
       << Proxy<String>(str_data_[right].name())
       << _T("\n");
  }
  // write data
  for(int irow = top; irow<bottom; irow++)
  {
    for(int iVar = left; iVar<right; iVar++)
    {
      try
      {
        os << std::setw(format[iVar]) << std::right
           << Proxy<Type>(str_data_[iVar].at(irow));
      }
      catch(...) // if an error occur, we put NA value
      { os << std::setw(format[iVar]) << std::right << stringNa;}
      os <<  delimiters_.at(0);
    }
    try
    {
      os << std::setw(format[right]) << std::right
         << Proxy<Type>(str_data_[right].at(irow));
    }
    catch(...) // if an error occur, we put NA value
    { os << std::setw(format[right]) << std::right << stringNa;}
     os << _T("\n");
  }
  for(int iVar = left; iVar<right; iVar++)
  {
    try
    {
      os << std::setw(format[iVar]) << std::right
         << Proxy<Type>(str_data_[iVar].at(bottom));
    }
    catch(...) // if an error occur, we put NA value
    { os << std::setw(format[iVar]) << std::right << stringNa;}
    os <<  delimiters_.at(0);
  }
  try
  {
    os << std::setw(format[right]) << std::right
       << Proxy<Type>(str_data_[right].at(bottom));
  }
  catch(...) // if an error occur, we put NA value
  { os << std::setw(format[right]) << std::right << stringNa;}
  if (with_last_nl_) { os << _T("\n");}
}
 
} // namespace STK
 
#endif // STK_READWRITECSV_H