#include <CSVParser.h>

Inheritance diagram for minsky::DataSpec:

Collaboration diagram for minsky::DataSpec:

Public Member Functions
std::size_t	nRowAxes () const
	start row of the data area More...

std::size_t	nColAxes () const
	start column of the data area More...

DataSpecSchema	toSchema ()

DataSpec &	operator= (const DataSpecSchema &x)

void	toggleDimension (std::size_t c)

void	setDataArea (std::size_t row, std::size_t col)
	set top left cell of the data area More...

void	guessFromStream (std::istream &file, uintmax_t fileSize=uintmax_t(-1))
	initial stab at dataspec from examining stream More...

void	guessFromFile (const std::string &fileName)
	initial stab at dataspec from examining file More...

void	populateFromRavelMetadata (const std::string &metadata, const std::string &horizontalName, std::size_t row)
	populates this spec from a "RavelHypercube" entry, row is the row being read, used to set the headerRow attribute If horizontalName is one of the dimensions, data is written in a tabular format More...

const std::vector< size_t > &	uniqueValues () const
	number of unique values in each column corrected for header row, so may be slightly inaccurate if header row contains one of the values More...

template<class TokenizerFunction >
void	givenTFguessRemainder (std::istream &initialInput, std::istream &remainingInput, const TokenizerFunction &tf, uintmax_t fileSize)

template<class TokenizerFunction , class UniqueVals >
bool	processChunk (std::istream &input, const TokenizerFunction &tf, size_t until, UniqueVals &uniqueVals)

Public Attributes
std::size_t	maxColumn =1000
	maximum number of columns that can be configured independently. Columns after this limit are treated as "data" More...

Public Attributes inherited from minsky::DataSpecSchema
std::size_t	dataRowOffset

std::size_t	dataColOffset

std::size_t	numCols =0
	number of columns in CSV. Must be > dataColOffset More...

char	separator =','

char	quote ='"'

char	escape ='\0'

char	decSeparator ='.'

bool	mergeDelimiters =false

bool	counter =false
	count data items, not read their values More...

bool	dontFail =false
	do not throw an error on corrupt data More...

double	missingValue =nan("")

std::size_t	headerRow =0
	number of header rows More...

std::string	horizontalDimName ="?"

civita::Dimension	horizontalDimension

DuplicateKeyAction	duplicateKeyAction =throwException

std::set< unsigned >	dimensionCols
	rows and columns that are comment lines to be ignored More...

std::set< unsigned >	dataCols

std::vector< civita::Dimension >	dimensions

std::vector< std::string >	dimensionNames

Private Member Functions
	CLASSDESC_ACCESS (DataSpec)

template<class T >
void	givenTFguessRemainder (std::istream &initialInput, std::istream &remainingInput, const T &tf, uintmax_t fileSize)
	try to fill in remainder of spec, given a tokenizer function tf eg boost::escaped_list_separator<char> tf(escape,separator,quote) More...

void	guessRemainder (std::istream &initialInput, std::istream &remainingInput, char separator, uintmax_t fileSize)
	figure out the tokenizer function and call givenTFguessRemainder More...

template<class T , class U >
bool	processChunk (std::istream &input, const T &tf, size_t until, U &)
	process chunk of input, updating guessed spec More...

Private Attributes
std::size_t	m_nRowAxes =0

std::size_t	m_nColAxes =0

std::vector< size_t >	starts

size_t	nCols =0

size_t	row =0

size_t	firstEmpty =std::numeric_limits<size_t>::max()

std::vector< size_t >	m_uniqueValues
	number of unique values in each column More...

Additional Inherited Members
Public Types inherited from minsky::DataSpecSchema
enum	DuplicateKeyAction { throwException, sum, product, min, max, av }
	what to do with duplicate keys More...

Detailed Description

Definition at line 37 of file CSVParser.h.

Member Function Documentation

◆ CLASSDESC_ACCESS()

minsky::DataSpec::CLASSDESC_ACCESS ( DataSpec )

private

◆ givenTFguessRemainder() [1/2]

template<class T >

void minsky::DataSpec::givenTFguessRemainder	(	std::istream &	initialInput,
		std::istream &	remainingInput,
		const T &	tf,
		uintmax_t	fileSize
	)

private

try to fill in remainder of spec, given a tokenizer function tf eg boost::escaped_list_separator<char> tf(escape,separator,quote)

◆ givenTFguessRemainder() [2/2]

template<class TokenizerFunction >

void minsky::DataSpec::givenTFguessRemainder	(	std::istream &	initialInput,
		std::istream &	remainingInput,
		const TokenizerFunction &	tf,
		uintmax_t	fileSize
	)

Definition at line 390 of file CSVParser.cc.

References minsky::minsky(), minsky::CSVDialog::numInitialLines, and minsky::ProgressUpdater::setProgress().

 {
   starts.clear();
   nCols=0;
   row=0;
   firstEmpty=numeric_limits<size_t>::max();
   m_nRowAxes=0;
 
   const BusyCursor busy(minsky());
   // we don't know how many times we'll be going around the loop here, so pick a largish number for the progress bar
   ProgressUpdater pu(minsky().progressState,"Guessing CSV format",100);
 
   vector<set<size_t,less<size_t>,LibCAllocator<size_t>>> uniqueVals;
   m_uniqueValues.clear(); // cleared in case of early return
   try
     {
       if (!processChunk(initialInput, tf, CSVDialog::numInitialLines, uniqueVals)) return;
       do
         {
           m_uniqueValues.resize(uniqueVals.size());
           for (size_t i=0; i<uniqueVals.size(); ++i) m_uniqueValues[i]=uniqueVals[i].size();
           if (fileSize==-1)
             ++minsky().progressState;
           else
             pu.setProgress(double(remainingInput.tellg())/fileSize);
         }
       while (!processChunk(remainingInput, tf, row+CSVDialog::numInitialLines, uniqueVals));
       m_uniqueValues.resize(uniqueVals.size());
       for (size_t i=0; i<uniqueVals.size(); ++i) m_uniqueValues[i]=uniqueVals[i].size();
       if (fileSize==-1)
         ++minsky().progressState;
       else
         pu.setProgress(double(remainingInput.tellg())/fileSize);
     }
   catch (std::exception&)
     {
       // progressState throws an exception on being cancelled by the user
       throw std::runtime_error("CSV format guess terminated by user, best guess specification used.");
     }
 }

Here is the call graph for this function:

◆ guessFromFile()

void minsky::DataSpec::guessFromFile ( const std::string & fileName )

inline

initial stab at dataspec from examining file

Definition at line 80 of file CSVParser.h.

References guessFromStream(), and minsky::stripByteOrderingMarker().

                                                   {
       std::ifstream is(fileName);
       stripByteOrderingMarker(is);
       guessFromStream(is, std::filesystem::file_size(fileName));
     }

Here is the call graph for this function:

◆ guessFromStream()

void DataSpec::guessFromStream	(	std::istream &	file,
		uintmax_t	fileSize = `uintmax_t(-1)`
	)

initial stab at dataspec from examining stream

Definition at line 517 of file CSVParser.cc.

References minsky::CSVDialog::numInitialLines.

Referenced by guessFromFile().

 {
   size_t numCommas=0, numSemicolons=0, numTabs=0;
   size_t row=0;
   string buf;
   ostringstream streamBuf;
   for (; getline(input, buf) && row<CSVDialog::numInitialLines; ++row, streamBuf<<buf<<endl)
     for (auto c:buf)
       switch (c)
         {
         case ',':
           numCommas++;
           break;
         case ';':
           numSemicolons++;
           break;
         case '\t':
           numTabs++;
           break;
         }
 
   {
     istringstream inputCopy(streamBuf.str());
     if (numCommas>0.9*row && numCommas>numSemicolons && numCommas>numTabs)
       guessRemainder(inputCopy,input,',',fileSize);
     else if (numSemicolons>0.9*row && numSemicolons>numTabs)
       guessRemainder(inputCopy,input,';',fileSize);
     else if (numTabs>0.9*row)
       guessRemainder(inputCopy,input,'\t',fileSize);
     else
       guessRemainder(inputCopy,input,' ',fileSize);
   }
 }

Here is the caller graph for this function:

◆ guessRemainder()

void DataSpec::guessRemainder	(	std::istream &	initialInput,
		std::istream &	remainingInput,
		char	separator,
		uintmax_t	fileSize
	)

private

figure out the tokenizer function and call givenTFguessRemainder

Definition at line 431 of file CSVParser.cc.

 {
   separator=sep;
   if (separator==' ')
     givenTFguessRemainder(initialInput, remainingInput, SpaceSeparatorParser(escape,separator,quote),fileSize); //assumes merged whitespace separators
   else
     givenTFguessRemainder(initialInput, remainingInput, Parser(escape,separator,quote),fileSize);
 }

◆ nColAxes()

std::size_t minsky::DataSpec::nColAxes ( ) const

inline

start column of the data area

Definition at line 48 of file CSVParser.h.

References m_nColAxes.

Referenced by minsky::ParseCSV< P >::parse(), and toSchema().

48 {return m_nColAxes;}

minsky::DataSpec::m_nColAxes

std::size_t m_nColAxes

Definition: CSVParser.h:39

Here is the caller graph for this function:

◆ nRowAxes()

std::size_t minsky::DataSpec::nRowAxes ( ) const

inline

start row of the data area

Definition at line 46 of file CSVParser.h.

References m_nRowAxes.

Referenced by minsky::ParseCSV< P >::parse(), minsky::reportFromCSVFileT(), and toSchema().

46 {return m_nRowAxes;}

minsky::DataSpec::m_nRowAxes

std::size_t m_nRowAxes

Definition: CSVParser.h:39

Here is the caller graph for this function:

◆ operator=()

DataSpec& minsky::DataSpec::operator= ( const DataSpecSchema & x )

inline

Definition at line 57 of file CSVParser.h.

References minsky::DataSpecSchema::dataColOffset, minsky::DataSpecSchema::dataCols, minsky::DataSpecSchema::dataRowOffset, minsky::DataSpecSchema::dimensionCols, and setDataArea().

                                                  {
       DataSpecSchema::operator=(x);
       setDataArea(dataRowOffset, dataColOffset);
       dimensionCols=x.dimensionCols; // revert clobber by setDataArea
       dataCols=x.dataCols; // revert clobber by setDataArea
       return *this;
     }

Here is the call graph for this function:

◆ populateFromRavelMetadata()

void DataSpec::populateFromRavelMetadata	(	const std::string &	metadata,
		const std::string &	horizontalName,
		std::size_t	row
	)

populates this spec from a "RavelHypercube" entry, row is the row being read, used to set the headerRow attribute If horizontalName is one of the dimensions, data is written in a tabular format

Definition at line 551 of file CSVParser.cc.

  {
    vector<NamedDimension> ravelMetadata;
    json(ravelMetadata,metadata);
    headerRow=row+2;
    setDataArea(headerRow, ravelMetadata.size());
    dimensionNames.clear();
    dimensions.clear();
    for (auto& i: ravelMetadata)
      if (i.name==horizontalName)
        {
          horizontalDimension=i.dimension;
          horizontalDimName=i.name;
        }
      else
        {
          dimensions.push_back(i.dimension);
          dimensionNames.push_back(i.name);
        }
    for (size_t i=0; i<dimensions.size(); ++i)
      dimensionCols.insert(i);
  }

◆ processChunk() [1/2]

template<class T , class U >

bool minsky::DataSpec::processChunk	(	std::istream &	input,
		const T &	tf,
		size_t	until,
		U &
	)

private

process chunk of input, updating guessed spec

Returns: true if there's no more work to be done

◆ processChunk() [2/2]

template<class TokenizerFunction , class UniqueVals >

bool minsky::DataSpec::processChunk	(	std::istream &	input,
		const TokenizerFunction &	tf,
		size_t	until,
		UniqueVals &	uniqueVals
	)

Definition at line 441 of file CSVParser.cc.

References anonymous_namespace{CSVParser.cc}::emptyTail(), and anonymous_namespace{CSVParser.cc}::firstNumerical().

 {
   string buf;
   const hash<string> h;
   for (; getline(input, buf) && row<until; ++row)
     {
       if (buf.empty()) continue;
       // remove trailing carriage returns
       if (buf.back()=='\r') buf=buf.substr(0,buf.size()-1);
       if (!buf.empty())
         {
           smatch match;
           static const regex ravelHypercube("\"RavelHypercube=(.*)\"");
           if (regex_match(buf, match, ravelHypercube))
             try
               {
                 string metadata=match[1];
                 // remove leaning toothpicks
                 metadata.erase(remove(metadata.begin(),metadata.end(),'\\'),metadata.end());
                 string horizontalName;
                 getline(input, buf);
                 static const regex re("HorizontalDimension=\"(.*)\"");
                 if (regex_match(buf, match, re))
                   {
                     horizontalName=match[1];
                     ++row;
                   }
                 populateFromRavelMetadata(metadata, horizontalName, row);
                 return false;
               }
             catch (...)
               {
                 continue; // in case of error, ignore the RavelHypercube line.
               }
         }
       const boost::tokenizer<TokenizerFunction> tok(buf.begin(),buf.end(), tf);
       const vector<string> line(tok.begin(), tok.end());
       if (line.size()>uniqueVals.size())
         uniqueVals.resize(std::min(maxColumn, line.size()));
       for (size_t i=0; i<std::min(line.size(), uniqueVals.size()); ++i)
         uniqueVals[i].insert(h(line[i]));
       starts.push_back(firstNumerical(line));
       nCols=std::max(nCols, line.size());
       if (starts.back()==line.size())
         m_nRowAxes=row;
       if (starts.size()-1 < firstEmpty && starts.back()<nCols && emptyTail(line, starts.back()))
         firstEmpty=starts.size()-1;
     }
   // compute average of starts, then look for first row that drops below average
   double sum=0;
   for (unsigned long i=0; i<starts.size(); ++i) 
     sum+=starts[i];
   const double av=sum/(starts.size());
   for (; starts.size()>m_nRowAxes && (starts[m_nRowAxes]>av); 
        ++m_nRowAxes);
   // if nRowAxes exceeds numInitialLines, assume first row is a header row, and that that is all there is.
   if (m_nRowAxes>=row-1) m_nRowAxes=1;
   m_nColAxes=0;
   for (size_t i=nRowAxes(); i<starts.size(); ++i)
     m_nColAxes=std::max(m_nColAxes,starts[i]);
   // if more than 1 data column, treat the first row as an axis row
   if (m_nRowAxes==0 && nCols-m_nColAxes>1)
     m_nRowAxes=1;
     
   if (firstEmpty==m_nRowAxes) ++m_nRowAxes; // allow for possible colAxes header line
   headerRow=nRowAxes()>0? nRowAxes()-1: 0;
   size_t i=0;
   dimensionCols.clear();
   for (; i<nColAxes() && i<maxColumn; ++i) dimensionCols.insert(i);
   dataCols.clear();
   for (; i<nCols && i<maxColumn; ++i) dataCols.insert(i);
   return !input;
 }

Here is the call graph for this function:

◆ setDataArea()

void DataSpec::setDataArea	(	std::size_t	row,
		std::size_t	col
	)

set top left cell of the data area

Definition at line 370 of file CSVParser.cc.

Referenced by operator=().

 {
   m_nRowAxes=row;
   m_nColAxes=std::min(col, maxColumn);
   numCols=std::max(numCols, m_nColAxes);
   if (headerRow>=row)
     headerRow=row>0? row-1: 0;
   if (dimensions.size()<nColAxes()) dimensions.resize(nColAxes());
   if (dimensionNames.size()<nColAxes()) dimensionNames.resize(nColAxes());
   // remove any dimensionCols > nColAxes
   dimensionCols.erase(dimensionCols.lower_bound(nColAxes()), dimensionCols.end());
   // adjust ignored columns
   for (unsigned i=0; i<m_nColAxes; ++i)
     dataCols.erase(i);
   for (unsigned i=m_nColAxes; i<numCols && i<maxColumn; ++i)
     dataCols.insert(i);
 }

Here is the caller graph for this function:

◆ toggleDimension()

void minsky::DataSpec::toggleDimension ( std::size_t c )

inline

Definition at line 65 of file CSVParser.h.

References minsky::DataSpecSchema::dimensionCols.

                                       {
       auto i=dimensionCols.find(c);
       if (i==dimensionCols.end())
         dimensionCols.insert(c);
       else
         dimensionCols.erase(i);
     }

◆ toSchema()

DataSpecSchema minsky::DataSpec::toSchema ( )

inline

Definition at line 51 of file CSVParser.h.

References minsky::DataSpecSchema::dataColOffset, minsky::DataSpecSchema::dataRowOffset, nColAxes(), and nRowAxes().

                               {
       dataRowOffset=nRowAxes();
       dataColOffset=nColAxes();
       return *this;
     }

Here is the call graph for this function:

◆ uniqueValues()

const std::vector<size_t>& minsky::DataSpec::uniqueValues ( ) const

inline

number of unique values in each column corrected for header row, so may be slightly inaccurate if header row contains one of the values

Definition at line 92 of file CSVParser.h.

References m_uniqueValues.

92 {return m_uniqueValues;}

minsky::DataSpec::m_uniqueValues

std::vector< size_t > m_uniqueValues

number of unique values in each column

Definition: CSVParser.h:109

Member Data Documentation

◆ firstEmpty

size_t minsky::DataSpec::firstEmpty =std::numeric_limits<size_t>::max()

private

Definition at line 106 of file CSVParser.h.

◆ m_nColAxes

std::size_t minsky::DataSpec::m_nColAxes =0

private

Definition at line 39 of file CSVParser.h.

Referenced by nColAxes().

◆ m_nRowAxes

std::size_t minsky::DataSpec::m_nRowAxes =0

private

Definition at line 39 of file CSVParser.h.

Referenced by nRowAxes().

◆ m_uniqueValues

std::vector<size_t> minsky::DataSpec::m_uniqueValues

private

number of unique values in each column

Definition at line 109 of file CSVParser.h.

Referenced by uniqueValues().

◆ maxColumn

std::size_t minsky::DataSpec::maxColumn =1000

maximum number of columns that can be configured independently. Columns after this limit are treated as "data"

Definition at line 43 of file CSVParser.h.

Referenced by minsky::ParseCSV< P >::parse().

◆ nCols

size_t minsky::DataSpec::nCols =0

private

Definition at line 104 of file CSVParser.h.

◆ row

size_t minsky::DataSpec::row =0

private

Definition at line 105 of file CSVParser.h.

◆ starts

std::vector<size_t> minsky::DataSpec::starts

private

Definition at line 103 of file CSVParser.h.

The documentation for this class was generated from the following files:

engine/CSVParser.h
engine/CSVParser.cc

Public Member Functions

Public Attributes

Private Member Functions

Private Attributes

Additional Inherited Members

Detailed Description

Member Function Documentation

◆ CLASSDESC_ACCESS()

◆ givenTFguessRemainder() [1/2]

◆ givenTFguessRemainder() [2/2]

◆ guessFromFile()

◆ guessFromStream()

◆ guessRemainder()

◆ nColAxes()

◆ nRowAxes()

◆ operator=()

◆ populateFromRavelMetadata()

◆ processChunk() [1/2]

◆ processChunk() [2/2]

◆ setDataArea()

◆ toggleDimension()

◆ toSchema()

◆ uniqueValues()

Member Data Documentation

◆ firstEmpty

◆ m_nColAxes

◆ m_nRowAxes

◆ m_uniqueValues

◆ maxColumn

◆ nCols

◆ row

◆ starts