nils/ebusd
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

reworked filereader

Browse Source
This commit is contained in:
john30
2017-03-19 21:18:38 +01:00
parent 380ba2d19d
commit f26dbe7b4f
11 changed files with 971 additions and 640 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/ebus/filereader.h
+145 -274
View File
@@ -19,13 +19,10 @@
#ifndef LIB_EBUS_FILEREADER_H_
#define LIB_EBUS_FILEREADER_H_
#include <climits>
#include <string>
#include <iostream>
#include <sstream>
#include <fstream>
#include <algorithm>
#include <vector>
#include <map>
#include <iomanip>
#include <mutex>
#include "lib/ebus/symbol.h"
#include "lib/ebus/result.h"
@@ -42,13 +39,10 @@ namespace ebusd {
*/
using std::string;
using std::map;
using std::ostream;
using std::ostringstream;
using std::istream;
using std::istringstream;
using std::ifstream;
using std::cout;
using std::endl;
using std::mutex;
/** the separator character used between fields. */
#define FIELD_SEPARATOR ','
@@ -62,11 +56,6 @@ using std::endl;
/** the separator character used between multiple values (in CSV only). */
#define VALUE_SEPARATOR ';'
extern void printErrorPos(ostream& out, vector<string>::iterator begin, const vector<string>::iterator end,
vector<string>::iterator pos, string filename, size_t lineNo, result_t result);
extern unsigned int parseInt(const char* str, int base, const unsigned int minValue, const unsigned int maxValue,
result_t& result, size_t* length);
/**
* An abstract class that support reading definitions from a file.
@@ -74,10 +63,9 @@ extern unsigned int parseInt(const char* str, int base, const unsigned int minVa
class FileReader {
public:
/**
* Construct a new instance.
* Constructor.
*/
explicit FileReader(bool supportsDefaults)
: m_supportsDefaults(supportsDefaults) {}
FileReader() {}
/**
* Destructor.
@@ -87,306 +75,189 @@ class FileReader {
/**
* Read the definitions from a file.
* @param filename the name of the file being read.
* @param errorDescription a string in which to store the error description in case of error.
* @param verbose whether to verbosely log problems.
* @param defaultDest the default destination address (may be overwritten by file name), or empty.
* @param defaultCircuit the default circuit name (may be overwritten by file name), or empty.
* @param defaultSuffix the default circuit name suffix (starting with a ".", may be overwritten by file name, or empty.
* @param defaults the default values by name (potentially overwritten by file name), or NULL to not use defaults.
* @param hash optional pointer to a @a size_t value for storing the hash of the file, or NULL.
* @param size optional pointer to a @a size_t value for storing the normalized size of the file, or NULL.
* @param time optional pointer to a @a time_t value for storing the modification time of the file, or NULL.
* @return @a RESULT_OK on success, or an error code.
*/
virtual result_t readFromFile(const string filename, bool verbose = false,
string defaultDest = "", string defaultCircuit = "", string defaultSuffix = "") {
ifstream ifs;
ifs.open(filename.c_str(), ifstream::in);
if (!ifs.is_open()) {
m_lastError = filename;
return RESULT_ERR_NOTFOUND;
}
size_t lastSep = filename.find_last_of('/');
if (lastSep != string::npos) { // potential destination address, matches "^ZZ."
// extract defaultDest, defaultCircuit, defaultSuffix from filename:
// ZZ.IDENT[.CIRCUIT][.SUFFIX].*csv
symbol_t checkDest;
string checkIdent, useCircuit, useSuffix;
unsigned int checkSw, checkHw;
if (extractDefaultsFromFilename(filename.substr(lastSep+1), checkDest, checkIdent, useCircuit, useSuffix,
checkSw, checkHw)) {
defaultDest = filename.substr(lastSep+1, 2);
if (!useCircuit.empty()) {
defaultCircuit = useCircuit;
}
if (!useSuffix.empty()) {
defaultSuffix = useSuffix;
}
}
}
unsigned int lineNo = 0;
vector<string> row;
vector< vector<string> > defaults;
while (splitFields(ifs, row, lineNo)) {
if (row.empty()) {
continue;
}
result_t result;
vector<string>::iterator it = row.begin();
const vector<string>::iterator end = row.end();
if (m_supportsDefaults) {
if (row[0][0] == '*') {
row[0] = row[0].substr(1);
result = addDefaultFromFile(defaults, row, it, defaultDest, defaultCircuit, defaultSuffix, filename, lineNo);
if (result == RESULT_OK) {
continue;
}
} else {
result = addFromFile(it, end, &defaults, defaultDest, defaultCircuit, defaultSuffix, filename, lineNo);
}
} else {
result = addFromFile(it, end, NULL, defaultDest, defaultCircuit, defaultSuffix, filename, lineNo);
}
if (result != RESULT_OK) {
if (!verbose) {
ifs.close();
ostringstream error;
error << filename << ":" << static_cast<unsigned>(lineNo);
if (m_lastError.length() > 0) {
error << ": " << m_lastError;
}
m_lastError = error.str();
return result;
}
if (m_lastError.length() > 0) {
cout << m_lastError << endl;
}
printErrorPos(cout, row.begin(), end, it, filename, lineNo, result);
} else if (!verbose) {
m_lastError = "";
}
}
ifs.close();
return RESULT_OK;
}
virtual result_t readFromFile(const string filename, string& errorDescription, bool verbose = false,
map<string, string>* defaults = NULL, size_t* hash = NULL, size_t* size = NULL, time_t* time = NULL);
/**
* Return a @a string describing the last error position.
* @return a @a string describing the last error position.
*/
virtual string getLastError() { return m_lastError; }
/**
* Add a default row that was read from a file.
* @param defaults the list to add the default row to.
* @param row the default row (initial star char removed).
* @param begin an iterator to the first column of the default row to read (for error reporting).
* @param defaultDest the valid destination address extracted from the file name (from ZZ part), or empty.
* @param defaultCircuit the valid circuit name extracted from the file name (from IDENT part), or empty.
* @param defaultSuffix the valid circuit name suffix (starting with a ".") extracted from the file name (number after after IDENT part and "."), or empty.
* Read a single line definition from the stream.
* @param stream the @a istream to read from.
* @param errorDescription a string in which to store the error description in case of error.
* @param filename the name of the file being read.
* @param lineNo the current line number in the file being read.
* @param lineNo the last line number (incremented with each line read).
* @param verbose whether to verbosely log problems.
* @param hash optional pointer to a @a size_t value for updating with the hash of the line, or NULL.
* @param size optional pointer to a @a size_t value for updating with the normalized length of the line, or NULL.
* @return @a RESULT_OK on success, or an error code.
*/
virtual result_t addDefaultFromFile(vector< vector<string> >& defaults, vector<string>& row,
vector<string>::iterator& begin, string defaultDest, string defaultCircuit, string defaultSuffix,
const string& filename, unsigned int lineNo) {
defaults.push_back(row);
begin = row.end();
return RESULT_OK;
}
virtual result_t readLineFromStream(istream& stream, string& errorDescription,
const string filename, unsigned int& lineNo, vector<string>& row, bool verbose = false,
size_t* hash = NULL, size_t* size = NULL);
/**
* Add a definition that was read from a file.
* @param begin an iterator to the first column of the definition row to read.
* @param end the end iterator of the definition row to read.
* @param defaults all previously read default rows (initial star char removed), or NULL if not supported.
* @param defaultDest the valid destination address extracted from the file name (from ZZ part), or empty.
* @param defaultCircuit the valid circuit name extracted from the file name (from IDENT part), or empty.
* @param defaultSuffix the valid circuit name suffix (starting with a ".") extracted from the file name (number after after IDENT part and "."), or empty.
* @param row the definition row.
* @param errorDescription a string in which to store the error description in case of error.
* @param filename the name of the file being read.
* @param lineNo the current line number in the file being read.
* @return @a RESULT_OK on success, or an error code.
*/
virtual result_t addFromFile(vector<string>::iterator& begin, const vector<string>::iterator end,
vector< vector<string> >* defaults, const string& defaultDest, const string& defaultCircuit,
const string& defaultSuffix, const string& filename, unsigned int lineNo) = 0;
virtual result_t addFromFile(vector<string>& row, string& errorDescription,
const string filename, unsigned int lineNo) = 0;
/**
* Left and right trim the string.
* @param str the @a string to trim.
*/
static void trim(string& str) {
size_t pos = str.find_first_not_of(" \t");
if (pos != string::npos) {
str.erase(0, pos);
}
pos = str.find_last_not_of(" \t");
if (pos != string::npos) {
str.erase(pos+1);
}
}
static void trim(string& str);
/**
* Convert all upper case characters in the string to lower case.
* @param str the @a string to convert.
*/
static void tolower(string& str) {
transform(str.begin(), str.end(), str.begin(), ::tolower);
}
static void tolower(string& str);
/**
* Split the next line(s) from the @a istring into fields.
* Split the next line(s) from the @a istream into fields.
* @param ifs the @a istream to read from.
* @param row the @a vector to which to add the fields. This will be empty for completely empty and comment lines.
* @param lineNo the current line number (incremented with each line read).
* @param hash optional pointer to a @a size_t value for combining the hash of the line with, or NULL.
* @param size optional pointer to a @a size_t value to add the trimmed line length to, or NULL.
* @return true if there are more lines to read, false when there are no more lines left.
*/
static bool splitFields(istream& ifs, vector<string>& row, unsigned int& lineNo) {
row.clear();
string line;
bool quotedText = false, wasQuoted = false;
ostringstream field;
char prev = FIELD_SEPARATOR;
bool empty = true, read = false;
while (getline(ifs, line)) {
read = true;
lineNo++;
trim(line);
static bool splitFields(istream& ifs, vector<string>& row, unsigned int& lineNo,
size_t* hash = NULL, size_t* size = NULL);
size_t length = line.length();
if (!quotedText && (length == 0 || line[0] == '#' || (line.length() > 1 && line[0] == '/' && line[1] == '/'))) {
continue; // skip empty lines and comments
}
for (size_t pos = 0; pos < length; pos++) {
char ch = line[pos];
switch (ch) {
case FIELD_SEPARATOR:
if (quotedText) {
field << ch;
} else {
string str = field.str();
trim(str);
empty &= str.empty();
row.push_back(str);
field.str("");
wasQuoted = false;
}
break;
case TEXT_SEPARATOR:
if (prev == TEXT_SEPARATOR && !quotedText) { // double dquote
field << ch;
quotedText = true;
} else if (quotedText) {
quotedText = false;
} else if (prev == FIELD_SEPARATOR) {
quotedText = wasQuoted = true;
} else {
field << ch;
}
break;
case '\r':
break;
default:
if (prev == TEXT_SEPARATOR && !quotedText && wasQuoted) {
field << TEXT_SEPARATOR; // single dquote in the middle of formerly quoted text
quotedText = true;
} else if (quotedText && pos == 0 && field.tellp() > 0 && *(field.str().end()-1) != VALUE_SEPARATOR) {
field << VALUE_SEPARATOR; // add separator in between multiline field parts
}
field << ch;
break;
}
prev = ch;
}
if (!quotedText) {
break;
}
}
string str = field.str();
trim(str);
if (empty && str.empty()) {
row.clear();
return read;
}
row.push_back(str);
return true;
/**
* Format the specified hash as 8 hex digits to the output stream.
* @param hash the hash code.
* @param str the @a ostream to write to.
*/
static void formatHash(size_t hash, ostream& str) {
str << std::hex << std::setw(8) << std::setfill('0') << (hash & 0xffffffff) << std::dec << std::setw(0);
}
};
class MappedFileReader : public FileReader {
public:
/**
* Constructor.
* @param supportsDefaults whether this instance supports rows with defaults (starting with a star).
*/
MappedFileReader(bool supportsDefaults) : FileReader(), m_supportsDefaults(supportsDefaults) {}
/**
* Destructor.
*/
virtual ~MappedFileReader() {
m_columnNames.clear();
m_lastDefaults.clear();
m_lastSubDefaults.clear();
}
// @copydoc
result_t readFromFile(const string filename, string& errorDescription, bool verbose = false,
map<string, string>* defaults = NULL, size_t* hash = NULL, size_t* size = NULL, time_t* time = NULL) override;
/**
* Extract default values from the file name.
* @param name the file name (without path) in the form "ZZ[.IDENT][.CIRCUIT][.SUFFIX][.SWXXXX][.HWXXXX][.*].csv".
* @param dest the output destination address ZZ (hex digits).
* @param ident the identification part IDENT (up to 5 characters, set to empty if not present).
* @param circuit the circuit part CIRCUIT (set to IDENT if not present).
* @param suffix the suffix part SUFFIX including the leading dot (decimal digit, set to empty if not present).
* @param software the software version part SWXXXX (BCD digits, set to @a UINT_MAX if not present).
* @param hardware the hardware version part HWXXXX (BCD digits, set to @a UINT_MAX if not present).
* @return true if at least the address and the identification part were extracted, false otherwise.
* @param name the name of the file (without path)
* @param defaults the default values by name to add to.
* @param software the variable in which to store the numeric software version, or NULL.
* @param hardware the variable in which to store the numeric software version, or NULL.
* @return true if the minimum parts were extracted, false otherwise.
*/
static bool extractDefaultsFromFilename(string name, symbol_t& dest, string& ident, string& circuit,
string& suffix, unsigned int& software, unsigned int& hardware) {
ident = circuit = suffix = "";
software = hardware = UINT_MAX;
if (name.length() > 4 && name.substr(name.length()-4) == ".csv") {
name = name.substr(0, name.length()-3); // including trailing "."
}
size_t pos = name.find('.');
if (pos != 2) {
return false; // missing "ZZ."
}
result_t result = RESULT_OK;
dest = (symbol_t)parseInt(name.substr(0, pos).c_str(), 16, 0, 0xff, result, NULL);
if (result != RESULT_OK || !isValidAddress(dest)) {
return false; // invalid "ZZ"
}
name.erase(0, pos);
if (name.length() > 1) {
pos = name.rfind(".SW"); // check for ".SWxxxx."
if (pos != string::npos && name.find(".", pos+1) == pos+7) {
software = parseInt(name.substr(pos+3, 4).c_str(), 10, 0, 9999, result, NULL);
if (result != RESULT_OK) {
return false; // invalid "SWxxxx"
}
name.erase(pos, 7);
}
}
if (name.length() > 1) {
pos = name.rfind(".HW"); // check for ".HWxxxx."
if (pos != string::npos && name.find(".", pos+1) == pos+7) {
hardware = parseInt(name.substr(pos+3, 4).c_str(), 10, 0, 9999, result, NULL);
if (result != RESULT_OK) {
return false; // invalid "HWxxxx"
}
name.erase(pos, 7);
}
}
if (name.length() > 1) {
pos = name.find('.', 1); // check for ".IDENT."
if (pos != string::npos && pos >= 1 && pos <= 6) {
// up to 5 chars between two "."s, immediately after "ZZ.", or ".."
ident = circuit = name.substr(1, pos-1);
name.erase(0, pos);
pos = name.find('.', 1); // check for ".CIRCUIT."
if (pos != string::npos && (pos>2 || name[1]<'0' || name[1]>'9')) {
circuit = name.substr(1, pos-1);
name.erase(0, pos);
pos = name.find('.', 1); // check for ".SUFFIX."
}
if (pos != string::npos && pos == 2 && name[1] >= '0' && name[1] <= '9') {
suffix = name.substr(0, 2);
name.erase(0, pos);
}
}
}
return true;
virtual bool extractDefaultsFromFilename(string filename, map<string, string>& defaults,
symbol_t* destAddress = NULL, unsigned int* software = NULL, unsigned int* hardware = NULL) {
return false;
}
// @copydoc
result_t addFromFile(vector<string>& row, string& errorDescription,
const string filename, unsigned int lineNo) override;
/**
* Get the field mapping from the given first line.
* @param row the first line from which to extract the field mapping, or empty to use the default mapping.
* @param begin an iterator to the first column of the first line to read (for error reporting).
* @return @a RESULT_OK on success, or an error code.
*/
virtual result_t getFieldMap(vector<string>& row, string& errorDescription) = 0;
/**
* Add a default row that was read from a file.
* @param row the default row by field name.
* @param subRows the sub default rows, each by field name.
* @param subRowDefaults the sub default values by type and field name to add to.
* @param filename the name of the file being read.
* @param lineNo the current line number in the file being read.
* @return @a RESULT_OK on success, or an error code.
*/
virtual result_t addDefaultFromFile(map<string, string>& row, vector< map<string, string> >& subRows,
string& errorDescription, const string filename, unsigned int lineNo) {
errorDescription = "defaults not supported";
return RESULT_ERR_INVALID_ARG;
}
/**
* Add a definition that was read from a file.
* @param row the main definition row by field name.
* @param subRows the sub definition rows, each by field name.
* @param rowDefaults all previously extracted default values by type and field name.
* @param subRowDefaults all previously extracted sub default values by type and field name.
* @param errorDescription a string in which to store the error description in case of error.
* @param filename the name of the file being read.
* @param lineNo the current line number in the file being read.
* @return @a RESULT_OK on success, or an error code.
*/
virtual result_t addFromFile(map<string, string>& row, vector< map<string, string> >& subRows,
string& errorDescription, const string filename, unsigned int lineNo) = 0;
/**
* @return a reference to all previously extracted default values by type and field name.
*/
virtual map<string, map<string, string> >& getDefaults() {
return m_lastDefaults;
}
/**
* @return a reference to all previously extracted sub default values by type and field name.
*/
virtual map<string, vector< map<string, string> > >& getSubDefaults() {
return m_lastSubDefaults;
}
/**
* Combine the row to a single string.
* @param row the mapped row.
* @return the combined string.
*/
static string combineRow(const map<string, string>& row);
private:
/** whether this instance supports rows with defaults (starting with a star). */
bool m_supportsDefaults;
const bool m_supportsDefaults;
/** a @a mutex for access to defaults. */
mutex m_mutex;
protected:
/** a @a string describing the last error position. */
string m_lastError;
/** the name of each column. */
vector<string> m_columnNames;
/** all previously extracted default values by type and field name. */
map<string, map<string, string> > m_lastDefaults;
/** all previously extracted sub default values by type and field name. */
map<string, vector< map<string, string> > > m_lastSubDefaults;
};
} // namespace ebusd