reworked filereader

src/lib/ebus/CMakeLists.txt

@@ -6,6 +6,7 @@ set(libebus_a_SOURCES
     symbol.cpp
     symbol.h
     filereader.h
+    filereader.cpp
     datatype.cpp
     datatype.h
     data.cpp

src/lib/ebus/Makefile.am

@@ -9,6 +9,7 @@ libebus_a_SOURCES = result.cpp \
 		    symbol.cpp \
 		    symbol.h \
 		    filereader.h \
+		    filereader.cpp \
 		    datatype.cpp \
 		    datatype.h \
 		    data.cpp \

src/lib/ebus/contrib/test/test_tem.cpp

@@ -24,6 +24,8 @@
 #include "lib/ebus/data.h"
 
 using namespace ebusd;
+using std::cout;
+using std::endl;
 
 static bool error = false;
 
@@ -47,6 +49,45 @@ void verify(bool expectFailMatch, string type, string input,
   }
 }
 
+class TestReader : public MappedFileReader {
+ public:
+  TestReader(DataFieldTemplates* templates, bool isSet, bool isMasterDest)
+  : MappedFileReader::MappedFileReader(true), m_templates(templates), m_isSet(isSet), m_isMasterDest(isMasterDest),
+    m_fields(NULL) {}
+  result_t getFieldMap(vector<string>& row, string& errorDescription) override {
+    if (row.empty()) {
+      row.push_back("*name");
+      row.push_back("part");
+      row.push_back("type");
+      row.push_back("divisor/values");
+      row.push_back("unit");
+      row.push_back("comment");
+      return RESULT_OK;
+    }
+    if (row[0][0] != '*') {
+      return RESULT_ERR_INVALID_ARG;
+    }
+    return RESULT_OK;  // leave it to DataField::create
+  }
+  result_t addFromFile(map<string, string>& row, vector< map<string, string> >& subRows,
+      string& errorDescription, const string filename, unsigned int lineNo) override {
+    if (!row.empty() || subRows.empty()) {
+      cout << "read line " << static_cast<unsigned>(lineNo) << ": read error: got "
+          << static_cast<unsigned>(row.size()) << "/0 main, " << static_cast<unsigned>(subRows.size())
+          << "/>=3 sub" << endl;
+      return RESULT_ERR_EOF;
+    }
+    cout << "read line " << static_cast<unsigned>(lineNo) << ": read OK" << endl;
+    return DataField::create(subRows, errorDescription, m_templates, m_fields, m_isSet, false, m_isMasterDest);
+  }
+ private:
+  DataFieldTemplates* m_templates;
+  const bool m_isSet;
+  const bool m_isMasterDest;
+ public:
+  DataField* m_fields;
+};
+
 int main() {
   DataType* type = DataTypeList::getInstance()->get("TEM_P");
   if (type == NULL) {
@@ -54,9 +95,10 @@ int main() {
     return 1;
   }
 
+  // entry: definition, decoded value, master data, slave data, flags
+  // definition: name,part,type[:len][,[divisor|values][,[unit][,[comment]]]]
+  unsigned int baseLine = __LINE__+1;
   string checks[][5] = {
-    // entry: definition, decoded value, master data, slave data, flags
-    // definition: name,part,type[:len][,[divisor|values][,[unit][,[comment]]]]
     {"x,,TEM_P", "04-033", "10fe0700020421", "00", ""},
     {"x,,TEM_P", "00-000", "10fe0700020000", "00", ""},
     {"x,,TEM_P", "31-127", "10fe0700021f7f", "00", ""},
@@ -72,6 +114,11 @@ int main() {
     {"x,,TEM_P", "00-128", "1015070000", "020080", "Rw"},
   };
   DataFieldTemplates* templates = new DataFieldTemplates();
+  unsigned int lineNo = 0;
+  istringstream dummystr("#");
+  string errorDescription;
+  vector<string> row;
+  templates->readLineFromStream(dummystr, errorDescription, "inline", lineNo, row);
   DataField* fields = NULL;
   for (size_t i = 0; i < sizeof(checks) / sizeof(checks[0]); i++) {
     string check[5] = checks[i];
@@ -98,20 +145,29 @@ int main() {
     bool failedWrite = flags.find('w') != string::npos;
     bool failedWriteMatch = flags.find('W') != string::npos;
     string item;
-    vector<string> entries;
-
-    while (getline(isstr, item, FIELD_SEPARATOR))
-      entries.push_back(item);
 
     if (fields != NULL) {
       delete fields;
       fields = NULL;
     }
-    vector<string>::iterator it = entries.begin();
+
-    result = DataField::create(it, entries.end(), templates, fields, isSet, false,
+    string errorDescription;
-        (mstr[1] == BROADCAST || isMaster(mstr[1])));
+    TestReader reader{templates, isSet, mstr[1] == BROADCAST || isMaster(mstr[1])};
+    lineNo = 0;
+    dummystr = istringstream("#");
+    result = reader.readLineFromStream(dummystr, errorDescription, "inline", lineNo, row);
     if (result != RESULT_OK) {
-      cout << "\"" << check[0] << "\": create error: " << getResultCode(result) << endl;
+      cout << "\"" << check[0] << "\": reader header error: " << getResultCode(result) << ", " << errorDescription
+          << endl;
+      error = true;
+      continue;
+    }
+    lineNo = baseLine + i;
+    result = reader.readLineFromStream(isstr, errorDescription, "", lineNo, row);
+    fields = reader.m_fields;
+
+    if (result != RESULT_OK) {
+      cout << "\"" << check[0] << "\": create error: " << getResultCode(result) << ", " << errorDescription << endl;
       error = true;
       continue;
     }
@@ -120,11 +176,6 @@ int main() {
       error = true;
       continue;
     }
-    if (it != entries.end()) {
-      cout << "\"" << check[0] << "\": create error: trailing input" << endl;
-      error = true;
-      continue;
-    }
     cout << "\"" << check[0] << "\"=\"";
     fields->dump(cout);
     cout << "\": create OK" << endl;

src/lib/ebus/datatype.cpp

@@ -39,56 +39,7 @@ using std::fixed;
 using std::setfill;
 using std::setprecision;
 using std::setw;
-
+using std::endl;
-void printErrorPos(ostream& out, vector<string>::iterator begin, const vector<string>::iterator end,
-    vector<string>::iterator pos, string filename, size_t lineNo, result_t result) {
-  if (pos > begin) {
-    pos--;
-  }
-  out << "Error reading \"" << filename << "\" line " << setw(0) << dec << static_cast<unsigned>(lineNo)
-      << " field " << static_cast<unsigned>(1+pos.base()-begin.base()) << " value \"" << *pos << "\": "
-      << getResultCode(result) << endl;
-  out << "Erroneous item is here:" << endl;
-  bool first = true;
-  int cnt = 0;
-  while (begin != end) {
-    if (first) {
-      first = false;
-    } else {
-      out << FIELD_SEPARATOR;
-      if (begin <= pos) {
-        cnt++;
-      }
-    }
-    string item = *begin;
-    size_t i = item.find(TEXT_SEPARATOR);
-    if (i != string::npos) {
-      do {
-        item.replace(i, 1, TEXT_SEPARATOR_STR TEXT_SEPARATOR_STR);
-        i = item.find(TEXT_SEPARATOR, i+sizeof(TEXT_SEPARATOR_STR)+sizeof(TEXT_SEPARATOR_STR));
-      } while (i != string::npos);
-      i = 0;
-    } else {
-      i = item.find(FIELD_SEPARATOR);
-    }
-    if (i != string::npos) {
-      out << TEXT_SEPARATOR << item << TEXT_SEPARATOR;
-      if (begin < pos) {
-        cnt += 2;
-      } else if (begin == pos) {
-        cnt++;
-      }
-    } else {
-      out << item;
-    }
-    if (begin < pos) {
-      cnt += (unsigned int)(item).length();
-    }
-    begin++;
-  }
-  out << endl;
-  out << setw(cnt) << " " << setw(0) << "^" << endl;
-}
 
 
 bool DataType::dump(ostream& output, const size_t length, const bool appendSeparatorDivisor) const {

src/lib/ebus/datatype.h

@@ -51,6 +51,8 @@ namespace ebusd {
 
 using std::map;
 using std::list;
+using std::istringstream;
+using std::ostringstream;
 
 /** the separator character used between base type name and length (in CSV only). */
 #define LENGTH_SEPARATOR ':'
@@ -146,20 +148,6 @@ enum PartType {
 #define CON 0x1000
 
 
-/**
- * Print the error position of the iterator.
- * @param out the @a ostream to print to.
- * @param begin the iterator to the beginning of the items.
- * @param end the iterator to the end of the items.
- * @param pos the iterator with the erroneous position.
- * @param filename the name of the file being read.
- * @param lineNo the current line number in the file being read.
- * @param result the result code.
- */
-void printErrorPos(ostream& out, vector<string>::iterator begin, const vector<string>::iterator end,
-    vector<string>::iterator pos, string filename, size_t lineNo, result_t result);
-
-
 /**
  * Base class for all kinds of data types.
  */

src/lib/ebus/filereader.cpp

@@ -0,0 +1,304 @@
+/*
+ * ebusd - daemon for communication with eBUS heating systems.
+ * Copyright (C) 2014-2017 John Baier <ebusd@ebusd.eu>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "lib/ebus/filereader.h"
+#include <sys/stat.h>
+#include <iostream>
+#include <iomanip>
+#include <climits>
+#include <fstream>
+#include <functional>
+
+namespace ebusd {
+
+using std::ifstream;
+using std::ostringstream;
+using std::cout;
+using std::endl;
+using std::setw;
+using std::dec;
+
+
+result_t FileReader::readFromFile(const string filename, string& errorDescription, bool verbose,
+    map<string, string>* defaults, size_t* hash, size_t* size, time_t* time) { //TODO use hash etc.
+  struct stat st;
+  if (stat(filename.c_str(), &st) != 0) {
+    errorDescription = filename;
+    return RESULT_ERR_NOTFOUND;
+  }
+  if (S_ISDIR(st.st_mode)) {
+    errorDescription = filename+" is a directory";
+    return RESULT_ERR_NOTFOUND;
+  }
+  ifstream ifs;
+  ifs.open(filename.c_str(), ifstream::in);
+  if (!ifs.is_open()) {
+    errorDescription = filename;
+    return RESULT_ERR_NOTFOUND;
+  }
+  if (hash) {
+    *hash = 0;
+  }
+  if (size) {
+    *size = 0;
+  }
+  if (time) {
+    *time = st.st_mtime;
+  }
+  unsigned int lineNo = 0;
+  vector<string> row;
+  result_t result = RESULT_OK;
+  while (ifs.peek() != EOF && result == RESULT_OK) {
+    result = readLineFromStream(ifs, errorDescription, filename, lineNo, row, verbose, hash, size);
+  }
+  ifs.close();
+  return result;
+}
+
+result_t FileReader::readLineFromStream(istream& stream, string& errorDescription,
+    const string filename, unsigned int& lineNo, vector<string>& row, bool verbose,
+    size_t* hash, size_t* size) {
+  if (!splitFields(stream, row, lineNo, hash, size)) {
+    return RESULT_ERR_EOF;
+  }
+  errorDescription = "";
+  result_t result = addFromFile(row, errorDescription, filename, lineNo);
+  if (result != RESULT_OK) {
+    if (!verbose) {
+      ostringstream error;
+      error << filename << ":" << lineNo;
+      if (errorDescription.length() > 0) {
+        error << ": " << errorDescription;
+      }
+      errorDescription = error.str();
+      return result;
+    }
+    if (!errorDescription.empty()) {
+      cout << "error reading " << filename << ":" << lineNo << ": " << getResultCode(result) << ", "
+          << errorDescription << endl;
+    }
+  } else if (!verbose) {
+    errorDescription = "";
+  }
+  return result;
+}
+
+void FileReader::trim(string& str) {
+  size_t pos = str.find_first_not_of(" \t");
+  if (pos != string::npos) {
+    str.erase(0, pos);
+  }
+  pos = str.find_last_not_of(" \t");
+  if (pos != string::npos) {
+    str.erase(pos+1);
+  }
+}
+
+void FileReader::tolower(string& str) {
+  transform(str.begin(), str.end(), str.begin(), ::tolower);
+}
+
+static std::hash<string> hashFunction;
+
+bool FileReader::splitFields(istream& ifs, vector<string>& row, unsigned int& lineNo,
+    size_t* hash, size_t* size) {
+  row.clear();
+  string line;
+  bool quotedText = false, wasQuoted = false;
+  ostringstream field;
+  char prev = FIELD_SEPARATOR;
+  bool empty = true, read = false;
+  while (getline(ifs, line)) {
+    read = true;
+    lineNo++;
+    trim(line);
+    size_t length = line.size();
+    if (size) {
+      *size += length + 1;  // normalized with trailing endl
+    }
+    if (hash) {
+      *hash ^= (hashFunction(line) << 1) ^ (length << ( 7 * (lineNo % 5)));
+    }
+    if (!quotedText && (length == 0 || line[0] == '#' || (line.length() > 1 && line[0] == '/' && line[1] == '/'))) {
+      if (lineNo == 1) {
+        break;  // keep empty first line for applying default header
+      }
+      continue;  // skip empty lines and comments
+    }
+    for (size_t pos = 0; pos < length; pos++) {
+      char ch = line[pos];
+      switch (ch) {
+      case FIELD_SEPARATOR:
+        if (quotedText) {
+          field << ch;
+        } else {
+          string str = field.str();
+          trim(str);
+          empty &= str.empty();
+          row.push_back(str);
+          field.str("");
+          wasQuoted = false;
+        }
+        break;
+      case TEXT_SEPARATOR:
+        if (prev == TEXT_SEPARATOR && !quotedText) {  // double dquote
+          field << ch;
+          quotedText = true;
+        } else if (quotedText) {
+          quotedText = false;
+        } else if (prev == FIELD_SEPARATOR) {
+          quotedText = wasQuoted = true;
+        } else {
+          field << ch;
+        }
+        break;
+      case '\r':
+        break;
+      default:
+        if (prev == TEXT_SEPARATOR && !quotedText && wasQuoted) {
+          field << TEXT_SEPARATOR;  // single dquote in the middle of formerly quoted text
+          quotedText = true;
+        } else if (quotedText && pos == 0 && field.tellp() > 0 && *(field.str().end()-1) != VALUE_SEPARATOR) {
+          field << VALUE_SEPARATOR;  // add separator in between multiline field parts
+        }
+        field << ch;
+        break;
+      }
+      prev = ch;
+    }
+    if (!quotedText) {
+      break;
+    }
+  }
+  string str = field.str();
+  trim(str);
+  if (empty && str.empty()) {
+    row.clear();
+    return read;
+  }
+  row.push_back(str);
+  return true;
+}
+
+
+result_t MappedFileReader::readFromFile(const string filename, string& errorDescription, bool verbose,
+    map<string, string>* defaults, size_t* hash, size_t* size, time_t* time) {
+  m_mutex.lock();
+  m_columnNames.clear();
+  m_lastDefaults.clear();
+  m_lastSubDefaults.clear();
+  if (defaults) {
+    m_lastDefaults[""] = *defaults;
+  }
+  size_t lastSep = filename.find_last_of('/');
+  string defaultsPart = lastSep == string::npos ? filename : filename.substr(lastSep+1);
+  extractDefaultsFromFilename(defaultsPart, m_lastDefaults[""]);
+  result_t result = FileReader::readFromFile(filename, errorDescription, verbose, defaults, hash, size, time);
+  m_mutex.unlock();
+  return result;
+}
+
+result_t MappedFileReader::addFromFile(vector<string>& row, string& errorDescription,
+    const string filename, unsigned int lineNo) {
+  result_t result;
+  if (lineNo == 1) {  // first line defines column names
+    result = getFieldMap(row, errorDescription);
+    if (result != RESULT_OK) {
+      return result;
+    }
+    if (row.empty()) {
+      errorDescription = "missing field map";
+      return RESULT_ERR_EOF;
+    }
+    m_columnNames = row;
+    return RESULT_OK;
+  }
+  if (row.empty()) {
+    return RESULT_OK;
+  }
+  if (m_columnNames.empty()) {
+    errorDescription = "missing field map";
+    return RESULT_ERR_INVALID_ARG;
+  }
+  map<string, string> rowMapped;
+  vector< map<string, string> > subRowsMapped;
+  vector<string>::iterator it = row.begin();
+  bool isDefault = m_supportsDefaults && !row[0].empty() && row[0][0] == '*';
+  if (isDefault) {
+    row[0] = row[0].substr(1);
+  }
+  size_t lastRepeatStart = UINT_MAX;
+  map<string, string>* lastMappedRow = &rowMapped;
+  bool empty = true;
+  for (size_t colIdx = 0, colNameIdx = 0; colIdx < row.size(); colIdx++, colNameIdx++) {
+    if (colNameIdx >= m_columnNames.size()) {
+      if (lastRepeatStart == UINT_MAX) {
+        errorDescription = "named columns exceeded";
+        return RESULT_ERR_INVALID_ARG;
+      }
+      colNameIdx = lastRepeatStart;
+    }
+    string columnName = m_columnNames[colNameIdx];
+    if (!columnName.empty() && columnName[0] == '*') {  // marker for next entry
+      if (empty) {
+        lastMappedRow->clear();
+      }
+      if (!empty || lastMappedRow == &rowMapped) {
+        subRowsMapped.resize(subRowsMapped.size() + 1);
+        lastMappedRow = &subRowsMapped[subRowsMapped.size() - 1];
+      }
+      columnName = columnName.substr(1);
+      lastRepeatStart = colNameIdx;
+      empty = true;
+    }
+    string value = row[colIdx];
+    empty &= value.empty();
+    (*lastMappedRow)[columnName] = value;
+  }
+  if (empty) {
+    lastMappedRow->clear();
+    if (lastMappedRow != &rowMapped) {
+      subRowsMapped.resize(subRowsMapped.size() - 1);
+    }
+  }
+/*cout<<"row:"<<MappedFileReader::combineRow(rowMapped);
+for (auto sub : subRowsMapped) {
+  cout<<std::endl<<"  subrow:"<<MappedFileReader::combineRow(sub);
+}
+cout<<std::endl;*/
+  if (isDefault) {
+    return addDefaultFromFile(rowMapped, subRowsMapped, errorDescription, filename, lineNo);
+  }
+  return addFromFile(rowMapped, subRowsMapped, errorDescription, filename, lineNo);
+}
+
+string MappedFileReader::combineRow(const map<string, string>& row) {
+  ostringstream ostream;
+  bool first = true;
+  for (auto entry : row) {
+    if (first) {
+      first = false;
+    } else {
+      ostream << ", ";
+    }
+    ostream << entry.first << ": \"" << entry.second << "\"";
+  }
+  return ostream.str();
+}
+
+}  // namespace ebusd

src/lib/ebus/filereader.h

@@ -19,13 +19,10 @@
 #ifndef LIB_EBUS_FILEREADER_H_
 #define LIB_EBUS_FILEREADER_H_
 
-#include <climits>
-#include <string>
-#include <iostream>
-#include <sstream>
-#include <fstream>
 #include <algorithm>
-#include <vector>
+#include <map>
+#include <iomanip>
+#include <mutex>
 #include "lib/ebus/symbol.h"
 #include "lib/ebus/result.h"
 
@@ -42,13 +39,10 @@ namespace ebusd {
  */
 
 using std::string;
+using std::map;
 using std::ostream;
-using std::ostringstream;
 using std::istream;
-using std::istringstream;
+using std::mutex;
-using std::ifstream;
-using std::cout;
-using std::endl;
 
 /** the separator character used between fields. */
 #define FIELD_SEPARATOR ','
@@ -62,11 +56,6 @@ using std::endl;
 /** the separator character used between multiple values (in CSV only). */
 #define VALUE_SEPARATOR ';'
 
-extern void printErrorPos(ostream& out, vector<string>::iterator begin, const vector<string>::iterator end,
-    vector<string>::iterator pos, string filename, size_t lineNo, result_t result);
-
-extern unsigned int parseInt(const char* str, int base, const unsigned int minValue, const unsigned int maxValue,
-    result_t& result, size_t* length);
 
 /**
  * An abstract class that support reading definitions from a file.
@@ -74,10 +63,9 @@ extern unsigned int parseInt(const char* str, int base, const unsigned int minVa
 class FileReader {
  public:
   /**
-   * Construct a new instance.
+   * Constructor.
    */
-  explicit FileReader(bool supportsDefaults)
+  FileReader() {}
-      : m_supportsDefaults(supportsDefaults) {}
 
   /**
    * Destructor.
@@ -87,306 +75,189 @@ class FileReader {
   /**
    * Read the definitions from a file.
    * @param filename the name of the file being read.
+   * @param errorDescription a string in which to store the error description in case of error.
    * @param verbose whether to verbosely log problems.
-   * @param defaultDest the default destination address (may be overwritten by file name), or empty.
+   * @param defaults the default values by name (potentially overwritten by file name), or NULL to not use defaults.
-   * @param defaultCircuit the default circuit name (may be overwritten by file name), or empty.
+   * @param hash optional pointer to a @a size_t value for storing the hash of the file, or NULL.
-   * @param defaultSuffix the default circuit name suffix (starting with a ".", may be overwritten by file name, or empty.
+   * @param size optional pointer to a @a size_t value for storing the normalized size of the file, or NULL.
+   * @param time optional pointer to a @a time_t value for storing the modification time of the file, or NULL.
    * @return @a RESULT_OK on success, or an error code.
    */
-  virtual result_t readFromFile(const string filename, bool verbose = false,
+  virtual result_t readFromFile(const string filename, string& errorDescription, bool verbose = false,
-    string defaultDest = "", string defaultCircuit = "", string defaultSuffix = "") {
+      map<string, string>* defaults = NULL, size_t* hash = NULL, size_t* size = NULL, time_t* time = NULL);
-    ifstream ifs;
-    ifs.open(filename.c_str(), ifstream::in);
-    if (!ifs.is_open()) {
-      m_lastError = filename;
-      return RESULT_ERR_NOTFOUND;
-    }
-    size_t lastSep = filename.find_last_of('/');
-    if (lastSep != string::npos) {  // potential destination address, matches "^ZZ."
-      // extract defaultDest, defaultCircuit, defaultSuffix from filename:
-      // ZZ.IDENT[.CIRCUIT][.SUFFIX].*csv
-      symbol_t checkDest;
-      string checkIdent, useCircuit, useSuffix;
-      unsigned int checkSw, checkHw;
-      if (extractDefaultsFromFilename(filename.substr(lastSep+1), checkDest, checkIdent, useCircuit, useSuffix,
-          checkSw, checkHw)) {
-        defaultDest = filename.substr(lastSep+1, 2);
-        if (!useCircuit.empty()) {
-          defaultCircuit = useCircuit;
-        }
-        if (!useSuffix.empty()) {
-          defaultSuffix = useSuffix;
-        }
-      }
-    }
-    unsigned int lineNo = 0;
-    vector<string> row;
-    vector< vector<string> > defaults;
-    while (splitFields(ifs, row, lineNo)) {
-      if (row.empty()) {
-        continue;
-      }
-      result_t result;
-      vector<string>::iterator it = row.begin();
-      const vector<string>::iterator end = row.end();
-      if (m_supportsDefaults) {
-        if (row[0][0] == '*') {
-          row[0] = row[0].substr(1);
-          result = addDefaultFromFile(defaults, row, it, defaultDest, defaultCircuit, defaultSuffix, filename, lineNo);
-          if (result == RESULT_OK) {
-            continue;
-          }
-        } else {
-          result = addFromFile(it, end, &defaults, defaultDest, defaultCircuit, defaultSuffix, filename, lineNo);
-        }
-      } else {
-        result = addFromFile(it, end, NULL, defaultDest, defaultCircuit, defaultSuffix, filename, lineNo);
-      }
-      if (result != RESULT_OK) {
-        if (!verbose) {
-          ifs.close();
-          ostringstream error;
-          error << filename << ":" << static_cast<unsigned>(lineNo);
-          if (m_lastError.length() > 0) {
-            error << ": " << m_lastError;
-          }
-          m_lastError = error.str();
-          return result;
-        }
-        if (m_lastError.length() > 0) {
-          cout << m_lastError << endl;
-        }
-        printErrorPos(cout, row.begin(), end, it, filename, lineNo, result);
-      } else if (!verbose) {
-        m_lastError = "";
-      }
-    }
-
-    ifs.close();
-    return RESULT_OK;
-  }
 
   /**
-   * Return a @a string describing the last error position.
+   * Read a single line definition from the stream.
-   * @return a @a string describing the last error position.
+   * @param stream the @a istream to read from.
-   */
+   * @param errorDescription a string in which to store the error description in case of error.
-  virtual string getLastError() { return m_lastError; }
-
-  /**
-   * Add a default row that was read from a file.
-   * @param defaults the list to add the default row to.
-   * @param row the default row (initial star char removed).
-   * @param begin an iterator to the first column of the default row to read (for error reporting).
-   * @param defaultDest the valid destination address extracted from the file name (from ZZ part), or empty.
-   * @param defaultCircuit the valid circuit name extracted from the file name (from IDENT part), or empty.
-   * @param defaultSuffix the valid circuit name suffix (starting with a ".") extracted from the file name (number after after IDENT part and "."), or empty.
    * @param filename the name of the file being read.
-   * @param lineNo the current line number in the file being read.
+   * @param lineNo the last line number (incremented with each line read).
+   * @param verbose whether to verbosely log problems.
+   * @param hash optional pointer to a @a size_t value for updating with the hash of the line, or NULL.
+   * @param size optional pointer to a @a size_t value for updating with the normalized length of the line, or NULL.
    * @return @a RESULT_OK on success, or an error code.
    */
-  virtual result_t addDefaultFromFile(vector< vector<string> >& defaults, vector<string>& row,
+  virtual result_t readLineFromStream(istream& stream, string& errorDescription,
-      vector<string>::iterator& begin, string defaultDest, string defaultCircuit, string defaultSuffix,
+      const string filename, unsigned int& lineNo, vector<string>& row, bool verbose = false,
-      const string& filename, unsigned int lineNo) {
+      size_t* hash = NULL, size_t* size = NULL);
-    defaults.push_back(row);
-    begin = row.end();
-    return RESULT_OK;
-  }
 
   /**
    * Add a definition that was read from a file.
-   * @param begin an iterator to the first column of the definition row to read.
+   * @param row the definition row.
-   * @param end the end iterator of the definition row to read.
+   * @param errorDescription a string in which to store the error description in case of error.
-   * @param defaults all previously read default rows (initial star char removed), or NULL if not supported.
-   * @param defaultDest the valid destination address extracted from the file name (from ZZ part), or empty.
-   * @param defaultCircuit the valid circuit name extracted from the file name (from IDENT part), or empty.
-   * @param defaultSuffix the valid circuit name suffix (starting with a ".") extracted from the file name (number after after IDENT part and "."), or empty.
    * @param filename the name of the file being read.
    * @param lineNo the current line number in the file being read.
    * @return @a RESULT_OK on success, or an error code.
    */
-  virtual result_t addFromFile(vector<string>::iterator& begin, const vector<string>::iterator end,
+  virtual result_t addFromFile(vector<string>& row, string& errorDescription,
-    vector< vector<string> >* defaults, const string& defaultDest, const string& defaultCircuit,
+      const string filename, unsigned int lineNo) = 0;
-    const string& defaultSuffix, const string& filename, unsigned int lineNo) = 0;
 
   /**
    * Left and right trim the string.
    * @param str the @a string to trim.
    */
-  static void trim(string& str) {
+  static void trim(string& str);
-    size_t pos = str.find_first_not_of(" \t");
-    if (pos != string::npos) {
-      str.erase(0, pos);
-    }
-    pos = str.find_last_not_of(" \t");
-    if (pos != string::npos) {
-      str.erase(pos+1);
-    }
-  }
 
   /**
    * Convert all upper case characters in the string to lower case.
    * @param str the @a string to convert.
    */
-  static void tolower(string& str) {
+  static void tolower(string& str);
-    transform(str.begin(), str.end(), str.begin(), ::tolower);
-  }
 
   /**
-   * Split the next line(s) from the @a istring into fields.
+   * Split the next line(s) from the @a istream into fields.
    * @param ifs the @a istream to read from.
    * @param row the @a vector to which to add the fields. This will be empty for completely empty and comment lines.
    * @param lineNo the current line number (incremented with each line read).
+   * @param hash optional pointer to a @a size_t value for combining the hash of the line with, or NULL.
+   * @param size optional pointer to a @a size_t value to add the trimmed line length to, or NULL.
    * @return true if there are more lines to read, false when there are no more lines left.
    */
-  static bool splitFields(istream& ifs, vector<string>& row, unsigned int& lineNo) {
+  static bool splitFields(istream& ifs, vector<string>& row, unsigned int& lineNo,
-    row.clear();
+      size_t* hash = NULL, size_t* size = NULL);
-    string line;
-    bool quotedText = false, wasQuoted = false;
-    ostringstream field;
-    char prev = FIELD_SEPARATOR;
-    bool empty = true, read = false;
-    while (getline(ifs, line)) {
-      read = true;
-      lineNo++;
-      trim(line);
 
-      size_t length = line.length();
+  /**
-      if (!quotedText && (length == 0 || line[0] == '#' || (line.length() > 1 && line[0] == '/' && line[1] == '/'))) {
+   * Format the specified hash as 8 hex digits to the output stream.
-        continue;  // skip empty lines and comments
+   * @param hash the hash code.
-      }
+   * @param str the @a ostream to write to.
-      for (size_t pos = 0; pos < length; pos++) {
+   */
-        char ch = line[pos];
+  static void formatHash(size_t hash, ostream& str) {
-        switch (ch) {
+    str << std::hex << std::setw(8) << std::setfill('0') << (hash & 0xffffffff) << std::dec << std::setw(0);
-        case FIELD_SEPARATOR:
-          if (quotedText) {
-            field << ch;
-          } else {
-            string str = field.str();
-            trim(str);
-            empty &= str.empty();
-            row.push_back(str);
-            field.str("");
-            wasQuoted = false;
-          }
-          break;
-        case TEXT_SEPARATOR:
-          if (prev == TEXT_SEPARATOR && !quotedText) {  // double dquote
-            field << ch;
-            quotedText = true;
-          } else if (quotedText) {
-            quotedText = false;
-          } else if (prev == FIELD_SEPARATOR) {
-            quotedText = wasQuoted = true;
-          } else {
-            field << ch;
-          }
-          break;
-        case '\r':
-          break;
-        default:
-          if (prev == TEXT_SEPARATOR && !quotedText && wasQuoted) {
-            field << TEXT_SEPARATOR;  // single dquote in the middle of formerly quoted text
-            quotedText = true;
-          } else if (quotedText && pos == 0 && field.tellp() > 0 && *(field.str().end()-1) != VALUE_SEPARATOR) {
-            field << VALUE_SEPARATOR;  // add separator in between multiline field parts
-          }
-          field << ch;
-          break;
-        }
-        prev = ch;
-      }
-      if (!quotedText) {
-        break;
-      }
-    }
-    string str = field.str();
-    trim(str);
-    if (empty && str.empty()) {
-      row.clear();
-      return read;
-    }
-    row.push_back(str);
-    return true;
   }
+};
+
+
+class MappedFileReader : public FileReader {
+ public:
+  /**
+   * Constructor.
+   * @param supportsDefaults whether this instance supports rows with defaults (starting with a star).
+   */
+  MappedFileReader(bool supportsDefaults) : FileReader(), m_supportsDefaults(supportsDefaults) {}
+
+  /**
+   * Destructor.
+   */
+  virtual ~MappedFileReader() {
+    m_columnNames.clear();
+    m_lastDefaults.clear();
+    m_lastSubDefaults.clear();
+  }
+
+  // @copydoc
+  result_t readFromFile(const string filename, string& errorDescription, bool verbose = false,
+      map<string, string>* defaults = NULL, size_t* hash = NULL, size_t* size = NULL, time_t* time = NULL) override;
 
   /**
    * Extract default values from the file name.
-   * @param name the file name (without path) in the form "ZZ[.IDENT][.CIRCUIT][.SUFFIX][.SWXXXX][.HWXXXX][.*].csv".
+   * @param name the name of the file (without path)
-   * @param dest the output destination address ZZ (hex digits).
+   * @param defaults the default values by name to add to.
-   * @param ident the identification part IDENT (up to 5 characters, set to empty if not present).
+   * @param software the variable in which to store the numeric software version, or NULL.
-   * @param circuit the circuit part CIRCUIT (set to IDENT if not present).
+   * @param hardware the variable in which to store the numeric software version, or NULL.
-   * @param suffix the suffix part SUFFIX including the leading dot (decimal digit, set to empty if not present).
+   * @return true if the minimum parts were extracted, false otherwise.
-   * @param software the software version part SWXXXX (BCD digits, set to @a UINT_MAX if not present).
-   * @param hardware the hardware version part HWXXXX (BCD digits, set to @a UINT_MAX if not present).
-   * @return true if at least the address and the identification part were extracted, false otherwise.
    */
-  static bool extractDefaultsFromFilename(string name, symbol_t& dest, string& ident, string& circuit,
+  virtual bool extractDefaultsFromFilename(string filename, map<string, string>& defaults,
-    string& suffix, unsigned int& software, unsigned int& hardware) {
+      symbol_t* destAddress = NULL, unsigned int* software = NULL, unsigned int* hardware = NULL) {
-    ident = circuit = suffix = "";
+    return false;
-    software = hardware = UINT_MAX;
-    if (name.length() > 4 && name.substr(name.length()-4) == ".csv") {
-      name = name.substr(0, name.length()-3);  // including trailing "."
-    }
-    size_t pos = name.find('.');
-    if (pos != 2) {
-      return false;  // missing "ZZ."
-    }
-    result_t result = RESULT_OK;
-    dest = (symbol_t)parseInt(name.substr(0, pos).c_str(), 16, 0, 0xff, result, NULL);
-    if (result != RESULT_OK || !isValidAddress(dest)) {
-      return false;  // invalid "ZZ"
-    }
-    name.erase(0, pos);
-    if (name.length() > 1) {
-      pos = name.rfind(".SW");  // check for ".SWxxxx."
-      if (pos != string::npos && name.find(".", pos+1) == pos+7) {
-        software = parseInt(name.substr(pos+3, 4).c_str(), 10, 0, 9999, result, NULL);
-        if (result != RESULT_OK) {
-          return false;  // invalid "SWxxxx"
-        }
-        name.erase(pos, 7);
-      }
-    }
-    if (name.length() > 1) {
-      pos = name.rfind(".HW");  // check for ".HWxxxx."
-      if (pos != string::npos && name.find(".", pos+1) == pos+7) {
-        hardware = parseInt(name.substr(pos+3, 4).c_str(), 10, 0, 9999, result, NULL);
-        if (result != RESULT_OK) {
-          return false;  // invalid "HWxxxx"
-        }
-        name.erase(pos, 7);
-      }
-    }
-    if (name.length() > 1) {
-      pos = name.find('.', 1);  // check for ".IDENT."
-      if (pos != string::npos && pos >= 1 && pos <= 6) {
-        // up to 5 chars between two "."s, immediately after "ZZ.", or ".."
-        ident = circuit = name.substr(1, pos-1);
-        name.erase(0, pos);
-        pos = name.find('.', 1);  // check for ".CIRCUIT."
-        if (pos != string::npos && (pos>2 || name[1]<'0' || name[1]>'9')) {
-          circuit = name.substr(1, pos-1);
-          name.erase(0, pos);
-          pos = name.find('.', 1);  // check for ".SUFFIX."
-        }
-        if (pos != string::npos && pos == 2 && name[1] >= '0' && name[1] <= '9') {
-          suffix = name.substr(0, 2);
-          name.erase(0, pos);
-        }
-      }
-    }
-    return true;
   }
 
+  // @copydoc
+  result_t addFromFile(vector<string>& row, string& errorDescription,
+      const string filename, unsigned int lineNo) override;
+
+  /**
+   * Get the field mapping from the given first line.
+   * @param row the first line from which to extract the field mapping, or empty to use the default mapping.
+   * @param begin an iterator to the first column of the first line to read (for error reporting).
+   * @return @a RESULT_OK on success, or an error code.
+   */
+  virtual result_t getFieldMap(vector<string>& row, string& errorDescription) = 0;
+
+  /**
+   * Add a default row that was read from a file.
+   * @param row the default row by field name.
+   * @param subRows the sub default rows, each by field name.
+   * @param subRowDefaults the sub default values by type and field name to add to.
+   * @param filename the name of the file being read.
+   * @param lineNo the current line number in the file being read.
+   * @return @a RESULT_OK on success, or an error code.
+   */
+  virtual result_t addDefaultFromFile(map<string, string>& row, vector< map<string, string> >& subRows,
+      string& errorDescription, const string filename, unsigned int lineNo) {
+    errorDescription = "defaults not supported";
+    return RESULT_ERR_INVALID_ARG;
+  }
+
+  /**
+   * Add a definition that was read from a file.
+   * @param row the main definition row by field name.
+   * @param subRows the sub definition rows, each by field name.
+   * @param rowDefaults all previously extracted default values by type and field name.
+   * @param subRowDefaults all previously extracted sub default values by type and field name.
+   * @param errorDescription a string in which to store the error description in case of error.
+   * @param filename the name of the file being read.
+   * @param lineNo the current line number in the file being read.
+   * @return @a RESULT_OK on success, or an error code.
+   */
+  virtual result_t addFromFile(map<string, string>& row, vector< map<string, string> >& subRows,
+      string& errorDescription, const string filename, unsigned int lineNo) = 0;
+
+  /**
+   * @return a reference to all previously extracted default values by type and field name.
+   */
+  virtual map<string, map<string, string> >& getDefaults() {
+    return m_lastDefaults;
+  }
+
+  /**
+   * @return a reference to all previously extracted sub default values by type and field name.
+   */
+  virtual map<string, vector< map<string, string> > >& getSubDefaults() {
+    return m_lastSubDefaults;
+  }
+
+  /**
+   * Combine the row to a single string.
+   * @param row the mapped row.
+   * @return the combined string.
+   */
+  static string combineRow(const map<string, string>& row);
 
  private:
   /** whether this instance supports rows with defaults (starting with a star). */
-  bool m_supportsDefaults;
+  const bool m_supportsDefaults;
 
+  /** a @a mutex for access to defaults. */
+  mutex m_mutex;
 
- protected:
+  /** the name of each column. */
-  /** a @a string describing the last error position. */
+  vector<string> m_columnNames;
-  string m_lastError;
+
+  /** all previously extracted default values by type and field name. */
+  map<string, map<string, string> > m_lastDefaults;
+
+  /** all previously extracted sub default values by type and field name. */
+  map<string, vector< map<string, string> > > m_lastSubDefaults;
 };
 
 }  // namespace ebusd

src/lib/ebus/test/test_data.cpp

@@ -23,6 +23,8 @@
 #include "lib/ebus/data.h"
 
 using namespace ebusd;
+using std::cout;
+using std::endl;
 
 static bool error = false;
 
@@ -31,25 +33,66 @@ void verify(bool expectFailMatch, string type, string input,
   match = match && expectStr == gotStr;
   if (expectFailMatch) {
     if (match) {
+      error = true;
       cout << "  failed " << type << " match >" << input
               << "< error: unexpectedly succeeded" << endl;
-      error = true;
     } else {
       cout << "  failed " << type << " match >" << input << "< OK" << endl;
     }
   } else if (match) {
     cout << "  " << type << " match >" << input << "< OK" << endl;
   } else {
+    error = true;
     cout << "  " << type << " match >" << input << "< error: got >"
             << gotStr << "<, expected >" << expectStr << "<" << endl;
-      error = true;
   }
 }
 
+class TestReader : public MappedFileReader {
+ public:
+  TestReader(DataFieldTemplates* templates, bool isSet, bool isMasterDest)
+  : MappedFileReader::MappedFileReader(true), m_templates(templates), m_isSet(isSet), m_isMasterDest(isMasterDest),
+    m_fields(NULL) {}
+  result_t getFieldMap(vector<string>& row, string& errorDescription) override {
+    if (row.empty()) {
+      row.push_back("*name");
+      row.push_back("part");
+      row.push_back("type");
+      row.push_back("divisor/values");
+      row.push_back("unit");
+      row.push_back("comment");
+      return RESULT_OK;
+    }
+    if (row[0][0] != '*') {
+      return RESULT_ERR_INVALID_ARG;
+    }
+    return RESULT_OK;  // leave it to DataField::create
+  }
+  result_t addFromFile(map<string, string>& row, vector< map<string, string> >& subRows,
+      string& errorDescription, const string filename, unsigned int lineNo) override {
+    if (!row.empty() || subRows.empty()) {
+      cout << "read line " << static_cast<unsigned>(lineNo) << ": read error: got "
+          << static_cast<unsigned>(row.size()) << "/0 main, " << static_cast<unsigned>(subRows.size())
+          << "/>=3 sub" << endl;
+      return RESULT_ERR_EOF;
+    }
+    cout << "read line " << static_cast<unsigned>(lineNo) << ": read OK" << endl;
+    return DataField::create(subRows, errorDescription, m_templates, m_fields, m_isSet, false, m_isMasterDest);
+  }
+ private:
+  DataFieldTemplates* m_templates;
+  const bool m_isSet;
+  const bool m_isMasterDest;
+ public:
+  DataField* m_fields;
+};
+
+
 int main() {
+  // entry: definition, decoded value, master data, slave data, flags
+  // definition: name,part,type[:len][,[divisor|values][,[unit][,[comment]]]]
+  unsigned int baseLine = __LINE__+1;
   string checks[][5] = {
-    // entry: definition, decoded value, master data, slave data, flags
-    // definition: name,part,type[:len][,[divisor|values][,[unit][,[comment]]]]
     {"x,,ign:10",  "",                              "10fe07000a00000000000000000000", "00", ""},
     {"x,,ign:*",   "",                              "10fe07000a00000000000000000000", "00", "W"},
     {"x,,ign,2",   "",                              "",                               "",   "c"},
@@ -59,8 +102,7 @@ int main() {
     {"x,,str:10",  "          ",                    "10fe07000a20202020202020202020", "00", ""},
     {"x,,str:10",  "",                              "10fe07000a20202020202020202020", "00", "R"},
     {"x,,str:11",  "",                              "10fe07000a20202020202020202020", "00", "rW"},
-    {"x,,str:24",  "abcdefghijklmnopqrstuvwx",      "10fe0700186162636465666768696a6b6c6d6e6f707172737475767778",
+    {"x,,str:24",  "abcdefghijklmnopqrstuvwx",      "10fe0700186162636465666768696a6b6c6d6e6f707172737475767778", "00", ""},
-        "00", ""},
     {"x,,str:*",   "abcde",                         "10fe0700056162636465",           "00", ""},
     {"x,,str,2",   "",                              "",                               "",   "c"},
     {"x,,str:10,=dummy", "",                        "10fe07000a48616c6c6f2044752120", "00", "W"},
@@ -74,8 +116,7 @@ int main() {
     {"x,,nts:10",  "",                              "10fe07000a00000000000000000000", "00", ""},
     {"x,,nts:10",  "abc",                           "10fe07000a6162630065666768696a", "00", "W"},
     {"x,,nts:11",  "",                              "10fe07000a20202020202020202020", "00", "rW"},
-    {"x,,nts:24",  "abcdefghijklmnopqrstuvwx",      "10fe0700186162636465666768696a6b6c6d6e6f707172737475767778",
+    {"x,,nts:24",  "abcdefghijklmnopqrstuvwx",      "10fe0700186162636465666768696a6b6c6d6e6f707172737475767778", "00", ""},
-        "00", ""},
     {"x,,nts:*",   "abcde",                         "10fe0700056162636465",           "00", "W"},
     {"x,,nts:*",   "abcde",                         "10fe070006616263646500",         "00", ""},
     {"x,,nts,2",   "",                              "",                               "",   "c"},
@@ -396,8 +437,7 @@ int main() {
     {"x,,ulg", "-",          "10feffff04ffffffff", "00", ""},
     {"x,,ulg,10", "3.8",      "10feffff0426000000", "00", ""},
     {"x,,ulg,-10", "380",     "10feffff0426000000", "00", ""},
-    {"x,,ulg,0x0FF0F00F   =  VRT 350 ;0x33CCCC33=VRT 360;0x3CC3C33C=SD 17;0x66999966=SD 37;0x69969669=VRT 360+",
+    {"x,,ulg,0x0FF0F00F   =  VRT 350 ;0x33CCCC33=VRT 360;0x3CC3C33C=SD 17;0x66999966=SD 37;0x69969669=VRT 360+", "VRT 350",         "10feffff040FF0F00F", "00", ""},
-          "VRT 350",         "10feffff040FF0F00F", "00", ""},
     {"x,,ulg,0x=test", "",   "10feffff040FF0F00F", "00", "c"},
     {"x,,ulr", "38",         "10feffff0400000026", "00", ""},
     {"x,,ulr", "0",          "10feffff0400000000", "00", ""},
@@ -433,19 +473,12 @@ int main() {
     {"x,,bi3:2,0=off;1=on,ja/nein,Wahrheitswert", "x=on ja/nein [Wahrheitswert]", "10feffff0108", "00", "vvv"},
     {"x,,bi3:2,0=off;1=on,ja/nein,Wahrheitswert", "x=1 ja/nein [Wahrheitswert]", "10feffff0108", "00", "vvvn"},
     {"x,,bi3:2,0=off;1=on,ja/nein,Wahrheitswert", "\n    \"x\": {\"value\": \"on\"}", "10feffff0108", "00", "vj"},
-    {",,bi3:2,0=off;1=on,ja/nein,Wahrheitswert",  "\n    \"0\": {\"name\": \"\", \"value\": \"on\"}", "10feffff0108",
+    {",,bi3:2,0=off;1=on,ja/nein,Wahrheitswert",  "\n    \"0\": {\"name\": \"\", \"value\": \"on\"}", "10feffff0108", "00", "vj"},
-        "00", "vj"},
+    {",,bi3:2,0=off;1=on,ja/nein,Wahrheitswert",  "\n    \"0\": {\"name\": \"\", \"value\": \"on\"}", "10feffff0108", "00", "j"},
-    {",,bi3:2,0=off;1=on,ja/nein,Wahrheitswert",  "\n    \"0\": {\"name\": \"\", \"value\": \"on\"}", "10feffff0108",
+    {"x,,bi3:2,0=off;1=on,ja/nein,Wahrheitswert", "\n    \"x\": {\"value\": \"on\", \"unit\": \"ja/nein\", \"comment\": \"Wahrheitswert\"}", "10feffff0108", "00", "vvvj"},
-        "00", "j"},
-    {"x,,bi3:2,0=off;1=on,ja/nein,Wahrheitswert",
-        "\n    \"x\": {\"value\": \"on\", \"unit\": \"ja/nein\", \"comment\": \"Wahrheitswert\"}", "10feffff0108",
-        "00", "vvvj"},
     {"x,,bi3:2,0=off;1=on,ja/nein,Wahrheitswert", "\n    \"x\": {\"value\": 1}", "10feffff0108", "00", "vnj"},
-    {"x,,bi3:2,0=off;1=on,ja/nein,Wahrheitswert",
+    {"x,,bi3:2,0=off;1=on,ja/nein,Wahrheitswert", "\n    \"x\": {\"value\": 1, \"unit\": \"ja/nein\", \"comment\": \"Wahrheitswert\"}", "10feffff0108", "00", "vvvnj"},
-        "\n    \"x\": {\"value\": 1, \"unit\": \"ja/nein\", \"comment\": \"Wahrheitswert\"}", "10feffff0108", "00",
+    {"x,,bi3:2,0=off;1=on,ja/nein,Wahrheitswert", "\n    \"0\": {\"name\": \"x\", \"value\": 1}", "10feffff0108", "00", "nj"},
-        "vvvnj"},
-    {"x,,bi3:2,0=off;1=on,ja/nein,Wahrheitswert", "\n    \"0\": {\"name\": \"x\", \"value\": 1}", "10feffff0108", "00",
-        "nj"},
     {"x,,uch,1=test;2=high;3=off;0x10=on", "on", "10feffff0110", "00", ""},
     {"x,s,uch", "3", "1050ffff00", "0103", ""},
     {"x,,d2b,,°C,Aussentemperatur", "x=18.004 °C [Aussentemperatur]", "10fe0700090112", "00", "vvv"},
@@ -468,10 +501,14 @@ int main() {
     {"x,,temp,,,,y,,d1c", "18.004;9.5", "10fe070003011213", "00", ""},  // reference to template, normal def
     {"x,,temp;HEX:2", "18.004;13 14", "10fe07000401121314", "00", ""},  // reference to template and base type
     {"x,,temp;HEX:2", "temp=18.004;=13 14", "10fe07000401121314", "00", "v"},  // reference to template and base type
-    // reference to template and base type
+    {"x,,temp:degrees;HEX:2", "degrees=18.004;=13 14", "10fe07000401121314", "00", "v"},  // reference to template and base type
-    {"x,,temp:degrees;HEX:2", "degrees=18.004;=13 14", "10fe07000401121314", "00", "v"},
   };
   DataFieldTemplates* templates = new DataFieldTemplates();
+  unsigned int lineNo = 0;
+  istringstream dummystr("#");
+  string errorDescription;
+  vector<string> row;
+  templates->readLineFromStream(dummystr, errorDescription, "inline", lineNo, row);
   DataField* fields = NULL;
   for (size_t i = 0; i < sizeof(checks) / sizeof(checks[0]); i++) {
     string check[5] = checks[i];
@@ -514,18 +551,34 @@ int main() {
     bool numeric = flags.find('n') != string::npos;
     bool isTemplate = flags.find('t') != string::npos;
     string item;
-    vector<string> entries;
-
-    while (getline(isstr, item, FIELD_SEPARATOR))
-      entries.push_back(item);
 
     if (fields != NULL) {
       delete fields;
       fields = NULL;
     }
-    vector<string>::iterator it = entries.begin();
+    if (isTemplate) {
-    result = DataField::create(it, entries.end(), templates, fields, isSet, isTemplate,
+      lineNo = baseLine + i;
-        !isTemplate && (mstr[1] == BROADCAST || isMaster(mstr[1])));
+      result = templates->readLineFromStream(isstr, errorDescription, "inline", lineNo, row, false);
+      if (result != RESULT_OK) {
+        cout << "\"" << check[0] << "\": template read error: " << getResultCode(result) << ", "
+            << errorDescription << endl;
+        error = true;
+      }
+      continue;
+    }
+    TestReader reader{templates, isSet, mstr[1] == BROADCAST || isMaster(mstr[1])};
+    lineNo = 0;
+    dummystr = istringstream("#");
+    result = reader.readLineFromStream(dummystr, errorDescription, "inline", lineNo, row);
+    if (result != RESULT_OK) {
+      cout << "\"" << check[0] << "\": reader header error: " << getResultCode(result) << ", " << errorDescription
+          << endl;
+      error = true;
+      continue;
+    }
+    lineNo = baseLine + i;
+    result = reader.readLineFromStream(isstr, errorDescription, "", lineNo, row);
+    fields = reader.m_fields;
     if (failedCreate) {
       if (result == RESULT_OK) {
         cout << "\"" << check[0] << "\": failed create error: unexpectedly succeeded" << endl;
@@ -536,7 +589,7 @@ int main() {
       continue;
     }
     if (result != RESULT_OK) {
-      cout << "\"" << check[0] << "\": create error: " << getResultCode(result) << endl;
+      cout << "\"" << check[0] << "\": create error: " << getResultCode(result) << ", " << errorDescription << endl;
       error = true;
       continue;
     }
@@ -545,26 +598,9 @@ int main() {
       error = true;
       continue;
     }
-    if (it != entries.end()) {
-      cout << "\"" << check[0] << "\": create error: trailing input" << endl;
-      error = true;
-      continue;
-    }
     cout << "\"" << check[0] << "\"=\"";
     fields->dump(cout);
     cout << "\": create OK" << endl;
-    if (isTemplate) {
-      // store new template
-      result = templates->add(fields, "", true);
-      if (result == RESULT_OK) {
-        fields = NULL;
-        cout << "  store template OK" << endl;
-      } else {
-        cout << "  store template error: " << getResultCode(result) << endl;
-        error = true;
-      }
-      continue;
-    }
 
     ostringstream output;
     MasterSymbolString writeMstr;

src/lib/ebus/test/test_filereader.cpp

@@ -47,50 +47,236 @@ void verify(bool expectFailMatch, string type, string input,
   }
 }
 
-int main() {
+string resultlines[][3] = {
-  istringstream ifs(
+  {"col 1", "col 2", "col 3"},
-    "line 1 col 1,line 1 col 2,line 1 col 3\n"
+  {"line 2 col 1", "line 2 col 2", "line 2 \"col 3\";default of col 3"},
-    "line 2 col 1,\"line 2 col 2\",\"line 2 \"\"col 3\"\"\"\n"
+  {"", "", ""},
-    "line 4 col 1,\"line 4 col 2 part 1\n"
+  {"line 4 col 1", "line 4 col 2 part 1;line 4 col 2 part 2", "line 4 col 3;default of col 3"},
-    "line 4 col 2 part 2\",line 4 col 3\n"
+  {"", "", ""},
-    ",,,\n"
+  {"line 6 col 1", "", "line 6 col 3;default of col 3"},
-    "line 6 col 1,,line 6 col 3\n"
+  {"", "", ""},
-    "line 8 col 1,\"line 8 col 2 part 1;\n"
+  {"line 8 col 1", "line 8 col 2 part 1;line 8 col 2 part 2", "line 8 col 3;default of col 3"},
-    "line 8 col 2 part 2\",line 8 col 3\n"
+};
-  );
-  string resultlines[][3] = {
-    {"line 1 col 1", "line 1 col 2", "line 1 col 3"},
-    {"line 2 col 1", "line 2 col 2", "line 2 \"col 3\""},
-    {"", "", ""},
-    {"line 4 col 1", "line 4 col 2 part 1;line 4 col 2 part 2", "line 4 col 3"},
-    {"", "", ""},
-    {"line 6 col 1", "", "line 6 col 3"},
-    {"", "", ""},
-    {"line 8 col 1", "line 8 col 2 part 1;line 8 col 2 part 2", "line 8 col 3"},
-  };
-  unsigned int lineNo = 0;
-  vector<string> row;
 
-  while (FileReader::splitFields(ifs, row, lineNo)) {
+string resultsublines[][2][4] = {
-    cout << "line " << static_cast<unsigned>(lineNo) << ": split OK" << endl;
+  {},
-    string resultline[3] = resultlines[lineNo-1];
+  {{"subcol 1", "line 2 subcol 1", "subcol 2", "line 2 subcol 2;default of sub 0 subcol 2"},{"subcol 2", "line 2 subcol 2", "subcol 3", "line 2 subcol 3"}},
+  {},
+  {{"subcol 1", "line 4 subcol 1", "subcol 2", "line 4 subcol 2;default of sub 0 subcol 2"},{"subcol 2", "line 4 subcol 2", "subcol 3", "line 4 subcol 3"}},
+  {},
+  {{"subcol 1", "line 6 subcol 1", "subcol 2", "line 6 subcol 2;default of sub 0 subcol 2"},{"subcol 2", "line 6 subcol 2", "subcol 3", "line 6 subcol 3"}},
+  {},
+  {{"subcol 1", "line 8 subcol 1", "subcol 2", "line 8 subcol 2;default of sub 0 subcol 2"},{"subcol 2", "line 8 subcol 2", "subcol 3", "line 8 subcol 3"}},
+};
+
+static unsigned int baseLine = 0;
+
+class NoopReader : public FileReader {
+ public:
+  result_t addFromFile(vector<string>& row, string& errorDescription,
+      const string filename, unsigned int lineNo) override {
+    return RESULT_OK;
+  }
+};
+
+class TestReader : public MappedFileReader {
+ public:
+  TestReader(size_t expectedCols) : MappedFileReader::MappedFileReader(false), m_expectedCols(expectedCols) {}
+  result_t getFieldMap(vector<string>& row, string& errorDescription) override {
+    if (row.size() == m_expectedCols) {
+      cout << "get field map: split OK" << endl;
+      return RESULT_OK;
+    }
+    cout << "get field map: error got " << static_cast<unsigned>(row.size()) << " columns, expected " <<
+        static_cast<unsigned>(m_expectedCols) << endl;
+    return RESULT_ERR_EOF;
+  }
+  result_t addFromFile(map<string, string>& row, vector< map<string, string> >& subRows,
+      string& errorDescription, const string filename, unsigned int lineNo) override {
+    if (row.empty() || (m_expectedCols == 3) != subRows.empty()) {
+      cout << "read line " << static_cast<unsigned>(baseLine + lineNo) << ": read error: got "
+          << static_cast<unsigned>(row.size()) << "/3 main, " << static_cast<unsigned>(subRows.size())
+          << (m_expectedCols == 3 ? "/0 sub" : "/>0 sub") << endl;
+      return RESULT_ERR_EOF;
+    }
+    if (lineNo < 2 || lineNo >= 1+sizeof(resultlines)/sizeof(string[3])) {
+      cout << "read line " << static_cast<unsigned>(baseLine + lineNo) << ": error invalid line" << endl;
+      return RESULT_ERR_INVALID_ARG;
+    }
+    cout << "read line " << static_cast<unsigned>(baseLine + lineNo) << ": split OK" << endl;
+    string resultline[3] = resultlines[lineNo - 1];
     if (row.empty()) {
       cout << "  result empty";
       if (resultline[0] == "") {
         cout << ": OK" << endl;
       } else {
         cout << ": error" << endl;
+        return RESULT_ERR_INVALID_ARG;
+      }
+      return RESULT_EMPTY;
+    }
+
+    bool error = false;
+    string colnames[3] = resultlines[0];
+    map<string, string>& defaults = getDefaults()[""];
+    for (size_t colIdx = 0; colIdx < 3; colIdx++) {
+      string col = colnames[colIdx];
+      string got = row[col] + defaults[col];
+      string expect = resultline[colIdx];
+      ostringstream type;
+      type << "line " << static_cast<unsigned>(baseLine + lineNo) << " column \"" << col << "\"";
+      bool match = got == expect;
+      verify(false, type.str(), expect, match, expect, got);
+      if (!match) {
         error = true;
       }
-      continue;
     }
-    for (vector<string>::iterator it = row.begin(); it != row.end(); it++) {
+    if (row.size() > 3) {
-      string got = *it;
-      string expect = resultline[distance(row.begin(), it)];
       ostringstream type;
-      type << "line " << static_cast<unsigned>(lineNo) << " col " << static_cast<size_t>(distance(row.begin(), it)+1);
+      type << "line " << static_cast<unsigned>(baseLine + lineNo);
-      verify(false, type.str(), expect, got == expect, expect, got);
+      verify(false, type.str(), "", false, "", "extra column");
+      error = true;
     }
+
+    for (size_t subIdx = 0; subIdx < subRows.size(); subIdx++) {
+      string resultsubline[4] = resultsublines[lineNo - 1][subIdx];
+      row = subRows[subIdx];
+      if (row.empty()) {
+        cout << "  sub " << subIdx << " result empty";
+        if (resultline[0] == "") {
+          cout << ": OK" << endl;
+        } else {
+          cout << ": error" << endl;
+          return RESULT_ERR_INVALID_ARG;
+        }
+        return RESULT_EMPTY;
+      }
+
+      vector< map<string, string> >& subDefaults = getSubDefaults()[""];
+      for (size_t colIdx = 0; colIdx < 2; colIdx++) {
+        string col = resultsubline[colIdx*2];
+        string got = row[col];
+        if (subIdx < subDefaults.size()) {
+          got += subDefaults[subIdx][col];
+        }
+        string expect = resultsubline[colIdx*2+1];
+        ostringstream type;
+        type << "line " << static_cast<unsigned>(baseLine + lineNo) << " sub " << subIdx << " column \"" << col << "\"";
+        bool match = got == expect;
+        verify(false, type.str(), expect, match, expect, got);
+        if (!match) {
+          error = true;
+        }
+      }
+      if (row.size() > 2) {
+        ostringstream type;
+        type << "line " << static_cast<unsigned>(baseLine + lineNo) << " sub " << subIdx;
+        verify(false, type.str(), "", false, "", "extra sub column");
+        error = true;
+      }
+    }
+    return error ? RESULT_ERR_INVALID_ARG : RESULT_OK;
+  }
+ private:
+  size_t m_expectedCols;
+};
+
+
+int main(int argc, char** argv) {
+  if (argc > 1) {
+    NoopReader reader;
+    for (int argpos = 1; argpos < argc; argpos++) {
+      size_t hash = 0, size = 0;
+      time_t time = 0;
+      string errorDescription;
+      result_t result = reader.readFromFile(argv[argpos], errorDescription, false, NULL, &hash, &size, &time);
+      cout << argv[argpos] << " ";
+      if (result != RESULT_OK) {
+        cout << getResultCode(result) << ", " << errorDescription << endl;
+        error = true;
+        continue;
+      }
+      FileReader::formatHash(hash, cout);
+      cout << " " << size << " " << time << endl;
+    }
+    return error ? 1 : 0;
+  }
+  baseLine = __LINE__+1;
+  istringstream ifs(
+    "col 1,col 2,col 3\n"
+    "line 2 col 1,\"line 2 col 2\",\"line 2 \"\"col 3\"\";default of col 3\"\n"
+    "line 4 col 1,\"line 4 col 2 part 1\n"
+    "line 4 col 2 part 2\",line 4 col 3;default of col 3\n"
+    ",,,\n"
+    "line 6 col 1,,line 6 col 3;default of col 3\n"
+    "line 8 col 1,\"line 8 col 2 part 1;\n"
+    "line 8 col 2 part 2\",line 8 col 3;default of col 3\n"
+  );
+  size_t hash = 0, size = 0, expectHash = 0xfd58724c2984595d, expectSize = 301;
+  TestReader reader{3};
+  unsigned int lineNo = 0;
+  vector<string> row;
+  string errorDescription;
+  while (ifs.peek() != EOF) {
+    istringstream str;
+    result_t result = reader.readLineFromStream(ifs, errorDescription, "", lineNo, row, true, &hash, &size);
+    if (result != RESULT_OK) {
+      cout << "  error " << getResultCode(result) << endl;
+      error = true;
+    }
+  }
+  if (hash == expectHash) {
+    cout << "hash OK" << endl;
+  } else {
+    cout << "hash error: got 0x" << hex << hash << ", expected 0x" << expectHash << dec << endl;
+    error = true;
+  }
+  if (size == expectSize) {
+    cout << "size OK" << endl;
+  } else {
+    cout << "size error: got " << size << ", expected " << expectSize << endl;
+    error = true;
+  }
+
+  ifs.clear();
+  baseLine = __LINE__+1;
+  ifs.str(
+    "col 1,col 2,col 3,*subcol 1,subcol 2,*subcol 2,subcol 3\n"
+    "line 2 col 1,\"line 2 col 2\",\"line 2 \"\"col 3\"\"\",line 2 subcol 1,line 2 subcol 2,line 2 subcol 2,line 2 subcol 3\n"
+    "line 4 col 1,\"line 4 col 2 part 1\n"
+    "line 4 col 2 part 2\",line 4 col 3,line 4 subcol 1,line 4 subcol 2,line 4 subcol 2,line 4 subcol 3\n"
+    ",,,\n"
+    "line 6 col 1,,line 6 col 3,line 6 subcol 1,line 6 subcol 2,line 6 subcol 2,line 6 subcol 3\n"
+    "line 8 col 1,\"line 8 col 2 part 1;\n"
+    "line 8 col 2 part 2\",line 8 col 3,line 8 subcol 1,line 8 subcol 2,line 8 subcol 2,line 8 subcol 3\n"
+  );
+  hash = 0, size = 0, expectHash = 0xec31914e32c6fb6f, expectSize = 527;
+  TestReader reader2{7};
+  lineNo = 0;
+  map<string, string> defaults;
+  reader2.getDefaults("")["col 3"] = ";default of col 3";
+  vector< map<string, string> >& subDefaults = reader2.getSubDefaults("");
+  subDefaults.resize(1);
+  subDefaults[0]["subcol 2"] = ";default of sub 0 subcol 2";
+  while (ifs.peek() != EOF) {
+    istringstream str;
+    result_t result = reader2.readLineFromStream(ifs, errorDescription, "", lineNo, row, true, &hash, &size);
+    if (result != RESULT_OK) {
+      cout << "  error " << getResultCode(result) << endl;
+      error = true;
+    }
+  }
+  if (hash == expectHash) {
+    cout << "hash OK" << endl;
+  } else {
+    cout << "hash error: got 0x" << hex << hash << ", expected 0x" << expectHash << dec << endl;
+    error = true;
+  }
+  if (size == expectSize) {
+    cout << "size OK" << endl;
+  } else {
+    cout << "size error: got " << size << ", expected " << expectSize << endl;
+    error = true;
   }
 
   return error ? 1 : 0;

src/lib/ebus/test/test_message.cpp

@@ -25,11 +25,16 @@
 #include "lib/ebus/message.h"
 
 using namespace ebusd;
+using std::cout;
+using std::endl;
+
+static bool error = false;
 
 void verify(bool expectFailMatch, string type, string input,
     bool match, string expectStr, string gotStr) {
   if (expectFailMatch) {
     if (match) {
+      error = true;
       cout << "  failed " << type << " match >" << input
               << "< error: unexpectedly succeeded" << endl;
     } else {
@@ -38,6 +43,7 @@ void verify(bool expectFailMatch, string type, string input,
   } else if (match) {
     cout << "  " << type << " match >" << input << "< OK" << endl;
   } else {
+    error = true;
     cout << "  " << type << " match >" << input << "< error: got >"
             << gotStr << "<, expected >" << expectStr << "<" << endl;
   }
@@ -46,6 +52,7 @@ void verify(bool expectFailMatch, string type, string input,
 DataFieldTemplates* templates = NULL;
 
 namespace ebusd {
+
 DataFieldTemplates* getTemplates(const string filename) {
   if (filename == "") {  // avoid compiler warning
     return templates;
@@ -59,8 +66,9 @@ int main() {
   // field:     name,part,type[:len][,[divisor|values][,[unit][,[comment]]]]
   // template:  name,type[:len][,[divisor|values][,[unit][,[comment]]]]
   // condition: name,circuit,messagename,[comment],[fieldname],[ZZ],values
+  // "message", "decoded", "master", "slave", "flags"
+  unsigned int baseLine = __LINE__+1;
   string checks[][5] = {
-    // "message", "decoded", "master", "slave", "flags"
     {"date,HDA:3,,,Datum", "", "", "", "template"},
     {"time,VTI,,,", "", "", "", "template"},
     {"dcfstate,UCH,0=nosignal;1=ok;2=sync;3=valid,,", "", "", "", "template"},
@@ -73,103 +81,75 @@ int main() {
     {"pumpstate,UCH,0=off;1=on;2=overrun,,Pumpenstatus", "", "", "", "template"},
     {"tempsensor,temp;sensor,,Temperatursensor", "", "", "", "template"},
     {"tempsensorc,temp;sensorc,,Temperatursensor", "", "", "", "template"},
-    {"r,,Status01,VL/RL/AussenTemp/VLWW/SpeicherTemp/Status,,08,B511,01,,,temp1;temp1;temp2;temp1;temp1;pumpstate",
+    {"r,,Status01,VL/RL/AussenTemp/VLWW/SpeicherTemp/Status,,08,B511,01,,,temp1;temp1;temp2;temp1;temp1;pumpstate", "28.0;24.0;4.938;35.0;41.0;4", "ff08b5110101", "093830f00446520400ff", "d"},
-        "28.0;24.0;4.938;35.0;41.0;4", "ff08b5110101", "093830f00446520400ff", "d"},
+    {"r,message circuit,message name,message comment,,25,B509,0d2800,,,tempsensor", "temp=-14.00 Temperatursensor [Temperatur];sensor=ok [Fühlerstatus]", "ff25b509030d2800", "0320ff00", "D"},
-    {"r,message circuit,message name,message comment,,25,B509,0d2800,,,tempsensor",
+    {"r,message circuit,message name,message comment,,25,B509,0d2800,,,tempsensor,,field unit,field comment", "temp=-14.00 field unit [field comment];sensor=ok [Fühlerstatus]", "ff25b509030d2800", "0320ff00", "D"},
-        "temp=-14.00 Temperatursensor [Temperatur];sensor=ok [Fühlerstatus]", "ff25b509030d2800", "0320ff00", "mD"},
+    {"r,message circuit,message name,message comment,,25,B509,0d2800,,,tempsensor,,field unit,field comment", "\n    \"temp\": {\"value\": -14.00},\n    \"sensor\": {\"value\": \"ok\"}", "ff25b509030d2800", "0320ff00", "j"},
-    {"r,message circuit,message name,message comment,,25,B509,0d2800,,,tempsensor,,field unit,field comment",
+    {"r,message circuit,message name,message comment,,25,B509,0d2800,,,tempsensor,,field unit,field comment", "\n    \"temp\": {\"value\": -14.00, \"unit\": \"field unit\", \"comment\": \"field comment\"},\n" "    \"sensor\": {\"value\": \"ok\", \"comment\": \"Fühlerstatus\"}", "ff25b509030d2800", "0320ff00", "J"},
-        "temp=-14.00 field unit [field comment];sensor=ok [Fühlerstatus]", "ff25b509030d2800", "0320ff00", "mD"},
+    {"r,message circuit,message name,message comment,,25,B509,0d2800,,,temp,,field unit,field comment,,,sensor", "temp=-14.00 field unit [field comment];sensor=ok [Fühlerstatus]", "ff25b509030d2800", "0320ff00", "D"},
-    {"r,message circuit,message name,message comment,,25,B509,0d2800,,,tempsensor,,field unit,field comment",
+    {"r,message circuit,message name,message comment,,25,B509,0d2800,,,D2C,,°C,Temperatur,,,sensor", "\n    \"0\": {\"name\": \"\", \"value\": -14.00},\n    \"1\": {\"name\": \"sensor\", \"value\": \"ok\"}", "ff25b509030d2800", "0320ff00", "j"},
-        "\n    \"temp\": {\"value\": -14.00},\n    \"sensor\": {\"value\": \"ok\"}", "ff25b509030d2800", "0320ff00",
+    {"r,,name,,,25,B509,0d2800,,,tempsensorc", "-14.00", "ff25b509030d2800", "0320ff55", ""},
-        "mj"},
+    {"r,,name,,,25,B509,0d28,,m,sensorc,,,,,,temp", "-14.00", "ff25b509030d2855", "0220ff", ""},
-    {"r,message circuit,message name,message comment,,25,B509,0d2800,,,tempsensor,,field unit,field comment",
-        "\n    \"temp\": {\"value\": -14.00, \"unit\": \"field unit\", \"comment\": \"field comment\"},\n"
-        "    \"sensor\": {\"value\": \"ok\", \"comment\": \"Fühlerstatus\"}", "ff25b509030d2800", "0320ff00", "mJ"},
-    {"r,message circuit,message name,message comment,,25,B509,0d2800,,,temp,,field unit,field comment,,,sensor",
-        "temp=-14.00 field unit [field comment];sensor=ok [Fühlerstatus]", "ff25b509030d2800", "0320ff00", "mD"},
-    {"r,message circuit,message name,message comment,,25,B509,0d2800,,,D2C,,°C,Temperatur,,,sensor",
-        "\n    \"0\": {\"name\": \"\", \"value\": -14.00},\n    \"1\": {\"name\": \"sensor\", \"value\": \"ok\"}",
-        "ff25b509030d2800", "0320ff00", "mj"},
-    {"r,,name,,,25,B509,0d2800,,,tempsensorc", "-14.00", "ff25b509030d2800", "0320ff55", "m"},
-    {"r,,name,,,25,B509,0d28,,m,sensorc,,,,,,temp", "-14.00", "ff25b509030d2855", "0220ff", "m"},
     {"u,,first,,,fe,0700,,x,,bda", "26.10.2014", "fffe07000426100614", "00", "p"},
     {"u,broadcast,hwStatus,,,fe,b505,27,,,UCH,,,,,,UCH,,,,,,UCH,,,", "0;19;0", "10feb505042700130097", "00", ""},
-    {"w,,first,,,15,b509,0400,date,,bda", "26.10.2014", "ff15b50906040026100614", "00", "m"},
+    {"w,,first,,,15,b509,0400,date,,bda", "26.10.2014", "ff15b50906040026100614", "00", ""},
-    {"w,,first,,,15,b509", "", "ff15b50900", "00", "m"},
+    {"w,,first,,,15,b509", "", "ff15b50900", "00", ""},
-    {"w,,,,,,b505,2d", "", "", "", "defaults"},
+    {"*w,,,,,,b505,2d", "", "", "", ""},
-    {"w,,offset,,,50,,,,,temp", "0.50", "ff50b505042d080000", "00", "md"},
+    {"w,,offset,,,50,,,,,temp", "0.50", "ff50b505042d080000", "00", "kd"},
-    {"r,ehp,time,,,08,b509,0d2800,,,time", "15:00:17", "ff08b509030d2800", "0311000f", "md"},
+    {"r,ehp,time,,,08,b509,0d2800,,,time", "15:00:17", "ff08b509030d2800", "0311000f", "d"},
     {"r,ehp,time,,,08;10,b509,0d2800,,,time", "", "", "", "c"},
-    {"r,ehp,time,,,08;09,b509,0d2800,,,time", "15:00:17", "ff08b509030d2800", "0311000f", "md*"},
+    {"r,ehp,time,,,08;09,b509,0d2800,,,time", "15:00:17", "ff08b509030d2800", "0311000f", "d*"},
-    {"r,ehp,date,,,08,b509,0d2900,,,date", "23.11.2014", "ff08b509030d2900", "03170b0e", "md"},
+    {"r,ehp,date,,,08,b509,0d2900,,,date", "23.11.2014", "ff08b509030d2900", "03170b0e", "d"},
-    {"r,700,date,,,15,b524,020000003400,,,IGN:4,,,,,,date", "23.11.2015", "ff15b52406020000003400", "0703003400170b0f",
+    {"r,700,date,,,15,b524,020000003400,,,IGN:4,,,,,,date", "23.11.2015", "ff15b52406020000003400", "0703003400170b0f", "d"},
-        "d"},
+    {"", "23.11.2015", "ff15b52406020000003400", "0703003400170b0f", "kd"},
-    {"r,700,time,,,15,b524,030000003500,,,IGN:4,,,,,,HTI", "12:29:06", "ff15b52406030000003500", "07030035000c1d06",
+    {"r,700,time,,,15,b524,030000003500,,,IGN:4,,,,,,HTI", "12:29:06", "ff15b52406030000003500", "07030035000c1d06", "d"},
-        "d"},
+    {"", "12:29:06", "ff15b52406030000003500", "07030035000c1d06", "kd"},
-    {"", "23.11.2015", "ff15b52406020000003400", "0703003400170b0f", "d"},
+    {"w,700,date,,,15,b524,020000003400,,,date", "23.11.2015", "ff15b52409020000003400170b0f", "00", ""},
-    {"", "12:29:06", "ff15b52406030000003500", "07030035000c1d06", "d"},
+    {"r,ehp,error,,,08,b509,0d2800,index,m,UCH,,,,,,time", "3;15:00:17", "ff08b509040d280003", "0311000f", "di"},
-    {"w,700,date,,,15,b524,020000003400,,,date", "23.11.2015", "ff15b52409020000003400170b0f", "00", "m"},
+    {"r,ehp,error,,,08,b509,0d2800,index,m,UCH,,,,,,time", "index=3;time=15:00:17", "ff08b509040d280003", "0311000f", "D"},
-    {"r,ehp,error,,,08,b509,0d2800,index,m,UCH,,,,,,time", "3;15:00:17", "ff08b509040d280003", "0311000f", "mdi"},
+    {"u,ehp,ActualEnvironmentPower,Energiebezug,,08,B509,29BA00,,s,IGN:2,,,,,s,power", "8", "1008b5090329ba00", "03ba0008", "p"},
-    {"r,ehp,error,,,08,b509,0d2800,index,m,UCH,,,,,,time", "index=3;time=15:00:17", "ff08b509040d280003", "0311000f",
+    {"uw,ehp,test,Test,,08,B5de,ab,,,power,,,,,s,hex:1", "8;39", "1008b5de02ab08", "0139", "p"},
-        "mD"},
-    {"u,ehp,ActualEnvironmentPower,Energiebezug,,08,B509,29BA00,,s,IGN:2,,,,,s,power", "8", "1008b5090329ba00",
-        "03ba0008", "pm"},
-    {"uw,ehp,test,Test,,08,B5de,ab,,,power,,,,,s,hex:1", "8;39", "1008b5de02ab08", "0139", "pm"},
     {"u,ehp,hwTankTemp,Speichertemperatur IST,,25,B509,290000,,,IGN:2,,,,,,tempsensor", "", "", "", "M"},
-    {"", "55.50;ok", "1025b50903290000", "050000780300", "d"},
+    {"", "55.50;ok", "1025b50903290000", "050000780300", "kd"},
-    {"r,ehp,datetime,Datum Uhrzeit,,50,B504,00,,,dcfstate,,,,time,,BTI,,,,date,,BDA,,,,temp,,temp2",
+    {"r,ehp,datetime,Datum Uhrzeit,,50,B504,00,,,dcfstate,,,,time,,BTI,,,,date,,BDA,,,,temp,,temp2", "valid;08:24:51;31.12.2014;-0.875", "1050b5040100", "0a035124083112031420ff", "d" },
-        "valid;08:24:51;31.12.2014;-0.875", "1050b5040100", "0a035124083112031420ff", "md" },
+    {"r,ehp,bad,invalid pos,,50,B5ff,000102,,m,HEX:8;tempsensor;tempsensor;tempsensor;tempsensor;power;power,,,", "", "", "", "c" },
-    {"r,ehp,bad,invalid pos,,50,B5ff,000102,,m,HEX:8;tempsensor;tempsensor;tempsensor;tempsensor;power;power,,,", "",
+    {"r,ehp,bad,invalid pos,,50,B5ff,,,s,HEX:8;tempsensor;tempsensor;tempsensor;tempsensor;tempsensor;power;power,,,", "", "", "", "c" },
-        "", "", "c" },
-    {"r,ehp,bad,invalid pos,,50,B5ff,,,s,HEX:8;tempsensor;tempsensor;tempsensor;tempsensor;tempsensor;power;power,,,",
-        "", "", "", "c" },
     {"r,ehp,ApplianceCode,,,08,b509,0d4301,,,UCH,", "9", "ff08b509030d4301", "0109", "d" },
-    {"r,ehp,,,,08,b509,0d", "", "", "", "defaults" },
+    {"*r,ehp,,,,08,b509,0d", "", "", "", "" },
-    {"w,ehp,,,,08,b509,0e", "", "", "", "defaults" },
+    {"*w,ehp,,,,08,b509,0e", "", "", "", "" },
-    {"[brinetowater],ehp,ApplianceCode,,,,4;6;8;9;10", "", "", "", "condition" },
+    {"*[brinetowater],ehp,ApplianceCode,,,,4;6;8;9;10", "", "", "", "" },
-    {"[airtowater]r,ehp,notavailable,,,,,0100,,,uch", "1", "", "", "c" },
+    {"[airtowater]r,ehp,notavailable,,,,,0100,,,uch", "1", "", "", "kc" },
-    {"[brinetowater]r,ehp,available,,,,,0100,,,uch", "1", "ff08b509030d0100", "0101", "d" },
+    {"[brinetowater]r,ehp,available,,,,,0100,,,uch", "1", "ff08b509030d0100", "0101", "kd" },
     {"r,,x,,,,,\"6800\",,,UCH,,,bit0=\"comment, continued comment", "", "", "", "c" },
-    {"r,,x,,,,,\"6800\",,,UCH,,\"\",\"bit0=\"comment, continued comment\"", "=1 [bit0=\"comment, continued comment]",
+    {"r,,x,,,,,\"6800\",,,UCH,,\"\",\"bit0=\"comment, continued comment\"", "=1 [bit0=\"comment, continued comment]", "ff08b509030d6800", "0101", "D" },
-        "ff08b509030d6800", "0101", "mD" },
+    {"r,ehp,multi,,,,,0001:5;0002;0003,longname,,STR:15", "ABCDEFGHIJKLMNO", "ff08b509030d0001;ff08b509030d0003;ff08b509030d0002", "054142434445;054b4c4d4e4f;05464748494a", "dC" },
-    {"r,ehp,multi,,,,,0001:5;0002;0003,longname,,STR:15", "ABCDEFGHIJKLMNO",
+    {"r,ehp,multi,,,,,01;02;03,longname,,STR:15", "ABCDEFGHIJKLMNO", "ff08b509020d01;ff08b509020d03;ff08b509020d02", "084142434445464748;054b4c4d4e4f;02494a", "dC" },
-        "ff08b509030d0001;ff08b509030d0003;ff08b509030d0002", "054142434445;054b4c4d4e4f;05464748494a", "mdC" },
+    {"w,ehp,multi,,,,,01:8;02:2;03,longname,,STR:15", "ABCDEFGHIJKLMNO", "ff08b5090a0e014142434445464748;ff08b509040e02494a;ff08b509070e034b4c4d4e4f", "00;00;00", "dC" },
-    {"r,ehp,multi,,,,,01;02;03,longname,,STR:15", "ABCDEFGHIJKLMNO", "ff08b509020d01;ff08b509020d03;ff08b509020d02",
+    {"w,ehp,multi,,,,,01:8;02:2;0304,longname,,STR:15", "ABCDEFGHIJKLMNO", "ff08b5090a0e014142434445464748;ff08b509040e02494a;ff08b509070e034b4c4d4e4f", "00;00;00", "cC" },
-        "084142434445464748;054b4c4d4e4f;02494a", "mdC" },
+    {"r,ehp,scan,chained scan,,08,B509,24:9;25;26;27,,,IGN,,,,id4,,STR:28", "21074500100027790000000000N8", "ff08b5090124;ff08b5090125;ff08b5090126;ff08b5090127", "09003231303734353030;09313030303237373930;09303030303030303030;024E38", "dC" },
-    {"w,ehp,multi,,,,,01:8;02:2;03,longname,,STR:15", "ABCDEFGHIJKLMNO",
+    {"r,,x,,,,,6900,,,UCH,10,bar,,Bit7,,BI7:1,0=B70;1=B71,,,Bit6,,BI6:1,0=B60;1=B61", "1.9;B71;B61", "ff08b509030d6900", "03138040", "d" },
-        "ff08b5090a0e014142434445464748;ff08b509040e02494a;ff08b509070e034b4c4d4e4f", "00;00;00", "mdC" },
+    {"r,,x,,,,,6900,,,UCH,10,bar,,Bit7,,BI7:1,0=B70;1=B71,,,Bit6,,BI6:1,0=B60;1=B61", "1.9;B71;B60", "ff08b509030d6900", "0313ffbf", "d" },
-    {"w,ehp,multi,,,,,01:8;02:2;0304,longname,,STR:15", "ABCDEFGHIJKLMNO",
+    {"r,,x,,,,,6900,,,UCH,10,bar,,Bit7,,BI7:1,0=B70;1=B71,,,Bit6,,BI6:1,0=B60;1=B61", "1.9;B70;B61", "ff08b509030d6900", "03137fff", "d" },
-        "ff08b5090a0e014142434445464748;ff08b509040e02494a;ff08b509070e034b4c4d4e4f", "00;00;00", "cC" },
+    {"r,,x,,,,,6900,,,UCH,10,bar,,Bit7,,BI7:1,0=B70;1=B71,,,Bit6,,BI6:1,0=B60;1=B61", "1.9;B70;B60", "ff08b509030d6900", "03137fbf", "d" },
-    {"r,ehp,scan,chained scan,,08,B509,24:9;25;26;27,,,IGN,,,,id4,,STR:28", "21074500100027790000000000N8",
+    {"r,,x,,,,,6a00,,,UCH,10,bar,,Bit6,,BI6:1,0=B60;1=B61,,,Bit7,,BI7:1,0=B70;1=B71", "1.9;B61;B71", "ff08b509030d6a00", "0213ff", "d" },
-        "ff08b5090124;ff08b5090125;ff08b5090126;ff08b5090127",
+    {"r,,x,,,,,6a00,,,UCH,10,bar,,Bit6,,BI6:1,0=B60;1=B61,,,Bit7,,BI7:1,0=B70;1=B71", "1.9;B60;B71", "ff08b509030d6a00", "0213bf", "d" },
-        "09003231303734353030;09313030303237373930;09303030303030303030;024E38", "mdC" },
+    {"r,,x,,,,,6a00,,,UCH,10,bar,,Bit6,,BI6:1,0=B60;1=B61,,,Bit7,,BI7:1,0=B70;1=B71", "1.9;B61;B70", "ff08b509030d6a00", "02137f", "d" },
-    {"r,,x,,,,,6900,,,UCH,10,bar,,Bit7,,BI7:1,0=B70;1=B71,,,Bit6,,BI6:1,0=B60;1=B61", "1.9;B71;B61",
+    {"r,,x,,,,,6a00,,,UCH,10,bar,,Bit6,,BI6:1,0=B60;1=B61,,,Bit7,,BI7:1,0=B70;1=B71", "1.9;B60;B70", "ff08b509030d6a00", "02133f", "d" },
-        "ff08b509030d6900", "03138040", "md" },
+    {"*r,cir*cuit#level,na*me,com*ment,ff,75,b509,0d", "", "", "", "" },
-    {"r,,x,,,,,6900,,,UCH,10,bar,,Bit7,,BI7:1,0=B70;1=B71,,,Bit6,,BI6:1,0=B60;1=B61", "1.9;B71;B60",
+    {"r,CIRCUIT,NAME,COMMENT,,,,0100,field,,UCH", "r,cirCIRCUITcuit,naNAMEme,comCOMMENTment,ff,75,b509,0d0100,field,s,UCH,,,: field=42", "ff75b509030d0100", "012a", "DN"},
-        "ff08b509030d6900", "0313ffbf", "md" },
-    {"r,,x,,,,,6900,,,UCH,10,bar,,Bit7,,BI7:1,0=B70;1=B71,,,Bit6,,BI6:1,0=B60;1=B61", "1.9;B70;B61",
-        "ff08b509030d6900", "03137fff", "md" },
-    {"r,,x,,,,,6900,,,UCH,10,bar,,Bit7,,BI7:1,0=B70;1=B71,,,Bit6,,BI6:1,0=B60;1=B61", "1.9;B70;B60",
-        "ff08b509030d6900", "03137fbf", "md" },
-    {"r,,x,,,,,6a00,,,UCH,10,bar,,Bit6,,BI6:1,0=B60;1=B61,,,Bit7,,BI7:1,0=B70;1=B71", "1.9;B61;B71",
-        "ff08b509030d6900", "0213ff", "md" },
-    {"r,,x,,,,,6a00,,,UCH,10,bar,,Bit6,,BI6:1,0=B60;1=B61,,,Bit7,,BI7:1,0=B70;1=B71", "1.9;B60;B71",
-        "ff08b509030d6900", "0213bf", "md" },
-    {"r,,x,,,,,6a00,,,UCH,10,bar,,Bit6,,BI6:1,0=B60;1=B61,,,Bit7,,BI7:1,0=B70;1=B71", "1.9;B61;B70",
-        "ff08b509030d6900", "02137f", "md" },
-    {"r,,x,,,,,6a00,,,UCH,10,bar,,Bit6,,BI6:1,0=B60;1=B61,,,Bit7,,BI7:1,0=B70;1=B71", "1.9;B60;B70",
-        "ff08b509030d6900", "02133f", "md" },
-    {"r,cir*cuit#level,na*me,com*ment,ff,75,b509,0d", "", "", "", "defaults" },
-    {"r,CIRCUIT,NAME,COMMENT,,,,0100,field,,UCH",
-        "r,cirCIRCUITcuit,naNAMEme,comCOMMENTment,ff,75,b509,0d0100,field,s,UCH,,,: field=42",
-        "ff08b509030d0100", "012a", "mDN"},
   };
   templates = new DataFieldTemplates();
+  unsigned int lineNo = 0;
+  istringstream dummystr("#");
+  string errorDescription;
+  vector<string> row;
+  templates->readLineFromStream(dummystr, errorDescription, "inline", lineNo, row, false);
+  lineNo = 0;
   MessageMap* messages = new MessageMap();
+  dummystr = istringstream("#");
+  messages->readLineFromStream(dummystr, errorDescription, "inline", lineNo, row, false);
   vector< vector<string> > defaultsRows;
-  map<string, Condition*> &conditions = messages->getConditions();
   Message* message = NULL;
-  vector<Message*> deleteMessages;
   vector<MasterSymbolString*> mstrs;
   vector<SlaveSymbolString*> sstrs;
   mstrs.resize(1);
@@ -179,9 +159,7 @@ int main() {
     string inputStr = check[1];
     string flags = check[4];
     bool isTemplate = flags == "template";
-    bool isCondition = flags == "condition";
+    bool keepMessages = flags.find('k') != string::npos;
-    bool isDefaults = isCondition || flags == "defaults";
-    bool dontMap = flags.find('m') != string::npos;
     bool onlyMap = flags.find('M') != string::npos;
     bool failedCreate = flags.find('c') != string::npos;
     bool isChain = flags.find('C') != string::npos;
@@ -194,60 +172,31 @@ int main() {
     bool multi = flags.find('*') != string::npos;
     bool withInput = flags.find('i') != string::npos;
     result_t result = RESULT_EMPTY;
-    vector<string> entries;
+    istringstream isstr(check[0]);
-    istringstream ifs(check[0]);
+    lineNo = baseLine + i;
-    unsigned int lineNo = 0;
+    cout << "line " << (lineNo+1) << " ";
-    if (!FileReader::splitFields(ifs, entries, lineNo)) {
-      entries.clear();
-    }
     if (isTemplate) {
-      // store new template
+      result = templates->readLineFromStream(isstr, errorDescription, "inline", lineNo, row);
-      DataField* fields = NULL;
-      vector<string>::iterator it = entries.begin();
-      result = DataField::create(it, entries.end(), templates, fields, false, true, false);
       if (result != RESULT_OK) {
-        cout << "\"" << check[0] << "\": template fields create error: " << getResultCode(result) << endl;
+        cout << "\"" << check[0] << "\": template read error: " << getResultCode(result) << ", " << errorDescription
-      } else if (it != entries.end()) {
+            << endl;
-        cout << "\"" << check[0] << "\": template fields create error: trailing input "
+        error = true;
-            << static_cast<unsigned>(entries.end()-it) << endl;
-      } else {
-        cout << "\"" << check[0] << "\": create template OK" << endl;
-        result = templates->add(fields, "", true);
-        if (result == RESULT_OK) {
-          cout << "  store template OK" << endl;
-        } else {
-          cout << "  store template error: " << getResultCode(result) << endl;
-          delete fields;
-        }
       }
+      cout << "\"" << check[0] << "\": template read OK" << endl;
       continue;
     }
-    if (isDefaults) {
+    if (!keepMessages) {
+      messages->clear();
+    }
+    if (isstr.peek() == '*') {
       // store defaults or condition
-      vector<string>::iterator it = entries.begin();
+      result = messages->readLineFromStream(isstr, errorDescription, "inline", lineNo, row);
-      size_t oldSize = conditions.size();
-      result = messages->addDefaultFromFile(defaultsRows, entries, it, "", "", "", "no file", 1);
       if (result != RESULT_OK) {
-        cout << "\"" << check[0] << "\": defaults read error: " << getResultCode(result) << endl;
+        cout << "\"" << check[0] << "\": default read error: " << getResultCode(result) << ", " << errorDescription << endl;
-      } else if (it != entries.end()) {
+        error = true;
-        cout << "\"" << check[0] << "\": defaults read error: trailing input "
+        continue;
-            << static_cast<unsigned>(entries.end()-it) << endl;
-      } else {
-        cout << "\"" << check[0] << "\": read defaults OK" << endl;
-        if (isCondition) {
-          if (conditions.size() == oldSize) {
-            cout << "  create condition error" << endl;
-          } else {
-            result = messages->resolveConditions();
-            if (result != RESULT_OK) {
-              cout << "  resolve conditions error: " << getResultCode(result) << " " << messages->getLastError()
-                  << endl;
-            } else {
-              cout << "  resolve conditions OK" << endl;
-            }
-          }
-        }
       }
+      cout << "\"" << check[0] << "\": default read OK" << endl;
       continue;
     }
     if (isChain) {
@@ -260,10 +209,11 @@ int main() {
         } else if (mstrs[pos] != NULL) {
           delete mstrs[pos];
         }
-        mstrs[pos] = new MasterSymbolString;
+        mstrs[pos] = new MasterSymbolString();
         result = mstrs[pos]->parseHex(token);
         if (result != RESULT_OK) {
           cout << "\"" << check[0] << "\": parse \"" << token << "\" error: " << getResultCode(result) << endl;
+          error = true;
           break;
         }
         pos++;
@@ -281,21 +231,24 @@ int main() {
         result = sstrs[pos]->parseHex(token);
         if (result != RESULT_OK) {
           cout << "\"" << check[0] << "\": parse \"" << token << "\" error: " << getResultCode(result) << endl;
+          error = true;
           break;
         }
         pos++;
       }
       if (result != RESULT_OK) {
+        error = true;
         continue;
       }
     } else {
       if (mstrs[0] != NULL) {
         delete mstrs[0];
       }
-      mstrs[0] = new MasterSymbolString;
+      mstrs[0] = new MasterSymbolString();
       result = mstrs[0]->parseHex(check[2]);
       if (result != RESULT_OK) {
         cout << "\"" << check[0] << "\": parse \"" << check[2] << "\" error: " << getResultCode(result) << endl;
+        error = true;
         continue;
       }
       if (sstrs[0] != NULL) {
@@ -305,95 +258,72 @@ int main() {
       result = sstrs[0]->parseHex(check[3]);
       if (result != RESULT_OK) {
         cout << "\"" << check[0] << "\": parse \"" << check[3] << "\" error: " << getResultCode(result) << endl;
+        error = true;
         continue;
       }
     }
 
-    if (deleteMessages.size() > 0) {
+    if (isstr.peek() == EOF) {
-      for (vector<Message*>::iterator it = deleteMessages.begin(); it != deleteMessages.end(); it++) {
-        Message* deleteMessage = *it;
-        delete deleteMessage;
-      }
-      deleteMessages.clear();
-    }
-    if (entries.size() == 0) {
       message = messages->find(*mstrs[0]);
       if (message == NULL) {
         cout << "\"" << check[2] << "\": find error: NULL" << endl;
+        error = true;
         continue;
       }
       cout << "\"" << check[2] << "\": find OK" << endl;
     } else {
-      vector<string>::iterator it = entries.begin();
+      result = messages->readLineFromStream(isstr, errorDescription, "inline", lineNo, row);
-      string types = *it;
-      Condition* condition = NULL;
-      result = messages->readConditions(types, "no file", condition);
-      if (result == RESULT_OK) {
-        *it = types;
-        result = Message::create(it, entries.end(), &defaultsRows, condition, "no file", templates, deleteMessages);
-      }
       if (failedCreate) {
         if (result == RESULT_OK) {
           cout << "\"" << check[0] << "\": failed create error: unexpectedly succeeded" << endl;
+          error = true;
         } else {
           cout << "\"" << check[0] << "\": failed create OK" << endl;
         }
         continue;
       }
       if (result != RESULT_OK) {
-        cout << "\"" << check[0] << "\": create error: "
+        cout << "\"" << check[0] << "\": create error: " << getResultCode(result) << ", " << errorDescription << endl;
-            << getResultCode(result) << endl;
+        error = true;
-        printErrorPos(cout, entries.begin(), entries.end(), it, "", 0, result);
         continue;
       }
-      if (deleteMessages.size() == 0) {
+      if (messages->size() == 0) {
         cout << "\"" << check[0] << "\": create error: NULL" << endl;
+        error = true;
         continue;
       }
-      if (it != entries.end()) {
+      if (multi && messages->size() == 1) {
-        cout << "\"" << check[0] << "\": create error: trailing input " << static_cast<unsigned>(entries.end()-it)
-            << endl;
-        continue;
-      }
-      if (multi && deleteMessages.size() == 1) {
         cout << "\"" << check[0] << "\": create error: single message instead of multiple" << endl;
+        error = true;
         continue;
       }
-      if (!multi && deleteMessages.size() > 1) {
+      if (!multi && messages->size() > 1) {
         cout << "\"" << check[0] << "\": create error: multiple messages instead of single" << endl;
+        error = true;
         continue;
       }
       cout << "\"" << check[0] << "\": create OK" << endl;
-      if (!dontMap) {
+      if (onlyMap) {
-        result_t result = RESULT_OK;
+        continue;
-        for (vector<Message*>::iterator it = deleteMessages.begin(); it != deleteMessages.end(); it++) {
+      }
-          Message* deleteMessage = *it;
+      deque<Message*> msgs = messages->findAll("", "", "*", false, true, true, true, true, false);
-          result_t result = messages->add(deleteMessage);
+      if (msgs.empty()) {
-          if (result != RESULT_OK) {
+        message = NULL;
-            cout << "\"" << check[0] << "\": add error: "
+        cout << "\"" << check[0] << "\": create error: message not found" << endl;
-                << getResultCode(result) << endl;
+        error = true;
-            break;
+        continue;
-          }
+      }
-        }
+      message = *msgs.begin();
-        if (result != RESULT_OK) {
+      Message* foundMessage = messages->find(*mstrs[0], false, true, true, true, false);
-          continue;
+      if (foundMessage == message) {
-        }
+        cout << "  find OK" << endl;
-        cout << "  map OK" << endl;
+      } else if (foundMessage == NULL) {
-        message = deleteMessages.front();
+        cout << "  find error: message not found by master " << mstrs[0]->getStr() << endl;
-        deleteMessages.clear();
+        error = true;
-        if (onlyMap) {
+        continue;
-          continue;
-        }
-        Message* foundMessage = messages->find(*mstrs[0]);
-        if (foundMessage == message) {
-          cout << "  find OK" << endl;
-        } else if (foundMessage == NULL) {
-          cout << "  find error: NULL" << endl;
-        } else {
-          cout << "  find error: different" << endl;
-        }
       } else {
-        message = deleteMessages.front();
+        cout << "  find error: different" << endl;
+        error = true;
       }
     }
 
@@ -411,6 +341,7 @@ int main() {
       if (result != RESULT_OK) {
         cout << "  \"" << check[2] << "\" / \"" << check[3] << "\": decode error: "
             << getResultCode(result) << endl;
+        error = true;
         continue;
       }
       cout << "  \"" << check[2] << "\" / \"" << check[3] <<  "\": decode OK" << endl;
@@ -424,6 +355,7 @@ int main() {
       if (failedPrepare) {
         if (result == RESULT_OK) {
           cout << "  \"" << inputStr << "\": failed prepare error: unexpectedly succeeded" << endl;
+          error = true;
         } else {
           cout << "  \"" << inputStr << "\": failed prepare OK" << endl;
         }
@@ -431,8 +363,8 @@ int main() {
       }
 
       if (result != RESULT_OK) {
-        cout << "  \"" << inputStr << "\": prepare error: "
+        cout << "  \"" << inputStr << "\": prepare error: " << getResultCode(result) << endl;
-            << getResultCode(result) << endl;
+        error = true;
         continue;
       }
       cout << "  \"" << inputStr << "\": prepare OK" << endl;
@@ -442,14 +374,6 @@ int main() {
     }
   }
 
-  if (deleteMessages.size() > 0) {
-    for (vector<Message*>::iterator it = deleteMessages.begin(); it != deleteMessages.end(); it++) {
-      Message* deleteMessage = *it;
-      delete deleteMessage;
-    }
-    deleteMessages.clear();
-  }
-
   delete templates;
   delete messages;
   for (vector<MasterSymbolString*>::iterator it = mstrs.begin(); it != mstrs.end(); it++) {
@@ -458,5 +382,5 @@ int main() {
   for (vector<SlaveSymbolString*>::iterator it = sstrs.begin(); it != sstrs.end(); it++) {
     delete *it;
   }
-  return 0;
+  return error ? 1 : 0;
 }

src/lib/ebus/test/test_symbol.cpp

@@ -19,6 +19,7 @@
 #include <iostream>
 #include <iomanip>
 #include <string>
+#include <unordered_map>
 #include "lib/ebus/symbol.h"
 
 using namespace std;
@@ -48,7 +49,6 @@ void verify(bool expectFailMatch, string type, string input,
 
 int main(int argc, char** argv) {
   MasterSymbolString mstr;
-
   if (argc > 1) {
     result_t result;
     if (argc > 2 && strcmp("escaped", argv[1]) == 0) {
@@ -119,5 +119,23 @@ int main(int argc, char** argv) {
     verify(false, "data size", "0427a90015a901", sstr.getDataSize() == 4, expectStr, gotStr);
   }
 
+  int masterCnt = 0, slaveCnt = 0;
+  for (int i=0; i<256; i++) {
+    if (isMaster(i)) {
+      masterCnt++;
+    } else if (isValidAddress(i, false)) {
+      slaveCnt++;
+    }
+  }
+  if (masterCnt == 25) {
+    cout << "count master addresses OK" << endl;
+  } else {
+    cout << "count master addresses error: found " << dec << masterCnt << " instead of 25" << endl;
+  }
+  if (slaveCnt == 228) {
+    cout << "count slave addresses OK" << endl;
+  } else {
+    cout << "count slave addresses error: found " << dec << slaveCnt << " instead of 228" << endl;
+  }
   return error ? 1 : 0;
 }