added support for multi language column selection in configuration files

src/ebusd/main.cpp

@@ -87,6 +87,7 @@ static struct options opt = {
   CONFIG_PATH,  // configPath
   false,  // scanConfig
   BROADCAST,  // initialScan
+  getenv("LANG"),  // preferLanguage
   false,  // checkConfig
   false,  // dumpConfig
   5,  // pollInterval
@@ -136,7 +137,8 @@ static const char argpdoc[] =
 
 #define O_INISND 1
 #define O_DEVLAT (O_INISND+1)
-#define O_CHKCFG (O_DEVLAT+1)
+#define O_CFGLNG (O_DEVLAT+1)
+#define O_CHKCFG (O_CFGLNG+1)
 #define O_DMPCFG (O_CHKCFG+1)
 #define O_POLINT (O_DMPCFG+1)
 #define O_ANSWER (O_POLINT+1)
@@ -177,6 +179,7 @@ static const struct argp_option argpoptions[] = {
       "\"none\" or empty for no initial scan message, \"full\" for full scan, or a single hex address to scan, "
       "default is broadcast ident message). If combined with --checkconfig, you can add scan message data as "
       "arguments for checking a particular scan configuration, e.g. \"FF08070400/0AB5454850303003277201\".", 0 },
+  {"configlang",     O_CFGLNG, "LANG",  0, "Prefer LANG in multilingual configuration files [system default language]", 0 },
   {"checkconfig",    O_CHKCFG, NULL,    0, "Check CSV config files, then stop", 0 },
   {"dumpconfig",     O_DMPCFG, NULL,    0, "Check and dump CSV config files, then stop", 0 },
   {"pollinterval",   O_POLINT, "SEC",   0, "Poll for data every SEC seconds (0=disable) [5]", 0 },
@@ -308,6 +311,9 @@ error_t parse_opt(int key, char *arg, struct argp_state *state) {
       }
     }
     break;
+  case O_CFGLNG:  // --configlang=LANG
+    opt->preferLanguage = arg;
+    break;
   case O_CHKCFG:  // --checkconfig
     opt->checkConfig = true;
     break;

src/ebusd/main.h

@@ -45,6 +45,7 @@ struct options {
   /** the initial address to scan for scanconfig
    * (@a ESC=none, 0xfe=broadcast ident, @a SYN=full scan, else: single slave address). */
   symbol_t initialScan;
+  string preferLanguage;  //!< preferred language in configuration files
   bool checkConfig;  //!< check CSV config files, then stop
   bool dumpConfig;   //!< dump CSV config files, then stop
   unsigned int pollInterval;  //!< poll interval in seconds, 0 to disable [5]

src/ebusd/mainloop.cpp

@@ -41,7 +41,7 @@ using std::ifstream;
 #define RECONNECT_MISSING_SIGNAL 60
 
 
-result_t UserList::getFieldMap(vector<string>& row, string& errorDescription) const {
+result_t UserList::getFieldMap(vector<string>& row, string& errorDescription, const string preferLanguage) const {
   // name,secret,level[,level]*
   if (row.empty()) {
     row.push_back("name");

src/ebusd/mainloop.h

@@ -62,7 +62,7 @@ class UserList : public UserInfo, public MappedFileReader {
   virtual ~UserList() {}
 
   // @copydoc
-  result_t getFieldMap(vector<string>& row, string& errorDescription) const override;
+  result_t getFieldMap(vector<string>& row, string& errorDescription, const string preferLanguage) const override;
 
   // @copydoc
   result_t addFromFile(map<string, string>& row, vector< map<string, string> >& subRows,

src/lib/ebus/data.cpp

@@ -46,7 +46,7 @@ static const char* defaultTemplateFieldMap[] = {
 
 
 
-string getDataFieldName(const string name) {
+string getDataFieldName(const string name, bool& supportsLanguage) {
   if (name.find("name") != string::npos || name.find("field") != string::npos) {
     return "name";
   }
@@ -58,10 +58,12 @@ string getDataFieldName(const string name) {
   }
   if (name.find("divisor") != string::npos) {
     if (name.find("values") != string::npos) {
+      supportsLanguage = true;
       return "divisor/values";
     }
     return "divisor";
   }
+  supportsLanguage = true;
   if (name == "values" || name == "unit") {
     return name;
   }
@@ -72,6 +74,12 @@ string getDataFieldName(const string name) {
 }
 
 
+const string AttributedItem::formatInt(size_t value) {
+  ostringstream stream;
+  stream << dec << static_cast<unsigned>(value);
+  return stream.str();
+}
+
 const string AttributedItem::pluck(map<string, string>& row, string key) {
   map<string, string>::iterator it = row.find(key);
   if (it == row.end()) {
@@ -174,12 +182,6 @@ bool AttributedItem::appendAttributes(ostringstream& output, OutputFormat output
 }
 
 
-string formatInt(size_t value) {
-  ostringstream stream;
-  stream << dec << static_cast<unsigned>(value);
-  return stream.str();
-}
-
 result_t DataField::create(vector< map<string, string> >& rows, string& errorDescription,
     DataFieldTemplates* templates, const DataField*& returnField,
     const bool isWriteMessage,
@@ -1120,7 +1122,8 @@ result_t DataFieldTemplates::add(const DataField* field, string name, bool repla
   return RESULT_OK;
 }
 
-result_t DataFieldTemplates::getFieldMap(vector<string>& row, string& errorDescription) const {
+result_t DataFieldTemplates::getFieldMap(vector<string>& row, string& errorDescription, const string preferLanguage)
+    const {
   // name[:usename],basetype[:len]|template[:usename][,[divisor|values][,[unit][,[comment]]]]
   if (row.empty()) {
     // default map does not include separate field name
@@ -1130,39 +1133,65 @@ result_t DataFieldTemplates::getFieldMap(vector<string>& row, string& errorDescr
     return RESULT_OK;
   }
   bool inDataFields = false;
-  map<string, string> seen;
+  map<string, size_t> seen;
-  for (auto &name : row) {
+  for (size_t col = 0; col < row.size(); col++) {
-    string useName = name;
+    string &name = row[col];
-    tolower(useName);
+    string lowerName = name;
+    tolower(lowerName);
+    trim(lowerName);
+    if (lowerName.empty()) {
+      errorDescription = "missing name in column " + AttributedItem::formatInt(col);
+      return RESULT_ERR_INVALID_ARG;
+    }
+    bool supportsLang = false;
+    string useName = getDataFieldName(lowerName, supportsLang);
+    bool unknown = useName.empty();
+    size_t langPos = supportsLang ? lowerName.find_last_of('.') : string::npos;
+    map<string, size_t>::iterator previous;
+    if (langPos == lowerName.length()-3) {
+      string lang = lowerName.substr(langPos+1);
+      if (unknown) {
+        useName = lowerName.substr(0, langPos);
+      }
+      previous = seen.find(useName);
+      if (previous != seen.end()) {
+        if (lang != preferLanguage) {
+          // skip this column
+          name = SKIP_COLUMN;
+          continue;
+        }
+        // replace previous
+        row[previous->second] = SKIP_COLUMN;
+        seen.erase(useName);
+        previous = seen.end();
+      }
+    } else {
+      if (unknown) {
+        useName = lowerName;
+      }
+      previous = seen.find(useName);
+    }
     if (inDataFields) {
-      string chkName = getDataFieldName(useName);
+      if (!unknown && previous != seen.end()) {
-      if (!chkName.empty()) {
-        useName = chkName;
-        if (seen.find(useName) != seen.end()) {
         if (seen.find("type") == seen.end()) {
-            errorDescription = "missing type";
+          errorDescription = "missing field type";
           return RESULT_ERR_EOF;  // require at least type
         }
         seen.clear();
       }
-      }
     } else {
-      if (useName == "name" && seen.find("name") == seen.end()) {
+      if (!unknown && useName == "name" && seen.find("name") == seen.end()) {
         // keep first name for template
-      } else {
+      } else if (!unknown) {
-        string chkName = getDataFieldName(useName);
-        if (!chkName.empty()) {
-          useName = chkName;
         if (seen.find("name") == seen.end()) {
-            errorDescription = "missing name";
+          errorDescription = "missing template name";
           return RESULT_ERR_EOF;  // require at least name
         }
         inDataFields = true;
         seen.clear();
       }
-      }
       if (!inDataFields && seen.find(useName) != seen.end()) {
-        errorDescription = "duplicate " + useName;
+        errorDescription = "duplicate template " + useName;
         return RESULT_ERR_INVALID_ARG;
       }
     }
@@ -1171,14 +1200,14 @@ result_t DataFieldTemplates::getFieldMap(vector<string>& row, string& errorDescr
     } else {
       name = useName;
     }
-    seen[useName] = useName;
+    seen[useName] = col;
   }
   if (!inDataFields) {
-    errorDescription = "missing fields";
+    errorDescription = "missing template fields";
     return RESULT_ERR_EOF;  // require at least one field
   }
   if (seen.find("type") == seen.end()) {
-    errorDescription = "missing type";
+    errorDescription = "missing field type";
     return RESULT_ERR_EOF;  // require at least type
   }
   return RESULT_OK;

src/lib/ebus/data.h

@@ -53,9 +53,10 @@ namespace ebusd {
 /**
  * Get the normalized data field name for the given name.
  * @param name the input field name.
+ * @param supportsLanguage set to true when the field supports multiple language.
  * @return the normalized data field name, or empty if unknown.
  */
-string getDataFieldName(const string name);
+string getDataFieldName(const string name, bool& supportsLanguage);
 
 class DataFieldTemplates;
 class SingleDataField;
@@ -86,6 +87,13 @@ class AttributedItem {
   virtual ~AttributedItem() {}
 
 
+  /**
+   * Format an int value to a string.
+   * @param value the int value.
+   * @return the formatted string.
+   */
+  static const string formatInt(size_t value);
+
   /**
    * Remove and return a certain value from a map.
    * @param row the map to remove the value from.
@@ -709,7 +717,7 @@ class DataFieldTemplates : public MappedFileReader {
   result_t add(const DataField* field, string name = "", bool replace = false);
 
   // @copydoc
-  result_t getFieldMap(vector<string>& row, string& errorDescription) const override;
+  result_t getFieldMap(vector<string>& row, string& errorDescription, const string preferLanguage) const override;
 
   // @copydoc
   result_t addFromFile(map<string, string>& row, vector< map<string, string> >& subRows,

src/lib/ebus/filereader.cpp

@@ -202,6 +202,25 @@ bool FileReader::splitFields(istream& ifs, vector<string>& row, unsigned int& li
 }
 
 
+string MappedFileReader::normalizeLanguage(string lang) {
+  tolower(lang);
+  if (lang.size() > 2) {
+    size_t pos = lang.find('.');
+    if (pos == string::npos) {
+      pos = lang.size();
+    }
+    size_t strip = lang.find('_');
+    if (strip == string::npos || strip > pos) {
+      strip = pos;
+    }
+    if (strip > 2) {
+      strip = 2;
+    }
+    lang = lang.substr(0, strip);
+  }
+  return lang;
+}
+
 result_t MappedFileReader::readFromFile(const string filename, string& errorDescription, bool verbose,
     map<string, string>* defaults, size_t* hash, size_t* size, time_t* time) {
   m_mutex.lock();
@@ -223,7 +242,7 @@ result_t MappedFileReader::addFromFile(vector<string>& row, string& errorDescrip
     const string filename, unsigned int lineNo) {
   result_t result;
   if (lineNo == 1) {  // first line defines column names
-    result = getFieldMap(row, errorDescription);
+    result = getFieldMap(row, errorDescription, m_preferLanguage);
     if (result != RESULT_OK) {
       return result;
     }
@@ -271,6 +290,8 @@ result_t MappedFileReader::addFromFile(vector<string>& row, string& errorDescrip
       columnName = columnName.substr(1);
       lastRepeatStart = colNameIdx;
       empty = true;
+    } else if (columnName == SKIP_COLUMN) {
+      continue;
     }
     string value = row[colIdx];
     empty &= value.empty();

src/lib/ebus/filereader.h

@@ -58,6 +58,8 @@ using std::mutex;
 /** the separator character used between multiple values (in CSV only). */
 #define VALUE_SEPARATOR ';'
 
+/** special marker string for skipping columns in @a MappedFileReader. */
+static const string SKIP_COLUMN = "\b";
 
 /**
  * An abstract class that support reading definitions from a file.
@@ -159,8 +161,11 @@ class MappedFileReader : public FileReader {
   /**
    * Constructor.
    * @param supportsDefaults whether this instance supports rows with defaults (starting with a star).
+   * @param preferLanguage the preferred language code, or empty.
    */
-  explicit MappedFileReader(bool supportsDefaults) : FileReader(), m_supportsDefaults(supportsDefaults) {}
+  explicit MappedFileReader(bool supportsDefaults, const string preferLanguage = "")
+  : FileReader(), m_supportsDefaults(supportsDefaults), m_preferLanguage(normalizeLanguage(preferLanguage)) {
+  }
 
   /**
    * Destructor.
@@ -171,6 +176,13 @@ class MappedFileReader : public FileReader {
     m_lastSubDefaults.clear();
   }
 
+  /**
+   * Normalize the language string to a lower case, max. 2 characters long language code.
+   * @param lang the language string to normalize.
+   * @return the normalized language code.
+   */
+  static string normalizeLanguage(string lang);
+
   // @copydoc
   result_t readFromFile(const string filename, string& errorDescription, bool verbose = false,
       map<string, string>* defaults = NULL, size_t* hash = NULL, size_t* size = NULL, time_t* time = NULL) override;
@@ -196,10 +208,13 @@ class MappedFileReader : public FileReader {
   /**
    * Get the field mapping from the given first line.
    * @param row the first line from which to extract the field mapping, or empty to use the default mapping.
+   * Columns set to @a SKIP_COLUMN are skipped in @a addFromFile(). Columns starting with a "*" mark the beginning of
+   * a repeated sub row.
    * @param errorDescription a string in which to store the error description in case of error.
+   * @param preferLanguage the preferred language code (up to 2 characters), or empty.
    * @return @a RESULT_OK on success, or an error code.
    */
-  virtual result_t getFieldMap(vector<string>& row, string& errorDescription) const = 0;
+  virtual result_t getFieldMap(vector<string>& row, string& errorDescription, const string preferLanguage) const = 0;
 
   /**
    * Add a default row that was read from a file.
@@ -253,6 +268,9 @@ class MappedFileReader : public FileReader {
   /** whether this instance supports rows with defaults (starting with a star). */
   const bool m_supportsDefaults;
 
+  /** the preferred language code (up to 2 characters), or empty. */
+  const string m_preferLanguage;
+
   /** a @a mutex for access to defaults. */
   mutex m_mutex;
 

src/lib/ebus/message.cpp

@@ -84,9 +84,10 @@ extern DataFieldTemplates* getTemplates(const string filename);
 /**
  * Get the normalized message field name for the given name.
  * @param name the input field name.
+ * @param supportsLanguage set to true when the field supports multiple language.
  * @return the normalized message field name, or empty if unknown.
  */
-string getMessageFieldName(const string name) {
+string getMessageFieldName(const string name, bool& supportsLanguage) {
   if (name.find("type") != string::npos) {
     return "type";
   }
@@ -96,6 +97,7 @@ string getMessageFieldName(const string name) {
   if (name.find("name") != string::npos) {
     return "name";
   }
+  supportsLanguage = true;
   if (name.find("comment") == 0) {
     return "comment";
   }
@@ -1775,7 +1777,7 @@ result_t MessageMap::add(Message* message, bool storeByName) {
   return RESULT_OK;
 }
 
-result_t MessageMap::getFieldMap(vector<string>& row, string& errorDescription) const {
+result_t MessageMap::getFieldMap(vector<string>& row, string& errorDescription, const string preferLanguage) const {
   // type (r[1-9];w;u),circuit,name,[comment],[QQ],ZZ,PBSB,[ID],field1,part (m/s),datatypes/templates,divider/values,
   //  unit,comment
   // minimum: type,name,PBSB,field,datatype
@@ -1786,30 +1788,66 @@ result_t MessageMap::getFieldMap(vector<string>& row, string& errorDescription)
     return RESULT_OK;
   }
   bool inDataFields = false;
-  map<string, string> seen;
+  map<string, size_t> seen;
-  for (auto &name : row) {
+  for (size_t col = 0; col < row.size(); col++) {
-    string useName = name;
+    string &name = row[col];
-    tolower(useName);
+    string lowerName = name;
+    tolower(lowerName);
+    trim(lowerName);
+    if (lowerName.empty()) {
+      errorDescription = "missing name in column " + AttributedItem::formatInt(col);
+      return RESULT_ERR_INVALID_ARG;
+    }
+    bool supportsLang = false, toDataFields = false;
+    string useName;
     if (inDataFields) {
-      string chkName = getDataFieldName(useName);
+      useName = getDataFieldName(lowerName, supportsLang);
-      if (!chkName.empty()) {
+    } else {
-        useName = chkName;
+      useName = getMessageFieldName(lowerName, supportsLang);
-        if (seen.find(useName) != seen.end()) {
+      if (useName.empty()) {
+        useName = getDataFieldName(lowerName, supportsLang);
+        toDataFields = !useName.empty();
+      }
+    }
+    bool unknown = useName.empty();
+    size_t langPos = supportsLang ? lowerName.find_last_of('.') : string::npos;
+    map<string, size_t>::iterator previous;
+    if (langPos == lowerName.length()-3) {
+      string lang = lowerName.substr(langPos+1);
+      if (unknown) {
+        useName = lowerName.substr(0, langPos);
+      }
+      previous = seen.find(useName);
+      if (previous != seen.end()) {
+        if (lang != preferLanguage) {
+          // skip this column
+          name = SKIP_COLUMN;
+          continue;
+        }
+        // replace previous
+        row[previous->second] = SKIP_COLUMN;
+        seen.erase(useName);
+        previous = seen.end();
+      }
+    } else {
+      if (unknown) {
+        useName = lowerName;
+      }
+      previous = seen.find(useName);
+    }
+    if (inDataFields) {
+      if (!unknown && previous != seen.end()) {
         if (seen.find("name") == seen.end() || seen.find("type") == seen.end()) {
           errorDescription = "missing field name/type as of already seen "+useName;
           return RESULT_ERR_EOF;  // require at least name and type
         }
         seen.clear();
       }
-      }
     } else {
-      string chkName = getMessageFieldName(useName);
+      /*if (!unknown && (useName != "name" || seen.find("name") == seen.end())) {
-      if (!chkName.empty() && (chkName != "name" || seen.find("name") == seen.end())) {
+        // keep first name for message
-        useName = chkName;
+      } else {*/
-      } else {
+      if (toDataFields) {
-        chkName = getDataFieldName(useName);
-        if (!chkName.empty()) {
-          useName = chkName;
         if (seen.find("type") == seen.end() || seen.find("name") == seen.end() || seen.find("pbsb") == seen.end()) {
           errorDescription = "missing message name/type/pbsb";
           return RESULT_ERR_EOF;  // require at least type, name, and pbsb
@@ -1817,7 +1855,6 @@ result_t MessageMap::getFieldMap(vector<string>& row, string& errorDescription)
         inDataFields = true;
         seen.clear();
       }
-      }
       if (!inDataFields && seen.find(useName) != seen.end()) {
         errorDescription = "duplicate message " + useName;
         return RESULT_ERR_INVALID_ARG;
@@ -1828,7 +1865,7 @@ result_t MessageMap::getFieldMap(vector<string>& row, string& errorDescription)
     } else {
       name = useName;
     }
-    seen[useName] = useName;
+    seen[useName] = col;
   }
   if (inDataFields) {
     if (seen.find("name") == seen.end() || seen.find("type") == seen.end()) {

src/lib/ebus/message.h

@@ -1213,8 +1213,9 @@ class MessageMap : public MappedFileReader {
   /**
    * Construct a new instance.
    * @param addAll whether to add all messages, even if duplicate.
+   * @param preferLanguage the preferred language to use, or empty.
    */
-  explicit MessageMap(const bool addAll = false) : MappedFileReader::MappedFileReader(true),
+  explicit MessageMap(const bool addAll = false, const string preferLanguage = "") : MappedFileReader::MappedFileReader(true),
     m_addAll(addAll), m_additionalScanMessages(false), m_maxIdLength(0), m_maxBroadcastIdLength(0),
     m_messageCount(0), m_conditionalMessageCount(0), m_passiveMessageCount(0) {
     m_scanMessage = Message::createScanMessage();
@@ -1246,7 +1247,7 @@ class MessageMap : public MappedFileReader {
   result_t add(Message* message, bool storeByName = true);
 
   // @copydoc
-  result_t getFieldMap(vector<string>& row, string& errorDescription) const override;
+  result_t getFieldMap(vector<string>& row, string& errorDescription, const string preferLanguage) const override;
 
   // @copydoc
   result_t addDefaultFromFile(map<string, string>& row, vector< map<string, string> >& subRows,

src/lib/ebus/test/test_data.cpp

@@ -53,7 +53,7 @@ class TestReader : public MappedFileReader {
   TestReader(DataFieldTemplates* templates, bool isSet, bool isMasterDest)
   : MappedFileReader::MappedFileReader(true), m_templates(templates), m_isSet(isSet), m_isMasterDest(isMasterDest),
     m_fields(NULL) {}
-  result_t getFieldMap(vector<string>& row, string& errorDescription) const override {
+  result_t getFieldMap(vector<string>& row, string& errorDescription, const string preferLanguage) const override {
     if (row.empty()) {
       row.push_back("*name");
       row.push_back("part");

src/lib/ebus/test/test_filereader.cpp

@@ -49,13 +49,13 @@ void verify(bool expectFailMatch, string type, string input,
 
 string resultlines[][3] = {
   {"col 1", "col 2", "col 3"},
-  {"line 2 col 1", "line 2 col 2", "line 2 \"col 3\";default of col 3"},
+  {"line 2 col 1 de", "line 2 col 2", "line 2 \"col 3\";default of col 3"},
   {"", "", ""},
-  {"line 4 col 1", "line 4 col 2 part 1;line 4 col 2 part 2", "line 4 col 3;default of col 3"},
+  {"line 4 col 1 de", "line 4 col 2 part 1;line 4 col 2 part 2", "line 4 col 3;default of col 3"},
   {"", "", ""},
-  {"line 6 col 1", "", "line 6 col 3;default of col 3"},
+  {"line 6 col 1 de", "", "line 6 col 3;default of col 3"},
   {"", "", ""},
-  {"line 8 col 1", "line 8 col 2 part 1;line 8 col 2 part 2", "line 8 col 3;default of col 3"},
+  {"line 8 col 1 de", "line 8 col 2 part 1;line 8 col 2 part 2", "line 8 col 3;default of col 3"},
 };
 
 string resultsublines[][2][4] = {
@@ -81,14 +81,20 @@ class NoopReader : public FileReader {
 
 class TestReader : public MappedFileReader {
  public:
-  TestReader(size_t expectedCols) : MappedFileReader::MappedFileReader(false), m_expectedCols(expectedCols) {}
+  TestReader(size_t expectedCols, size_t langCols)
-  result_t getFieldMap(vector<string>& row, string& errorDescription) const override {
+  : MappedFileReader::MappedFileReader(false, ""), m_expectedCols(expectedCols), m_langCols(langCols) {}
-    if (row.size() == m_expectedCols) {
+  result_t getFieldMap(vector<string>& row, string& errorDescription, const string preferLanguage) const override {
+    if (row.size() == m_expectedCols+m_langCols) {
       cout << "get field map: split OK" << endl;
+      if (m_langCols == 1) {
+        row[0] = SKIP_COLUMN;
+        size_t pos = row[1].find_last_of('.');
+        row[1] = row[1].substr(0, pos);
+      }
       return RESULT_OK;
     }
     cout << "get field map: error got " << static_cast<unsigned>(row.size()) << " columns, expected " <<
-        static_cast<unsigned>(m_expectedCols) << endl;
+        static_cast<unsigned>(m_expectedCols+m_langCols) << endl;
     return RESULT_ERR_EOF;
   }
   result_t addFromFile(map<string, string>& row, vector< map<string, string> >& subRows,
@@ -179,6 +185,7 @@ class TestReader : public MappedFileReader {
   }
  private:
   size_t m_expectedCols;
+  size_t m_langCols;
 };
 
 
@@ -203,17 +210,17 @@ int main(int argc, char** argv) {
   }
   baseLine = __LINE__+1;
   istringstream ifs(
-    "col 1,col 2,col 3\n"
+    "col 1.en,col 1.de,col 2,col 3\n"
-    "line 2 col 1,\"line 2 col 2\",\"line 2 \"\"col 3\"\";default of col 3\"\n"
+    "line 2 col 1 en,line 2 col 1 de,\"line 2 col 2\",\"line 2 \"\"col 3\"\";default of col 3\"\n"
-    "line 4 col 1,\"line 4 col 2 part 1\n"
+    "line 4 col 1 en,line 4 col 1 de,\"line 4 col 2 part 1\n"
     "line 4 col 2 part 2\",line 4 col 3;default of col 3\n"
     ",,,\n"
-    "line 6 col 1,,line 6 col 3;default of col 3\n"
+    "line 6 col 1 en,line 6 col 1 de,,line 6 col 3;default of col 3\n"
-    "line 8 col 1,\"line 8 col 2 part 1;\n"
+    "line 8 col 1 en,line 8 col 1 de,\"line 8 col 2 part 1;\n"
     "line 8 col 2 part 2\",line 8 col 3;default of col 3\n"
   );
-  size_t hash = 0, size = 0, expectHash = 0xfd58724c2984595d, expectSize = 301;
+  size_t hash = 0, size = 0, expectHash = 0x5e5e086475ab3bd9, expectSize = 389;
-  TestReader reader{3};
+  TestReader reader{3, 1};
   unsigned int lineNo = 0;
   vector<string> row;
   string errorDescription;
@@ -242,16 +249,16 @@ int main(int argc, char** argv) {
   baseLine = __LINE__+1;
   ifs.str(
     "col 1,col 2,col 3,*subcol 1,subcol 2,*subcol 2,subcol 3\n"
-    "line 2 col 1,\"line 2 col 2\",\"line 2 \"\"col 3\"\"\",line 2 subcol 1,line 2 subcol 2,line 2 subcol 2,line 2 subcol 3\n"
+    "line 2 col 1 de,\"line 2 col 2\",\"line 2 \"\"col 3\"\"\",line 2 subcol 1,line 2 subcol 2,line 2 subcol 2,line 2 subcol 3\n"
-    "line 4 col 1,\"line 4 col 2 part 1\n"
+    "line 4 col 1 de,\"line 4 col 2 part 1\n"
     "line 4 col 2 part 2\",line 4 col 3,line 4 subcol 1,line 4 subcol 2,line 4 subcol 2,line 4 subcol 3\n"
     ",,,\n"
-    "line 6 col 1,,line 6 col 3,line 6 subcol 1,line 6 subcol 2,line 6 subcol 2,line 6 subcol 3\n"
+    "line 6 col 1 de,,line 6 col 3,line 6 subcol 1,line 6 subcol 2,line 6 subcol 2,line 6 subcol 3\n"
-    "line 8 col 1,\"line 8 col 2 part 1;\n"
+    "line 8 col 1 de,\"line 8 col 2 part 1;\n"
     "line 8 col 2 part 2\",line 8 col 3,line 8 subcol 1,line 8 subcol 2,line 8 subcol 2,line 8 subcol 3\n"
   );
-  hash = 0, size = 0, expectHash = 0xec31914e32c6fb6f, expectSize = 527;
+  hash = 0, size = 0, expectHash = 0x9a675169d5837bb5, expectSize = 539;
-  TestReader reader2{7};
+  TestReader reader2{7, 0};
   lineNo = 0;
   map<string, string> defaults;
   reader2.getDefaults()[""]["col 3"] = ";default of col 3";