moved field defintion to message/data, reworked filereader

src/ebusd/mainloop.cpp

@@ -34,68 +34,61 @@ using std::dec;
 using std::hex;
 using std::setfill;
 using std::setw;
+using std::endl;
+using std::ifstream;
 
 /** the number of seconds of permanent missing signal after which to reconnect the device. */
 #define RECONNECT_MISSING_SIGNAL 60
 
-/** the known column names (pairs of full length name and short length name). */
-static const char* columnNames[] = {
-  "type", "t",
-  "circuit", "c",
-  "level", "l",
-  "name", "n",
-  "comment", "co",
-  "qq", "q",
-  "zz", "z",
-  "pbsb", "p",
-  "id", "i",
-  "fields", "f",
-};
 
-/** the known column IDs according to @a columnNames. */
-static const column_t columnIds[] = {
-  COLUMN_TYPE, COLUMN_TYPE,
-  COLUMN_CIRCUIT, COLUMN_CIRCUIT,
-  COLUMN_LEVEL, COLUMN_LEVEL,
-  COLUMN_NAME, COLUMN_NAME,
-  COLUMN_COMMENT, COLUMN_COMMENT,
-  COLUMN_QQ, COLUMN_QQ,
-  COLUMN_ZZ, COLUMN_ZZ,
-  COLUMN_PBSB, COLUMN_PBSB,
-  COLUMN_ID, COLUMN_ID,
-  COLUMN_FIELDS, COLUMN_FIELDS,
-};
-
-/** the number of known column names. */
-static const size_t columnCount = sizeof(columnNames) / sizeof(char*);
-
-
-result_t UserList::addFromFile(vector<string>::iterator& begin, const vector<string>::iterator end,
-    vector< vector<string> >* defaults, const string& defaultDest, const string& defaultCircuit,
-    const string& defaultSuffix, const string& filename, unsigned int lineNo) {
+result_t UserList::getFieldMap(vector<string>& row, string& errorDescription) {
   // name,secret,level[,level]*
-  if (begin == end) {
-    return RESULT_ERR_EOF;
+  if (row.empty()) {
+    row.push_back("name");
+    row.push_back("secret");
+    row.push_back("*level"); // TODO last repeat is repeated as often as necessary in addFromFile...
+    return RESULT_OK;
   }
-  string name = *begin++;
-  if (begin == end) {
-    return RESULT_ERR_EOF;
+  map<string, string> seen;
+  for (auto &name : row) {
+    tolower(name);
+    if (name == "name" || name == "secret") {
+      if (seen.find(name) != seen.end()) {
+        errorDescription = "duplicate field " + name;
+        return RESULT_ERR_INVALID_ARG;
+      }
+    } else if (name == "level") {
+      name = "*level";
+    } else {
+      errorDescription = "unknown field " + name;
+      return RESULT_ERR_INVALID_ARG;
+    }
+    seen[name] = name;
   }
+  if (seen.find("name") == seen.end() || seen.find("secret") == seen.end()) {
+    return RESULT_ERR_EOF;  // require at least name and secret
+  }
+  return RESULT_OK;
+}
+
+result_t UserList::addFromFile(map<string, string>& row, vector< map<string, string> >& subRows,
+    string& errorDescription, const string filename, unsigned int lineNo) {
+  string name = row["name"];
+  string secret = row["secret"];
   if (name.empty()) {
     return RESULT_ERR_INVALID_ARG;
   }
-  if (name == "*") {  // default levels
-    name = "";
+  if (name == "*") {
+    name = "";  // default levels
   }
-  const string secret = *begin++;
-  if (begin == end) {
-    return RESULT_ERR_EOF;
-  }
-  string levels = *begin++;
-  while (begin != end) {
-    string level = *begin++;
+  string levels;
+  for (auto entry : subRows) {
+    string level = entry["level"];
     if (!level.empty()) {
-      levels += VALUE_SEPARATOR + level;
+      if (!levels.empty()) {
+        levels += VALUE_SEPARATOR;
+      }
+      levels += level;
     }
   }
   m_userSecrets[name] = secret;
@@ -128,7 +121,8 @@ MainLoop::MainLoop(const struct options opt, Device *device, MessageMap* message
   }
   m_logRawEnabled = opt.logRaw;
   if (opt.aclFile[0]) {
-    result_t result = m_userList.readFromFile(opt.aclFile);
+    string errorDescription;
+    result_t result = m_userList.readFromFile(opt.aclFile, errorDescription);
     if (result != RESULT_OK) {
       logError(lf_main, "error reading ACL file \"%s\": %s", opt.aclFile, getResultCode(result));
     }
@@ -1088,7 +1082,7 @@ string MainLoop::executeFind(vector<string> &args, string levels) {
   bool configFormat = false, exact = false, withRead = true, withWrite = false, withPassive = true, first = true,
       onlyWithData = false, hexFormat = false, userLevel = true;
   OutputFormat verbosity = 0;
-  vector<column_t> columns;
+  vector<size_t> fieldIds;
   string circuit;
   vector<symbol_t> id;
   while (args.size() > argPos && args[argPos][0] == '-') {
@@ -1120,23 +1114,7 @@ string MainLoop::executeFind(vector<string> &args, string levels) {
         argPos = 0;  // print usage
         break;
       }
-      istringstream input(args[argPos]);
-      string column;
-      while (getline(input, column, ',')) {
-        size_t idx = columnCount;
-        for (size_t i = 0; i < columnCount; i++) {
-          if (strcasecmp(columnNames[i], column.c_str()) == 0) {
-            idx = i;
-            break;
-          }
-        }
-        if (idx == columnCount) {
-          argPos = 0;  // print usage
-          break;
-        }
-        columns.push_back(columnIds[idx]);
-      }
-      if (columns.empty()) {
+      if (!Message::extractFieldIds(args[argPos], fieldIds)) {
         argPos = 0;  // print usage
         break;
       }
@@ -1246,11 +1224,11 @@ string MainLoop::executeFind(vector<string> &args, string levels) {
         result << endl;
       }
       message->dump(result);
-    } else if (!columns.empty()) {
+    } else if (!fieldIds.empty()) {
       if (found) {
         result << endl;
       }
-      message->dump(result, &columns);
+      message->dump(result, &fieldIds);
     } else {
       if (found) {
         result << endl;

src/ebusd/mainloop.h

@@ -40,13 +40,13 @@ namespace ebusd {
 /**
  * Helper class for user authentication.
  */
-class UserList : public UserInfo, public FileReader {
+class UserList : public UserInfo, public MappedFileReader {
  public:
   /**
    * Constructor.
    * @param defaultLevels the default access levels.
    */
-  explicit UserList(const string defaultLevels) : FileReader::FileReader(false) {
+  explicit UserList(const string defaultLevels) : MappedFileReader::MappedFileReader(false) {
     if (!defaultLevels.empty()) {
       string levels = defaultLevels;
       transform(levels.begin(), levels.end(), levels.begin(), [](unsigned char c) {
@@ -62,9 +62,11 @@ class UserList : public UserInfo, public FileReader {
   virtual ~UserList() {}
 
   // @copydoc
-  result_t addFromFile(vector<string>::iterator& begin, const vector<string>::iterator end,
-      vector< vector<string> >* defaults, const string& defaultDest, const string& defaultCircuit,
-      const string& defaultSuffix, const string& filename, unsigned int lineNo) override;
+  result_t getFieldMap(vector<string>& row, string& errorDescription) override;
+
+  // @copydoc
+  result_t addFromFile(map<string, string>& row, vector< map<string, string> >& subRows,
+      string& errorDescription, const string filename, unsigned int lineNo) override;
 
   // @copydoc
   bool hasUser(const string user) override {

src/ebusd/mqtthandler.cpp

@@ -169,56 +169,56 @@ DataHandler* mqtthandler_register(UserInfo* userInfo, BusHandler* busHandler, Me
   return new MqttHandler(userInfo, busHandler, messages);
 }
 
-/** the known topic column names. */
-static const char* columnNames[] = {
+/** the known topic field names. */
+static const char* knownFieldNames[] = {
   "circuit",
   "name",
   "field",
 };
 
-/** the known topic column IDs. */
-static const column_t columnIds[] = {
-  COLUMN_CIRCUIT,
-  COLUMN_NAME,
-  COLUMN_FIELDS,
+/** the known topic field IDs. */
+static const size_t knownFieldIds[] = {
+  MESSAGEFIELD_CIRCUIT,
+  MESSAGEFIELD_NAME,
+  MESSAGEFIELD_DATAFIELDS,
 };
 
-/** the number of known column names. */
-static const size_t columnCount = sizeof(columnNames) / sizeof(char*);
+/** the number of known field names. */
+static const size_t knownFieldCount = sizeof(knownFieldNames) / sizeof(char*);
 
 
 /**
  * Parse the topic template.
  * @param topic the topic template.
  * @param strs the @a vector to which the string parts shall be added.
- * @param cols the @a vector to which the column parts shall be added.
+ * @param fields the @a vector to which the field parts shall be added.
  * @return true on success, false on malformed topic template.
  */
-bool parseTopic(const string topic, vector<string> &strs, vector<column_t> &cols) {
+bool parseTopic(const string topic, vector<string> &strs, vector<size_t> &fields) {
   size_t lastpos = 0;
   size_t end = topic.length();
   vector<string> columns;
   for (size_t pos=topic.find('%', lastpos); pos != string::npos; ) {
-    size_t idx = columnCount;
+    size_t idx = knownFieldCount;
     size_t len = 0;
-    for (size_t i = 0; i < columnCount; i++) {
-      len = strlen(columnNames[i]);
-      if (topic.substr(pos+1, len) == columnNames[i]) {
+    for (size_t i = 0; i < knownFieldCount; i++) {
+      len = strlen(knownFieldNames[i]);
+      if (topic.substr(pos+1, len) == knownFieldNames[i]) {
         idx = i;
         break;
       }
     }
-    if (idx== columnCount) {
+    if (idx== knownFieldCount) {
       return false;
     }
-    column_t col = columnIds[idx];
-    for (vector<column_t>::iterator it=cols.begin(); it != cols.end(); it++) {
-      if (*it == col) {
+    size_t fieldId = knownFieldIds[idx];
+    for (vector<size_t>::iterator it=fields.begin(); it != fields.end(); it++) {
+      if (*it == fieldId) {
         return false;  // duplicate column
       }
     }
     strs.push_back(topic.substr(lastpos, pos-lastpos));
-    cols.push_back(col);
+    fields.push_back(fieldId);
     lastpos = pos+1+len;
     pos = topic.find('%', lastpos);
   }
@@ -265,7 +265,7 @@ MqttHandler::MqttHandler(UserInfo* userInfo, BusHandler* busHandler, MessageMap*
   bool enabled = g_port != 0;
   m_publishByField = false;
   m_mosquitto = NULL;
-  if (enabled && !parseTopic(g_topic, m_topicStrs, m_topicCols)) {
+  if (enabled && !parseTopic(g_topic, m_topicStrs, m_topicFields)) {
     logOtherError("mqtt", "malformed topic %s", g_topic);
     return;
   }
@@ -278,7 +278,7 @@ MqttHandler::MqttHandler(UserInfo* userInfo, BusHandler* busHandler, MessageMap*
     logOtherError("mqtt", "invalid mosquitto version %d instead of %d", major, LIBMOSQUITTO_MAJOR);
     return;
   }
-  if (m_topicCols.empty()) {
+  if (m_topicFields.empty()) {
     if (m_topicStrs.empty()) {
       m_topicStrs.push_back("");
     } else {
@@ -287,12 +287,12 @@ MqttHandler::MqttHandler(UserInfo* userInfo, BusHandler* busHandler, MessageMap*
         m_topicStrs[0] = str+"/";
       }
     }
-    m_topicCols.push_back(COLUMN_CIRCUIT);  // circuit
+    m_topicFields.push_back(MESSAGEFIELD_CIRCUIT);  // circuit
     m_topicStrs.push_back("/");
-    m_topicCols.push_back(COLUMN_NAME);  // name
+    m_topicFields.push_back(MESSAGEFIELD_NAME);  // name
   } else {
-    for (size_t i = 0; i < m_topicCols.size(); i++) {
-      if (m_topicCols[i] == COLUMN_FIELDS) {  // fields
+    for (size_t i = 0; i < m_topicFields.size(); i++) {
+      if (m_topicFields[i] == MESSAGEFIELD_DATAFIELDS) {  // fields
         m_publishByField = true;
         break;
       }
@@ -421,7 +421,7 @@ void MqttHandler::notifyTopic(string topic, string data) {
       if (pos == string::npos) {
         return;
       }
-    } else if (idx-1 < m_topicCols.size()) {
+    } else if (idx-1 < m_topicFields.size()) {
       pos = remain.size();
     } else if (last < remain.size()) {
       return;
@@ -438,14 +438,14 @@ void MqttHandler::notifyTopic(string topic, string data) {
       if (field.empty()) {
         return;
       }
-      switch (m_topicCols[idx-1]) {
-      case COLUMN_CIRCUIT:
+      switch (m_topicFields[idx-1]) {
+      case MESSAGEFIELD_CIRCUIT:
         circuit = field;
         break;
-      case COLUMN_NAME:
+      case MESSAGEFIELD_NAME:
         name = field;
         break;
-      case COLUMN_FIELDS:
+      case MESSAGEFIELD_DATAFIELDS:
         //field = field;  // TODO add support for writing a single field
         break;
       default:
@@ -567,11 +567,11 @@ string MqttHandler::getTopic(Message* message, ssize_t fieldIndex) {
     if (!message) {
       break;
     }
-    if (i < m_topicCols.size()) {
-      if (m_topicCols[i] == COLUMN_FIELDS && fieldIndex >= 0) {
-        ret << message->getFieldName(fieldIndex);
+    if (i < m_topicFields.size()) {
+      if (m_topicFields[i] == MESSAGEFIELD_DATAFIELDS && fieldIndex >= 0) {
+        ret << message->getFieldName(fieldIndex); // TODO skip ignored fields
       } else {
-        message->dumpColumn(ret, m_topicCols[i]);
+        message->dumpField(ret, m_topicFields[i]);
       }
     }
   }

src/ebusd/mqtthandler.h

@@ -121,8 +121,8 @@ class MqttHandler : public DataSink, public DataSource, public Thread {
   /** the MQTT topic string parts. */
   vector<string> m_topicStrs;
 
-  /** the MQTT topic column parts. */
-  vector<column_t> m_topicCols;
+  /** the MQTT topic field parts. */
+  vector<size_t> m_topicFields;
 
   /** the global topic prefix. */
   string m_globalTopic;

src/lib/ebus/data.cpp

@@ -38,147 +38,165 @@ using std::setw;
 /** the week day names. */
 static const char* dayNames[] = {"Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"};
 
+size_t getDataFieldId(const string name) {
+  if (name == "name" || name.find("field") != string::npos) {
+    return DATAFIELD_NAME;
+  }
+  if (name.find("part") != string::npos) {
+    return DATAFIELD_PART;
+  }
+  if (name.find("type") != string::npos) {
+    return DATAFIELD_TYPE;
+  }
+  if (name.find("divisor") != string::npos || name.find("values") != string::npos) {
+    return DATAFIELD_DIVISORVALUES;
+  }
+  if (name == "unit") {
+    return DATAFIELD_UNIT;
+  }
+  if (name == "comment") {
+    return DATAFIELD_COMMENT;
+  }
+  return UINT_MAX;
+}
 
-result_t DataField::create(vector<string>::iterator& it,
-    const vector<string>::iterator end,
+string getDataFieldName(const size_t fieldId) {
+  switch (fieldId) {
+  case DATAFIELD_NAME:
+    return "name";
+  case DATAFIELD_PART:
+    return "part";
+  case DATAFIELD_TYPE:
+    return "type";
+  case DATAFIELD_DIVISORVALUES:
+    return "divisor/values";
+  case DATAFIELD_UNIT:
+    return "unit";
+  case DATAFIELD_COMMENT:
+    return "comment";
+  default:
+    return "";
+  };
+}
+
+result_t DataField::create(vector< map<string, string> >& rows, string& errorDescription,
     DataFieldTemplates* templates, DataField*& returnField,
     const bool isWriteMessage,
     const bool isTemplate, const bool isBroadcastOrMasterDestination,
     const size_t maxFieldLength) {
+  // template: name,[,part]basetype[:len]|template[:name][,[divisor|values][,[unit][,[comment]]]]
+  // std: name,part,basetype[:len]|template[:name][,[divisor|values][,[unit][,[comment]]]]
   vector<SingleDataField*> fields;
-  string firstName, firstComment;
+  string firstName;
   result_t result = RESULT_OK;
-  if (it == end) {
+  if (rows.empty()) {
     return RESULT_ERR_EOF;
   }
-  while (it != end && result == RESULT_OK) {
-    string unit, comment;
-    PartType partType;
-    int divisor = 0;
-    bool hasPartStr = false;
-    string token;
-
-    // template: name,basetype[:len]|template[:name][,[divisor|values][,[unit][,[comment]]]]
-    // std: name,part,basetype[:len]|template[:name][,[divisor|values][,[unit][,[comment]]]]
-    const string name = *it++;  // name
-    if (it == end) {
-      if (!name.empty()) {
-        result = RESULT_ERR_MISSING_TYPE;
-      }
+  for (auto row : rows) {
+    if (result != RESULT_OK) {
       break;
     }
-
+    const string name = row["name"];
+    PartType partType;
+    int divisor = 0;
+    bool hasPart = false;
     if (isTemplate) {
       partType = pt_any;
     } else {
-      const char* partStr = (*it++).c_str();  // part
-      hasPartStr = partStr[0] != 0;
-      if (it == end) {
-        if (!name.empty() || hasPartStr) {
-          result = RESULT_ERR_MISSING_TYPE;
-        }
-        break;
+      string part = row["part"];
+      hasPart = !part.empty();
+      if (hasPart) {
+        FileReader::tolower(part);
       }
       if (isBroadcastOrMasterDestination
-        || (isWriteMessage && !hasPartStr)
-        || strcasecmp(partStr, "M") == 0) {  // master data
+        || (isWriteMessage && !hasPart)
+        || part == "m") {  // master data
         partType = pt_masterData;
-      } else if ((!isWriteMessage && !hasPartStr)
-        || strcasecmp(partStr, "S") == 0) {  // slave data
+      } else if ((!isWriteMessage && !hasPart)
+        || part == "s") {  // slave data
         partType = pt_slaveData;
       } else {
-        result = RESULT_ERR_INVALID_PART;
+        errorDescription = "part "+part+" in "+MappedFileReader::combineRow(row);
+        result = hasPart ? RESULT_ERR_INVALID_ARG : RESULT_ERR_MISSING_ARG;
         break;
       }
     }
-
+    string comment = row["comment"];
+    if (comment == NULL_VALUE) {
+      comment = "";
+    }
     if (fields.empty()) {
       firstName = name;
-      firstComment = comment;
     }
 
-    const string typeStr = *it++;  // basetype[:len]|template[:name]
-    vector<string>::iterator typePos = it;
+    const string typeStr = row["type"];  // basetype[:len]|template[:name]
     if (typeStr.empty()) {
-      if (!name.empty() || hasPartStr) {
-        result = RESULT_ERR_MISSING_TYPE;
-      }
+      errorDescription = "field type in "+MappedFileReader::combineRow(row);
+      result = RESULT_ERR_MISSING_ARG;
       break;
     }
 
     map<unsigned int, string> values;
     string constantValue;
     bool verifyValue = false;
-    if (it != end) {
-      const string divisorStr = *it++;  // [divisor|values]
-      if (!divisorStr.empty()) {
-        size_t equalPos = divisorStr.find('=');
-        if (equalPos == string::npos) {
-          divisor = parseSignedInt(divisorStr.c_str(), 10, -MAX_DIVISOR, MAX_DIVISOR, result);
-        } else if (equalPos == 0 && divisorStr.length() > 1) {
-          verifyValue = divisorStr[1] == '=';  // == forced verification of constant value
-          if (verifyValue && divisorStr.length() == 1) {
+    const string divisorStr = row["divisor/values"];  // [divisor|values]
+    if (!divisorStr.empty()) {
+      size_t equalPos = divisorStr.find('=');
+      if (equalPos == string::npos) {
+        divisor = parseSignedInt(divisorStr.c_str(), 10, -MAX_DIVISOR, MAX_DIVISOR, result);
+        if (result != RESULT_OK) {
+          errorDescription = "divisor "+divisorStr+" in "+MappedFileReader::combineRow(row);
+        }
+      } else if (equalPos == 0 && divisorStr.length() > 1) {
+        verifyValue = divisorStr[1] == '=';  // == forced verification of constant value
+        if (verifyValue && divisorStr.length() == 1) {
+          errorDescription = "divisor "+divisorStr+" in "+MappedFileReader::combineRow(row);
+          result = RESULT_ERR_INVALID_LIST;
+          break;
+        }
+        constantValue = divisorStr.substr(equalPos+(verifyValue?2:1));
+      } else {
+        string token;
+        istringstream stream(divisorStr);
+        while (getline(stream, token, VALUE_SEPARATOR)) {
+          FileReader::trim(token);
+          const char* str = token.c_str();
+          char* strEnd = NULL;
+          unsigned long id;
+          if (strncasecmp(str, "0x", 2) == 0) {
+            str += 2;
+            id = strtoul(str, &strEnd, 16);  // hexadecimal
+          } else {
+            id = strtoul(str, &strEnd, 10);  // decimal
+          }
+          if (strEnd == NULL || strEnd == str || id > MAX_VALUE) {
+            errorDescription = "value "+token+" in "+MappedFileReader::combineRow(row);
             result = RESULT_ERR_INVALID_LIST;
             break;
           }
-          constantValue = divisorStr.substr(equalPos+(verifyValue?2:1));
-        } else {
-          istringstream stream(divisorStr);
-          while (getline(stream, token, VALUE_SEPARATOR)) {
-            FileReader::trim(token);
-            const char* str = token.c_str();
-            char* strEnd = NULL;
-            unsigned long id;
-            if (strncasecmp(str, "0x", 2) == 0) {
-              str += 2;
-              id = strtoul(str, &strEnd, 16);  // hexadecimal
-            } else {
-              id = strtoul(str, &strEnd, 10);  // decimal
-            }
-            if (strEnd == NULL || strEnd == str || id > MAX_VALUE) {
-              result = RESULT_ERR_INVALID_LIST;
-              break;
-            }
-            // remove blanks around '=' sign
-            while (*strEnd == ' ') strEnd++;
-            if (*strEnd != '=') {
-              result = RESULT_ERR_INVALID_LIST;
-              break;
-            }
-            token = string(strEnd + 1);
-            FileReader::trim(token);
-            values[(unsigned int)id] = token;
+          // remove blanks around '=' sign
+          while (*strEnd == ' ') strEnd++;
+          if (*strEnd != '=') {
+            errorDescription = "value "+token+" in "+MappedFileReader::combineRow(row);
+            result = RESULT_ERR_INVALID_LIST;
+            break;
           }
+          token = string(strEnd + 1);
+          FileReader::trim(token);
+          values[(unsigned int)id] = token;
         }
-        if (result != RESULT_OK) {
-          break;
-        }
+      }
+      if (result != RESULT_OK) {
+        break;
       }
     }
 
-    if (it == end) {
+    string unit = row["unit"];
+    if (unit == NULL_VALUE) {
       unit = "";
-    } else {
-      const string str = *it++;  // [unit]
-      if (strcasecmp(str.c_str(), NULL_VALUE) == 0) {
-        unit = "";
-      } else {
-        unit = str;
-      }
     }
-
-    if (it == end) {
-      comment = "";
-    } else {
-      const string str = *it++;  // [comment]
-      if (strcasecmp(str.c_str(), NULL_VALUE) == 0) {
-        comment = "";
-      } else {
-        comment = str;
-      }
-    }
-
     bool firstType = true;
+    string token;
     istringstream stream(typeStr);
     while (result == RESULT_OK && getline(stream, token, VALUE_SEPARATOR)) {
       FileReader::trim(token);
@@ -199,6 +217,7 @@ result_t DataField::create(vector<string>::iterator& it,
           } else {
             length = (size_t)parseInt(token.substr(pos+1).c_str(), 10, 1, (unsigned int)maxFieldLength, result);
             if (result != RESULT_OK) {
+              errorDescription = "field type "+token+" in "+MappedFileReader::combineRow(row);
               break;
             }
           }
@@ -211,13 +230,15 @@ result_t DataField::create(vector<string>::iterator& it,
         if (add != NULL) {
           fields.push_back(add);
         } else {
-          it = typePos;  // back to type
           if (result == RESULT_OK) {
+            errorDescription = "field type " + typeName+" in "+MappedFileReader::combineRow(row);
             result = RESULT_ERR_NOTFOUND;  // type not found
+          } else {
+            errorDescription = "create field in "+MappedFileReader::combineRow(row);
           }
         }
       } else if (!constantValue.empty()) {
-        it = typePos;  // back to type
+        errorDescription = "constant value "+constantValue+" in "+MappedFileReader::combineRow(row);
         result = RESULT_ERR_INVALID_ARG;  // invalid value list
       } else {  // template[:name]
         string fieldName;
@@ -230,7 +251,7 @@ result_t DataField::create(vector<string>::iterator& it,
         result = templ->derive(fieldName, firstType ? comment : "", firstType ? unit : "", partType, divisor, values,
             fields);
         if (result != RESULT_OK) {
-          it = typePos;  // back to type
+          errorDescription = "derive field "+fieldName+" in "+MappedFileReader::combineRow(row);
         }
       }
       firstType = false;
@@ -248,7 +269,7 @@ result_t DataField::create(vector<string>::iterator& it,
   if (fields.size() == 1) {
     returnField = fields[0];
   } else {
-    returnField = new DataFieldSet(firstName, firstComment, fields);
+    returnField = new DataFieldSet(firstName, "", fields);
   }
   return RESULT_OK;
 }
@@ -1005,7 +1026,7 @@ result_t DataFieldSet::write(istringstream& input, SymbolString& data,
 
 
 DataFieldTemplates::DataFieldTemplates(DataFieldTemplates& other)
-  : FileReader::FileReader(false) {
+    : MappedFileReader::MappedFileReader(false) {
   for (map<string, DataField*>::iterator it = other.m_fieldsByName.begin(); it != other.m_fieldsByName.end(); it++) {
     m_fieldsByName[it->first] = it->second->clone();
   }
@@ -1037,26 +1058,103 @@ result_t DataFieldTemplates::add(DataField* field, string name, bool replace) {
   return RESULT_OK;
 }
 
-result_t DataFieldTemplates::addFromFile(vector<string>::iterator& begin, const vector<string>::iterator end,
-    vector< vector<string> >* defaults, const string& defaultDest, const string& defaultCircuit,
-    const string& defaultSuffix, const string& filename, unsigned int lineNo) {
-  vector<string>::iterator restart = begin;
-  DataField* field = NULL;
-  string name;
-  if (begin != end) {
-    size_t colon = begin->find(':');
-    if (colon != string::npos) {
-      name = begin->substr(0, colon);
-      begin->erase(0, colon+1);
+result_t DataFieldTemplates::getFieldMap(vector<string>& row, string& errorDescription) {
+  // name[:usename],basetype[:len]|template[:usename][,[divisor|values][,[unit][,[comment]]]]
+  if (row.empty()) {
+    // default map does not include separate field name
+    row.push_back("name");
+    for (size_t cnt = 0; cnt < 2; cnt++) {
+      bool first = true;
+      for (size_t fieldId = DATAFIELD_RANGE_MIN; fieldId <= DATAFIELD_RANGE_MAX; fieldId++) {
+        if (cnt == 0 && fieldId == DATAFIELD_NAME) {
+          continue;
+        }
+        if (fieldId == DATAFIELD_PART) {  // not included in default map
+          continue;
+        }
+        // subsequent fields start with field name
+        if (first) {
+          first = false;
+          row.push_back("*"+getDataFieldName(fieldId));
+        } else {
+          row.push_back(getDataFieldName(fieldId));
+        }
+      }
     }
+    return RESULT_OK;
   }
-  result_t result = DataField::create(begin, end, this, field, false, true, false);
+  bool inDataFields = false;
+  map<string, string> seen;
+  for (auto &name : row) {
+    tolower(name);
+    size_t fieldId;
+    if (inDataFields) {
+      fieldId = getDataFieldId(name);
+      if (fieldId == UINT_MAX) {
+        errorDescription = "unknown field " + name;
+        return RESULT_ERR_INVALID_ARG;
+      }
+      if (seen.find(name) != seen.end()) {
+        if (seen.find("type") == seen.end()) {
+          return RESULT_ERR_EOF;  // require at least type
+        }
+        seen.clear();
+        name = "*" + getDataFieldName(fieldId);  // data field repetition
+      } else {
+        name = getDataFieldName(fieldId);
+      }
+    } else if (name == "name") {
+      if (seen.find(name) != seen.end()) {
+        errorDescription = "duplicate field " + name;
+        return RESULT_ERR_INVALID_ARG;
+      }
+      name = "name";
+    } else {
+      fieldId = getDataFieldId(name);
+      if (fieldId == UINT_MAX) {
+        errorDescription = "unknown field " + name;
+        return RESULT_ERR_INVALID_ARG;
+      }
+      if (seen.find("name") == seen.end()) {
+        return RESULT_ERR_EOF;  // require at least name
+      }
+      inDataFields = true;
+      seen.clear();
+      name = "*" + getDataFieldName(fieldId);
+    }
+    seen[name] = name;
+  }
+  if (!inDataFields) {
+    return RESULT_ERR_EOF;  // require at least one field
+  }
+  if (seen.find("name") == seen.end() || seen.find("type") == seen.end()) {
+    return RESULT_ERR_EOF;  // require at least name and type
+  }
+  return RESULT_OK;
+}
+
+result_t DataFieldTemplates::addFromFile(map<string, string>& row, vector< map<string, string> >& subRows,
+    string& errorDescription, const string filename, unsigned int lineNo) {
+  string name = row["name"];  // required
+  string firstFieldName;
+  size_t colon = name.find(':');
+  if (colon == string::npos) {
+    firstFieldName = name;
+  } else {
+    firstFieldName = name.substr(colon+1);
+    name = name.substr(0, colon);
+  }
+  DataField* field = NULL;
+  if (!subRows.empty() && subRows[0].find("name") == subRows[0].end()) {
+    subRows[0]["name"] = firstFieldName;
+  }
+  result_t result = DataField::create(subRows, errorDescription, this, field, false, true, false);
   if (result != RESULT_OK) {
     return result;
   }
   result = add(field, name, true);
   if (result == RESULT_ERR_DUPLICATE_NAME) {
-    begin = restart+1;  // mark name as invalid
+    errorDescription = name;
   }
   if (result != RESULT_OK) {
     delete field;

src/lib/ebus/data.h

@@ -25,6 +25,7 @@
 #include <fstream>
 #include <vector>
 #include <map>
+#include <climits>
 #include "lib/ebus/symbol.h"
 #include "lib/ebus/result.h"
 #include "lib/ebus/filereader.h"
@@ -48,6 +49,45 @@ namespace ebusd {
  * class.
  */
 
+/** the field ID for the field name. */
+#define DATAFIELD_NAME 0
+
+/** the field ID for the part filter. */
+#define DATAFIELD_PART (DATAFIELD_NAME+1)
+
+/** the field ID for the data type. */
+#define DATAFIELD_TYPE (DATAFIELD_PART+1)
+
+/** the field ID for the divisor/values. */
+#define DATAFIELD_DIVISORVALUES (DATAFIELD_TYPE+1)
+
+/** the field ID for the unit. */
+#define DATAFIELD_UNIT (DATAFIELD_DIVISORVALUES+1)
+
+/** the field ID for the comment. */
+#define DATAFIELD_COMMENT (DATAFIELD_UNIT+1)
+
+/** the marker field ID for the minimum field ID. */
+#define DATAFIELD_RANGE_MIN DATAFIELD_NAME
+
+/** the marker field ID for the maximum field ID. */
+#define DATAFIELD_RANGE_MAX DATAFIELD_COMMENT
+
+
+/**
+ * Get the data field ID for the given field name.
+ * @param name the field name.
+ * @return the field ID, or @a UINT_MAX if not found.
+ */
+size_t getDataFieldId(const string name);
+
+/**
+ * Get the data field name for the given field ID.
+ * @param fieldId the field ID.
+ * @return the field name, or empty if not found.
+ */
+string getDataFieldName(const size_t fieldId);
+
 class DataFieldTemplates;
 class SingleDataField;
 
@@ -88,7 +128,7 @@ class DataField {
    * @return @a RESULT_OK on success, or an error code.
    * Note: the caller needs to free the created instance.
    */
-  static result_t create(vector<string>::iterator& it, const vector<string>::iterator end,
+  static result_t create(vector< map<string, string> >& rows, string& errorDescription,
       DataFieldTemplates* templates, DataField*& returnField,
       const bool isWriteMessage,
       const bool isTemplate, const bool isBroadcastOrMasterDestination,
@@ -498,8 +538,7 @@ class DataFieldSet : public DataField {
    */
   DataFieldSet(const string name, const string comment,
       const vector<SingleDataField*> fields)
-    : DataField(name, comment),
-      m_fields(fields) {
+    : DataField(name, comment), m_fields(fields) {
     bool uniqueNames = true;
     map<string, string> names;
     for (vector<SingleDataField*>::const_iterator it = fields.begin(); it != fields.end(); it++) {
@@ -602,12 +641,12 @@ class DataFieldSet : public DataField {
 /**
  * A map of template @a DataField instances.
  */
-class DataFieldTemplates : public FileReader {
+class DataFieldTemplates : public MappedFileReader {
  public:
   /**
    * Constructs a new instance.
    */
-  DataFieldTemplates() : FileReader::FileReader(false) {}
+  DataFieldTemplates() : MappedFileReader::MappedFileReader(false) {}
 
   /**
    * Constructs a new copied instance.
@@ -638,9 +677,11 @@ class DataFieldTemplates : public FileReader {
   result_t add(DataField* field, string name = "", bool replace = false);
 
   // @copydoc
-  result_t addFromFile(vector<string>::iterator& begin, const vector<string>::iterator end,
-    vector< vector<string> >* defaults, const string& defaultDest, const string& defaultCircuit,
-    const string& defaultSuffix, const string& filename, unsigned int lineNo) override;
+  result_t getFieldMap(vector<string>& row, string& errorDescription) override;
+
+  // @copydoc
+  result_t addFromFile(map<string, string>& row, vector< map<string, string> >& subRows,
+      string& errorDescription, const string filename, unsigned int lineNo) override;
 
   /**
    * Gets the template @a DataField instance with the specified name.