added decode/encode commands, use [DD]* instead of Dx in help

ChangeLog.md

@@ -15,6 +15,8 @@
 * changed logging for messages from "updated" to "received" and "sent" depending on who initiated the request
 * added support for serving .csv files and new argument maxage to HTTP/JSON port
 * added timeout argument and error result code to ebusctl
+* added "-def" option to "read" and "write" commands and "define" command to test message defintions and add new definitions during runtime and disabled all by default (use "--enabledefine" command line option).
+* added "encode" and "decode" commands for testing field definitions
 
 
 # 3.1 (2017-12-26)
@@ -24,20 +26,20 @@
 * corrected dump of chained messages
 * corrected missing default MQTT topic
 * corrected dump of value list and constant fields
-* fix for input string vanishing on certain compiler versions in write commands
+* fix for input string vanishing on certain compiler versions in "write" command
 * fix for potential endless waits
 * corrected initial dumping and raw data logging
-* corrected address check with "-s" or "-d" argument in read and write commands
+* corrected address check with "-s" or "-d" argument in "read" and "write" commands
 * fixed timeout for update check
 * corrected wrong initial scan load message
 * corrected "read" command with field name and MQTT output by field when master and slave part both carry fields
 
 ## Features
-* added helper script for reading all Vaillant registers for a single slave via hex command
+* added helper script for reading all Vaillant registers for a single slave via "hex" command
 * check all message definitions independent of any condition when checking/dumping configuration
 * added option to disable automatic update check
 * added scan config mode to start log entry
-* added measurement and logging of min/max send-receive latency and include values in info command and JSON output
+* added measurement and logging of min/max send-receive latency and include values in "info" command and JSON output
 * drop initial potential garbage for UDP/TCP devices
 * better support for cygwin builds
 * added measurement of arbitration delay
@@ -90,11 +92,11 @@
 * allow invalid SW and HW fields and remove all but alpha-numeric chars and underscore from ident when looking for scan config file
 * added "auth" command and user/access information to "info" command output
 * added user and secret arguments to HTTP/JSON port and added options to retrieve message definition and raw messages
-* use ACL in read/write/find/scan commands
+* use ACL in "read"/"write"/"find"/"scan" commands
 * added -a and -l options to "find" command
 * added possibility to define different log level per area on command line and with "log" command
 * added decode option to "grab" command presenting decoding hints for unknown messages
-* added option to use different source address QQ to read/write/hex commands
+* added option to use different source address QQ to "read"/"write"/"hex" commands
 * enhanced scan for individual slave
 * wait for broadcast scan answers before doing individual scans
 * enhanced SymbolString to be aware of master/slave and get rid of CRC (instead calculate while sending/receiving)
@@ -190,7 +192,7 @@
 * added support for UDP connected devices
 
 ## Bug fixes
-* require "-c" in write command
+* require "-c" in "write" command
 * corrected duplicate messages check with condition
 * corrected maximum values for BCD and HCD types
 * corrected weekday calculation (for smaller systems mainly)
@@ -280,7 +282,7 @@ https://github.com/john30/ebusd/compare/v1.3.0...v2.0
 * trim CSV lines and fields
 * corrected derivation of field name, comment and unit in templates
 * check invalid divisor/values for string and value list based types
-* fix for overriding destination address in read command
+* fix for overriding destination address in "read" command
 * fix for cached master data
 
 ## Features

src/ebusd/mainloop.cpp

@@ -558,12 +558,12 @@ result_t MainLoop::decodeMessage(const string &data, bool isHttp, bool* connecte
     *ostream << "ERR: command not enabled";
     return RESULT_OK;
   }
-  /*if (cmd == "D" || cmd == "DECODE") {
+  if (cmd == "D" || cmd == "DECODE") {
     return executeDecode(args, ostream);
   }
   if (cmd == "E" || cmd == "ENCODE") {
     return executeEncode(args, ostream);
-  }*/
+  }
   if (cmd == "SCAN") {
     return executeScan(args, getUserLevels(*user), ostream);
   }
@@ -749,7 +749,7 @@ result_t MainLoop::executeRead(const vector<string>& args, const string& levels,
         "usage: read [-f] [-m SECONDS] [-s QQ] [-d ZZ] [-c CIRCUIT] [-p PRIO] [-v|-V] [-n|-N] [-i VALUE[;VALUE]*]"
         " NAME [FIELD[.N]]\n"
         "  or:  read [-f] [-m SECONDS] [-s QQ] [-d ZZ] [-v|-V] [-n|-N] [-i VALUE[;VALUE]*] -def DEFINITION\n"
-        "  or:  read [-f] [-m SECONDS] [-s QQ] [-c CIRCUIT] -h ZZPBSBNNDx\n"
+        "  or:  read [-f] [-m SECONDS] [-s QQ] [-c CIRCUIT] -h ZZPBSBNN[DD]*\n"
         " Read value(s) or hex message.\n"
         "  -f           force reading from the bus (same as '-m 0')\n"
         "  -m SECONDS   only return cached value if age is less than SECONDS [300]\n"
@@ -775,7 +775,7 @@ result_t MainLoop::executeRead(const vector<string>& args, const string& levels,
         "    ZZ         destination address\n"
         "    PB SB      primary/secondary command byte\n"
         "    NN         number of following data bytes\n"
-        "    Dx         data byte(s) to send";
+        "    DD         data byte(s) to send";
     return RESULT_OK;
   }
   time_t now;
@@ -999,7 +999,7 @@ result_t MainLoop::executeWrite(const vector<string>& args, const string levels,
   if (argPos == 0 || (!newDefinition && (circuit.empty() || (args.size() != argPos + 2 && args.size() != argPos + 1)))) {
     *ostream << "usage: write [-s QQ] [-d ZZ] -c CIRCUIT NAME [VALUE[;VALUE]*]\n"
         "  or:  write [-s QQ] [-d ZZ] -def DEFINITION [VALUE[;VALUE]*]\n"
-        "  or:  write [-s QQ] [-c CIRCUIT] -h ZZPBSBNNDx\n"
+        "  or:  write [-s QQ] [-c CIRCUIT] -h ZZPBSBNN[DD]*\n"
         " Write value(s) or hex message.\n"
         "  -s QQ        override source address QQ\n"
         "  -d ZZ        override destination address ZZ\n"
@@ -1012,7 +1012,7 @@ result_t MainLoop::executeWrite(const vector<string>& args, const string levels,
         "    ZZ         destination address\n"
         "    PB SB      primary/secondary command byte\n"
         "    NN         number of following data bytes\n"
-        "    Dx         data byte(s) to send";
+        "    DD         data byte(s) to send";
     return RESULT_OK;
   }
 
@@ -1185,13 +1185,13 @@ result_t MainLoop::executeHex(const vector<string>& args, ostringstream* ostream
     return ret;
   }
 
-  *ostream << "usage: hex [-s QQ] ZZPBSBNNDx\n"
+  *ostream << "usage: hex [-s QQ] ZZPBSBNN[DD]*\n"
               " Send arbitrary data in hex (only if enabled).\n"
               "  -s QQ  override source address QQ\n"
               "  ZZ     destination address\n"
               "  PB SB  primary/secondary command byte\n"
               "  NN     number of following data bytes\n"
-              "  Dx     data byte(s) to send";
+              "  DD     data byte(s) to send";
   return RESULT_OK;
 }
 
@@ -1501,7 +1501,7 @@ result_t MainLoop::executeDefine(const vector<string>& args, ostringstream* ostr
 }
 
 
-/*result_t MainLoop::executeDecode(const vector<string>& args, ostringstream* ostream) {
+result_t MainLoop::executeDecode(const vector<string>& args, ostringstream* ostream) {
   size_t argPos = 1;
   bool numeric = false, valueName = false;
   OutputFormat verbosity = 0;
@@ -1539,14 +1539,14 @@ result_t MainLoop::executeDefine(const vector<string>& args, ostringstream* ostr
 
   if (argPos == 0 || args.size() != argPos + 2) {
     *ostream <<
-        "usage: decode [-v|-V] [-n|-N] DEFINITION Dx\n"
+        "usage: decode [-v|-V] [-n|-N] DEFINITION DD[DD]*\n"
-        " Decode a field by definition and hex data.\n"
+        " Decode field(s) by definition and hex data.\n"
         "  -v          increase verbosity (include names/units/comments)\n"
         "  -V          be very verbose (include names, units, and comments)\n"
         "  -n          use numeric value of value=name pairs\n"
         "  -N          use numeric and named value of value=name pairs\n"
-        "  DEFINITION  field definition (type,divisor/values,unit,comment)\n"
+        "  DEFINITION  field definition (type,divisor/values,unit,comment,...)\n"
-        "  Dx          data byte(s) to decode";
+        "  DD          data byte(s) to decode";
     return RESULT_OK;
   }
 
@@ -1567,7 +1567,7 @@ result_t MainLoop::executeDefine(const vector<string>& args, ostringstream* ostr
   if (ret != RESULT_OK) {
     return ret;
   }
-  slave[0] = slave.size() - 1;  // adjust length
+  slave.adjustHeader();
   return fields.read(slave, 0, false, NULL, -1, verbosity, -1, ostream);
 }
 
@@ -1576,9 +1576,9 @@ result_t MainLoop::executeEncode(const vector<string>& args, ostringstream* ostr
   size_t argPos = 1;
   if (argPos == 0 || args.size() != argPos + 2) {
     *ostream <<
-        "usage: encode DEFINITION VALUE\n"
+        "usage: encode DEFINITION VALUE[;VALUE]*\n"
-        " Encode a field by definition and decoded value.\n"
+        " Encode field(s) by definition and decoded value(s).\n"
-        "  DEFINITION  field definition (type,divisor/values,unit,comment)\n"
+        "  DEFINITION  field definition (type,divisor/values,unit,comment,...)\n"
         "  VALUE       single field VALUE to encode";
     return RESULT_OK;
   }
@@ -1595,13 +1595,13 @@ result_t MainLoop::executeEncode(const vector<string>& args, ostringstream* ostr
   }
   istr = istringstream(args[argPos+1]);
   SlaveSymbolString slave;
-  ret = fields.write(FIELD_SEPARATOR, 0, &istr, &slave, NULL);
+  ret = fields.write(UI_FIELD_SEPARATOR, 0, &istr, &slave, NULL);
   if (ret != RESULT_OK) {
     return ret;
   }
   *ostream << slave.getStr(1);
   return ret;
-}*/
+}
 
 
 result_t MainLoop::executeScan(const vector<string>& args, const string& levels, ostringstream* ostream) {
@@ -1787,12 +1787,12 @@ result_t MainLoop::executeHelp(ostringstream* ostream) {
       " [-i VALUE[;VALUE]*] NAME [FIELD[.N]]\n"
       "           Read by new defintion: read [-f] [-m SECONDS] [-s QQ] [-d ZZ] [-v|-V] [-n|-N]"
       " [-i VALUE[;VALUE]*] -def DEFINITION\n"
-      "           Read hex message:      read [-f] [-m SECONDS] [-s QQ] [-c CIRCUIT] -h ZZPBSBNNDx\n"
+      "           Read hex message:      read [-f] [-m SECONDS] [-s QQ] [-c CIRCUIT] -h ZZPBSBNN[DD]*\n"
       " write|w   Write value(s):        write [-s QQ] [-d ZZ] -c CIRCUIT NAME [VALUE[;VALUE]*]\n"
       "           Write by new def.:     write [-s QQ] [-d ZZ] -def DEFINITION [VALUE[;VALUE]*]\n"
-      "           Write hex message:     write [-s QQ] [-c CIRCUIT] -h ZZPBSBNNDx\n"
+      "           Write hex message:     write [-s QQ] [-c CIRCUIT] -h ZZPBSBNN[DD]*\n"
       " auth|a    Authenticate user:     auth USER SECRET\n"
-      " hex       Send hex data:         hex [-s QQ] ZZPBSBNNDx\n"
+      " hex       Send hex data:         hex [-s QQ] ZZPBSBNN[DD]*\n"
       " find|f    Find message(s):       find [-v|-V] [-r] [-w] [-p] [-a] [-d] [-h] [-i ID] [-f] [-F COL[,COL]*] [-e]"
       " [-c CIRCUIT] [-l LEVEL] [NAME]\n"
       " listen|l  Listen for updates:    listen [stop]\n"
@@ -1801,8 +1801,8 @@ result_t MainLoop::executeHelp(ostringstream* ostream) {
       " grab|g    Grab messages:         grab [stop]\n"
       "           Report the messages:   grab result [all]\n"
       " define    Define new message:    define [-r] DEFINITION\n"
-      //" decode|d  Decode a field:        decode [-v|-V] [-n|-N] DEFINITION Dx\n"
+      " decode|d  Decode field(s):       decode [-v|-V] [-n|-N] DEFINITION DD[DD]*\n"
-      //" encode|e  Encode a field:        encode DEFINITION VALUE\n"
+      " encode|e  Encode field(s):       encode DEFINITION VALUE[;VALUE]*\n"
       " scan      Scan slaves:           scan [full|ZZ]\n"
       "           Report scan result:    scan result\n"
       " log       Set log area level:    log [AREA[,AREA]* LEVEL]\n"

src/ebusd/mainloop.h

@@ -254,7 +254,7 @@ class MainLoop : public Thread, DeviceListener {
    * @param ostream the @a ostringstream to format the result string to.
    * @return the result code.
    */
-  //result_t executeDecode(const vector<string>& args, ostringstream* ostream);
+  result_t executeDecode(const vector<string>& args, ostringstream* ostream);
 
   /**
    * Execute the encode command.
@@ -262,7 +262,7 @@ class MainLoop : public Thread, DeviceListener {
    * @param ostream the @a ostringstream to format the result string to.
    * @return the result code.
    */
-  //result_t executeEncode(const vector<string>& args, ostringstream* ostream);
+  result_t executeEncode(const vector<string>& args, ostringstream* ostream);
 
   /**
    * Execute the scan command.

src/lib/ebus/data.cpp

@@ -44,6 +44,11 @@ static const char* defaultTemplateFieldMap[] = {
     "*name", "type", "divisor/values", "unit", "comment",
 };
 
+/** the default field map for fields only. */
+static const char* defaultFieldsFieldMap[] = {
+    "*type", "divisor/values", "unit", "comment",
+};
+
 
 string AttributedItem::formatInt(size_t value) {
   ostringstream stream;
@@ -1129,6 +1134,124 @@ result_t DataFieldSet::write(char separator, size_t offset, istringstream* input
 }
 
 
+result_t LoadableDataFieldSet::getFieldMap(const string& preferLanguage, vector<string>* row,
+    string* errorDescription) const {
+  // *type,divisor/values,unit,comment
+  if (row->empty()) {
+    for (const auto& col : defaultFieldsFieldMap) {
+      row->push_back(col);
+    }
+    return RESULT_OK;
+  }
+  map<string, size_t> seen;
+  for (size_t col = 0; col < row->size(); col++) {
+    string &name = (*row)[col];
+    string lowerName = name;
+    tolower(&lowerName);
+    trim(&lowerName);
+    bool toDataFields;
+    if (!lowerName.empty() && lowerName[0] == '*') {
+      lowerName.erase(0, 1);
+      toDataFields = true;
+    } else {
+      toDataFields = col == 0;
+    }
+    if (lowerName.empty()) {
+      *errorDescription = "missing name in column " + AttributedItem::formatInt(col);
+      return RESULT_ERR_INVALID_ARG;
+    }
+    if (toDataFields) {
+      if (!seen.empty() && seen.find("type") == seen.end()) {
+        *errorDescription = "missing field type";
+        return RESULT_ERR_EOF;  // require at least name and type
+      }
+      seen.clear();
+    }
+    size_t langPos = lowerName.find_last_of('.');
+    if (langPos != string::npos && langPos > 0 && langPos == lowerName.length()-3) {
+      string lang = lowerName.substr(langPos+1);
+      lowerName.erase(langPos);
+      map<string, size_t>::iterator previous = seen.find(lowerName);
+      if (previous != seen.end()) {
+        if (lang != preferLanguage) {
+          // skip this column
+          name = SKIP_COLUMN;
+          continue;
+        }
+        // replace previous
+        (*row)[previous->second] = SKIP_COLUMN;
+        seen.erase(lowerName);
+      }
+    } else {
+      map<string, size_t>::iterator previous = seen.find(lowerName);
+      if (previous != seen.end()) {
+        *errorDescription = "duplicate field " + name;
+        return RESULT_ERR_INVALID_ARG;
+      }
+    }
+    name = toDataFields ? "*"+lowerName : lowerName;
+    seen[lowerName] = col;
+  }
+  if (seen.find("type") == seen.end()) {
+    *errorDescription = "missing field type";
+    return RESULT_ERR_EOF;  // require at least type
+  }
+  return RESULT_OK;
+}
+
+result_t LoadableDataFieldSet::addFromFile(const string& filename, unsigned int lineNo, map<string, string>* row,
+    vector< map<string, string> >* subRows, string* errorDescription, bool replace) {
+  const DataField* field = NULL;
+  result_t result = DataField::create(false, false, false, MAX_POS, m_templates, subRows, errorDescription, &field);
+  if (result != RESULT_OK) {
+    return result;
+  }
+  if (!field) {
+    return RESULT_ERR_INVALID_ARG;
+  }
+  map<string, string> names;
+  for (auto check : m_fields) {
+    if (check->isIgnored()) {
+      continue;
+    }
+    string name = check->getName(-1);
+    if (!name.empty()) {
+      names[name] = name;
+    }
+  }
+  if (field->isSet()) {
+    const DataFieldSet* fieldSet = dynamic_cast<const DataFieldSet*>(field);
+    for (auto sfield : fieldSet->m_fields) {
+      m_fields.push_back(sfield);
+      if (sfield->isIgnored()) {
+        m_ignoredCount++;
+        continue;
+      }
+      string name = sfield->getName(-1);
+      if (name.empty() || names.find(name) != names.end()) {
+        m_uniqueNames = false;
+      } else {
+        names[name] = name;
+      }
+    }
+  } else {
+    const SingleDataField* sfield = dynamic_cast<const SingleDataField*>(field);
+    m_fields.push_back(sfield);
+    if (sfield->isIgnored()) {
+      m_ignoredCount++;
+    } else {
+      string name = sfield->getName(-1);
+      if (name.empty() || names.find(name) != names.end()) {
+        m_uniqueNames = false;
+      } else {
+        names[name] = name;
+      }
+    }
+  }
+  return result;
+}
+
+
 DataFieldTemplates::DataFieldTemplates(const DataFieldTemplates& other)
 : MappedFileReader::MappedFileReader(false) {
   for (const auto it : other.m_fieldsByName) {

src/lib/ebus/data.h

@@ -203,6 +203,12 @@ class DataField : public AttributedItem {
    */
   virtual const DataField* clone() const = 0;
 
+  /**
+   * Return whether this is a @a DataFieldSet.
+   * @return true if this is a @a DataFieldSet.
+   */
+  virtual bool isSet() const { return false; };
+
   /**
    * Factory method for creating new instances.
    * @param isWriteMessage whether the field is part of a write message (default false).
@@ -589,6 +595,7 @@ class ConstantDataField : public SingleDataField {
  * A set of @a DataField instances.
  */
 class DataFieldSet : public DataField {
+ friend class LoadableDataFieldSet;
  public:
   /**
    * Get the @a DataFieldSet for parsing the identification message (service 0x07 0x04).
@@ -636,6 +643,9 @@ class DataFieldSet : public DataField {
   // @copydoc
   const DataFieldSet* clone() const override;
 
+  // @copydoc
+  bool isSet() const override { return true; };
+
   // @copydoc
   size_t getLength(PartType partType, size_t maxLength) const override;
 
@@ -692,8 +702,9 @@ class DataFieldSet : public DataField {
   /** the @a DataFieldSet containing the ident message @a SingleDataField instances, or NULL. */
   static DataFieldSet* s_identFields;
 
+ protected:
   /** the @a vector of @a SingleDataField instances part of this set. */
-  const vector<const SingleDataField*> m_fields;
+  vector<const SingleDataField*> m_fields;
 
   /** whether all fields have a unique name. */
   bool m_uniqueNames;
@@ -703,6 +714,32 @@ class DataFieldSet : public DataField {
 };
 
 
+/**
+ * A special @a DataFieldSet that supports loading via @a MappedFileReader.
+ */
+class LoadableDataFieldSet : public DataFieldSet, public MappedFileReader {
+ public:
+  /**
+   * Constructs a new instance.
+   * @param name the field name.
+   * @param fields the @a vector of @a SingleDataField instances part of this set.
+   */
+  LoadableDataFieldSet(const string& name, DataFieldTemplates* templates)
+    : DataFieldSet(name, vector<const SingleDataField*>()), MappedFileReader(false), m_templates(templates) {
+  }
+
+  // @copydoc
+  result_t getFieldMap(const string& preferLanguage, vector<string>* row, string* errorDescription) const override;
+
+  // @copydoc
+  result_t addFromFile(const string& filename, unsigned int lineNo, map<string, string>* row,
+      vector< map<string, string> >* subRows, string* errorDescription, bool replace) override;
+
+ private:
+  DataFieldTemplates* m_templates;
+};
+
+
 /**
  * A map of template @a DataField instances.
  */