enhanced JSON interface to retrieve definition and raw messages, fixed for JSON strings with quotes

contrib/html/swagger.yaml

@@ -7,6 +7,7 @@ schemes:
   - http
 produces:
   - 'application/json;charset=utf-8'
+host: 'l:8080'
 paths:
   /data:
     get:
@@ -34,6 +35,8 @@ paths:
     - $ref: '#/parameters/fullQuery'
     - $ref: '#/parameters/requiredQuery'
     - $ref: '#/parameters/writeQuery'
+    - $ref: '#/parameters/rawQuery'
+    - $ref: '#/parameters/defQuery'
     - $ref: '#/parameters/userQuery'
     - $ref: '#/parameters/secretQuery'
   /data/{circuit}:
@@ -64,6 +67,8 @@ paths:
     - $ref: '#/parameters/fullQuery'
     - $ref: '#/parameters/requiredQuery'
     - $ref: '#/parameters/writeQuery'
+    - $ref: '#/parameters/rawQuery'
+    - $ref: '#/parameters/defQuery'
     - $ref: '#/parameters/userQuery'
     - $ref: '#/parameters/secretQuery'
   /data/{circuit}/{message}:
@@ -89,6 +94,8 @@ paths:
     - $ref: '#/parameters/fullQuery'
     - $ref: '#/parameters/requiredQuery'
     - $ref: '#/parameters/writeQuery'
+    - $ref: '#/parameters/rawQuery'
+    - $ref: '#/parameters/defQuery'
     - $ref: '#/parameters/userQuery'
     - $ref: '#/parameters/secretQuery'
   /{file}:
@@ -115,6 +122,7 @@ paths:
           - image/jpeg
           - image/svg+xml
           - 'application/json;charset=utf-8'
+          - 'application/yaml;charset=utf-8'
     head:
       summary: Retrieve headers for a certain file.
       responses:
@@ -157,6 +165,12 @@ definitions:
         description: the maximum symbol rate on the bus seen since start.
         minimum: 0
         example: 167
+      qq:
+        type: integer
+        description: the ebusd master address (only if not readonly).
+        minimum: 0
+        maximum: 255
+        example: 49
       reconnects:
         type: integer
         description: the number of reconnects to the device since start.
@@ -186,39 +200,87 @@ definitions:
     - lastup
   Circuit:
     type: object
-    items:
+    properties:
-      $ref: '#/definitions/Message'
+      name:
+        type: string
+        description: the circuit name.
+      zz:
+        type: integer
+        description: the circuit slave address.
+        minimum: 0
+        maximum: 255
+        example: 8
+      messages:
+        type: object
+        additionalProperties:
+          $ref: '#/definitions/Message'
   Message:
     type: object
     properties:
       name:
         type: string
         description: the message name.
-      lastup:
-        type: integer
-        description: the time in UTC seconds of the last update of the message.
-        minimum: 0
-      zz:
-        type: string
-        description: destination bus address.
-        pattern: '[0-9a-f]{2}'
-      comment:
-        type: string
-        description: the message comment (only with verbose).
-      fields:
-        type: object
-        items:
-          $ref: '#/definitions/Field'
       passive:
         type: boolean
         description: true when this is a passive message.
       write:
         type: boolean
         description: true for a write message, false for a read message.
+      lastup:
+        type: integer
+        description: the time in UTC seconds of the last update of the message (0 for never).
+        minimum: 0
+      qq:
+        type: integer
+        description: limited source master address (only with def).
+        minimum: 0
+        maximum: 255
+        example: 49
+      zz:
+        type: integer
+        description: destination master or slave address.
+        minimum: 0
+        maximum: 255
+        example: 8
+      id:
+        type: array
+        description: the message ID composed of PBSB and further master data bytes (only with def).
+        items:
+          type: integer
+          minimum: 0
+          maximum: 255
+      comment:
+        type: string
+        description: the message comment (only with verbose).
+      master:
+        type: array
+        description: the last seen master data bytes (only with raw and if available).
+        items:
+          type: integer
+          minimum: 0
+          maximum: 255
+      slave:
+        type: array
+        description: the last seen slave data bytes (only with raw and if available).
+        items:
+          type: integer
+          minimum: 0
+          maximum: 255
+      fields:
+        type: object
+        description: the decoded fields the message is composed of (only if available).
+        additionalProperties:
+          $ref: '#/definitions/Field'
+      fielddefs:
+        type: array
+        description: the field definitions the message is composed of (only with def).
+        items:
+          $ref: '#/definitions/FieldDef'
     required:
-    - lastup
+    - name
     - passive
     - write
+    - lastup
   Field:
     type: object
     properties:
@@ -233,12 +295,57 @@ definitions:
       comment:
         type: string
         description: the field comment (only with verbose).
+  FieldDef:
+    type: object
+    properties:
+      name:
+        type: string
+        description: the field name.
+      slave:
+        type: boolean
+        description: whether the field is part of the slave data.
+      type:
+        type: string
+        description: the field type.
+      isbits:
+        type: boolean
+        description: true when the length is in bits.
+      length:
+        type: number
+        description: the field length in bytes (-1 for remainder, number of bits when isbits is true).
+      divisor:
+        type: number
+        description: the extra divisor applied to the raw value (only if applicable).
+      value:
+        type: string
+        description: the constant value (only if applicable).
+      verify:
+        type: boolean
+        description: whether the constant value is verified when decoding (only if applicable).
+      values:
+        type: object
+        description: the raw numeric value to name assignments (only if applicable).
+        additionalProperties:
+          type: string
+          description: the value name.
+      unit:
+        type: string
+        description: the field unit.
+      comment:
+        type: string
+        description: the field comment.
+    required:
+      - name
+      - slave
+      - type
+      - isbits
+      - length
   Data:
     type: object
     properties:
       global:
         $ref: '#/definitions/Global'
-    items:
+    additionalProperties:
       $ref: '#/definitions/Circuit'
     required:
     - global
@@ -282,7 +389,7 @@ parameters:
   indexedQuery:
     name: indexed
     in: query
-    description: return field indexes instead of names.
+    description: always return field indexes instead of names.
     type: boolean
     allowEmptyValue: false
   numericQuery:
@@ -315,6 +422,18 @@ parameters:
     description: retrieve write messages in addition to read/poll messages.
     type: boolean
     allowEmptyValue: false
+  rawQuery:
+    name: raw
+    in: query
+    description: include raw master/slave data.
+    type: boolean
+    allowEmptyValue: false
+  defQuery:
+    name: def
+    in: query
+    description: include message and field definition.
+    type: boolean
+    allowEmptyValue: false
   userQuery:
     name: user
     in: query

src/ebusd/bushandler.cpp

@@ -234,7 +234,7 @@ bool decodeType(const DataType* type, const SymbolString& input, size_t length,
       first = false;
       *output << endl << " ";
       ostringstream::pos_type cnt = output->tellp();
-      type->dump(length, false, output);
+      type->dump(false, length, false, output);
       cnt = output->tellp() - cnt;
       while (cnt < 5) {
         *output << " ";

src/ebusd/mainloop.cpp

@@ -1616,7 +1616,8 @@ result_t MainLoop::executeHelp(ostringstream* ostream) {
 }
 
 result_t MainLoop::executeGet(const vector<string>& args, bool* connected, ostringstream* ostream) {
-  bool numeric = false, valueName = false, required = false, full = false, withWrite = false;
+  bool numeric = false, valueName = false, required = false, full = false, withWrite = false, raw = false;
+  bool withDefinition = false;
   OutputFormat verbosity = OF_NAMES;
   size_t argPos = 1;
   string uri = args[argPos++];
@@ -1673,6 +1674,10 @@ result_t MainLoop::executeGet(const vector<string>& args, bool* connected, ostri
           required = value.length() == 0 || value == "1" || value == "true";
         } else if (qname == "write") {
           withWrite = value.length() == 0 || value == "1" || value == "true";
+        } else if (qname == "raw") {
+          raw = value.length() == 0 || value == "1" || value == "true";
+        } else if (qname == "def") {
+          withDefinition = value.length() == 0 || value == "1" || value == "true";
         } else if (qname == "user") {
           user = value;
         } else if (qname == "secret") {
@@ -1692,7 +1697,8 @@ result_t MainLoop::executeGet(const vector<string>& args, bool* connected, ostri
     time_t maxLastUp = 0;
     if (ret == RESULT_OK) {
       bool first = true;
-      verbosity |= (valueName ? OF_VALUENAME : numeric ? OF_NUMERIC : 0) | OF_JSON | (full ? OF_ALL_ATTRS : 0);
+      verbosity |= (valueName ? OF_VALUENAME : numeric ? OF_NUMERIC : 0) | OF_JSON | (full ? OF_ALL_ATTRS : 0)
+                   | (withDefinition ? OF_DEFINTION : 0);
       deque<Message*> messages;
       m_messages->findAll(circuit, name, getUserLevels(user), exact, true, withWrite, true, true, true, 0, 0,
           &messages);
@@ -1725,15 +1731,16 @@ result_t MainLoop::executeGet(const vector<string>& args, bool* connected, ostri
         }
         if (message->getCircuit() != lastCircuit) {
           if (lastCircuit.length() > 0) {
-            *ostream << "\n },";
+            *ostream << "\n  }\n },";
           }
           lastCircuit = message->getCircuit();
           *ostream << "\n \"" << lastCircuit << "\": {";
-          first = true;
           if (full && m_messages->decodeCircuit(lastCircuit, verbosity, ostream)) {  // add circuit specific values
-            first = false;
+            *ostream << ",";
           }
+          *ostream << "\n  \"messages\": {";
           lastName = "";
+          first = true;
         }
         name = message->getName();
         bool same = name == lastName;
@@ -1741,16 +1748,16 @@ result_t MainLoop::executeGet(const vector<string>& args, bool* connected, ostri
           Message* next = *(it+1);
           same = next->getCircuit() == lastCircuit && next->getName() == name;
         }
-        message->decode(!first, same, verbosity, ostream);
+        message->decodeJson(!first, same, raw, verbosity, ostream);
         lastName = name;
         first = false;
       }
 
       if (lastCircuit.length() > 0) {
-        *ostream << "\n },";
+        *ostream << "\n  }\n },";
       }
       *ostream << "\n \"global\": {"
-               << "\n  \"version\": \"" << PACKAGE_VERSION "." REVISION "\"";
+               << "\n  \"version\": \"" << PACKAGE_VERSION "." REVISION "\"" << setw(0) << dec;
       if (!m_updateCheck.empty()) {
         *ostream << ",\n  \"updatecheck\": \"" << m_updateCheck << "\"";
       }
@@ -1766,10 +1773,13 @@ result_t MainLoop::executeGet(const vector<string>& args, bool* connected, ostri
         *ostream << ",\n  \"symbolrate\": " << m_busHandler->getSymbolRate()
                  << ",\n  \"maxsymbolrate\": " << m_busHandler->getMaxSymbolRate();
       }
+      if (!m_device->isReadOnly()) {
+        *ostream << ",\n  \"qq\": " << static_cast<unsigned>(m_address);
+      }
       *ostream << ",\n  \"reconnects\": " << m_reconnectCount
                << ",\n  \"masters\": " << m_busHandler->getMasterCount()
                << ",\n  \"messages\": " << m_messages->size()
-               << ",\n  \"lastup\": " << setw(0) << dec << static_cast<unsigned>(maxLastUp)
+               << ",\n  \"lastup\": " << static_cast<unsigned>(maxLastUp)
                << "\n }"
                << "\n}";
       type = 6;
@@ -1802,6 +1812,8 @@ result_t MainLoop::executeGet(const vector<string>& args, bool* connected, ostri
         type = 5;
       } else if (ext == "json") {
         type = 6;
+      } else if (ext == "yaml") {
+        type = 7;
       }
     }
     if (type < 0) {
@@ -1848,6 +1860,9 @@ result_t MainLoop::formatHttpResult(result_t ret, int type, ostringstream* ostre
     case 6:
       *ostream << "application/json;charset=utf-8";
       break;
+    case 7:
+      *ostream << "application/yaml;charset=utf-8";
+      break;
     default:
       *ostream << "text/html";
       break;

src/lib/ebus/contrib/test/test_tem.cpp

@@ -178,7 +178,7 @@ int main() {
       continue;
     }
     cout << "\"" << check[0] << "\"=\"";
-    fields->dump(&cout);
+    fields->dump(false, false, &cout);
     cout << "\": create OK" << endl;
 
     ostringstream output;

src/lib/ebus/data.cpp

@@ -45,35 +45,6 @@ static const char* defaultTemplateFieldMap[] = {
 };
 
 
-
-string getDataFieldName(const string& name, bool* supportsLanguage) {
-  if (name.find("name") != string::npos || name.find("field") != string::npos) {
-    return "name";
-  }
-  if (name.find("part") != string::npos) {
-    return "part";
-  }
-  if (name.find("type") != string::npos) {
-    return "type";
-  }
-  if (name.find("divisor") != string::npos) {
-    if (name.find("values") != string::npos) {
-      *supportsLanguage = true;
-      return "divisor/values";
-    }
-    return "divisor";
-  }
-  *supportsLanguage = true;
-  if (name == "values" || name == "unit") {
-    return name;
-  }
-  if (name.find("comment") != string::npos) {
-    return "comment";
-  }
-  return "";
-}
-
-
 string AttributedItem::formatInt(size_t value) {
   ostringstream stream;
   stream << dec << static_cast<unsigned>(value);
@@ -102,8 +73,8 @@ void AttributedItem::dumpString(bool prependFieldSeparator, const string& str, o
 }
 
 void AttributedItem::appendJson(bool prependFieldSeparator, const string& name, const string& value,
-    bool asString, ostream* output) {
+    bool forceString, ostream* output) {
-  bool plain = !asString;
+  bool plain = !forceString && !value.empty();
   if (plain) {
     plain = value == "false" || value == "true";
     if (!plain) {
@@ -120,7 +91,15 @@ void AttributedItem::appendJson(bool prependFieldSeparator, const string& name,
   if (plain) {
     *output << value;
   } else {
-    *output << '"' << value << '"';
+    *output << '"';
+    if (value.find_first_of('"') == string::npos) {  // check for nested '"'
+      *output << value;
+    } else {
+      string replaced = value;
+      replace(replaced.begin(), replaced.end(), '"', '\'');
+      *output << replaced;
+    }
+    *output << '"';
   }
 }
 
@@ -133,8 +112,13 @@ void AttributedItem::mergeAttributes(map<string, string>* attributes) const {
   }
 }
 
-void AttributedItem::dumpAttribute(bool prependFieldSeparator, const string& name, ostream* output) const {
+void AttributedItem::dumpAttribute(bool prependFieldSeparator, bool asJson, const string& name, ostream* output)
-  dumpString(prependFieldSeparator, getAttribute(name), output);
+    const {
+  if (asJson) {
+    appendJson(prependFieldSeparator, name, getAttribute(name), false, output);
+  } else {
+    dumpString(prependFieldSeparator, getAttribute(name), output);
+  }
 }
 
 bool AttributedItem::appendAttribute(OutputFormat outputFormat, const string& name, bool onlyIfNonEmpty,
@@ -164,10 +148,19 @@ bool AttributedItem::appendAttributes(OutputFormat outputFormat, ostream* output
     for (const auto entry : m_attributes) {
       ret = true;
       if (!entry.second.empty() && entry.first != "unit" && entry.first != "comment") {
+        const string& key = entry.first;
         if (outputFormat & OF_JSON) {
-          appendJson(true, entry.first, entry.second, false, output);
+          if (key == "zz" || key == "qq") {
+            result_t result = RESULT_EMPTY;
+            size_t addr = parseInt(entry.second.c_str(), 16, 0, 255, &result);
+            if (result == RESULT_OK) {
+              *output << FIELD_SEPARATOR << " \"" << key << "\": " << addr;
+              continue;
+            }
+          }
+          appendJson(true, key, entry.second, false, output);
         } else {
-          *output << " " << entry.first << "=" << entry.second;
+          *output << " " << key << "=" << entry.second;
         }
       }
     }
@@ -459,27 +452,44 @@ result_t SingleDataField::create(const string& name, const map<string, string>&
   return RESULT_OK;
 }
 
-void SingleDataField::dumpPrefix(ostream* output) const {
+void SingleDataField::dumpPrefix(bool prependFieldSeparator, bool asJson, ostream* output) const {
   *output << setw(0) << dec;  // initialize formatting
-  dumpString(false, m_name, output);
+  if (asJson) {
-  *output << FIELD_SEPARATOR;
+    if (prependFieldSeparator) {
-  if (m_partType == pt_masterData) {
+      *output << FIELD_SEPARATOR;
-    *output << "m";
+    }
-  } else if (m_partType == pt_slaveData) {
+    *output << "\n     {";
-    *output << "s";
+    appendJson(false, "name", m_name, true, output);
+  } else {
+    dumpString(prependFieldSeparator, m_name, output);
   }
   *output << FIELD_SEPARATOR;
+  if (asJson) {
+    *output << " \"slave\": " << (m_partType == pt_slaveData ? "true" : "false") << ", ";
+  } else {
+    if (m_partType == pt_masterData) {
+      *output << "m";
+    } else if (m_partType == pt_slaveData) {
+      *output << "s";
+    }
+  }
+  if (!asJson) {
+    *output << FIELD_SEPARATOR;
+  }
+  m_dataType->dump(asJson, m_length, true, output);
 }
 
-void SingleDataField::dumpSuffix(ostream* output) const {
+void SingleDataField::dumpSuffix(bool asJson, ostream* output) const {
-  dumpAttribute(true, "unit", output);
+  dumpAttribute(true, asJson, "unit", output);
-  dumpAttribute(true, "comment", output);
+  dumpAttribute(true, asJson, "comment", output);
+  if (asJson) {
+    *output << "}";
+  }
 }
 
-void SingleDataField::dump(ostream* output) const {
+void SingleDataField::dump(bool prependFieldSeparator, bool asJson, ostream* output) const {
-  dumpPrefix(output);
+  dumpPrefix(prependFieldSeparator, asJson, output);
-  m_dataType->dump(m_length, true, output);
+  dumpSuffix(asJson, output);
-  dumpSuffix(output);
 }
 
 result_t SingleDataField::read(const SymbolString& data, size_t offset,
@@ -676,10 +686,19 @@ result_t ValueListDataField::derive(const string& name, PartType partType, int d
   return RESULT_OK;
 }
 
-void ValueListDataField::dump(ostream* output) const {
+void ValueListDataField::dump(bool prependFieldSeparator, bool asJson, ostream* output) const {
-  dumpPrefix(output);
+  dumpPrefix(prependFieldSeparator, asJson, output);
-  if (!m_dataType->dump(m_length, true, output)) {  // no divisor appended
+  // no divisor appended since it is not allowed for ValueListDataField
-    bool first = true;
+  bool first = true;
+  if (asJson) {
+    *output << ", \"values\": {";
+    for (const auto it : m_values) {
+      appendJson(!first, formatInt(it.first), it.second, true, output);  // TODO optimize?
+      first = false;
+    }
+    *output << " }";
+  } else {
+    *output << FIELD_SEPARATOR;
     for (const auto it : m_values) {
       if (first) {
         first = false;
@@ -688,8 +707,8 @@ void ValueListDataField::dump(ostream* output) const {
       }
       *output << it.first << "=" << it.second;
     }
-  }  // else: impossible since divisor is not allowed for ValueListDataField
+  }
-  dumpSuffix(output);
+  dumpSuffix(asJson, output);
 }
 
 result_t ValueListDataField::readSymbols(const SymbolString& input, size_t offset,
@@ -783,12 +802,16 @@ result_t ConstantDataField::derive(const string& name, PartType partType, int di
   return RESULT_OK;
 }
 
-void ConstantDataField::dump(ostream* output) const {
+void ConstantDataField::dump(bool prependFieldSeparator, bool asJson, ostream* output) const {
-  dumpPrefix(output);
+  dumpPrefix(prependFieldSeparator, asJson, output);
-  if (!m_dataType->dump(m_length, true, output)) {  // no divisor appended
+  // no divisor appended since it is not allowed for ConstantDataField
-    *output << (m_verify?"==":"=") << m_value;
+  if (asJson) {
-  }  // else: impossible since divisor is not allowed for ConstantDataField
+    appendJson(false, "value", m_value, true, output);
-  dumpSuffix(output);
+    *output << ", \"verify\":" << (m_verify ? "true" : "false");
+  } else {
+    *output << FIELD_SEPARATOR << (m_verify?"==":"=") << m_value;
+  }
+  dumpSuffix(asJson, output);
 }
 
 result_t ConstantDataField::readSymbols(const SymbolString& input, size_t offset,
@@ -953,15 +976,10 @@ bool DataFieldSet::hasField(const char* fieldName, bool numeric) const {
   return false;
 }
 
-void DataFieldSet::dump(ostream* output) const {
+void DataFieldSet::dump(bool prependFieldSeparator, bool asJson, ostream* output) const {
-  bool first = true;
   for (const auto field : m_fields) {
-    if (first) {
+    field->dump(prependFieldSeparator, asJson, output);
-      first = false;
+    prependFieldSeparator = true;
-    } else {
-      *output << FIELD_SEPARATOR;
-    }
-    field->dump(output);
   }
 }
 

src/lib/ebus/data.h

@@ -50,14 +50,6 @@ namespace ebusd {
  */
 
 
-/**
- * Get the normalized data field name for the given name.
- * @param name the input field name.
- * @param supportsLanguage set to true when the field supports multiple language.
- * @return the normalized data field name, or empty if unknown.
- */
-string getDataFieldName(const string& name, bool* supportsLanguage);
-
 class DataFieldTemplates;
 class SingleDataField;
 
@@ -115,11 +107,11 @@ class AttributedItem {
    * @param prependFieldSeparator whether to start with a @a FIELD_SEPARATOR.
    * @param name the name of the attribute.
    * @param value the value of the attribute.
-   * @param asString true to force writing as string, false to detect the type from the value.
+   * @param forceString true to force writing as string, false to detect the type from the value.
    * @param output the @a ostream to append to.
    */
   static void appendJson(bool prependFieldSeparator, const string& name, const string& value,
-      bool asString, ostream* output);
+      bool forceString, ostream* output);
 
   /**
    * Merge this instance's additional named attributes into the specified attributes.
@@ -130,10 +122,11 @@ class AttributedItem {
   /**
    * Dump the attribute optionally embedded in @a TEXT_SEPARATOR to the output.
    * @param prependFieldSeparator whether to start with a @a FIELD_SEPARATOR.
+   * @param asJson whether to output in JSON format.
    * @param name the name of the attribute to dump.
    * @param output the @a ostream to dump to.
    */
-  void dumpAttribute(bool prependFieldSeparator, const string& name, ostream* output) const;
+  void dumpAttribute(bool prependFieldSeparator, bool asJson, const string& name, ostream* output) const;
 
   /**
    * Append the attribute value to the output.
@@ -265,9 +258,11 @@ class DataField : public AttributedItem {
 
   /**
    * Dump the field settings to the output.
+   * @param prependFieldSeparator whether to start with a @a FIELD_SEPARATOR.
+   * @param asJson whether to output in JSON format.
    * @param output the @a ostream to dump to.
    */
-  virtual void dump(ostream* output) const = 0;
+  virtual void dump(bool prependFieldSeparator, bool asJson, ostream* output) const = 0;
 
   /**
    * Return whether the field is available.
@@ -404,18 +399,21 @@ class SingleDataField : public DataField {
 
   /**
    * Dump the common prefix field settings to the output (name and part type).
+   * @param prependFieldSeparator whether to start with a @a FIELD_SEPARATOR.
+   * @param asJson whether to output in JSON format.
    * @param output the @a ostream to dump to.
    */
-  void dumpPrefix(ostream* output) const;
+  void dumpPrefix(bool prependFieldSeparator, bool asJson, ostream* output) const;
 
   /**
    * Dump the common suffix field settings to the output (optional unit and comment).
+   * @param asJson whether to output in JSON format.
    * @param output the @a ostream to dump to.
    */
-  void dumpSuffix(ostream* output) const;
+  void dumpSuffix(bool asJson, ostream* output) const;
 
   // @copydoc
-  void dump(ostream* output) const override;
+  void dump(bool prependFieldSeparator, bool asJson, ostream* output) const override;
 
   // @copydoc
   bool hasField(const char* fieldName, bool numeric) const override;
@@ -501,7 +499,7 @@ class ValueListDataField : public SingleDataField {
       vector<const SingleDataField*>* fields) const override;
 
   // @copydoc
-  void dump(ostream* output) const override;
+  void dump(bool prependFieldSeparator, bool asJson, ostream* output) const override;
 
 
  protected:
@@ -554,7 +552,7 @@ class ConstantDataField : public SingleDataField {
       vector<const SingleDataField*>* fields) const override;
 
   // @copydoc
-  void dump(ostream* output) const override;
+  void dump(bool prependFieldSeparator, bool asJson, ostream* output) const override;
 
 
  protected:
@@ -660,7 +658,7 @@ class DataFieldSet : public DataField {
   bool hasField(const char* fieldName, bool numeric) const override;
 
   // @copydoc
-  void dump(ostream* output) const override;
+  void dump(bool prependFieldSeparator, bool asJson, ostream* output) const override;
 
   // @copydoc
   result_t read(const SymbolString& data, size_t offset,

src/lib/ebus/datatype.cpp

@@ -42,17 +42,27 @@ using std::setw;
 using std::endl;
 
 
-bool DataType::dump(size_t length, bool appendSeparatorDivisor, ostream* output) const {
+bool DataType::dump(bool asJson, size_t length, bool appendDivisor, ostream* output) const {
-  *output << m_id;
+  if (asJson) {
-  if (isAdjustableLength()) {
+    *output << "\"type\": \"" << m_id << "\"" << FIELD_SEPARATOR << " \"isbits\": "
-    if (length == REMAIN_LEN) {
+            << (getBitCount() < 8 ? "true" : "false") << FIELD_SEPARATOR << " \"length\": ";
-      *output << ":*";
+    if (isAdjustableLength() && length == REMAIN_LEN) {
+      *output << "-1";
     } else {
-      *output << ":" << static_cast<unsigned>(length);
+      *output << static_cast<unsigned>(length);
+    }
+  } else {
+    *output << m_id;
+    if (isAdjustableLength()) {
+      if (length == REMAIN_LEN) {
+        *output << ":*";
+      } else {
+        *output << ":" << static_cast<unsigned>(length);
+      }
+    }
+    if (appendDivisor) {
+      *output << FIELD_SEPARATOR;
     }
-  }
-  if (appendSeparatorDivisor) {
-    *output << FIELD_SEPARATOR;
   }
   return false;
 }
@@ -502,21 +512,27 @@ size_t NumberDataType::calcPrecision(int divisor) {
   return precision;
 }
 
-bool NumberDataType::dump(size_t length, bool appendSeparatorDivisor, ostream* output) const {
+bool NumberDataType::dump(bool asJson, size_t length, bool appendDivisor, ostream* output) const {
   if (m_bitCount < 8) {
-    DataType::dump(m_bitCount, appendSeparatorDivisor, output);
+    DataType::dump(asJson, m_bitCount, appendDivisor, output);
   } else {
-    DataType::dump(length, appendSeparatorDivisor, output);
+    DataType::dump(asJson, length, appendDivisor, output);
   }
-  if (!appendSeparatorDivisor) {
+  if (!appendDivisor) {
     return false;
   }
   if (m_baseType) {
     if (m_baseType->m_divisor != m_divisor) {
+      if (asJson) {
+        *output << ", \"divisor\": ";
+      }
       *output << (m_divisor / m_baseType->m_divisor);
       return true;
     }
   } else if (m_divisor != 1) {
+    if (asJson) {
+      *output << ", \"divisor\": ";
+    }
     *output << m_divisor;
     return true;
   }

src/lib/ebus/datatype.h

@@ -106,6 +106,9 @@ typedef int OutputFormat;
 /** bit flag for @a OutputFormat: include all attributes. */
 #define OF_ALL_ATTRS 0x80
 
+/** bit flag for @a OutputFormat: include message/field definition. */
+#define OF_DEFINTION 0x100
+
 /** the message part in which a data field is stored. */
 enum PartType {
   pt_any,          //!< stored in any data (master or slave)
@@ -216,12 +219,13 @@ class DataType {
   /**
    * Dump the type identifier with the specified length and optionally the
    * divisor to the output.
+   * @param asJson whether to output in JSON format.
    * @param length the number of symbols to read/write.
-   * @param appendSeparatorDivisor whether to append a @a FIELD_SEPARATOR followed by the divisor (if available).
+   * @param appendDivisor whether to append the divisor (if available).
    * @param output the @a ostream to dump to.
    * @return true when a non-default divisor was written to the output.
    */
-  virtual bool dump(size_t length, bool appendSeparatorDivisor, ostream* output) const;
+  virtual bool dump(bool asJson, size_t length, bool appendDivisor, ostream* output) const;
 
   /**
    * Internal method for reading the numeric raw value from a @a SymbolString.
@@ -432,7 +436,7 @@ class NumberDataType : public DataType {
   static size_t calcPrecision(int divisor);
 
   // @copydoc
-  bool dump(size_t length, bool appendSeparatorDivisor, ostream* output) const override;
+  bool dump(bool asJson, size_t length, bool appendDivisor, ostream* output) const override;
 
   /**
    * Derive a new @a NumberDataType from this.

src/lib/ebus/message.cpp

@@ -81,29 +81,6 @@ static const char* defaultMessageFieldMap[] = {  // access level not included in
 
 extern DataFieldTemplates* getTemplates(const string& filename);
 
-/**
- * Get the normalized message field name for the given name.
- * @param name the input field name.
- * @param supportsLanguage set to true when the field supports multiple language.
- * @return the normalized message field name, or empty if unknown.
- */
-string getMessageFieldName(const string& name, bool* supportsLanguage) {
-  if (name.find("type") != string::npos) {
-    return "type";
-  }
-  if (name == "circuit" || name == "level" || name == "qq" || name == "zz" || name == "pbsb" || name == "id") {
-    return name;
-  }
-  if (name.find("name") != string::npos) {
-    return "name";
-  }
-  *supportsLanguage = true;
-  if (name.find("comment") == 0) {
-    return "comment";
-  }
-  return "";
-}
-
 
 Message::Message(const string& circuit, const string& level, const string& name,
     bool isWrite, bool isPassive, const map<string, string>& attributes,
@@ -921,18 +898,32 @@ void Message::dumpField(const string& fieldName, bool withConditions, ostream* o
     return;
   }
   if (fieldName == "fields") {
-    m_data->dump(output);
+    m_data->dump(false, false, output);
     return;
   }
-  dumpAttribute(false, fieldName, output);
+  dumpAttribute(false, false, fieldName, output);
 }
 
-void Message::decode(bool leadingSeparator, bool appendDirection, OutputFormat outputFormat, ostringstream* output)
+void addData(const SymbolString& data, ostringstream* output) {
-    const {
+  if (data.size() == 0) {
+    return;
+  }
+  *output << ",\n    \"" << (data.isMaster() ? "master" : "slave") << "\": [";
+  for (size_t pos = 0; pos < data.size(); pos++) {
+    if (pos > 0) {
+      *output << ", ";
+    }
+    *output << dec << static_cast<unsigned>(data[pos]);
+  }
+  *output << "]";
+}
+
+void Message::decodeJson(bool leadingSeparator, bool appendDirection, bool addRaw, OutputFormat outputFormat,
+    ostringstream* output) const {
   if (leadingSeparator) {
     *output << ",";
   }
-  *output << "\n  \"" << getName();
+  *output << "\n   \"" << getName();
   if (appendDirection) {
     if (isPassive()) {
       *output << "-u";
@@ -940,27 +931,54 @@ void Message::decode(bool leadingSeparator, bool appendDirection, OutputFormat o
       *output << "-w";
     }
   }
+  bool withDefinition = (outputFormat & OF_DEFINTION) != 0;
   *output << "\": {"
-          << "\n   \"name\": \"" << getName() << "\""
+          << "\n    \"name\": \"" << getName() << "\""
-          << ",\n   \"lastup\": " << setw(0) << dec << static_cast<unsigned>(getLastUpdateTime());
+          << ",\n    \"passive\": " << (isPassive() ? "true" : "false")
-  if (getLastUpdateTime() != 0) {
+          << ",\n    \"write\": " << (isWrite() ? "true" : "false")
-    *output << ",\n   \"zz\": \"" << setfill('0') << setw(2) << hex << static_cast<unsigned>(getDstAddress()) << "\"";
+          << ",\n    \"lastup\": " << setw(0) << dec << m_lastUpdateTime;
+  bool hasData = getLastUpdateTime() != 0;
+  if (hasData || withDefinition) {
+    if (withDefinition && m_srcAddress != SYN) {
+      *output << ",\n    \"qq\": " << dec << static_cast<unsigned>(m_srcAddress);
+    }
+    *output << ",\n    \"zz\": " << dec << static_cast<unsigned>(m_dstAddress);
+    if (withDefinition) {
+      *output << ",\n    \"id\": [" << dec;
+      for (auto it = m_id.begin(); it < m_id.end(); it++) {
+        if (it > m_id.begin()) {
+          *output << ", ";
+        }
+        *output << dec << static_cast<unsigned>(*it);
+      }
+      *output << "]";
+    }
     appendAttributes(OF_JSON | outputFormat, output);
-    size_t pos = (size_t)output->tellp();
+    if (hasData) {
-    *output << ",\n   \"fields\": {";
+      if (addRaw) {
-    result_t dret = decodeLastData(false, NULL, -1, outputFormat, output);
+        addData(m_lastMasterData, output);
-    if (dret == RESULT_OK) {
+        addData(m_lastSlaveData, output);
-      *output << "\n   }";
+        *output << dec;
-    } else {
+      }
-      string prefix = output->str().substr(0, pos);
+      size_t pos = (size_t)output->tellp();
-      output->str("");
+      *output << ",\n    \"fields\": {";
-      output->clear();  // remove written fields
+      result_t dret = decodeLastData(false, NULL, -1, outputFormat, output);
-      *output << prefix << ",\n   \"decodeerror\": \"" << getResultCode(dret) << "\"";
+      if (dret == RESULT_OK) {
+        *output << "\n    }";
+      } else {
+        string prefix = output->str().substr(0, pos);
+        output->str("");
+        output->clear();  // remove written fields
+        *output << prefix << ",\n    \"decodeerror\": \"" << getResultCode(dret) << "\"";
+      }
+    }
+    if (withDefinition) {
+      *output << ",\n    \"fielddefs\": [";
+      m_data->dump(false, true, output);
+      *output << "\n    ]";
     }
   }
-  *output << ",\n   \"passive\": " << (isPassive() ? "true" : "false")
+  *output << "\n   }";
-          << ",\n   \"write\": " << (isWrite() ? "true" : "false")
-          << "\n  }";
 }
 
 
@@ -2091,6 +2109,16 @@ result_t MessageMap::readFromFile(const string& filename, bool verbose, map<stri
     time = &localTime;
   }
   result_t result = MappedFileReader::readFromFile(filename, verbose, defaults, errorDescription, hash, size, time);
+  if (defaults) {
+    string circuit = AttributedItem::pluck("circuit", defaults);
+    if (!circuit.empty() && m_circuitData.find(circuit) == m_circuitData.end()) {
+      string name = AttributedItem::pluck("name", defaults);
+      if (!name.empty() || !defaults->empty()) {
+        AttributedItem::pluck("suffix", defaults);
+        m_circuitData[circuit] = new AttributedItem(name, *defaults);
+      }
+    }
+  }
   if (result == RESULT_OK) {
     const string file = getRelativePath(filename);
     m_loadedFileInfos[file].m_hash = *hash;
@@ -2542,7 +2570,7 @@ bool MessageMap::decodeCircuit(const string& circuit, OutputFormat outputFormat,
     return false;
   }
   if (outputFormat & OF_JSON) {
-    *output << "\"name\": \"" << it->second->getName() << "\"";
+    *output << "\n  \"name\": \"" << it->second->getName() << "\"";
   } else {
     *output << it->second->getName() << "=";
   }

src/lib/ebus/message.h

@@ -552,11 +552,12 @@ class Message : public AttributedItem {
    * Decode the message from the last stored data in JSON format.
    * @param leadingSeparator whether to prepend a separator before the first value.
    * @param appendDirection whether to append the direction to the name key (for passive and write).
+   * @param addRaw whether to add the raw symbols as well.
    * @param outputFormat the @a OutputFormat options to use.
    * @param output the @a ostringstream to append the decoded value(s) to.
    */
-  virtual void decode(bool leadingSeparator, bool appendDirection, OutputFormat outputFormat, ostringstream* output)
+  virtual void decodeJson(bool leadingSeparator, bool appendDirection, bool addRaw, OutputFormat outputFormat,
-      const;
+      ostringstream* output) const;
 
  protected:
   /** the optional circuit name. */

src/lib/ebus/test/test_data.cpp

@@ -609,7 +609,7 @@ int main() {
       continue;
     }
     cout << "\"" << check[0] << "\"=\"";
-    fields->dump(&cout);
+    fields->dump(false, false, &cout);
     cout << "\": create OK" << endl;
 
     ostringstream output;

src/lib/ebus/test/test_message.cpp

@@ -145,6 +145,25 @@ int main() {
     {"r,,x,,,,,6a00,,,UCH,10,bar,,Bit6,,BI6:1,0=B60;1=B61,,,Bit7,,BI7:1,0=B70;1=B71", "1.9;B60;B70", "ff08b509030d6a00", "02133f", "d" },
     {"*r,cir*cuit#level,na*me,com*ment,ff,75,b509,0d", "", "", "", "" },
     {"r,CIRCUIT,NAME,COMMENT,,,,0100,field,,UCH", "r,cirCIRCUITcuit,naNAMEme,comCOMMENTment,ff,75,b509,0d0100,field,s,UCH,,,: field=42", "ff75b509030d0100", "012a", "DN"},
+    {"r,CIRCUIT,NAME,COMMENT,,,,0100,field,,UCH",
+        // "\"naNAMEme\": {r,cirCIRCUITcuit,naNAMEme,comCOMMENTment,ff,75,b509,0d0100,field,s,UCH,,,: field=42"
+      "\n"
+      "   \"naNAMEme\": {\n"
+      "    \"name\": \"naNAMEme\",\n"
+      "    \"passive\": false,\n"
+      "    \"write\": false,\n"
+      "    \"lastup\": *,\n"
+      "    \"qq\": 255,\n"
+      "    \"zz\": 117,\n"
+      "    \"id\": [181, 9, 13, 1, 0],\n"
+      "    \"fields\": {\n"
+      "    \"0\": {\"name\": \"field\", \"value\": 42}\n"
+      "    },\n"
+      "    \"fielddefs\": [\n"
+      "     { \"name\": \"field\", \"slave\": true, \"type\": \"UCH\", \"isbits\": false, \"length\": 1, \"unit\": \"\", \"comment\": \"\"}\n"
+      "    ]\n"
+      "   }: \n"
+      "    \"field\": {\"value\": 42}", "ff75b509030d0100", "012a", "jN"},
   };
   templates = new DataFieldTemplates();
   unsigned int lineNo = 0;
@@ -349,8 +368,23 @@ int main() {
         message->storeLastData(*mstrs[index], *sstrs[index]);
       }
       ostringstream output;
-      if (withMessageDump && !decodeJson) {
+      if (withMessageDump) {
-        message->dump(NULL, true, &output);
+        if (decodeJson) {
+          message->decodeJson(false, false, false, OF_JSON|OF_DEFINTION, &output);
+          string str = output.str();
+          size_t start = str.find("\"lastup\": ");
+          if (start != string::npos) {
+            start += 10;
+            size_t end = str.find(",", start);
+            if (end != string::npos) {
+              str = str.substr(0, start)+"*"+str.substr(end);
+            }
+          }
+          output.str("");
+          output << str;
+        } else {
+          message->dump(NULL, true, &output);
+        }
         output << ": ";
       }
       result = message->decodeLastData(false, NULL, -1,
@@ -362,6 +396,7 @@ int main() {
         continue;
       }
       cout << "  \"" << check[2] << "\" / \"" << check[3] <<  "\": decode OK" << endl;
+      string outStr = output.str();
       bool match = inputStr == output.str();
       verify(false, "decode", check[2] + "/" + check[3], match, inputStr, output.str());
       if (checkUpdateTime || checkSameChangeTime) {