add /raw endpoint

ChangeLog.md

@@ -9,6 +9,8 @@
 * fix for message after debian install
 * fix for replacing already existing message definitions
 * fix missing length in CSV dump for some data types
+* fix some missing data type lengths in "grab decode" result
+* fix for injecting several messages via command line args
 
 ## Features
 * added DTM and BDZ data types
@@ -19,7 +21,7 @@
 * added some PIC calibration data to "ebuspicloader" verbose output
 * added support for upcoming adapter 3 firmware enhancements
 * added config override path
-* added support for adding message definition via HTTP port
+* added support for adding message definition, retrieving data types and raw messages to HTTP port
 * added "--mqttverbose" option
 
 ## Breaking Changes

contrib/html/openapi.yaml

@@ -145,6 +145,18 @@ paths:
         500:
           description: General error.
           content: { }
+  /raw:
+    get:
+      summary: Retrieve raw data from grabbed and/or decoded messages.
+      parameters:
+        - $ref: '#/components/parameters/sinceQuery'
+      responses:
+        200:
+          description: Success
+          content:
+            application/json;charset=utf-8:
+              schema:
+                $ref: '#/components/schemas/RawMessages'
   /{file}:
     get:
       summary: Retrieve a particular file.
@@ -384,19 +396,11 @@ components:
           type: string
           description: the message comment (only with verbose).
         master:
-          type: array
+          $ref: '#/components/schemas/Symbols'
           description: the last seen master data bytes (only with raw and if available).
-          items:
-            maximum: 255
-            minimum: 0
-            type: integer
         slave:
-          type: array
+          $ref: '#/components/schemas/Symbols'
           description: the last seen slave data bytes (only with raw and if available).
-          items:
-            maximum: 255
-            minimum: 0
-            type: integer
         fields:
           type: object
           additionalProperties:
@@ -411,6 +415,15 @@ components:
             def).
           items:
             $ref: '#/components/schemas/FieldDef'
+    Symbols:
+      description: master or slave data bytes.
+      type: array
+      minItems: 1
+      maximum: 32
+      items:
+        maximum: 255
+        minimum: 0
+        type: integer
     Field:
       type: object
       properties:
@@ -531,6 +544,38 @@ components:
           $ref: '#/components/schemas/Global'
       additionalProperties:
         $ref: '#/components/schemas/Circuit'
+    RawMessage:
+      type: object
+      description: raw message seen on the bus.
+      properties:
+        master:
+          $ref: '#/components/schemas/Symbols'
+          description: the last seen master data bytes (only with raw and if available).
+        slave:
+          $ref: '#/components/schemas/Symbols'
+          description: the last seen slave data bytes (only with raw and if available).
+        lastup:
+          minimum: 0
+          type: integer
+          description: the time in UTC seconds of the last update of the message (0
+            for never).
+        count:
+          type: number
+          description: number of times the master part was seen.
+        circuit:
+          type: string
+          description: name of the circuit in case of an already associated message definition.
+        name:
+          type: string
+          description: name of the message in case of an already associated message definition.
+      required:
+        - master
+        - count
+    RawMessages:
+      type: array
+      description: raw messages seen on the bus.
+      items:
+        $ref: '#/components/schemas/RawMessage'
   responses:
     BadRequest:
       description: Invalid request parameters.

src/ebusd/bushandler.cpp

@@ -253,65 +253,87 @@ bool decodeType(const DataType* type, const SymbolString& input, size_t length,
   return !first;
 }
 
-bool GrabbedMessage::dump(bool unknown, MessageMap* messages, bool first, bool decode, ostringstream* output,
+bool GrabbedMessage::dump(bool unknown, MessageMap* messages, bool first, OutputFormat outputFormat,
-    bool isDirectMode) const {
+    ostringstream* output, bool isDirectMode) const {
   Message* message = messages->find(m_lastMaster);
   if (unknown && message) {
     return false;
   }
   if (!first) {
-    *output << endl;
+    if (outputFormat & OF_JSON) {
+      *output << ",";
+    } else {
+      *output << endl;
+    }
   }
   symbol_t dstAddress = m_lastMaster[1];
-  *output << m_lastMaster.getStr();
+  if (outputFormat & OF_JSON) {
-  if (dstAddress != BROADCAST && !isMaster(dstAddress)) {
+    *output << "\n{";
-    *output << (isDirectMode ? " " : " / ") << m_lastSlave.getStr();
+    if (m_lastMaster.dumpJson(false, output)) {
-  }
+      *output << ", ";
-  if (!isDirectMode) {
+      if (dstAddress != BROADCAST && !isMaster(dstAddress) && m_lastSlave.dumpJson(false, output)) {
-    *output << " = " << m_count;
+        *output << ", ";
+      }
+    }
+    *output << "\"count\": " << static_cast<unsigned>(m_count);
+    *output << ", \"lastup\": " << setw(0) << dec << m_lastTime;
     if (message) {
-      *output << ": " << message->getCircuit() << " " << message->getName();
+      *output << ", \"circuit\": \"" << message->getCircuit() << "\""
+              << ", \"name\": \"" << message->getName() << "\"";
+    }
+    *output << "}";
+  } else {
+    *output << m_lastMaster.getStr();
+    if (dstAddress != BROADCAST && !isMaster(dstAddress)) {
+      *output << (isDirectMode ? " " : " / ") << m_lastSlave.getStr();
+    }
+    if (!isDirectMode) {
+      *output << " = " << m_count;
+      if (message) {
+        *output << ": " << message->getCircuit() << " " << message->getName();
+      }
     }
   }
-  if (decode) {
+  if (!(outputFormat & OF_DEFINITION) || (outputFormat & OF_JSON)) {
-    DataTypeList *types = DataTypeList::getInstance();
+    return true;
-    if (!types) {
+  }
-      return true;
+  DataTypeList *types = DataTypeList::getInstance();
+  if (!types) {
+    return true;
+  }
+  bool master = isMaster(dstAddress) || dstAddress == BROADCAST || m_lastSlave.getDataSize() <= 0;
+  size_t remain = master ? m_lastMaster.getDataSize() : m_lastSlave.getDataSize();
+  if (remain == 0) {
+    return true;
+  }
+  for (const auto& it : *types) {
+    const DataType* baseType = it.second;
+    if ((baseType->getBitCount() % 8) != 0 || baseType->isIgnored() || baseType->hasFlag(DUP)) {  // skip bit and ignored types
+      continue;
     }
-    bool master = isMaster(dstAddress) || dstAddress == BROADCAST || m_lastSlave.getDataSize() <= 0;
+    size_t maxLength = baseType->getBitCount()/8;
-    size_t remain = master ? m_lastMaster.getDataSize() : m_lastSlave.getDataSize();
+    bool firstOnly = maxLength >= 8;
-    if (remain == 0) {
+    if (maxLength > remain) {
-      return true;
+      maxLength = remain;
     }
-    for (const auto& it : *types) {
+    if (baseType->isAdjustableLength()) {
-      const DataType* baseType = it.second;
+      for (size_t length = maxLength; length >= 1; length--) {
-      if ((baseType->getBitCount() % 8) != 0 || baseType->isIgnored() || baseType->hasFlag(DUP)) {  // skip bit and ignored types
+        const DataType* type = types->get(baseType->getId(), length);
-        continue;
+        bool decoded;
-      }
-      size_t maxLength = baseType->getBitCount()/8;
-      bool firstOnly = maxLength >= 8;
-      if (maxLength > remain) {
-        maxLength = remain;
-      }
-      if (baseType->isAdjustableLength()) {
-        for (size_t length = maxLength; length >= 1; length--) {
-          const DataType* type = types->get(baseType->getId(), length);
-          bool decoded;
-          if (master) {
-            decoded = decodeType(type, m_lastMaster, length, remain-length, firstOnly, output);
-          } else {
-            decoded = decodeType(type, m_lastSlave, length, remain-length, firstOnly, output);
-          }
-          if (decoded && firstOnly) {
-            break;  // only a single offset with maximum length when adjustable maximum size is at least 8 bytes
-          }
-        }
-      } else if (maxLength > 0) {
         if (master) {
-          decodeType(baseType, m_lastMaster, maxLength, remain-maxLength, false, output);
+          decoded = decodeType(type, m_lastMaster, length, remain-length, firstOnly, output);
         } else {
-          decodeType(baseType, m_lastSlave, maxLength, remain-maxLength, false, output);
+          decoded = decodeType(type, m_lastSlave, length, remain-length, firstOnly, output);
         }
+        if (decoded && firstOnly) {
+          break;  // only a single offset with maximum length when adjustable maximum size is at least 8 bytes
+        }
+      }
+    } else if (maxLength > 0) {
+      if (master) {
+        decodeType(baseType, m_lastMaster, maxLength, remain-maxLength, false, output);
+      } else {
+        decodeType(baseType, m_lastSlave, maxLength, remain-maxLength, false, output);
       }
     }
   }
@@ -1610,10 +1632,10 @@ bool BusHandler::enableGrab(bool enable) {
   return true;
 }
 
-void BusHandler::formatGrabResult(bool unknown, bool decode, ostringstream* output, bool isDirectMode,
+void BusHandler::formatGrabResult(bool unknown, OutputFormat outputFormat, ostringstream* output, bool isDirectMode,
     time_t since, time_t until) const {
   if (!m_grabMessages) {
-    if (!isDirectMode) {
+    if (!isDirectMode && !(outputFormat & OF_JSON)) {
       *output << "grab disabled";
     }
     return;
@@ -1624,7 +1646,7 @@ void BusHandler::formatGrabResult(bool unknown, bool decode, ostringstream* outp
     || (until > 0 && it.second.getLastTime() >= until)) {
       continue;
     }
-    if (it.second.dump(unknown, m_messages, first, decode, output, isDirectMode)) {
+    if (it.second.dump(unknown, m_messages, first, outputFormat, output, isDirectMode)) {
       first = false;
     }
   }

src/ebusd/bushandler.h

@@ -334,12 +334,12 @@ class GrabbedMessage {
    * @param unknown whether to dump only if this message is unknown.
    * @param messages the @a MessageMap instance for resolving known @a Message instances.
    * @param first whether this is the first message to be added to the output.
-   * @param decode whether to add decoding hints.
+   * @param outputFormat the @a OutputFormat options to use.
    * @param output the @a ostringstream to format the messages to.
    * @param isDirectMode true for direct mode, false for grab command.
    * @return whether the message was added to the output.
    */
-  bool dump(bool unknown, MessageMap* messages, bool first, bool decode, ostringstream* output,
+  bool dump(bool unknown, MessageMap* messages, bool first, OutputFormat outputFormat, ostringstream* output,
       bool isDirectMode = false) const;
 
 
@@ -538,13 +538,13 @@ class BusHandler : public WaitThread {
   /**
    * Format the grabbed messages to the @a ostringstream.
    * @param unknown whether to dump only unknown messages.
-   * @param decode whether to add decoding hints.
+   * @param outputFormat the @a OutputFormat options to use.
    * @param output the @a ostringstream to format the messages to.
    * @param isDirectMode true for direct mode, false for grab command.
    * @param since the start time from which to add received messages (inclusive), or 0 for all.
    * @param until the end time to which to add received messages (exclusive), or 0 for all.
    */
-  void formatGrabResult(bool unknown, bool decode, ostringstream* output, bool isDirectMode = false,
+  void formatGrabResult(bool unknown, OutputFormat outputFormat, ostringstream* output, bool isDirectMode = false,
       time_t since = 0, time_t until = 0) const;
 
   /**

src/ebusd/mainloop.cpp

@@ -434,14 +434,14 @@ void MainLoop::run() {
       }
       if (settings.listenWithUnknown || settings.listenOnlyUnknown) {
         if (m_busHandler->isGrabEnabled()) {
-          m_busHandler->formatGrabResult(true, false, &ostream, true, since, now);
+          m_busHandler->formatGrabResult(true, OF_NONE, &ostream, true, since, now);
         } else {
           m_busHandler->enableGrab(true);  // needed for listening to all messages
         }
       }
     } else if (settings.mode == cm_direct) {
       if (m_busHandler->isGrabEnabled()) {
-        m_busHandler->formatGrabResult(false, false, &ostream, true, since, now);
+        m_busHandler->formatGrabResult(false, OF_NONE, &ostream, true, since, now);
       }
     }
     // send result to client
@@ -1639,7 +1639,7 @@ result_t MainLoop::executeGrab(const vector<string>& args, ostringstream* ostrea
       }
     }
     if (!invalid) {
-      m_busHandler->formatGrabResult(onlyUnknown, decode, ostream);
+      m_busHandler->formatGrabResult(onlyUnknown, decode ? OF_DEFINITION : OF_NONE, ostream);
       return RESULT_OK;
     }
   }
@@ -2218,6 +2218,47 @@ result_t MainLoop::executeGet(const vector<string>& args, bool* connected, ostri
     return formatHttpResult(ret, type, ostream);
   }
 
+  if (uri == "/raw") {
+    time_t since = 0, until = 0;
+    bool onlyUnknown = false;
+    if (args.size() > argPos) {
+      string query = args[argPos];
+      istringstream stream(query);
+      string token;
+      while (getline(stream, token, '&')) {
+        size_t pos = token.find('=');
+        string qname, value;
+        if (pos != string::npos) {
+          qname = token.substr(0, pos);
+          value = token.substr(pos + 1);
+        } else {
+          qname = token;
+        }
+        if (qname == "since") {
+          since = parseInt(value.c_str(), 10, 0, 0xffffffff, &ret);
+        } else if (qname == "unknown") {
+          onlyUnknown = parseBoolQuery(value);
+        }
+        if (ret != RESULT_OK) {
+          break;
+        }
+      }
+    }
+    if (ret == RESULT_OK) {
+      *ostream << "[";
+      if (since > 0) {
+        time_t now;
+        time(&now);
+        until = now-1;
+      }
+      m_busHandler->formatGrabResult(onlyUnknown, OF_JSON, ostream, false, since, until);
+      *ostream << "\n]";
+      type = 6;
+    }
+    *connected = false;
+    return formatHttpResult(ret, type, ostream);
+  }
+
   if (uri.length() < 1 || uri[0] != '/' || uri.find("//") != string::npos || uri.find("..") != string::npos) {
     ret = RESULT_ERR_INVALID_ARG;
   } else {