made http client more flexible and support content-length response header

src/lib/utils/httpclient.cpp

@@ -29,20 +29,81 @@ using std::ostringstream;
 using std::dec;
 using std::hex;
 
+bool HttpClient::parseUrl(const string& url, string& proto, string& host, uint16_t& port, string& uri) {
+  size_t hostPos = url.find("://");
+  if (hostPos == string::npos) {
+    return false;
+  }
+  proto = url.substr(0, hostPos);
+  hostPos += 3;
+  if (proto != "http") {
+    return false;
+  }
+  size_t pos = url.find('/', hostPos);
+  if (pos == string::npos || pos == hostPos) {
+    return false;
+  }
+  host = url.substr(hostPos, pos - hostPos);
+  uri = url.substr(pos);
+  if (uri[uri.length()-1] != '/') {
+    uri += "/";
+  }
+  pos = host.find(':');
+  if (pos == 0) {
+    return false;
+  }
+  port = 80;
+  if (pos != string::npos) {
+    char* strEnd = NULL;
+    unsigned long value = strtoul(host.c_str()+pos+1, &strEnd, 10);
+    if (strEnd == NULL || *strEnd != '\0' || value < 1 || value > 65535) {
+      return false;
+    }
+    port = static_cast<uint16_t>(value);
+    host = host.substr(0, pos);
+  }
+  return true;
+}
+
 bool HttpClient::connect(const string& host, const uint16_t port, const string& userAgent, const int timeout) {
+  disconnect();
+  m_socket = m_client.connect(host, port);
+  if (!m_socket) {
+    return false;
+  }
+  m_socket->setTimeout(timeout);
+  m_host = host;
+  m_port = port;
+  m_timeout = timeout;
+  m_userAgent = userAgent;
+  return true;
+}
+
+bool HttpClient::reconnect() {
+  disconnect();
+  if (m_host.empty() || !m_port) {
+    return false;
+  }
+  m_socket = m_client.connect(m_host, m_port);
+  if (!m_socket) {
+    return false;
+  }
+  m_socket->setTimeout(m_timeout);
+  return true;
+}
+
+bool HttpClient::ensureConnected() {
+  if (m_socket && m_socket->isValid()) {
+    return true;
+  }
+  return reconnect();
+}
+
+void HttpClient::disconnect() {
   if (m_socket) {
     delete m_socket;
     m_socket = nullptr;
   }
-  TCPSocket* socket = m_client.connect(host, port);
-  if (!socket) {
-    return false;
-  }
-  socket->setTimeout(timeout);
-  m_socket = socket;
-  m_host = host;
-  m_userAgent = userAgent;
-  return true;
 }
 
 bool HttpClient::get(const string& uri, const string& body, string& response) {
@@ -54,7 +115,7 @@ bool HttpClient::post(const string& uri, const string& body, string& response) {
 }
 
 bool HttpClient::request(const string& method, const string& uri, const string& body, string& response) {
-  if (!m_socket) {
+  if (!ensureConnected()) {
     response = "not connected";
     return false;
   }
@@ -78,55 +139,75 @@ bool HttpClient::request(const string& method, const string& uri, const string&
   for (size_t pos = 0; pos < len; ) {
     ssize_t sent = m_socket->send(cstr + pos, len - pos);
     if (sent < 0) {
+      disconnect();
       response = "send error";
       return false;
     }
     pos += sent;
   }
-  if (!m_buffer) {
+  string result;
-    m_buffer = (char *) malloc(1024);
+  size_t pos = readUntil(" ", 4 * 1024, result); // max 4k headers
-    if (!m_buffer) {
+  if (pos == string::npos || pos > 8 || result.substr(0, 5) != "HTTP/") {
-      response = "memory allocation";
+    disconnect();
-      return false;
-    }
-    m_bufferSize = 1024;
-  }
-  ssize_t received = m_socket->recv(m_buffer, m_bufferSize);
-  if (received <= 0) {
-    response = "receive error";
-    return false;
-  }
-  string result = string(m_buffer, 0, static_cast<unsigned>(received)); // expect "HTTP/1.1 200 OK"
-  size_t pos = result.find(' ');
-  if (result.substr(0, 5) != "HTTP/" || pos == string::npos || pos > 8) {
     response = "receive error (headers)";
     return false;
   }
   if (result.substr(pos+1, 6) != "200 OK") {
+    disconnect();
     size_t endpos = result.find("\r\n", pos+1);
     response = "receive error: " + result.substr(pos+1, endpos == string::npos ? endpos : endpos-pos-1);
     return false;
   }
-  pos = result.find("\r\n\r\n");
+  pos = readUntil("\r\n\r\n", 4 * 1024, result); // max 4k headers
-  while (pos == string::npos && result.length() < 256*1024) {
+  if (pos == string::npos) {
-    received = m_socket->recv(m_buffer, m_bufferSize);
+    disconnect();
+    response = "receive error (headers)";
+    return false;
+  }
+  string headers = result.substr(0, pos+2); // including final \r\n
+  response = result.substr(pos+4);
+  pos = headers.find("Content-Length: "); // 16 chars
+  if (pos == string::npos) {
+    disconnect();
+    return true;
+  }
+  char* strEnd = NULL;
+  unsigned long length = strtoul(headers.c_str()+pos+16, &strEnd, 10);
+  if (strEnd == NULL || *strEnd != '\r') {
+    disconnect();
+    response = "invalid content length ";
+    return false;
+  }
+  pos = readUntil("", length, response);
+  disconnect();
+  return pos == length;
+}
+
+size_t HttpClient::readUntil(const string& delim, const size_t length, string& result) {
+  if (!m_buffer) {
+    m_buffer = (char*)malloc(1024);
+    if (!m_buffer) {
+      return string::npos;
+    }
+    m_bufferSize = 1024;
+  }
+  bool findDelim = !delim.empty();
+  size_t pos = findDelim ? result.find(delim) : string::npos;
+  while (pos == string::npos && result.length() < length) {
+    ssize_t received = m_socket->recv(m_buffer, m_bufferSize);
     if (received < 0) {
-      response = "receive error";
+      return string::npos;
-      return false;
     }
     if (received == 0) {
       break;
     }
     size_t oldLength = result.length();
     result += string(m_buffer, 0, static_cast<unsigned>(received));
-    pos = result.find("\r\n\r\n", oldLength - 3);
+    if (findDelim) {
+      pos = result.find(delim, oldLength - (delim.length() - 1));
+    }
   }
-  if (pos == string::npos) {
+  return findDelim ? pos : result.length();
-    response = "receive error (headers)";
-    return false;
-  }
-  response = result.substr(pos+4);
-  return true;
 }
 
 }  // namespace ebusd

src/lib/utils/httpclient.h

@@ -44,22 +44,30 @@ class HttpClient {
   /**
    * Constructor.
    */
-  HttpClient() : m_socket(nullptr), m_bufferSize(0), m_buffer(nullptr) {}
+  HttpClient() : m_port(0), m_timeout(0), m_socket(nullptr), m_bufferSize(0), m_buffer(nullptr) {}
 
   /**
    * Destructor.
    */
   ~HttpClient() {
-    if (m_socket) {
+    disconnect();
-      delete m_socket;
-      m_socket = nullptr;
-    }
     if (m_buffer) {
       free(m_buffer);
       m_buffer = nullptr;
     }
   }
 
+  /**
+   * Parse an HTTP URL.
+   * @param url the URL to parse.
+   * @param proto the extracted protocol.
+   * @param host the extracted host name.
+   * @param port the extracted port (or default).
+   * @param uri the extracted URI starting with '/'.
+   * @return true on success, false on failure.
+   */
+  static bool parseUrl(const string& url, string& proto, string& host, uint16_t& port, string& uri);
+
   /**
    * Connect to the specified server.
    * @param host the host name to connect to.
@@ -70,6 +78,23 @@ class HttpClient {
    */
   bool connect(const string& host, uint16_t port, const string& userAgent = "", int timeout = 5);
 
+  /**
+   * Re-connect to the last specified server.
+   * @return true on success, false on connect failure.
+   */
+  bool reconnect();
+
+  /**
+   * Ensure the client is connected to the last specified server.
+   * @return true if still connected or connection was re-established successfully, false on connect failure.
+   */
+  bool ensureConnected();
+
+  /**
+   * Disconnect from the servier.
+   */
+  void disconnect();
+
   /**
    * Execute a GET request.
    * @param uri the URI string.
@@ -98,11 +123,38 @@ class HttpClient {
   bool request(const string& method, const string& uri, const string& body, string& response);
 
 private:
+  /**
+   * Read from the connected socket until the specified delimiter is found or the specified number of bytes was received.
+   * @param delim the delimiter to find, or empty for reading the specified number of bytes.
+   * @param length the maximum number of bytes to receive.
+   * @param result the string to append the read data to and in which to find the delimiter.
+   * @return the position of the delimiter if delimiter was set or the number of bytes received, or string::npos if not found.
+   */
+  size_t readUntil(const string& delim, const size_t length, string& result);
+
+private:
+  /** the @a TCPClient handling the traffic. */
   TCPClient m_client;
+
+  /** the name of the host last successfully connected to. */
   string m_host;
+
+  /** the port last successfully connected to. */
+  uint16_t m_port;
+
+  /** the timeout in seconds. */
+  int m_timeout;
+
+  /** the optional user agent to send in the request header. */
   string m_userAgent;
+
+  /** the currently connected socket. */
   TCPSocket* m_socket;
+
+  /** the size of the @a m_buffer. */
   size_t m_bufferSize;
+
+  /** the buffer for preparing/receiving data. */
   char* m_buffer;
 };