From e419c950d850ade0e2c085606825a6be1aa7d772 Mon Sep 17 00:00:00 2001 From: John Date: Thu, 18 May 2023 12:24:50 +0200 Subject: [PATCH] separate resolving+scanning to own interface+class and avoid use of extern --- src/ebusd/CMakeLists.txt | 1 + src/ebusd/bushandler.cpp | 5 +- src/ebusd/bushandler.h | 9 +- src/ebusd/main.cpp | 581 ++--------------------------- src/ebusd/main.h | 51 --- src/ebusd/mainloop.cpp | 25 +- src/ebusd/mainloop.h | 7 +- src/ebusd/scan.cpp | 510 +++++++++++++++++++++++++ src/ebusd/scan.h | 205 ++++++++++ src/lib/ebus/message.cpp | 16 +- src/lib/ebus/message.h | 53 ++- src/lib/ebus/test/test_message.cpp | 37 +- 12 files changed, 860 insertions(+), 640 deletions(-) create mode 100644 src/ebusd/scan.cpp create mode 100644 src/ebusd/scan.h diff --git a/src/ebusd/CMakeLists.txt b/src/ebusd/CMakeLists.txt index 325663c2..95eb8933 100644 --- a/src/ebusd/CMakeLists.txt +++ b/src/ebusd/CMakeLists.txt @@ -5,6 +5,7 @@ set(ebusd_SOURCES datahandler.h datahandler.cpp network.h network.cpp mainloop.h mainloop.cpp + scan.h scan.cpp main.h main.cpp ) diff --git a/src/ebusd/bushandler.cpp b/src/ebusd/bushandler.cpp index 524a3e2c..ded986d7 100644 --- a/src/ebusd/bushandler.cpp +++ b/src/ebusd/bushandler.cpp @@ -22,7 +22,6 @@ #include "ebusd/bushandler.h" #include -#include "ebusd/main.h" #include "lib/utils/log.h" namespace ebusd { @@ -1637,14 +1636,14 @@ result_t BusHandler::scanAndWait(symbol_t dstAddress, bool loadScanConfig, bool bool timedOut = result == RESULT_ERR_TIMEOUT; bool loadFailed = false; if (timedOut || result == RESULT_OK) { - result = loadScanConfigFile(m_messages, dstAddress, false, &file); // try to load even if one message timed out + result = m_scanHelper->loadScanConfigFile(dstAddress, &file); // try to load even if one message timed out loadFailed = result != RESULT_OK; if (timedOut && loadFailed) { result = RESULT_ERR_TIMEOUT; // back to previous result } } if (result == RESULT_OK) { - executeInstructions(m_messages); + m_scanHelper->executeInstructions(this); setScanConfigLoaded(dstAddress, file); if (!hasAdditionalScanMessages && m_messages->hasAdditionalScanMessages()) { // additional scan messages now available diff --git a/src/ebusd/bushandler.h b/src/ebusd/bushandler.h index e9a9e7d3..a54eb1ec 100755 --- a/src/ebusd/bushandler.h +++ b/src/ebusd/bushandler.h @@ -25,6 +25,7 @@ #include #include #include +#include "ebusd/scan.h" #include "lib/ebus/message.h" #include "lib/ebus/data.h" #include "lib/ebus/symbol.h" @@ -367,6 +368,7 @@ class BusHandler : public WaitThread { * Construct a new instance. * @param device the @a Device instance for accessing the bus. * @param messages the @a MessageMap instance with all known @a Message instances. + * @param scanHelper the @a ScanHelper instance. * @param ownAddress the own master address. * @param answer whether to answer queries for the own master/slave address. * @param busLostRetries the number of times a send is repeated due to lost arbitration. @@ -377,13 +379,13 @@ class BusHandler : public WaitThread { * @param generateSyn whether to enable AUTO-SYN symbol generation. * @param pollInterval the interval in seconds in which poll messages are cycled, or 0 if disabled. */ - BusHandler(Device* device, MessageMap* messages, + BusHandler(Device* device, MessageMap* messages, ScanHelper* scanHelper, symbol_t ownAddress, bool answer, unsigned int busLostRetries, unsigned int failedSendRetries, unsigned int busAcquireTimeout, unsigned int slaveRecvTimeout, unsigned int lockCount, bool generateSyn, unsigned int pollInterval) - : WaitThread(), m_device(device), m_reconnect(false), m_messages(messages), + : WaitThread(), m_device(device), m_reconnect(false), m_messages(messages), m_scanHelper(scanHelper), m_ownMasterAddress(ownAddress), m_ownSlaveAddress(getSlaveAddress(ownAddress)), m_answer(answer), m_addressConflict(false), m_busLostRetries(busLostRetries), m_failedSendRetries(failedSendRetries), @@ -676,6 +678,9 @@ class BusHandler : public WaitThread { /** the @a MessageMap instance with all known @a Message instances. */ MessageMap* m_messages; + /** the @a ScanHelper instance. */ + ScanHelper* m_scanHelper; + /** the own master address. */ const symbol_t m_ownMasterAddress; diff --git a/src/ebusd/main.cpp b/src/ebusd/main.cpp index 52611bea..9e5b7b6e 100644 --- a/src/ebusd/main.cpp +++ b/src/ebusd/main.cpp @@ -33,6 +33,7 @@ #include "ebusd/mainloop.h" #include "lib/utils/log.h" #include "lib/utils/httpclient.h" +#include "ebusd/scan.h" /** the version string of the program. */ @@ -139,24 +140,15 @@ static struct options s_opt = { /** the @a MessageMap instance, or nullptr. */ static MessageMap* s_messageMap = nullptr; +/** the @a ScanHelper instance, or nullptr. */ +static ScanHelper* s_scanHelper = nullptr; + /** the @a MainLoop instance, or nullptr. */ static MainLoop* s_mainLoop = nullptr; /** the (optionally corrected) config path for retrieving configuration files from. */ static string s_configPath = CONFIG_PATH; -/** the path prefix (including trailing "/") for retrieving configuration files from local files (empty for HTTPS). */ -static string s_configLocalPrefix = ""; - -/** the URI prefix (including trailing "/") for retrieving configuration files from HTTPS (empty for local files). */ -static string s_configUriPrefix = ""; - -/** the optional language query part for retrieving configuration files from HTTPS (empty for local files). */ -static string s_configLangQuery = ""; - -/** the @a HttpClient for retrieving configuration files from HTTPS. */ -static HttpClient* s_configHttpClient = nullptr; - /** the documentation of the program. */ static const char argpdoc[] = "A daemon for communication with eBUS heating systems."; @@ -278,15 +270,6 @@ static const struct argp_option argpoptions[] = { {nullptr, 0, nullptr, 0, nullptr, 0 }, }; -/** the global @a DataFieldTemplates. */ -static DataFieldTemplates s_globalTemplates; - -/** - * the loaded @a DataFieldTemplates by relative path (may also carry - * @a globalTemplates as replacement for missing file). - */ -static map s_templatesByPath; - /** * The program argument parsing function. * @param key the key from @a argpoptions. @@ -760,16 +743,9 @@ void cleanup() { delete s_messageMap; s_messageMap = nullptr; } - // free templates - for (const auto& it : s_templatesByPath) { - if (it.second != &s_globalTemplates) { - delete it.second; - } - } - s_templatesByPath.clear(); - if (s_configHttpClient) { - delete s_configHttpClient; - s_configHttpClient = nullptr; + if (s_scanHelper) { + delete s_scanHelper; + s_scanHelper = nullptr; } } @@ -833,503 +809,6 @@ void signalHandler(int sig) { } } -/** - * Lazy create the s_configHttpClient if not already done. - * @return true (always). - */ -bool lazyHttpClient() { - if (!s_configHttpClient) { - s_configHttpClient = new HttpClient(s_opt.caFile, s_opt.caPath); - } - return true; -} - -/** - * Collect configuration files matching the prefix and extension from the specified path. - * @param relPath the relative path from which to collect the files (without trailing "/"). - * @param prefix the filename prefix the files have to match, or empty. - * @param extension the filename extension the files have to match. - * @param files the @a vector to which to add the matching files. - * @param query the query string suffix for HTTPS retrieval starting with "&", or empty. - * @param dirs the @a vector to which to add found directories (without any name check), or nullptr to ignore. - * @param hasTemplates the bool to set when the templates file was found in the path, or nullptr to ignore. - * @return the result code. - */ -static result_t collectConfigFiles(const string& relPath, const string& prefix, const string& extension, - vector* files, - bool ignoreAddressPrefix = false, const string& query = "", - vector* dirs = nullptr, bool* hasTemplates = nullptr) { - const string relPathWithSlash = relPath.empty() ? "" : relPath + "/"; - if (!s_configUriPrefix.empty()) { - string uri = s_configUriPrefix + relPathWithSlash + s_configLangQuery + (s_configLangQuery.empty() ? "?" : "&") - + "t=" + extension.substr(1) + query; - string names; - if (!lazyHttpClient() || !s_configHttpClient->get(uri, "", &names)) { - return RESULT_ERR_NOTFOUND; - } - istringstream stream(names); - string name; - while (getline(stream, name)) { - if (name.empty()) { - continue; - } - if (name == "_templates"+extension) { - if (hasTemplates) { - *hasTemplates = true; - } - continue; - } - if (prefix.length() == 0 ? (!ignoreAddressPrefix || name.length() < 3 || name.find_first_of('.') != 2) - : (name.length() >= prefix.length() && name.substr(0, prefix.length()) == prefix)) { - files->push_back(relPathWithSlash + name); - } - } - return RESULT_OK; - } - const string path = s_configLocalPrefix + relPathWithSlash; - logDebug(lf_main, "reading directory %s", path.c_str()); - DIR* dir = opendir(path.c_str()); - if (dir == nullptr) { - return RESULT_ERR_NOTFOUND; - } - dirent* d; - while ((d = readdir(dir)) != nullptr) { - string name = d->d_name; - if (name == "." || name == "..") { - continue; - } - const string p = path + name; - struct stat stat_buf = {}; - if (stat(p.c_str(), &stat_buf) != 0) { - logError(lf_main, "unable to stat file %s", p.c_str()); - continue; - } - logDebug(lf_main, "file type of %s is %s", p.c_str(), - S_ISDIR(stat_buf.st_mode) ? "dir" : S_ISREG(stat_buf.st_mode) ? "file" : "other"); - if (S_ISDIR(stat_buf.st_mode)) { - if (dirs != nullptr) { - dirs->push_back(relPathWithSlash + name); - } - } else if (S_ISREG(stat_buf.st_mode) && name.length() >= extension.length() - && name.substr(name.length()-extension.length()) == extension) { - if (name == "_templates"+extension) { - if (hasTemplates) { - *hasTemplates = true; - } - continue; - } - if (prefix.length() == 0 ? (!ignoreAddressPrefix || name.length() < 3 || name.find_first_of('.') != 2) - : (name.length() >= prefix.length() && name.substr(0, prefix.length()) == prefix)) { - files->push_back(relPathWithSlash + name); - } - } - } - closedir(dir); - - return RESULT_OK; -} - -DataFieldTemplates* getTemplates(const string& filename) { - if (filename == "*") { - unsigned long maxLength = 0; - DataFieldTemplates* best = nullptr; - for (auto it : s_templatesByPath) { - if (it.first.size() > maxLength) { - best = it.second; - } - } - if (best) { - return best; - } - } else { - string path; - size_t pos = filename.find_last_of('/'); - if (pos != string::npos) { - path = filename.substr(0, pos); - } - const auto it = s_templatesByPath.find(path); - if (it != s_templatesByPath.end()) { - return it->second; - } - } - return &s_globalTemplates; -} - -/** - * Read the @a DataFieldTemplates for the specified path if necessary. - * @param relPath the relative path from which to read the files (without trailing "/"). - * @param extension the filename extension of the files to read. - * @param available whether the templates file is available in the path. - * @param verbose whether to verbosely log problems. - * @return false when the templates for the path were already loaded before, true when the templates for the path were added (independent from @a available). - * @return the @a DataFieldTemplates. - */ -static bool readTemplates(const string relPath, const string extension, bool available, bool verbose = false) { - const auto it = s_templatesByPath.find(relPath); - if (it != s_templatesByPath.end()) { - return false; - } - DataFieldTemplates* templates; - if (relPath.empty() || !available) { - templates = &s_globalTemplates; - } else { - templates = new DataFieldTemplates(s_globalTemplates); - } - s_templatesByPath[relPath] = templates; - if (!available) { - // global templates are stored as replacement in order to determine whether the directory was already loaded - return true; - } - string errorDescription; - string logPath = relPath.empty() ? "/" : relPath; - logInfo(lf_main, "reading templates %s", logPath.c_str()); - string file = (relPath.empty() ? "" : relPath + "/") + "_templates" + extension; - result_t result = loadDefinitionsFromConfigPath(templates, file, verbose, nullptr, &errorDescription, true); - if (result == RESULT_OK) { - logInfo(lf_main, "read templates in %s", logPath.c_str()); - return true; - } - logError(lf_main, "error reading templates in %s: %s, last error: %s", logPath.c_str(), getResultCode(result), - errorDescription.c_str()); - return false; -} - -/** - * Read the configuration files from the specified path. - * @param relPath the relative path from which to read the files (without trailing "/"). - * @param extension the filename extension of the files to read. - * @param messages the @a MessageMap to load the messages into. - * @param recursive whether to load all files recursively. - * @param verbose whether to verbosely log problems. - * @param errorDescription a string in which to store the error description in case of error. - * @return the result code. - */ -static result_t readConfigFiles(const string& relPath, const string& extension, bool recursive, - bool verbose, string* errorDescription, MessageMap* messages) { - vector files, dirs; - bool hasTemplates = false; - result_t result = collectConfigFiles(relPath, "", extension, &files, false, "", &dirs, &hasTemplates); - if (result != RESULT_OK) { - return result; - } - readTemplates(relPath, extension, hasTemplates, verbose); - for (const auto& name : files) { - logInfo(lf_main, "reading file %s", name.c_str()); - result = loadDefinitionsFromConfigPath(messages, name, verbose, nullptr, errorDescription); - if (result != RESULT_OK) { - return result; - } - logInfo(lf_main, "successfully read file %s", name.c_str()); - } - if (recursive) { - for (const auto& name : dirs) { - logInfo(lf_main, "reading dir %s", name.c_str()); - result = readConfigFiles(name, extension, true, verbose, errorDescription, messages); - if (result != RESULT_OK) { - return result; - } - logInfo(lf_main, "successfully read dir %s", name.c_str()); - } - } - return RESULT_OK; -} - -/** - * Helper method for immediate reading of a @a Message from the bus. - * @param message the @a Message to read. - */ -void readMessage(Message* message) { - if (!s_mainLoop || !message) { - return; - } - BusHandler* busHandler = s_mainLoop->getBusHandler(); - result_t result = busHandler->readFromBus(message, ""); - if (result != RESULT_OK) { - logError(lf_main, "error reading message %s %s: %s", message->getCircuit().c_str(), message->getName().c_str(), - getResultCode(result)); - } -} - -result_t executeInstructions(MessageMap* messages, bool verbose) { - string errorDescription; - result_t result = messages->resolveConditions(verbose, &errorDescription); - if (result != RESULT_OK) { - logError(lf_main, "error resolving conditions: %s, last error: %s", getResultCode(result), - errorDescription.c_str()); - } - ostringstream log; - result = messages->executeInstructions(readMessage, &log); - if (result != RESULT_OK) { - logError(lf_main, "error executing instructions: %s, last error: %s", getResultCode(result), - log.str().c_str()); - } else if (verbose && log.tellp() > 0) { - logInfo(lf_main, log.str().c_str()); - } - logNotice(lf_main, "found messages: %d (%d conditional on %d conditions, %d poll, %d update)", messages->size(), - messages->sizeConditional(), messages->sizeConditions(), messages->sizePoll(), messages->sizePassive()); - return result; -} - -result_t loadDefinitionsFromConfigPath(FileReader* reader, const string& filename, bool verbose, - map* defaults, string* errorDescription, bool replace) { - istream* stream = nullptr; - time_t mtime = 0; - if (s_configUriPrefix.empty()) { - stream = FileReader::openFile(s_configLocalPrefix + filename, errorDescription, &mtime); - } else { - string content; - if (lazyHttpClient() - && s_configHttpClient->get(s_configUriPrefix + filename + s_configLangQuery, "", &content, &mtime)) { - stream = new istringstream(content); - } - } - result_t result; - if (stream) { - result = reader->readFromStream(stream, filename, mtime, verbose, defaults, errorDescription, replace); - delete(stream); - } else { - result = RESULT_ERR_NOTFOUND; - } - return result; -} - -result_t loadConfigFiles(MessageMap* messages, bool verbose, bool denyRecursive) { - logInfo(lf_main, "loading configuration files from %s", s_configPath.c_str()); - messages->lock(); - messages->clear(); - s_globalTemplates.clear(); - for (auto& it : s_templatesByPath) { - if (it.second != &s_globalTemplates) { - delete it.second; - } - it.second = nullptr; - } - s_templatesByPath.clear(); - - string errorDescription; - result_t result = readConfigFiles("", ".csv", - (!s_opt.scanConfig || s_opt.checkConfig) && !denyRecursive, verbose, &errorDescription, messages); - if (result == RESULT_OK) { - logInfo(lf_main, "read config files, got %d messages", messages->size()); - } else { - logError(lf_main, "error reading config files from %s: %s, last error: %s", s_configPath.c_str(), - getResultCode(result), errorDescription.c_str()); - } - messages->unlock(); - return s_opt.checkConfig ? result : RESULT_OK; -} - -result_t loadScanConfigFile(MessageMap* messages, symbol_t address, bool verbose, string* relativeFile) { - Message* message = messages->getScanMessage(address); - if (!message || message->getLastUpdateTime() == 0) { - return RESULT_ERR_NOTFOUND; - } - const SlaveSymbolString& data = message->getLastSlaveData(); - if (data.getDataSize() < 1+5+2+2) { - logError(lf_main, "unable to load scan config %2.2x: slave part too short (%d)", address, data.getDataSize()); - return RESULT_EMPTY; - } - DataFieldSet* identFields = DataFieldSet::getIdentFields(); - string manufStr, addrStr, ident; // path: cfgpath/MANUFACTURER, prefix: ZZ., ident: C[C[C[C[C]]]], SW: xxxx, HW: xxxx - unsigned int sw = 0, hw = 0; - ostringstream out; - size_t offset = 0; - size_t field = 0; - bool fromLocal = s_configUriPrefix.empty(); - // manufacturer name - result_t result = (*identFields)[field]->read(data, offset, false, nullptr, -1, OF_NONE, -1, &out); - if (result == RESULT_ERR_NOTFOUND && fromLocal) { - result = (*identFields)[field]->read(data, offset, false, nullptr, -1, OF_NUMERIC, -1, &out); // manufacturer name - } - if (result == RESULT_OK) { - manufStr = out.str(); - transform(manufStr.begin(), manufStr.end(), manufStr.begin(), ::tolower); - out.str(""); - out << setw(2) << hex << setfill('0') << nouppercase << static_cast(address); - addrStr = out.str(); - out.str(""); - out.clear(); - offset += (*identFields)[field++]->getLength(pt_slaveData, MAX_LEN); - result = (*identFields)[field]->read(data, offset, false, nullptr, -1, OF_NONE, -1, &out); // identification string - } - if (result == RESULT_OK) { - ident = out.str(); - out.str(""); - out.clear(); - offset += (*identFields)[field++]->getLength(pt_slaveData, MAX_LEN); - result = (*identFields)[field]->read(data, offset, nullptr, -1, &sw); // software version number - if (result == RESULT_ERR_OUT_OF_RANGE) { - sw = (data.dataAt(offset) << 16) | data.dataAt(offset+1); // use hex value instead - result = RESULT_OK; - } - } - if (result == RESULT_OK) { - offset += (*identFields)[field++]->getLength(pt_slaveData, MAX_LEN); - result = (*identFields)[field]->read(data, offset, nullptr, -1, &hw); // hardware version number - if (result == RESULT_ERR_OUT_OF_RANGE) { - hw = (data.dataAt(offset) << 16) | data.dataAt(offset+1); // use hex value instead - result = RESULT_OK; - } - } - if (result != RESULT_OK) { - logError(lf_main, "unable to load scan config %2.2x: decode field %s %s", address, - identFields->getName(field).c_str(), getResultCode(result)); - return result; - } - bool hasTemplates = false; - string best; - map bestDefaults; - vector files; - auto it = ident.begin(); - while (it != ident.end()) { - if (*it != '_' && !::isalnum(*it)) { - it = ident.erase(it); - } else { - *it = static_cast(::tolower(*it)); - it++; - } - } - // find files matching MANUFACTURER/ZZ.*csv in cfgpath - string query; - if (!fromLocal) { - out << "&a=" << addrStr << "&i=" << ident << "&h=" << dec << static_cast(hw) << "&s=" << dec - << static_cast(sw); - query = out.str(); - out.str(""); - out.clear(); - } - result = collectConfigFiles(manufStr, addrStr + ".", ".csv", &files, false, query, nullptr, &hasTemplates); - if (result != RESULT_OK) { - logError(lf_main, "unable to load scan config %2.2x: list files in %s %s", address, manufStr.c_str(), - getResultCode(result)); - return result; - } - if (files.empty()) { - logError(lf_main, "unable to load scan config %2.2x: no file from %s with prefix %s found", address, - manufStr.c_str(), addrStr.c_str()); - return RESULT_ERR_NOTFOUND; - } - logDebug(lf_main, "found %d matching scan config files from %s with prefix %s: %s", files.size(), manufStr.c_str(), - addrStr.c_str(), getResultCode(result)); - // complete name: cfgpath/MANUFACTURER/ZZ[.C[C[C[C[C]]]]][.circuit][.suffix][.*][.SWxxxx][.HWxxxx][.*].csv - size_t bestMatch = 0; - for (const auto& name : files) { - symbol_t checkDest; - unsigned int checkSw, checkHw; - map defaults; - const string filename = name.substr(manufStr.length()+1); - if (!messages->extractDefaultsFromFilename(filename, &defaults, &checkDest, &checkSw, &checkHw)) { - continue; - } - if (address != checkDest || (checkSw != UINT_MAX && sw != checkSw) || (checkHw != UINT_MAX && hw != checkHw)) { - continue; - } - size_t match = 1; - string checkIdent = defaults["name"]; - if (!checkIdent.empty()) { - string remain = ident; - bool matches = false; - while (remain.length() > 0 && remain.length() >= checkIdent.length()) { - if (checkIdent == remain) { - matches = true; - break; - } - if (!::isdigit(remain[remain.length()-1])) { - break; - } - remain.erase(remain.length()-1); // remove trailing digit - } - if (!matches) { - continue; // IDENT mismatch - } - match += remain.length(); - } - if (match >= bestMatch) { - bestMatch = match; - best = name; - bestDefaults = defaults; - } - } - - if (best.empty()) { - logError(lf_main, - "unable to load scan config %2.2x: no file from %s with prefix %s matches ID \"%s\", SW%4.4d, HW%4.4d", - address, manufStr.c_str(), addrStr.c_str(), ident.c_str(), sw, hw); - return RESULT_ERR_NOTFOUND; - } - - // found the right file. load the templates if necessary, then load the file itself - bool readCommon = readTemplates(manufStr, ".csv", hasTemplates, s_opt.checkConfig); - if (readCommon) { - result = collectConfigFiles(manufStr, "", ".csv", &files, true, "&a=-"); - if (result == RESULT_OK && !files.empty()) { - for (const auto& name : files) { - string baseName = name.substr(manufStr.length()+1, name.length()-manufStr.length()-strlen(".csv")); // *. - if (baseName == "_templates.") { // skip templates - continue; - } - if (baseName.length() < 3 || baseName.find_first_of('.') != 2) { // different from the scheme "ZZ." - string errorDescription; - result = loadDefinitionsFromConfigPath(messages, name, verbose, nullptr, &errorDescription); - if (result == RESULT_OK) { - logNotice(lf_main, "read common config file %s", name.c_str()); - } else { - logError(lf_main, "error reading common config file %s: %s, %s", name.c_str(), getResultCode(result), - errorDescription.c_str()); - } - } - } - } - } - bestDefaults["name"] = ident; - string errorDescription; - result = loadDefinitionsFromConfigPath(messages, best, verbose, &bestDefaults, &errorDescription); - if (result != RESULT_OK) { - logError(lf_main, "error reading scan config file %s for ID \"%s\", SW%4.4d, HW%4.4d: %s, %s", best.c_str(), - ident.c_str(), sw, hw, getResultCode(result), errorDescription.c_str()); - return result; - } - logNotice(lf_main, "read scan config file %s for ID \"%s\", SW%4.4d, HW%4.4d", best.c_str(), ident.c_str(), sw, hw); - *relativeFile = best; - return RESULT_OK; -} - -/** - * Helper method for parsing a master/slave message pair from a command line argument. - * @param arg the argument to parse. - * @param onlyMasterSlave true to parse only a MS message, false to also parse MM and BC message. - * @param master the @a MasterSymbolString to parse into. - * @param slave the @a SlaveSymbolString to parse into. - * @return true when the argument was valid, false otherwise. - */ -bool parseMessage(const string& arg, bool onlyMasterSlave, MasterSymbolString* master, SlaveSymbolString* slave) { - size_t pos = arg.find_first_of('/'); - if (pos == string::npos) { - logError(lf_main, "invalid message %s: missing \"/\"", arg.c_str()); - return false; - } - result_t result = master->parseHex(arg.substr(0, pos)); - if (result == RESULT_OK) { - result = slave->parseHex(arg.substr(pos+1)); - } - if (result != RESULT_OK) { - logError(lf_main, "invalid message %s: %s", arg.c_str(), getResultCode(result)); - return false; - } - if (master->size() < 5) { // skip QQ ZZ PB SB NN - logError(lf_main, "invalid message %s: master part too short", arg.c_str()); - return false; - } - if (!isMaster((*master)[0])) { - logError(lf_main, "invalid message %s: QQ is no master", arg.c_str()); - return false; - } - if (!isValidAddress((*master)[1], !onlyMasterSlave) || (onlyMasterSlave && isMaster((*master)[1]))) { - logError(lf_main, "invalid message %s: ZZ is invalid", arg.c_str()); - return false; - } - return true; -} /** * Main function. @@ -1395,8 +874,13 @@ int main(int argc, char* argv[], char* envp[]) { if (!s_configPath.empty() && s_configPath[s_configPath.length()-1] != '/') { s_configPath += "/"; } + const string lang = MappedFileReader::normalizeLanguage( + s_opt.preferLanguage == nullptr || !s_opt.preferLanguage[0] ? "" : s_opt.preferLanguage + ); + string configLocalPrefix, configUriPrefix; + HttpClient* configHttpClient = nullptr; if (s_configPath.find("://") == string::npos) { - s_configLocalPrefix = s_configPath; + configLocalPrefix = s_configPath; } else { if (!s_opt.scanConfig) { logError(lf_main, "invalid configpath without scanconfig"); @@ -1411,7 +895,7 @@ int main(int argc, char* argv[], char* envp[]) { } uint16_t configPort = 80; string proto, configHost; - if (!HttpClient::parseUrl(s_configPath, &proto, &configHost, &configPort, &s_configUriPrefix)) { + if (!HttpClient::parseUrl(s_configPath, &proto, &configHost, &configPort, &configUriPrefix)) { #ifndef HAVE_SSL if (proto == "https") { logError(lf_main, "invalid configPath URL (HTTPS not supported)"); @@ -1421,23 +905,19 @@ int main(int argc, char* argv[], char* envp[]) { logError(lf_main, "invalid configPath URL"); return EINVAL; } - if (!lazyHttpClient() || ( + configHttpClient = new HttpClient(s_opt.caFile, s_opt.caPath); + if ( // check with low timeout of 1 second initially: - !s_configHttpClient->connect(configHost, configPort, proto == "https", PACKAGE_NAME "/" PACKAGE_VERSION, 1) + !configHttpClient->connect(configHost, configPort, proto == "https", PACKAGE_NAME "/" PACKAGE_VERSION, 1) // if that did not work, issue a single retry with default timeout: - && !s_configHttpClient->connect(configHost, configPort, proto == "https", PACKAGE_NAME "/" PACKAGE_VERSION) - )) { + && !configHttpClient->connect(configHost, configPort, proto == "https", PACKAGE_NAME "/" PACKAGE_VERSION) + ) { logError(lf_main, "invalid configPath URL (connect)"); + delete configHttpClient; cleanup(); return EINVAL; } - s_configHttpClient->disconnect(); - } - const string lang = MappedFileReader::normalizeLanguage( - s_opt.preferLanguage == nullptr || !s_opt.preferLanguage[0] ? "" : s_opt.preferLanguage - ); - if (!lang.empty()) { - s_configLangQuery = "?l=" + lang; + configHttpClient->disconnect(); } if (!s_opt.readOnly && s_opt.scanConfig && s_opt.initialScan == 0) { s_opt.initialScan = BROADCAST; @@ -1448,16 +928,19 @@ int main(int argc, char* argv[], char* envp[]) { } s_messageMap = new MessageMap(s_opt.checkConfig, lang); + s_scanHelper = new ScanHelper(s_messageMap, s_configPath, configLocalPrefix, configUriPrefix, + lang.empty() ? lang : "?l=" + lang, configHttpClient, s_opt.checkConfig); + s_messageMap->setResolver(s_scanHelper); if (s_opt.checkConfig) { logNotice(lf_main, PACKAGE_STRING "." REVISION " performing configuration check..."); - result_t result = loadConfigFiles(s_messageMap, true, s_opt.scanConfig && arg_index < argc); - result_t overallResult = executeInstructions(s_messageMap, true); + result_t result = s_scanHelper->loadConfigFiles(!s_opt.scanConfig || arg_index >= argc); + result_t overallResult = s_scanHelper->executeInstructions(nullptr); MasterSymbolString master; SlaveSymbolString slave; while (result == RESULT_OK && s_opt.scanConfig && arg_index < argc) { // check scan config for each passed ident message - if (!parseMessage(argv[arg_index++], true, &master, &slave)) { + if (!s_scanHelper->parseMessage(argv[arg_index++], true, &master, &slave)) { continue; } symbol_t address = master[1]; @@ -1470,8 +953,8 @@ int main(int argc, char* argv[], char* envp[]) { } else { message->storeLastData(master, slave); string file; - result_t res = loadScanConfigFile(s_messageMap, address, true, &file); - result_t instrRes = executeInstructions(s_messageMap, true); + result_t res = s_scanHelper->loadScanConfigFile(address, &file); + result_t instrRes = s_scanHelper->executeInstructions(nullptr); if (res == RESULT_OK) { logInfo(lf_main, "scan config %2.2x: file %s loaded", address, file.c_str()); } else if (overallResult == RESULT_OK) { @@ -1543,17 +1026,17 @@ int main(int argc, char* argv[], char* envp[]) { device->getName()); // load configuration files - loadConfigFiles(s_messageMap); + s_scanHelper->loadConfigFiles(s_messageMap); // create the MainLoop and start it - s_mainLoop = new MainLoop(s_opt, device, s_messageMap); + s_mainLoop = new MainLoop(s_opt, device, s_messageMap, s_scanHelper); if (s_opt.injectMessages) { BusHandler* busHandler = s_mainLoop->getBusHandler(); while (arg_index < argc) { // add each passed message MasterSymbolString master; SlaveSymbolString slave; - if (!parseMessage(argv[arg_index++], false, &master, &slave)) { + if (!s_scanHelper->parseMessage(argv[arg_index++], false, &master, &slave)) { continue; } busHandler->injectMessage(master, slave); diff --git a/src/ebusd/main.h b/src/ebusd/main.h index c83e6660..09f3174e 100644 --- a/src/ebusd/main.h +++ b/src/ebusd/main.h @@ -23,7 +23,6 @@ #include #include #include "lib/ebus/data.h" -#include "lib/ebus/message.h" #include "lib/ebus/result.h" #include "lib/utils/log.h" @@ -91,56 +90,6 @@ struct options { bool dumpFlush; //!< flush each byte }; -/** - * Get the @a DataFieldTemplates for the specified configuration file. - * @param filename the full name of the configuration file, or "*" to get the non-root templates with the longest name - * or the root templates if not available. - * @return the @a DataFieldTemplates. - */ -DataFieldTemplates* getTemplates(const string& filename); - -/** - * Load the message definitions from configuration files. - * @param messages the @a MessageMap to load the messages into. - * @param verbose whether to verbosely log problems. - * @param denyRecursive whether to avoid loading all files recursively (e.g. for scan config check). - * @return the result code. - */ -result_t loadConfigFiles(MessageMap* messages, bool verbose = false, bool denyRecursive = false); - -/** - * Load the message definitions from a configuration file matching the scan result. - * @param messages the @a MessageMap to load the messages into. - * @param address the address of the scan participant - * (either master for broadcast master data or slave for read slave data). - * @param data the scan @a SlaveSymbolString for which to load the configuration file. - * @param verbose whether to verbosely log problems. - * @param relativeFile the string in which the name of the configuration file is stored on success. - * @return the result code. - */ -result_t loadScanConfigFile(MessageMap* messages, symbol_t address, bool verbose, string* relativeFile); - -/** - * Helper method for executing all loaded and resolvable instructions. - * @param messages the @a MessageMap instance. - * @param verbose whether to verbosely log all problems. - * @return the result code. - */ -result_t executeInstructions(MessageMap* messages, bool verbose = false); - -/** - * Helper method for loading definitions from a relative file from the config path/URL. - * @param reader the @a FileReader instance to load with the definitions. - * @param filename the relative name of the file being read. - * @param verbose whether to verbosely log problems. - * @param defaults the default values by name (potentially overwritten by file name), or nullptr to not use defaults. - * @param errorDescription a string in which to store the error description in case of error. - * @param replace whether to replace an already existing entry. - * @return @a RESULT_OK on success, or an error code. - */ -result_t loadDefinitionsFromConfigPath(FileReader* reader, const string& filename, bool verbose, - map* defaults, string* errorDescription, bool replace = false); - } // namespace ebusd #endif // EBUSD_MAIN_H_ diff --git a/src/ebusd/mainloop.cpp b/src/ebusd/mainloop.cpp index c87d6ed9..8cd22f84 100644 --- a/src/ebusd/mainloop.cpp +++ b/src/ebusd/mainloop.cpp @@ -21,6 +21,7 @@ #endif #include "ebusd/mainloop.h" +#include "ebusd/scan.h" #include #include #include @@ -104,8 +105,8 @@ result_t UserList::addFromFile(const string& filename, unsigned int lineNo, map< #define VERBOSITY_4 (VERBOSITY_3 | OF_ALL_ATTRS) -MainLoop::MainLoop(const struct options& opt, Device *device, MessageMap* messages) - : Thread(), m_device(device), m_reconnectCount(0), m_userList(opt.accessLevel), m_messages(messages), +MainLoop::MainLoop(const struct options& opt, Device *device, MessageMap* messages, ScanHelper* scanHelper) + : Thread(), m_device(device), m_reconnectCount(0), m_userList(opt.accessLevel), m_messages(messages), m_scanHelper(scanHelper), m_address(opt.address), m_scanConfig(opt.scanConfig), m_initialScan(opt.readOnly ? ESC : opt.initialScan), m_polling(opt.pollInterval > 0), m_enableHex(opt.enableHex), m_shutdown(false), m_runUpdateCheck(opt.updateCheck), m_httpClient(opt.caFile, opt.caPath) { @@ -148,7 +149,7 @@ MainLoop::MainLoop(const struct options& opt, Device *device, MessageMap* messag } } // create BusHandler - m_busHandler = new BusHandler(m_device, m_messages, + m_busHandler = new BusHandler(m_device, m_messages, scanHelper, m_address, opt.answer, opt.acquireRetries, opt.sendRetries, opt.acquireTimeout, opt.receiveTimeout, @@ -166,7 +167,12 @@ MainLoop::MainLoop(const struct options& opt, Device *device, MessageMap* messag } else { logError(lf_main, "error registering data handlers"); } - m_newlyDefinedMessages = opt.enableDefine ? new MessageMap(true, "", false) : nullptr; + if (opt.enableDefine) { + m_newlyDefinedMessages = new MessageMap(true, "", false); + m_newlyDefinedMessages->setResolver(scanHelper); + } else { + m_newlyDefinedMessages = nullptr; + } } MainLoop::~MainLoop() { @@ -330,7 +336,7 @@ void MainLoop::run() { } else if (reload && m_busHandler->hasSignal()) { reload = false; // execute initial instructions - executeInstructions(m_messages); + m_scanHelper->executeInstructions(m_busHandler); if (m_messages->sizeConditions() > 0 && !m_polling) { logError(lf_main, "conditions require a poll interval > 0"); } @@ -1749,7 +1755,7 @@ result_t MainLoop::executeDecode(const vector& args, ostringstream* ostr time(&now); istringstream defstr("#\n" + args[argPos]); // ensure first line is not used for determining col names string errorDescription; - DataFieldTemplates* templates = getTemplates("*"); + DataFieldTemplates* templates = m_scanHelper->getTemplates("*"); LoadableDataFieldSet fields("", templates); result_t ret = fields.readFromStream(&defstr, "temporary", now, true, nullptr, &errorDescription); if (ret != RESULT_OK) { @@ -1781,7 +1787,7 @@ result_t MainLoop::executeEncode(const vector& args, ostringstream* ostr time(&now); istringstream defstr("#\n" + args[argPos]); // ensure first line is not used for determining col names string errorDescription; - DataFieldTemplates* templates = getTemplates("*"); + DataFieldTemplates* templates = m_scanHelper->getTemplates("*"); LoadableDataFieldSet fields("", templates); result_t ret = fields.readFromStream(&defstr, "temporary", now, true, nullptr, &errorDescription); if (ret != RESULT_OK) { @@ -1922,7 +1928,8 @@ result_t MainLoop::executeReload(const vector& args, ostringstream* ostr return RESULT_OK; } m_busHandler->clear(); - return loadConfigFiles(m_messages); + m_scanHelper->loadConfigFiles(!m_scanConfig); + return RESULT_OK; } result_t MainLoop::executeInfo(const vector& args, const string& user, ostringstream* ostream) { @@ -2343,7 +2350,7 @@ result_t MainLoop::executeGet(const vector& args, bool* connected, ostri time(&now); istringstream defstr("#\n" + def); // ensure first line is not used for determining col names string errorDescription; - DataFieldTemplates* templates = getTemplates("*"); + DataFieldTemplates* templates = m_scanHelper->getTemplates("*"); LoadableDataFieldSet fields("", templates); ret = fields.readFromStream(&defstr, "temporary", now, true, nullptr, &errorDescription); if (ret == RESULT_OK && fields.size()) { diff --git a/src/ebusd/mainloop.h b/src/ebusd/mainloop.h index 1092182c..b76797ef 100644 --- a/src/ebusd/mainloop.h +++ b/src/ebusd/mainloop.h @@ -27,6 +27,7 @@ #include "ebusd/bushandler.h" #include "ebusd/datahandler.h" #include "ebusd/network.h" +#include "ebusd/scan.h" #include "lib/ebus/filereader.h" #include "lib/ebus/message.h" #include "lib/utils/rotatefile.h" @@ -105,8 +106,9 @@ class MainLoop : public Thread, DeviceListener { * @param opt the program options. * @param device the @a Device instance. * @param messages the @a MessageMap instance. + * @param scanHelper the @a ScanHelper instance. */ - MainLoop(const struct options& opt, Device *device, MessageMap* messages); + MainLoop(const struct options& opt, Device *device, MessageMap* messages, ScanHelper* scanHelper); /** * Destructor. @@ -406,6 +408,9 @@ class MainLoop : public Thread, DeviceListener { /** the @a MessageMap instance. */ MessageMap* m_messages; + /** the @a ScanHelper instance. */ + ScanHelper* m_scanHelper; + /** the own master address for sending on the bus. */ const symbol_t m_address; diff --git a/src/ebusd/scan.cpp b/src/ebusd/scan.cpp new file mode 100644 index 00000000..64452cfc --- /dev/null +++ b/src/ebusd/scan.cpp @@ -0,0 +1,510 @@ +/* + * ebusd - daemon for communication with eBUS heating systems. + * Copyright (C) 2014-2023 John Baier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifdef HAVE_CONFIG_H +# include +#endif + +#include "ebusd/scan.h" +#include "ebusd/bushandler.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include "lib/utils/log.h" + + +namespace ebusd { + +using std::dec; +using std::hex; +using std::setfill; +using std::setw; +using std::nouppercase; +using std::cout; + + +ScanHelper::~ScanHelper() { + // free templates + for (const auto& it : m_templatesByPath) { + if (it.second != &m_globalTemplates) { + delete it.second; + } + } + m_templatesByPath.clear(); + if (m_configHttpClient) { + delete m_configHttpClient; + m_configHttpClient = nullptr; + } +} + +result_t ScanHelper::collectConfigFiles(const string& relPath, const string& prefix, const string& extension, + vector* files, + bool ignoreAddressPrefix, const string& query, + vector* dirs, bool* hasTemplates) { + const string relPathWithSlash = relPath.empty() ? "" : relPath + "/"; + if (!m_configUriPrefix.empty()) { + string uri = m_configUriPrefix + relPathWithSlash + m_configLangQuery + (m_configLangQuery.empty() ? "?" : "&") + + "t=" + extension.substr(1) + query; + string names; + if (!m_configHttpClient->get(uri, "", &names)) { + return RESULT_ERR_NOTFOUND; + } + istringstream stream(names); + string name; + while (getline(stream, name)) { + if (name.empty()) { + continue; + } + if (name == "_templates"+extension) { + if (hasTemplates) { + *hasTemplates = true; + } + continue; + } + if (prefix.length() == 0 ? (!ignoreAddressPrefix || name.length() < 3 || name.find_first_of('.') != 2) + : (name.length() >= prefix.length() && name.substr(0, prefix.length()) == prefix)) { + files->push_back(relPathWithSlash + name); + } + } + return RESULT_OK; + } + const string path = m_configLocalPrefix + relPathWithSlash; + logDebug(lf_main, "reading directory %s", path.c_str()); + DIR* dir = opendir(path.c_str()); + if (dir == nullptr) { + return RESULT_ERR_NOTFOUND; + } + dirent* d; + while ((d = readdir(dir)) != nullptr) { + string name = d->d_name; + if (name == "." || name == "..") { + continue; + } + const string p = path + name; + struct stat stat_buf = {}; + if (stat(p.c_str(), &stat_buf) != 0) { + logError(lf_main, "unable to stat file %s", p.c_str()); + continue; + } + logDebug(lf_main, "file type of %s is %s", p.c_str(), + S_ISDIR(stat_buf.st_mode) ? "dir" : S_ISREG(stat_buf.st_mode) ? "file" : "other"); + if (S_ISDIR(stat_buf.st_mode)) { + if (dirs != nullptr) { + dirs->push_back(relPathWithSlash + name); + } + } else if (S_ISREG(stat_buf.st_mode) && name.length() >= extension.length() + && name.substr(name.length()-extension.length()) == extension) { + if (name == "_templates"+extension) { + if (hasTemplates) { + *hasTemplates = true; + } + continue; + } + if (prefix.length() == 0 ? (!ignoreAddressPrefix || name.length() < 3 || name.find_first_of('.') != 2) + : (name.length() >= prefix.length() && name.substr(0, prefix.length()) == prefix)) { + files->push_back(relPathWithSlash + name); + } + } + } + closedir(dir); + + return RESULT_OK; +} + +DataFieldTemplates* ScanHelper::getTemplates(const string& filename) { + if (filename == "*") { + unsigned long maxLength = 0; + DataFieldTemplates* best = nullptr; + for (auto it : m_templatesByPath) { + if (it.first.size() > maxLength) { + best = it.second; + } + } + if (best) { + return best; + } + } else { + string path; + size_t pos = filename.find_last_of('/'); + if (pos != string::npos) { + path = filename.substr(0, pos); + } + const auto it = m_templatesByPath.find(path); + if (it != m_templatesByPath.end()) { + return it->second; + } + } + return &m_globalTemplates; +} + +bool ScanHelper::readTemplates(const string relPath, const string extension, bool available) { + const auto it = m_templatesByPath.find(relPath); + if (it != m_templatesByPath.end()) { + return false; + } + DataFieldTemplates* templates; + if (relPath.empty() || !available) { + templates = &m_globalTemplates; + } else { + templates = new DataFieldTemplates(m_globalTemplates); + } + m_templatesByPath[relPath] = templates; + if (!available) { + // global templates are stored as replacement in order to determine whether the directory was already loaded + return true; + } + string errorDescription; + string logPath = relPath.empty() ? "/" : relPath; + logInfo(lf_main, "reading templates %s", logPath.c_str()); + string file = (relPath.empty() ? "" : relPath + "/") + "_templates" + extension; + result_t result = loadDefinitionsFromConfigPath(templates, file, nullptr, &errorDescription, true); + if (result == RESULT_OK) { + logInfo(lf_main, "read templates in %s", logPath.c_str()); + return true; + } + logError(lf_main, "error reading templates in %s: %s, last error: %s", logPath.c_str(), getResultCode(result), + errorDescription.c_str()); + return false; +} + +result_t ScanHelper::readConfigFiles(const string& relPath, const string& extension, bool recursive, + string* errorDescription) { + vector files, dirs; + bool hasTemplates = false; + result_t result = collectConfigFiles(relPath, "", extension, &files, false, "", &dirs, &hasTemplates); + if (result != RESULT_OK) { + return result; + } + readTemplates(relPath, extension, hasTemplates); + for (const auto& name : files) { + logInfo(lf_main, "reading file %s", name.c_str()); + result = loadDefinitionsFromConfigPath(m_messages, name, nullptr, errorDescription); + if (result != RESULT_OK) { + return result; + } + logInfo(lf_main, "successfully read file %s", name.c_str()); + } + if (recursive) { + for (const auto& name : dirs) { + logInfo(lf_main, "reading dir %s", name.c_str()); + result = readConfigFiles(name, extension, true, errorDescription); + if (result != RESULT_OK) { + return result; + } + logInfo(lf_main, "successfully read dir %s", name.c_str()); + } + } + return RESULT_OK; +} + +static BusHandler* executeInstructionsBusHandlerInstance = nullptr; + +/** + * Helper method for immediate reading of a @a Message from the bus. + * @param message the @a Message to read. + */ +static void readMessage(Message* message) { + if (!executeInstructionsBusHandlerInstance || !message) { + return; + } + result_t result = executeInstructionsBusHandlerInstance->readFromBus(message, ""); + if (result != RESULT_OK) { + logError(lf_main, "error reading message %s %s: %s", message->getCircuit().c_str(), message->getName().c_str(), + getResultCode(result)); + } +} + +result_t ScanHelper::executeInstructions(BusHandler* busHandler) { + string errorDescription; + result_t result = m_messages->resolveConditions(m_verbose, &errorDescription); + if (result != RESULT_OK) { + logError(lf_main, "error resolving conditions: %s, last error: %s", getResultCode(result), + errorDescription.c_str()); + } + ostringstream log; + executeInstructionsBusHandlerInstance = busHandler; + result = m_messages->executeInstructions(&readMessage, &log); + if (result != RESULT_OK) { + logError(lf_main, "error executing instructions: %s, last error: %s", getResultCode(result), + log.str().c_str()); + } else if (m_verbose && log.tellp() > 0) { + logInfo(lf_main, log.str().c_str()); + } + logNotice(lf_main, "found messages: %d (%d conditional on %d conditions, %d poll, %d update)", m_messages->size(), + m_messages->sizeConditional(), m_messages->sizeConditions(), m_messages->sizePoll(), m_messages->sizePassive()); + return result; +} + +result_t ScanHelper::loadDefinitionsFromConfigPath(FileReader* reader, const string& filename, + map* defaults, string* errorDescription, bool replace) { + istream* stream = nullptr; + time_t mtime = 0; + if (m_configUriPrefix.empty()) { + stream = FileReader::openFile(m_configLocalPrefix + filename, errorDescription, &mtime); + } else { + string content; + if (m_configHttpClient + && m_configHttpClient->get(m_configUriPrefix + filename + m_configLangQuery, "", &content, &mtime)) { + stream = new istringstream(content); + } + } + result_t result; + if (stream) { + result = reader->readFromStream(stream, filename, mtime, m_verbose, defaults, errorDescription, replace); + delete(stream); + } else { + result = RESULT_ERR_NOTFOUND; + } + return result; +} + +result_t ScanHelper::loadConfigFiles(bool recursive) { + logInfo(lf_main, "loading configuration files from %s", m_configPath.c_str()); + m_messages->lock(); + m_messages->clear(); + m_globalTemplates.clear(); + for (auto& it : m_templatesByPath) { + if (it.second != &m_globalTemplates) { + delete it.second; + } + it.second = nullptr; + } + m_templatesByPath.clear(); + + string errorDescription; + result_t result = readConfigFiles("", ".csv", recursive, &errorDescription); + if (result == RESULT_OK) { + logInfo(lf_main, "read config files, got %d messages", m_messages->size()); + } else { + logError(lf_main, "error reading config files from %s: %s, last error: %s", m_configPath.c_str(), + getResultCode(result), errorDescription.c_str()); + } + m_messages->unlock(); + return result; +} + +result_t ScanHelper::loadScanConfigFile(symbol_t address, string* relativeFile) { + Message* message = m_messages->getScanMessage(address); + if (!message || message->getLastUpdateTime() == 0) { + return RESULT_ERR_NOTFOUND; + } + const SlaveSymbolString& data = message->getLastSlaveData(); + if (data.getDataSize() < 1+5+2+2) { + logError(lf_main, "unable to load scan config %2.2x: slave part too short (%d)", address, data.getDataSize()); + return RESULT_EMPTY; + } + DataFieldSet* identFields = DataFieldSet::getIdentFields(); + string manufStr, addrStr, ident; // path: cfgpath/MANUFACTURER, prefix: ZZ., ident: C[C[C[C[C]]]], SW: xxxx, HW: xxxx + unsigned int sw = 0, hw = 0; + ostringstream out; + size_t offset = 0; + size_t field = 0; + bool fromLocal = m_configUriPrefix.empty(); + // manufacturer name + result_t result = (*identFields)[field]->read(data, offset, false, nullptr, -1, OF_NONE, -1, &out); + if (result == RESULT_ERR_NOTFOUND && fromLocal) { + result = (*identFields)[field]->read(data, offset, false, nullptr, -1, OF_NUMERIC, -1, &out); // manufacturer name + } + if (result == RESULT_OK) { + manufStr = out.str(); + transform(manufStr.begin(), manufStr.end(), manufStr.begin(), ::tolower); + out.str(""); + out << setw(2) << hex << setfill('0') << nouppercase << static_cast(address); + addrStr = out.str(); + out.str(""); + out.clear(); + offset += (*identFields)[field++]->getLength(pt_slaveData, MAX_LEN); + result = (*identFields)[field]->read(data, offset, false, nullptr, -1, OF_NONE, -1, &out); // identification string + } + if (result == RESULT_OK) { + ident = out.str(); + out.str(""); + out.clear(); + offset += (*identFields)[field++]->getLength(pt_slaveData, MAX_LEN); + result = (*identFields)[field]->read(data, offset, nullptr, -1, &sw); // software version number + if (result == RESULT_ERR_OUT_OF_RANGE) { + sw = (data.dataAt(offset) << 16) | data.dataAt(offset+1); // use hex value instead + result = RESULT_OK; + } + } + if (result == RESULT_OK) { + offset += (*identFields)[field++]->getLength(pt_slaveData, MAX_LEN); + result = (*identFields)[field]->read(data, offset, nullptr, -1, &hw); // hardware version number + if (result == RESULT_ERR_OUT_OF_RANGE) { + hw = (data.dataAt(offset) << 16) | data.dataAt(offset+1); // use hex value instead + result = RESULT_OK; + } + } + if (result != RESULT_OK) { + logError(lf_main, "unable to load scan config %2.2x: decode field %s %s", address, + identFields->getName(field).c_str(), getResultCode(result)); + return result; + } + bool hasTemplates = false; + string best; + map bestDefaults; + vector files; + auto it = ident.begin(); + while (it != ident.end()) { + if (*it != '_' && !::isalnum(*it)) { + it = ident.erase(it); + } else { + *it = static_cast(::tolower(*it)); + it++; + } + } + // find files matching MANUFACTURER/ZZ.*csv in cfgpath + string query; + if (!fromLocal) { + out << "&a=" << addrStr << "&i=" << ident << "&h=" << dec << static_cast(hw) << "&s=" << dec + << static_cast(sw); + query = out.str(); + out.str(""); + out.clear(); + } + result = collectConfigFiles(manufStr, addrStr + ".", ".csv", &files, false, query, nullptr, &hasTemplates); + if (result != RESULT_OK) { + logError(lf_main, "unable to load scan config %2.2x: list files in %s %s", address, manufStr.c_str(), + getResultCode(result)); + return result; + } + if (files.empty()) { + logError(lf_main, "unable to load scan config %2.2x: no file from %s with prefix %s found", address, + manufStr.c_str(), addrStr.c_str()); + return RESULT_ERR_NOTFOUND; + } + logDebug(lf_main, "found %d matching scan config files from %s with prefix %s: %s", files.size(), manufStr.c_str(), + addrStr.c_str(), getResultCode(result)); + // complete name: cfgpath/MANUFACTURER/ZZ[.C[C[C[C[C]]]]][.circuit][.suffix][.*][.SWxxxx][.HWxxxx][.*].csv + size_t bestMatch = 0; + for (const auto& name : files) { + symbol_t checkDest; + unsigned int checkSw, checkHw; + map defaults; + const string filename = name.substr(manufStr.length()+1); + if (!m_messages->extractDefaultsFromFilename(filename, &defaults, &checkDest, &checkSw, &checkHw)) { + continue; + } + if (address != checkDest || (checkSw != UINT_MAX && sw != checkSw) || (checkHw != UINT_MAX && hw != checkHw)) { + continue; + } + size_t match = 1; + string checkIdent = defaults["name"]; + if (!checkIdent.empty()) { + string remain = ident; + bool matches = false; + while (remain.length() > 0 && remain.length() >= checkIdent.length()) { + if (checkIdent == remain) { + matches = true; + break; + } + if (!::isdigit(remain[remain.length()-1])) { + break; + } + remain.erase(remain.length()-1); // remove trailing digit + } + if (!matches) { + continue; // IDENT mismatch + } + match += remain.length(); + } + if (match >= bestMatch) { + bestMatch = match; + best = name; + bestDefaults = defaults; + } + } + + if (best.empty()) { + logError(lf_main, + "unable to load scan config %2.2x: no file from %s with prefix %s matches ID \"%s\", SW%4.4d, HW%4.4d", + address, manufStr.c_str(), addrStr.c_str(), ident.c_str(), sw, hw); + return RESULT_ERR_NOTFOUND; + } + + // found the right file. load the templates if necessary, then load the file itself + bool readCommon = readTemplates(manufStr, ".csv", hasTemplates); + if (readCommon) { + result = collectConfigFiles(manufStr, "", ".csv", &files, true, "&a=-"); + if (result == RESULT_OK && !files.empty()) { + for (const auto& name : files) { + string baseName = name.substr(manufStr.length()+1, name.length()-manufStr.length()-strlen(".csv")); // *. + if (baseName == "_templates.") { // skip templates + continue; + } + if (baseName.length() < 3 || baseName.find_first_of('.') != 2) { // different from the scheme "ZZ." + string errorDescription; + result = loadDefinitionsFromConfigPath(m_messages, name, nullptr, &errorDescription); + if (result == RESULT_OK) { + logNotice(lf_main, "read common config file %s", name.c_str()); + } else { + logError(lf_main, "error reading common config file %s: %s, %s", name.c_str(), getResultCode(result), + errorDescription.c_str()); + } + } + } + } + } + bestDefaults["name"] = ident; + string errorDescription; + result = loadDefinitionsFromConfigPath(m_messages, best, &bestDefaults, &errorDescription); + if (result != RESULT_OK) { + logError(lf_main, "error reading scan config file %s for ID \"%s\", SW%4.4d, HW%4.4d: %s, %s", best.c_str(), + ident.c_str(), sw, hw, getResultCode(result), errorDescription.c_str()); + return result; + } + logNotice(lf_main, "read scan config file %s for ID \"%s\", SW%4.4d, HW%4.4d", best.c_str(), ident.c_str(), sw, hw); + *relativeFile = best; + return RESULT_OK; +} + +bool ScanHelper::parseMessage(const string& arg, bool onlyMasterSlave, MasterSymbolString* master, SlaveSymbolString* slave) { + size_t pos = arg.find_first_of('/'); + if (pos == string::npos) { + logError(lf_main, "invalid message %s: missing \"/\"", arg.c_str()); + return false; + } + result_t result = master->parseHex(arg.substr(0, pos)); + if (result == RESULT_OK) { + result = slave->parseHex(arg.substr(pos+1)); + } + if (result != RESULT_OK) { + logError(lf_main, "invalid message %s: %s", arg.c_str(), getResultCode(result)); + return false; + } + if (master->size() < 5) { // skip QQ ZZ PB SB NN + logError(lf_main, "invalid message %s: master part too short", arg.c_str()); + return false; + } + if (!isMaster((*master)[0])) { + logError(lf_main, "invalid message %s: QQ is no master", arg.c_str()); + return false; + } + if (!isValidAddress((*master)[1], !onlyMasterSlave) || (onlyMasterSlave && isMaster((*master)[1]))) { + logError(lf_main, "invalid message %s: ZZ is invalid", arg.c_str()); + return false; + } + return true; +} + +} // namespace ebusd diff --git a/src/ebusd/scan.h b/src/ebusd/scan.h new file mode 100644 index 00000000..d5372f35 --- /dev/null +++ b/src/ebusd/scan.h @@ -0,0 +1,205 @@ +/* + * ebusd - daemon for communication with eBUS heating systems. + * Copyright (C) 2014-2023 John Baier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef EBUSD_SCAN_H_ +#define EBUSD_SCAN_H_ + +#include +#include +#include +#include "lib/ebus/data.h" +#include "lib/ebus/message.h" +#include "lib/ebus/result.h" +#include "lib/utils/httpclient.h" +#include "lib/utils/log.h" + +namespace ebusd { + +/** \file ebusd/scan.h + * Helpers for handling device scanning and config loading. + */ + +class BusHandler; + +/** + * Helper class for handling device scanning and config loading. + */ +class ScanHelper : public Resolver { + + public: + /** + * Constructor. + * @param messages the @a MessageMap to load the messages into. + * @param configPath the (optionally corrected) config path for retrieving configuration files from. + * @param configLocalPrefix the path prefix (including trailing "/") for retrieving configuration files from local files (empty for HTTPS). + * @param configUriPrefix the URI prefix (including trailing "/") for retrieving configuration files from HTTPS (empty for local files). + * @param configLangQuery the optional language query part for retrieving configuration files from HTTPS (empty for local files). + * @param configHttpClient the @a HttpClient for retrieving configuration files from HTTPS. + * @param verbose whether to verbosely log problems. + */ + ScanHelper(MessageMap* messages, + const string configPath, const string configLocalPrefix, + const string configUriPrefix, const string configLangQuery, + HttpClient* configHttpClient, bool verbose) + : Resolver(), m_messages(messages), + m_configPath(configPath), m_configLocalPrefix(configLocalPrefix), + m_configUriPrefix(configUriPrefix), m_configLangQuery(configLangQuery), + m_configHttpClient(configHttpClient), m_verbose(verbose) {} + + /** + * Destructor. + */ + virtual ~ScanHelper(); + + /** + * Try to connect to the specified server. + * @param host the host name to connect to. + * @param port the port to connect to. + * @param https true for HTTPS, false for HTTP. + * @param timeout the timeout in seconds, defaults to 5 seconds. + * @return true on success, false on connect failure. + */ + bool connect(const string& host, uint16_t port, bool https = false, int timeout = 5); + + /** + * Get the @a DataFieldTemplates for the specified configuration file. + * @param filename the full name of the configuration file, or "*" to get the non-root templates with the longest name + * or the root templates if not available. + * @return the @a DataFieldTemplates. + */ + virtual DataFieldTemplates* getTemplates(const string& filename); + + /** + * Load the message definitions from configuration files. + * @param recursive whether to load all files recursively. + * @return the result code. + */ + result_t loadConfigFiles(bool recursive = true); + + /** + * Load the message definitions from a configuration file matching the scan result. + * @param address the address of the scan participant + * (either master for broadcast master data or slave for read slave data). + * @param data the scan @a SlaveSymbolString for which to load the configuration file. + * @param relativeFile the string in which the name of the configuration file is stored on success. + * @return the result code. + */ + result_t loadScanConfigFile(symbol_t address, string* relativeFile); + + /** + * Helper method for executing all loaded and resolvable instructions. + * @param busHandler the @a BusHandler instance. + * @return the result code. + */ + result_t executeInstructions(BusHandler* busHandler); + + /** + * Helper method for loading definitions from a relative file from the config path/URL. + * @param reader the @a FileReader instance to load with the definitions. + * @param filename the relative name of the file being read. + * @param defaults the default values by name (potentially overwritten by file name), or nullptr to not use defaults. + * @param errorDescription a string in which to store the error description in case of error. + * @param replace whether to replace an already existing entry. + * @return @a RESULT_OK on success, or an error code. + */ + virtual result_t loadDefinitionsFromConfigPath(FileReader* reader, const string& filename, + map* defaults, string* errorDescription, bool replace = false); + + /** + * Helper method for parsing a master/slave message pair from a command line argument. + * @param arg the argument to parse. + * @param onlyMasterSlave true to parse only a MS message, false to also parse MM and BC message. + * @param master the @a MasterSymbolString to parse into. + * @param slave the @a SlaveSymbolString to parse into. + * @return true when the argument was valid, false otherwise. + */ + bool parseMessage(const string& arg, bool onlyMasterSlave, MasterSymbolString* master, SlaveSymbolString* slave); + + + private: + /** + * Collect configuration files matching the prefix and extension from the specified path. + * @param relPath the relative path from which to collect the files (without trailing "/"). + * @param prefix the filename prefix the files have to match, or empty. + * @param extension the filename extension the files have to match. + * @param files the @a vector to which to add the matching files. + * @param query the query string suffix for HTTPS retrieval starting with "&", or empty. + * @param dirs the @a vector to which to add found directories (without any name check), or nullptr to ignore. + * @param hasTemplates the bool to set when the templates file was found in the path, or nullptr to ignore. + * @return the result code. + */ + result_t collectConfigFiles(const string& relPath, const string& prefix, const string& extension, + vector* files, + bool ignoreAddressPrefix = false, const string& query = "", + vector* dirs = nullptr, bool* hasTemplates = nullptr); + + /** + * Read the @a DataFieldTemplates for the specified path if necessary. + * @param relPath the relative path from which to read the files (without trailing "/"). + * @param extension the filename extension of the files to read. + * @param available whether the templates file is available in the path. + * @return false when the templates for the path were already loaded before, true when the templates for the path were added (independent from @a available). + * @return the @a DataFieldTemplates. + */ + bool readTemplates(const string relPath, const string extension, bool available); + + /** + * Read the configuration files from the specified path. + * @param relPath the relative path from which to read the files (without trailing "/"). + * @param extension the filename extension of the files to read. + * @param recursive whether to load all files recursively. + * @param errorDescription a string in which to store the error description in case of error. + * @return the result code. + */ + result_t readConfigFiles(const string& relPath, const string& extension, bool recursive, + string* errorDescription); + + /** the @a MessageMap instance. */ + MessageMap* m_messages; + + /** the (optionally corrected) config path for retrieving configuration files from. */ + const string m_configPath; + + /** the path prefix (including trailing "/") for retrieving configuration files from local files (empty for HTTPS). */ + const string m_configLocalPrefix; + + /** the URI prefix (including trailing "/") for retrieving configuration files from HTTPS (empty for local files). */ + const string m_configUriPrefix; + + /** the optional language query part for retrieving configuration files from HTTPS (empty for local files). */ + const string m_configLangQuery; + + /** the @a HttpClient for retrieving configuration files from HTTPS. */ + HttpClient* m_configHttpClient; + + /** whether to verbosely log problems. */ + const bool m_verbose; + + /** the global @a DataFieldTemplates. */ + DataFieldTemplates m_globalTemplates; + + /** + * the loaded @a DataFieldTemplates by relative path (may also carry + * @a globalTemplates as replacement for missing file). + */ + map m_templatesByPath; +}; + +} // namespace ebusd + +#endif // EBUSD_SCAN_H_ diff --git a/src/lib/ebus/message.cpp b/src/lib/ebus/message.cpp index 7bd4eaa3..1bcb20cd 100644 --- a/src/lib/ebus/message.cpp +++ b/src/lib/ebus/message.cpp @@ -94,11 +94,6 @@ static const char* defaultMessageFieldMap[] = { // access level not included in /** the m_pollOrder of the last polled message. */ static unsigned int g_lastPollOrder = 0; -extern DataFieldTemplates* getTemplates(const string& filename); - -extern result_t loadDefinitionsFromConfigPath(FileReader* reader, const string& filename, bool verbose, - map* defaults, string* errorDescription, bool replace = false); - Message::Message(const string& filename, const string& circuit, const string& level, const string& name, bool isWrite, bool isPassive, const map& attributes, @@ -1738,7 +1733,11 @@ void Instruction::getDestination(ostringstream* ostream) const { result_t LoadInstruction::execute(MessageMap* messages, ostringstream* log) { string errorDescription; - result_t result = loadDefinitionsFromConfigPath(messages, m_filename, false, &m_defaults, &errorDescription); + Resolver* resolver = messages->getResolver(); + if (!resolver) { + return RESULT_ERR_MISSING_ARG; + } + result_t result = resolver->loadDefinitionsFromConfigPath(messages, m_filename, &m_defaults, &errorDescription); if (log->tellp() > 0) { *log << ", "; } @@ -2306,7 +2305,10 @@ result_t MessageMap::addFromFile(const string& filename, unsigned int lineNo, ma return RESULT_ERR_INVALID_ARG; } result = RESULT_ERR_EOF; - DataFieldTemplates* templates = getTemplates(filename); + if (!m_resolver) { + return result; + } + DataFieldTemplates* templates = m_resolver->getTemplates(filename); bool hasMulti = types.find(VALUE_SEPARATOR) != string::npos; istringstream stream(types); string type; diff --git a/src/lib/ebus/message.h b/src/lib/ebus/message.h index 9ce3d284..9bd0be5a 100644 --- a/src/lib/ebus/message.h +++ b/src/lib/ebus/message.h @@ -1255,6 +1255,43 @@ class LoadedFileInfo { }; +/** + * Interface for resolving templates and loading additional message definitions. + */ +class Resolver { + public: + /** + * Constructor. + */ + Resolver() {} + + /** + * Destructor. + */ + virtual ~Resolver() {} + + /** + * Get the @a DataFieldTemplates for the specified configuration file. + * @param filename the full name of the configuration file, or "*" to get the non-root templates with the longest name + * or the root templates if not available. + * @return the @a DataFieldTemplates. + */ + virtual DataFieldTemplates* getTemplates(const string& filename) = 0; + + /** + * Load definitions from a relative file from the config path/URL. + * @param reader the @a FileReader instance to load with the definitions. + * @param filename the relative name of the file being read. + * @param defaults the default values by name (potentially overwritten by file name), or nullptr to not use defaults. + * @param errorDescription a string in which to store the error description in case of error. + * @param replace whether to replace an already existing entry. + * @return @a RESULT_OK on success, or an error code. + */ + virtual result_t loadDefinitionsFromConfigPath(FileReader* reader, const string& filename, + map* defaults, string* errorDescription, bool replace = false) = 0; +}; + + /** * Holds a map of all known @a Message instances. */ @@ -1267,7 +1304,7 @@ class MessageMap : public MappedFileReader { * @param deleteData whether to delete the scan message @a DataField during @a Message destruction. */ explicit MessageMap(bool addAll = false, const string& preferLanguage = "", bool deleteData = true) - : MappedFileReader::MappedFileReader(true, preferLanguage), + : MappedFileReader::MappedFileReader(true, preferLanguage), m_resolver(nullptr), m_addAll(addAll), m_additionalScanMessages(false), m_maxIdLength(0), m_maxBroadcastIdLength(0), m_messageCount(0), m_conditionalMessageCount(0), m_passiveMessageCount(0) { m_scanMessage = Message::createScanMessage(false, deleteData); @@ -1289,6 +1326,17 @@ class MessageMap : public MappedFileReader { } } + /** + * Set the @a Resolver instance. + * @param the @a Resolver instance. + */ + void setResolver(Resolver* resolver) { m_resolver = resolver; } + + /** + * @return the @a Resolver instance. + */ + Resolver* getResolver() const { return m_resolver; } + /** * Add a @a Message instance to this set. * @param message the @a Message instance to add. @@ -1575,6 +1623,9 @@ class MessageMap : public MappedFileReader { /** empty vector for @a getLoadedFiles(). */ static vector s_noFiles; + /** the @a Resolver instance. */ + Resolver* m_resolver; + /** whether to add all messages, even if duplicate. */ const bool m_addAll; diff --git a/src/lib/ebus/test/test_message.cpp b/src/lib/ebus/test/test_message.cpp index 7b46295a..26019392 100644 --- a/src/lib/ebus/test/test_message.cpp +++ b/src/lib/ebus/test/test_message.cpp @@ -54,27 +54,29 @@ DataFieldTemplates* templates = nullptr; namespace ebusd { -DataFieldTemplates* getTemplates(const string& filename) { - if (filename == "") { // avoid compiler warning +class TestResolver : public Resolver { + public: + virtual DataFieldTemplates* getTemplates(const string& filename) { + if (filename == "") { // avoid compiler warning + return templates; + } return templates; } - return templates; -} -result_t loadDefinitionsFromConfigPath(FileReader* reader, const string& filename, bool verbose, - map* defaults, string* errorDescription, bool replace = false) { - time_t mtime = 0; - istream* stream = FileReader::openFile(filename, errorDescription, &mtime); - result_t result; - if (stream) { - result = reader->readFromStream(stream, filename, mtime, verbose, defaults, errorDescription); - delete(stream); - } else { - result = RESULT_ERR_NOTFOUND; + virtual result_t loadDefinitionsFromConfigPath(FileReader* reader, const string& filename, + map* defaults, string* errorDescription, bool replace = false) { + time_t mtime = 0; + istream* stream = FileReader::openFile(filename, errorDescription, &mtime); + result_t result; + if (stream) { + result = reader->readFromStream(stream, filename, mtime, false, defaults, errorDescription); + delete(stream); + } else { + result = RESULT_ERR_NOTFOUND; + } + return result; } - return result; -} - +}; } // namespace ebusd @@ -208,6 +210,7 @@ int main() { templates->readLineFromStream(&dummystr, __FILE__, false, &lineNo, &row, &errorDescription, false, nullptr, nullptr); lineNo = 0; MessageMap* messages = new MessageMap(""); + messages->setResolver(new TestResolver()); dummystr.clear(); dummystr.str("#"); messages->readLineFromStream(&dummystr, __FILE__, false, &lineNo, &row, &errorDescription, false, nullptr, nullptr);