allow using newlines within quoted fields (special treat for @wfr2000)

src/lib/ebus/data.h

@@ -60,9 +60,6 @@
 
 using namespace std;
 
-/** the separator character used between multiple values (in CSV only). */
-#define VALUE_SEPARATOR ';'
-
 /** the separator character used between base type name and length (in CSV only). */
 #define LENGTH_SEPARATOR ':'
 

src/lib/ebus/filereader.h

@@ -49,6 +49,9 @@ using namespace std;
 /** the separator character as string used to quote text having the @a FIELD_SEPARATOR in it. */
 #define TEXT_SEPARATOR_STR "\""
 
+/** the separator character used between multiple values (in CSV only). */
+#define VALUE_SEPARATOR ';'
+
 extern void printErrorPos(ostream& out, vector<string>::iterator begin, const vector<string>::iterator end, vector<string>::iterator pos, string filename, size_t lineNo, result_t result);
 
 extern unsigned int parseInt(const char* str, int base, const unsigned int minValue, const unsigned int maxValue, result_t& result, unsigned int* length);
@@ -85,7 +88,6 @@ public:
 			m_lastError = filename;
 			return RESULT_ERR_NOTFOUND;
 		}
-		string line;
 		size_t lastSep = filename.find_last_of('/');
 		size_t firstDot = filename.find_first_of('.', lastSep+1);
 		string defaultDest = "";
@@ -117,16 +119,15 @@ public:
 		unsigned int lineNo = 0;
 		vector<string> row;
 		vector< vector<string> > defaults;
-		while (getline(ifs, line) != 0) {
+		while (splitFields(ifs, row, lineNo)) {
-			lineNo++;
+			if (row.empty())
-			if (!splitFields(line, row) || row.empty())
 				continue;
 
 			result_t result;
 			vector<string>::iterator it = row.begin();
 			const vector<string>::iterator end = row.end();
 			if (m_supportsDefaults) {
-				if (line[0] == '*') {
+				if (row[0][0] == '*') {
 					row[0] = row[0].substr(1);
 					result = addDefaultFromFile(defaults, row, it, defaultDest, defaultCircuit, defaultSuffix, filename, lineNo);
 					if (result == RESULT_OK)
@@ -226,68 +227,80 @@ public:
 	}
 
 	/**
-	 * Split the line into fields.
+	 * Split the next line(s) from the @a istring into fields.
-	 * @param line the @a string with the line to split.
+	 * @param in the @a istream to read from.
-	 * @param row the @a vector to which to add the fields.
+	 * @param row the @a vector to which to add the fields. This will be empty for completely empty and comment lines.
-	 * @return true if the line was split, false if the line was completely empty or a comment line.
+	 * @param lineNo the current line number (incremented with each line read).
+	 * @return true if there are more lines to read, false when there are no more lines left.
 	 */
-	static bool splitFields(string& line, vector<string>& row)
+	static bool splitFields(istream& ifs, vector<string>& row, unsigned int& lineNo)
 	{
 		row.clear();
-		trim(line);
+		string line;
-		// skip empty lines and comments
-		size_t length = line.length();
-		if (length == 0 || line[0] == '#' || (line.length() > 1 && line[0] == '/' && line[1] == '/'))
-			return false;
-
 		bool quotedText = false, wasQuoted = false;
 		ostringstream field;
 		char prev = FIELD_SEPARATOR;
-		bool empty = true;
+		bool empty = true, read = false;
-		for (size_t pos = 0; pos < length; pos++) {
+		while (getline(ifs, line) != 0) {
-			char ch = line[pos];
+			read = true;
-			switch (ch)
+			lineNo++;
-			{
+			trim(line);
-			case FIELD_SEPARATOR:
+
-				if (quotedText) {
+			size_t length = line.length();
+			if (!quotedText && (length == 0 || line[0] == '#' || (line.length() > 1 && line[0] == '/' && line[1] == '/')))
+				continue; // skip empty lines and comments
+
+			for (size_t pos = 0; pos < length; pos++) {
+				char ch = line[pos];
+				switch (ch)
+				{
+				case FIELD_SEPARATOR:
+					if (quotedText) {
+						field << ch;
+					} else {
+						string str = field.str();
+						trim(str);
+						empty &= str.empty();
+						row.push_back(str);
+						field.str("");
+						wasQuoted = false;
+					}
+					break;
+				case TEXT_SEPARATOR:
+					if (prev == TEXT_SEPARATOR && !quotedText) { // double dquote
+						field << ch;
+						quotedText = true;
+					} else if (quotedText) {
+						quotedText = false;
+					} else if (prev == FIELD_SEPARATOR) {
+						quotedText = wasQuoted = true;
+					} else {
+						field << ch;
+					}
+					break;
+				case '\r':
+					break;
+				default:
+					if (prev==TEXT_SEPARATOR && !quotedText && wasQuoted) {
+						field << TEXT_SEPARATOR; // single dquote in the middle of formerly quoted text
+						quotedText = true;
+					} else if (quotedText && pos==0 && field.tellp()>0 && *(field.str().end()-1)!=VALUE_SEPARATOR) {
+						field << VALUE_SEPARATOR;
+					}
 					field << ch;
-				} else {
+					break;
-					string str = field.str();
-					trim(str);
-					empty &= str.empty();
-					row.push_back(str);
-					field.str("");
-					wasQuoted = false;
 				}
-				break;
+				prev = ch;
-			case TEXT_SEPARATOR:
-				if (prev == TEXT_SEPARATOR && !quotedText) { // double dquote
-					field << ch;
-					quotedText = true;
-				} else if (quotedText) {
-					quotedText = false;
-				} else if (prev == FIELD_SEPARATOR) {
-					quotedText = wasQuoted = true;
-				} else {
-					field << ch;
-				}
-				break;
-			case '\r':
-				break;
-			default:
-				if (prev==TEXT_SEPARATOR && !quotedText && wasQuoted) {
-					field << TEXT_SEPARATOR; // single dquote in the middle of formerly quoted text
-					quotedText = true;
-				}
-				field << ch;
-				break;
 			}
-			prev = ch;
+			if (!quotedText)
+				break;
 		}
 		string str = field.str();
 		trim(str);
-		if (empty && str.empty())
+		if (empty && str.empty()) {
-			return false;
+			row.clear();
+			return read;
+		}
 		row.push_back(str);
 		return true;
 	}

src/lib/ebus/test/Makefile.am

@@ -3,11 +3,15 @@ AM_CXXFLAGS = -fpic \
 	      -Wextra \
 	      -isystem$(top_srcdir)/src/lib/ebus
 
-noinst_PROGRAMS = test_device \
+noinst_PROGRAMS = test_filereader \
+		  test_device \
 		  test_symbol \
 		  test_data \
 		  test_message
 
+test_filereader_SOURCES = test_filereader.cpp
+test_filereader_LDADD = ../libebus.a
+
 test_device_SOURCES = test_device.cpp
 test_device_LDADD = ../libebus.a
 

src/lib/ebus/test/test_filereader.cpp

@@ -0,0 +1,89 @@
+/*
+ * ebusd - daemon for communication with eBUS heating systems.
+ * Copyright (C) 2014-2016 John Baier <ebusd@ebusd.eu>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "filereader.h"
+#include <iostream>
+#include <iomanip>
+
+using namespace std;
+
+void verify(bool expectFailMatch, string type, string input,
+		bool match, string expectStr, string gotStr)
+{
+	match = match && expectStr == gotStr;
+	if (expectFailMatch) {
+		if (match)
+			cout << "  failed " << type << " match >" << input
+			        << "< error: unexpectedly succeeded" << endl;
+		else
+			cout << "  failed " << type << " match >" << input << "< OK" << endl;
+	}
+	else if (match)
+		cout << "  " << type << " match >" << input << "< OK" << endl;
+	else
+		cout << "  " << type << " match >" << input << "< error: got >"
+		        << gotStr << "<, expected >" << expectStr << "<" << endl;
+}
+
+int main()
+{
+	istringstream ifs(
+		"line 1 col 1,line 1 col 2,line 1 col 3\n"
+		"line 2 col 1,\"line 2 col 2\",\"line 2 \"\"col 3\"\"\"\n"
+		"line 4 col 1,\"line 4 col 2 part 1\n"
+		"line 4 col 2 part 2\",line 4 col 3\n"
+		",,,\n"
+		"line 6 col 1,,line 6 col 3\n"
+		"line 8 col 1,\"line 8 col 2 part 1;\n"
+		"line 8 col 2 part 2\",line 8 col 3\n"
+	);
+	string resultlines[][3] = {
+		{"line 1 col 1", "line 1 col 2", "line 1 col 3"},
+		{"line 2 col 1", "line 2 col 2", "line 2 \"col 3\""},
+		{"", "", ""},
+		{"line 4 col 1", "line 4 col 2 part 1;line 4 col 2 part 2", "line 4 col 3"},
+		{"", "", ""},
+		{"line 6 col 1", "", "line 6 col 3"},
+		{"", "", ""},
+		{"line 8 col 1", "line 8 col 2 part 1;line 8 col 2 part 2", "line 8 col 3"},
+	};
+	unsigned int lineNo = 0;
+	vector<string> row;
+
+	while (FileReader::splitFields(ifs, row, lineNo)) {
+		cout << "line " << static_cast<unsigned>(lineNo) << ": split OK" << endl;
+		string resultline[3] = resultlines[lineNo-1];
+		if (row.empty()) {
+			cout << "  result empty";
+			if (resultline[0] == "")
+				cout << ": OK" << endl;
+			else
+				cout << ": error" << endl;
+			continue;
+		}
+		for (vector<string>::iterator it = row.begin(); it != row.end(); it++) {
+			string got = *it;
+			string expect = resultline[distance(row.begin(), it)];
+			ostringstream type;
+			type << "line " << static_cast<unsigned>(lineNo) << " col " << static_cast<size_t>(distance(row.begin(), it)+1);
+			verify(false, type.str(), expect, got == expect, expect, got);
+		}
+	}
+
+	return 0;
+}

src/lib/ebus/test/test_message.cpp

@@ -44,6 +44,8 @@ DataFieldTemplates* templates = NULL;
 
 DataFieldTemplates* getTemplates(const string filename)
 {
+	if (filename=="") // avoid compiler warning
+		return templates;
 	return templates;
 }
 
@@ -192,10 +194,12 @@ int main()
 				continue;
 			}
 		}
-		string item;
 		vector<string> entries;
-
+		istringstream ifs(check[0]);
-		FileReader::splitFields(check[0], entries);
+		unsigned int lineNo = 0;
+		if (!FileReader::splitFields(ifs, entries, lineNo)) {
+			entries.clear();
+		}
 
 		if (deleteMessages.size()>0) {
 			for (vector<Message*>::iterator it = deleteMessages.begin(); it != deleteMessages.end(); it++) {