move string_tokenize out of i2p_stream and add unit test for it

include/libtorrent/string_util.hpp

@@ -59,6 +59,15 @@ namespace libtorrent
 	// x cannot be greater than 7
 	void* align_pointer(void* p);
 
+	// searches for separator in the string 'last'. the pointer last points to
+	// is set to point to the first character following the separator.
+	// returns a pointer to a null terminated string starting at last, ending
+	// at the separator (the string is mutated to replace the separator with
+	// a '\0' character). If there is no separator, but the end of the string,
+	// the pointer next points to is set to the last null terminator, which will
+	// make the following invocation return NULL, to indicate the end of the
+	// string.
+	TORRENT_EXTRA_EXPORT char* string_tokenize(char* last, char sep, char** next);
 }
 
 #endif

src/i2p_stream.cpp

@@ -34,6 +34,7 @@ POSSIBILITY OF SUCH DAMAGE.
 #include "libtorrent/i2p_stream.hpp"
 #include "libtorrent/assert.hpp"
 #include "libtorrent/error_code.hpp"
+#include "libtorrent/string_util.hpp"
 
 #if TORRENT_USE_I2P
 
@@ -271,27 +272,6 @@ namespace libtorrent
 			, boost::bind(&i2p_stream::read_line, this, _1, h));
 	}
 
-	char* string_tokenize(char* last, char sep, char** next)
-	{
-		if (last == 0) return 0;
-		if (last[0] == '"')
-		{
-			*next = strchr(last + 1, '"');
-			// consume the actual separator as well.
-			if (*next != NULL)
-				*next = strchr(*next, sep);
-		}
-		else
-		{
-			*next = strchr(last, sep);
-		}
-		if (*next == 0) return last;
-		**next = 0;
-		++(*next);
-		while (**next == sep && **next) ++(*next);
-		return last;
-	}
-
 	void i2p_stream::read_line(error_code const& e, boost::shared_ptr<handler_type> h)
 	{
 		TORRENT_ASSERT(m_magic == 0x1337);

src/string_util.cpp

@@ -143,5 +143,27 @@ namespace libtorrent
 		// 8 - offset.
 		return static_cast<char*>(p) + (8 - offset);
 	}
+
+	char* string_tokenize(char* last, char sep, char** next)
+	{
+		if (last == 0) return 0;
+		if (last[0] == '"')
+		{
+			*next = strchr(last + 1, '"');
+			// consume the actual separator as well.
+			if (*next != NULL)
+				*next = strchr(*next, sep);
+		}
+		else
+		{
+			*next = strchr(last, sep);
+		}
+		if (*next == 0) return last;
+		**next = 0;
+		++(*next);
+		while (**next == sep && **next) ++(*next);
+		return last;
+	}
+
 }
 

test/test_string.cpp

@@ -208,6 +208,31 @@ int test_main()
 	TEST_CHECK(url_has_argument("http://127.0.0.1/test?foo=24&bar=23&a=e", "a") == "e");
 	TEST_CHECK(url_has_argument("http://127.0.0.1/test?foo=24&bar=23&a=e", "b") == "");
 
+	// test string_tokenize
+
+	char test_tokenize[] = "a b c \"foo bar\" d\ne f";
+	char* next = test_tokenize;
+	char* ptr = string_tokenize(next, ' ', &next);
+	TEST_EQUAL(ptr, std::string("a"));
+
+	ptr = string_tokenize(next, ' ', &next);
+	TEST_EQUAL(ptr, std::string("b"));
+
+	ptr = string_tokenize(next, ' ', &next);
+	TEST_EQUAL(ptr, std::string("c"));
+
+	ptr = string_tokenize(next, ' ', &next);
+	TEST_EQUAL(ptr, std::string("\"foo bar\""));
+
+	ptr = string_tokenize(next, ' ', &next);
+	TEST_EQUAL(ptr, std::string("d\ne"));
+
+	ptr = string_tokenize(next, ' ', &next);
+	TEST_EQUAL(ptr, std::string("f"));
+
+	ptr = string_tokenize(next, ' ', &next);
+	TEST_EQUAL(ptr, NULL);
+
 	return 0;
 }