Added tokenize() functions to Readable_String and various subclasses.

General fixes to CString.

Jupiter/CString.h

@@ -183,16 +183,38 @@ namespace Jupiter
 		*/
 		static CString_Type<T> gotoWord(const Jupiter::Readable_String<T> &in, size_t pos, const T *whitespace);
 
+		/**
+		* @brief Tokenizes a string, based on an input token separator
+		*
+		* @param separator Separator to split tokens by
+		* @return TokenizeResult containing the results of the tokenization process.
+		*/
+		TokenizeResult<Jupiter::CString_Type> tokenize(const T &separator) const;
+		TokenizeResult<Jupiter::CString_Type> tokenize(const Jupiter::Readable_String<T> &separator) const;
+		TokenizeResult<Jupiter::CString_Type> tokenize(const T *separator, size_t separator_size) const;
+
+		/**
+		* @brief Tokenizes a string, based on an input token separator
+		*
+		* @param in String to split into tokens
+		* @param separator Separator to split tokens by
+		* @return TokenizeResult containing the results of the tokenization process.
+		*/
+		static TokenizeResult<Jupiter::CString_Type> tokenize(const Jupiter::Readable_String<T> &in, const T &separator);
+		static TokenizeResult<Jupiter::CString_Type> tokenize(const Jupiter::Readable_String<T> &in, const Jupiter::Readable_String<T> &separator);
+		static TokenizeResult<Jupiter::CString_Type> tokenize(const Jupiter::Readable_String<T> &in, const T *separator, size_t separator_size);
+
 		/**
 		* @brief Copies the data from the input string to the CString.
 		*
 		* @param in String containing the data to be copied.
 		* @return New size of the CString.
 		*/
-		size_t set(const Readable_String<T> &in);
-		size_t set(const std::basic_string<T> &in);
-		size_t set(const T *in);
-		size_t set(const T in);
+		size_t set(const T *in, size_t inSize) override;
+		size_t set(const Jupiter::Readable_String<T> &in) override;
+		size_t set(const std::basic_string<T> &in) override;
+		size_t set(const T *in) override;
+		size_t set(const T &in) override;
 
 		/**
 		* @brief Copies the data from the input string and concatenates it to the end of CString.
@@ -200,10 +222,11 @@ namespace Jupiter
 		* @param in String containing the data to be concatenated.
 		* @return New size of the CString.
 		*/
-		size_t concat(const Readable_String<T> &in);
-		size_t concat(const std::basic_string<T> &in);
-		size_t concat(const T *in);
-		size_t concat(const T in);
+		size_t concat(const T *in, size_t inSize) override;
+		size_t concat(const Readable_String<T> &in) override;
+		size_t concat(const std::basic_string<T> &in) override;
+		size_t concat(const T *in) override;
+		size_t concat(const T &in) override;
 
 		/** Default Constructor */
 		CString_Type();
@@ -380,6 +403,27 @@ namespace Jupiter
 		*/
 		static CString_Loose<T> gotoWord(const Jupiter::Readable_String<T> &in, size_t pos, const T *whitespace);
 
+		/**
+		* @brief Tokenizes a string, based on an input token separator
+		*
+		* @param separator Separator to split tokens by
+		* @return TokenizeResult containing the results of the tokenization process.
+		*/
+		TokenizeResult<Jupiter::CString_Loose> tokenize(const T &separator);
+		TokenizeResult<Jupiter::CString_Loose> tokenize(const Jupiter::Readable_String<T> &separator);
+		TokenizeResult<Jupiter::CString_Loose> tokenize(const T *separator, size_t separator_size);
+
+		/**
+		* @brief Tokenizes a string, based on an input token separator
+		*
+		* @param in String to split into tokens
+		* @param separator Separator to split tokens by
+		* @return TokenizeResult containing the results of the tokenization process.
+		*/
+		static TokenizeResult<Jupiter::CString_Loose> tokenize(const Jupiter::Readable_String<T> &in, const T &separator);
+		static TokenizeResult<Jupiter::CString_Loose> tokenize(const Jupiter::Readable_String<T> &in, const Jupiter::Readable_String<T> &separator);
+		static TokenizeResult<Jupiter::CString_Loose> tokenize(const Jupiter::Readable_String<T> &in, const T *separator, size_t separator_size);
+
 		/** Default constructor */
 		CString_Loose();
 

Jupiter/CString_Imp.h

@@ -264,6 +264,8 @@ template<typename T> Jupiter::CString_Type<T> Jupiter::CString_Type<T>::Format(c
 	return r;
 }
 
+// substring
+
 template<typename T> Jupiter::CString_Type<T> Jupiter::CString_Type<T>::substring(size_t pos) const
 {
 	return Jupiter::CString_Type<T>::substring(*this, pos);
@@ -306,6 +308,8 @@ template<typename T> Jupiter::CString_Type<T> Jupiter::CString_Type<T>::substrin
 	return r;
 }
 
+// getWord
+
 template<typename T> Jupiter::CString_Type<T> Jupiter::CString_Type<T>::getWord(size_t pos, const T *whitespace) const
 {
 	return Jupiter::CString_Type<T>::getWord(*this, pos, whitespace);
@@ -321,6 +325,8 @@ template<typename T> Jupiter::CString_Type<T> Jupiter::CString_Type<T>::getWord(
 	return Jupiter::Readable_String<T>::getWord<Jupiter::CString_Type>(in, pos, whitespace);
 }
 
+// gotoWord
+
 template<typename T> Jupiter::CString_Type<T> Jupiter::CString_Type<T>::gotoWord(size_t pos, const T *whitespace) const
 {
 	return Jupiter::CString_Type<T>::gotoWord(*this, pos, whitespace);
@@ -331,22 +337,54 @@ template<typename T> Jupiter::CString_Type<T> Jupiter::CString_Type<T>::gotoWord
 	return Jupiter::Readable_String<T>::gotoWord<Jupiter::CString_Type>(in, pos, whitespace);
 }
 
-template<typename T> size_t Jupiter::CString_Type<T>::set(const Jupiter::Readable_String<T> &in)
+// tokenize
+
+template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::CString_Type> Jupiter::CString_Type<T>::tokenize(const T &separator) const
 {
-	this->setBufferSizeNoCopy(in.size());
-	for (Jupiter::String_Type<T>::length = 0; Jupiter::String_Type<T>::length != in.size() && in.get(Jupiter::String_Type<T>::length) != 0; Jupiter::String_Type<T>::length++)
-		Jupiter::String_Type<T>::str[Jupiter::String_Type<T>::length] = in.get(Jupiter::String_Type<T>::length);
-	Jupiter::String_Type<T>::str[Jupiter::String_Type<T>::length] = 0;
+	return Jupiter::CString_Type<T>::tokenize(*this, separator);
+}
+
+template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::CString_Type> Jupiter::CString_Type<T>::tokenize(const Jupiter::Readable_String<T> &separator) const
+{
+	return Jupiter::CString_Type<T>::tokenize(*this, separator);
+}
+
+template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::CString_Type> Jupiter::CString_Type<T>::tokenize(const T *separator, size_t separator_size) const
+{
+	return Jupiter::CString_Type<T>::tokenize(*this, separator, separator_size);
+}
+
+template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::CString_Type> Jupiter::CString_Type<T>::tokenize(const Jupiter::Readable_String<T> &in, const T &token)
+{
+	return Jupiter::Readable_String<T>::tokenize<Jupiter::CString_Type>(in, token);
+}
+
+template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::CString_Type> Jupiter::CString_Type<T>::tokenize(const Jupiter::Readable_String<T> &in, const Jupiter::Readable_String<T> &separator)
+{
+	return Jupiter::Readable_String<T>::tokenize<Jupiter::CString_Type>(in, separator);
+}
+
+template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::CString_Type> Jupiter::CString_Type<T>::tokenize(const Jupiter::Readable_String<T> &in, const T *separator, size_t separator_size)
+{
+	return Jupiter::Readable_String<T>::tokenize<Jupiter::CString_Type>(in, separator, separator_size);
+}
+
+// set
+
+template<typename T> size_t Jupiter::CString_Type<T>::set(const T *in, size_t in_size)
+{
+	Jupiter::String_Type<T>::str[Jupiter::String_Type<T>::set(in, in_size)] = 0;
 	return Jupiter::String_Type<T>::length;
 }
 
+template<typename T> size_t Jupiter::CString_Type<T>::set(const Jupiter::Readable_String<T> &in)
+{
+	return this->set(in.ptr(), in.size());
+}
+
 template<typename T> size_t Jupiter::CString_Type<T>::set(const std::basic_string<T> &in)
 {
-	this->setBufferSizeNoCopy(in.size());
-	for (Jupiter::String_Type<T>::length = 0; Jupiter::String_Type<T>::length != in.size() && in.at(Jupiter::String_Type<T>::length) != 0; Jupiter::String_Type<T>::length++)
-		Jupiter::String_Type<T>::str[Jupiter::String_Type<T>::length] = in.at(Jupiter::String_Type<T>::length);
-	Jupiter::String_Type<T>::str[Jupiter::String_Type<T>::length];
-	return Jupiter::String_Type<T>::length;
+	return this->set(in.data(), in.size());
 }
 
 template<typename T> size_t Jupiter::CString_Type<T>::set(const T *in)
@@ -355,7 +393,7 @@ template<typename T> size_t Jupiter::CString_Type<T>::set(const T *in)
 	return Jupiter::String_Type<T>::length;
 }
 
-template<typename T> size_t Jupiter::CString_Type<T>::set(const T in)
+template<typename T> size_t Jupiter::CString_Type<T>::set(const T &in)
 {
 	if (in == 0)
 	{
@@ -369,34 +407,22 @@ template<typename T> size_t Jupiter::CString_Type<T>::set(const T in)
 	return Jupiter::String_Type<T>::length = 1;
 }
 
-template<typename T> size_t Jupiter::CString_Type<T>::concat(const Jupiter::Readable_String<T> &in)
-{
-	size_t nSize = Jupiter::String_Type<T>::length + in.size();
-	const T *inData = in.ptr();
-	this->setBufferSize(nSize);
-	while (Jupiter::String_Type<T>::length != nSize && *inData != 0)
+// concat
+
+template<typename T> size_t Jupiter::CString_Type<T>::concat(const T *in, size_t in_size)
 {
-		Jupiter::String_Type<T>::str[Jupiter::String_Type<T>::length] = *inData;
-		Jupiter::String_Type<T>::length++;
-		inData++;
-	}
-	Jupiter::String_Type<T>::str[Jupiter::String_Type<T>::length] = 0;
+	Jupiter::String_Type<T>::str[Jupiter::String_Type<T>::concat(in, in_size)] = 0;
 	return Jupiter::String_Type<T>::length;
 }
 
-template<typename T> size_t Jupiter::CString_Type<T>::concat(const std::basic_string<T> &in)
-{
-	size_t nSize = Jupiter::String_Type<T>::length + in.size();
-	const T *inData = in.data();
-	this->setBufferSize(nSize);
-	while (Jupiter::String_Type<T>::length != nSize && *inData != 0)
+template<typename T> size_t Jupiter::CString_Type<T>::concat(const Jupiter::Readable_String<T> &in)
 {
-		Jupiter::String_Type<T>::str[Jupiter::String_Type<T>::length] = *inData;
-		Jupiter::String_Type<T>::length++;
-		inData++;
+	return this->concat(in.ptr(), in.size());
 }
-	Jupiter::String_Type<T>::str[Jupiter::String_Type<T>::length] = 0;
-	return Jupiter::String_Type<T>::length;
+
+template<typename T> size_t Jupiter::CString_Type<T>::concat(const std::basic_string<T> &in)
+{
+	return this->concat(in.data(), in.size());
 }
 
 template<typename T> size_t Jupiter::CString_Type<T>::concat(const T *in)
@@ -407,15 +433,16 @@ template<typename T> size_t Jupiter::CString_Type<T>::concat(const T *in)
 	return Jupiter::String_Type<T>::length = nSize;
 }
 
-template<typename T> size_t Jupiter::CString_Type<T>::concat(const T c)
+template<typename T> size_t Jupiter::CString_Type<T>::concat(const T &in)
 {
 	this->setBufferSize(Jupiter::String_Type<T>::length + 1);
-	Jupiter::String_Type<T>::str[Jupiter::String_Type<T>::length] = c;
+	Jupiter::String_Type<T>::str[Jupiter::String_Type<T>::length] = in;
 	Jupiter::String_Type<T>::str[++Jupiter::String_Type<T>::length] = 0;
 	return Jupiter::String_Type<T>::length;
 }
 
 // Operators
+
 template<typename T> inline Jupiter::CString_Type<T> Jupiter::CString_Type<T>::operator+(const Jupiter::CString_Type<T> &rhs) const
 {
 	return Jupiter::CString_Type<T>::operator+(reinterpret_cast<const Jupiter::Readable_String<T> &>(rhs));
@@ -668,7 +695,40 @@ template<typename T> Jupiter::CString_Loose<T> Jupiter::CString_Loose<T>::gotoWo
 	return Jupiter::Readable_String<T>::gotoWord<Jupiter::CString_Loose>(in, pos, whitespace);
 }
 
+// tokenize
+
+template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::CString_Loose> Jupiter::CString_Loose<T>::tokenize(const T &token)
+{
+	return Jupiter::CString_Loose<T>::tokenize(*this, token);
+}
+
+template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::CString_Loose> Jupiter::CString_Loose<T>::tokenize(const Jupiter::Readable_String<T> &separator)
+{
+	return Jupiter::CString_Loose<T>::tokenize(*this, separator);
+}
+
+template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::CString_Loose> Jupiter::CString_Loose<T>::tokenize(const T *separator, size_t separator_size)
+{
+	return Jupiter::CString_Loose<T>::tokenize(*this, separator, separator_size);
+}
+
+template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::CString_Loose> Jupiter::CString_Loose<T>::tokenize(const Jupiter::Readable_String<T> &in, const T &token)
+{
+	return Jupiter::Readable_String<T>::tokenize<Jupiter::CString_Loose>(in, token);
+}
+
+template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::CString_Loose> Jupiter::CString_Loose<T>::tokenize(const Jupiter::Readable_String<T> &in, const Jupiter::Readable_String<T> &separator)
+{
+	return Jupiter::Readable_String<T>::tokenize<Jupiter::CString_Loose>(in, separator);
+}
+
+template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::CString_Loose> Jupiter::CString_Loose<T>::tokenize(const Jupiter::Readable_String<T> &in, const T *separator, size_t separator_size)
+{
+	return Jupiter::Readable_String<T>::tokenize<Jupiter::CString_Loose>(in, separator, separator_size);
+}
+
 // Operators
+
 template<typename T> inline Jupiter::CString_Loose<T> Jupiter::CString_Loose<T>::operator+(const Jupiter::CString_Loose<T> &rhs) const
 {
 	return Jupiter::CString_Loose<T>::operator+(reinterpret_cast<const Jupiter::Readable_String<T> &>(rhs));

Jupiter/Jupiter.h

@@ -35,7 +35,7 @@
 #define JUPITER_API
 #endif // _WIN32
 
-#define JUPITER_VERSION "Jupiter 0.16" /** Version of this program at compile time. */
+#define JUPITER_VERSION "Jupiter 0.17-dev" /** Version of this program at compile time. */
 
 #if defined __cplusplus
 extern "C"

Jupiter/Readable_String.h

@@ -315,6 +315,65 @@ namespace Jupiter
 		template<template<typename> class R> static R<T> gotoToken(const Jupiter::Readable_String<T> &in, size_t pos, const T &token);
 		template<template<typename> class R> static R<T> gotoToken(const Jupiter::Readable_String<T> &in, size_t pos, const Jupiter::Readable_String<T> &token);
 
+		/**
+		* @brief Contains the results from a call to tokenize().
+		*/
+		template<template<typename> class R> struct TokenizeResult
+		{
+			/** Underlying type used to represent each token */
+			typedef R<T> token_type;
+
+			/** Array of tokens, with a length of 'token_count' */
+			R<T> *tokens;
+
+			/** Length of 'tokens' array */
+			size_t token_count;
+
+			/**
+			* @brief Default constructor for the TokenizeResult class.
+			* Note: 'tokens' assigned nullptr, and 'token_count' is assigned 0.
+			*/
+			TokenizeResult();
+
+			/**
+			* @brief Allocation constructor for the TokenizeResult class.
+			*/
+			TokenizeResult(size_t in_token_count);
+
+			/**
+			* @brief Capture constructor for the TokenizeResult class.
+			*/
+			TokenizeResult(R<T> *in_tokens, size_t in_token_count);
+
+			/**
+			* @brief Copy constructor for the TokenizeResult class.
+			*/
+			TokenizeResult(const TokenizeResult &source);
+
+			/**
+			* @brief Move constructor for the TokenizeResult class.
+			*/
+			TokenizeResult(const TokenizeResult &&source);
+
+			/**
+			* @brief Destructor for the TokenizeResult class.
+			*/
+			~TokenizeResult();
+		};
+
+		/**
+		* @brief Tokenizes a string, based on an input token separator
+		*
+		* @param R String type to use when constructing results
+		*
+		* @param in String to split into tokens
+		* @param separator Separator to split tokens by
+		* @return TokenizeResult containing the results of the tokenization process.
+		*/
+		template<template<typename> class R> static TokenizeResult<R> tokenize(const Jupiter::Readable_String<T> &in, const T &separator);
+		template<template<typename> class R> static TokenizeResult<R> tokenize(const Jupiter::Readable_String<T> &in, const Jupiter::Readable_String<T> &separator);
+		template<template<typename> class R> static TokenizeResult<R> tokenize(const Jupiter::Readable_String<T> &in, const T *separator, size_t separator_size);
+
 		/** Access operator */
 		inline const T &operator[](size_t index) const { return this->get(index); };
 

Jupiter/Readable_String_Imp.h

@@ -1301,6 +1301,199 @@ template<typename T> template<template<typename> class R> R<T> Jupiter::Readable
 	return R<T>();
 }
 
+/** tokenization */
+
+template<typename T> template<template<typename> class R> Jupiter::Readable_String<T>::TokenizeResult<R>::TokenizeResult()
+{
+	Jupiter::Readable_String<T>::TokenizeResult<R>::tokens = nullptr;
+	Jupiter::Readable_String<T>::TokenizeResult<R>::token_count = 0;
+}
+
+template<typename T> template<template<typename> class R> Jupiter::Readable_String<T>::TokenizeResult<R>::TokenizeResult(size_t in_token_count)
+{
+	Jupiter::Readable_String<T>::TokenizeResult<R>::token_count = in_token_count;
+	Jupiter::Readable_String<T>::TokenizeResult<R>::tokens = new R<T>[Jupiter::Readable_String<T>::TokenizeResult<R>::token_count];
+}
+
+template<typename T> template<template<typename> class R> Jupiter::Readable_String<T>::TokenizeResult<R>::TokenizeResult(R<T> *in_tokens, size_t in_token_count)
+{
+	Jupiter::Readable_String<T>::TokenizeResult<R>::tokens = in_tokens;
+	Jupiter::Readable_String<T>::TokenizeResult<R>::token_count = in_token_count;
+}
+
+template<typename T> template<template<typename> class R> Jupiter::Readable_String<T>::TokenizeResult<R>::TokenizeResult(const TokenizeResult &source)
+{
+	Jupiter::Readable_String<T>::TokenizeResult<R>::tokens = new R<T>[source.token_count];
+	Jupiter::Readable_String<T>::TokenizeResult<R>::token_count = 0;
+	while (Jupiter::Readable_String<T>::TokenizeResult<R>::token_count != source.token_count)
+	{
+		Jupiter::Readable_String<T>::TokenizeResult<R>::tokens[Jupiter::Readable_String<T>::TokenizeResult<R>::token_count] = source.tokens[Jupiter::Readable_String<T>::TokenizeResult<R>::token_count];
+		++Jupiter::Readable_String<T>::TokenizeResult<R>::token_count;
+	}
+}
+
+template<typename T> template<template<typename> class R> Jupiter::Readable_String<T>::TokenizeResult<R>::TokenizeResult(const TokenizeResult &&source)
+{
+	Jupiter::Readable_String<T>::TokenizeResult<R>::tokens = source.tokens;
+	Jupiter::Readable_String<T>::TokenizeResult<R>::token_count = source.token_count;
+	source.tokens = nullptr;
+	source.token_count = 0;
+}
+
+template<typename T> template<template<typename> class R> Jupiter::Readable_String<T>::TokenizeResult<R>::~TokenizeResult()
+{
+	if (Jupiter::Readable_String<T>::TokenizeResult<R>::tokens != nullptr)
+		delete[] Jupiter::Readable_String<T>::TokenizeResult<R>::tokens;
+}
+
+// tokenize
+
+template<typename T> template<template<typename> class R> typename Jupiter::Readable_String<T>::TokenizeResult<R> Jupiter::Readable_String<T>::tokenize(const Jupiter::Readable_String<T> &in, const T &token)
+{
+	// special case: no input
+	if (in.isEmpty())
+		return TokenizeResult<R>();
+
+	size_t length = in.tokenCount(token);
+	R<T> *tokens = new R<T>[length];
+
+	// special case: only 1 token
+	if (length == 1)
+	{
+		*tokens = in;
+		return TokenizeResult<R>(tokens, length);
+	}
+
+	R<T> *itr = tokens;
+
+	const T *in_itr = in.ptr();
+	const T *in_end = in.ptr() + in.size();
+	const T *token_start = in_itr;
+
+	while (in_itr != in_end)
+	{
+		loop_start:
+		if (*in_itr == token)
+		{
+			// push token to output
+			itr->set(token_start, in_itr - token_start);
+			++itr;
+
+			// increment over separator
+			++in_itr;
+
+			// iterator is at end of string
+			if (in_itr == in_end)
+				return TokenizeResult<R>(tokens, length);
+
+			// start reading new token
+			token_start = in_itr;
+			goto loop_start;
+		}
+		++in_itr;
+	}
+
+	itr->set(token_start, in_end - token_start);
+	return TokenizeResult<R>(tokens, length);
+}
+
+template<typename T> template<template<typename> class R> typename Jupiter::Readable_String<T>::TokenizeResult<R> Jupiter::Readable_String<T>::tokenize(const Jupiter::Readable_String<T> &in, const Jupiter::Readable_String<T> &separator)
+{
+	return Jupiter::Readable_String<T>::tokenize<R>(in, separator.ptr(), separator.size());
+}
+
+template<typename T> template<template<typename> class R> typename Jupiter::Readable_String<T>::TokenizeResult<R> Jupiter::Readable_String<T>::tokenize(const Jupiter::Readable_String<T> &in, const T *separator, size_t separator_size)
+{
+	// special case: separator is a single element
+	if (separator_size == 1)
+		return Jupiter::Readable_String<T>::tokenize<R>(in, *separator);
+
+	// special case: no input
+	if (in.isEmpty())
+		return TokenizeResult<R>();
+
+	R<T> *tokens;
+
+	// special case: no separator
+	if (separator_size == 0)
+	{
+		tokens = new R<T>[1];
+		*tokens = in;
+		return TokenizeResult<R>(tokens, 1);
+	}
+
+	// special case: only enough space for 1 token, or 2 empty tokens
+	if (separator_size >= in.size())
+	{
+		if (in.equals(separator, separator_size))
+			return TokenizeResult<R>(2);
+
+		tokens = new R<T>[1];
+		*tokens = in;
+		return TokenizeResult<R>(tokens, 1);
+	}
+
+	size_t length = in.tokenCount(separator);
+	tokens = new R<T>[length];
+
+	// special case: only 1 token
+	if (length == 1)
+	{
+		*tokens = in;
+		return TokenizeResult<R>(tokens, length);
+	}
+
+	R<T> *itr = tokens;
+
+	const T *in_itr = in.ptr();
+	const T *in_end = in.ptr() + in.size();
+	const T *token_start = in_itr;
+
+	auto is_separator = [](const T *sep, size_t sep_size, const T *ptr)
+	{
+		while (*sep == *ptr)
+		{
+			if (--sep_size == 0)
+				return true;
+
+			++sep, ++ptr;
+		}
+		return false;
+	};
+
+	while (in_itr + separator_size - 1 != in_end)
+	{
+	loop_start:
+		if (is_separator(separator, separator_size, in_itr))
+		{
+			// push token to output
+			itr->set(token_start, in_itr - token_start);
+			++itr;
+
+			in_itr += separator_size;
+			token_start = in_itr;
+
+			// not enough room for another separator
+			if (in_itr + separator_size > in_end)
+				break;
+
+			// only enough room for 1 separator (empty token), or 1 token
+			if (in_itr + separator_size == in_end)
+			{
+				if (is_separator(separator, separator_size, in_itr))
+					return TokenizeResult<R>(tokens, length);
+				break;
+			}
+
+			goto loop_start;
+		}
+		++in_itr;
+	}
+
+	itr->set(token_start, in_end - token_start);
+	return TokenizeResult<R>(tokens, length);
+}
+
 // Jupiter::DataBuffer specialization
 
 template<> struct _Jupiter_DataBuffer_partial_specialization_impl<Jupiter::Readable_String>

Jupiter/Reference_String.h

@@ -234,6 +234,27 @@ namespace Jupiter
 		static Reference_String<T> gotoToken(const Jupiter::Readable_String<T> &in, size_t pos, const T &token);
 		static Reference_String<T> gotoToken(const Jupiter::Readable_String<T> &in, size_t pos, const Jupiter::Readable_String<T> &token);
 
+		/**
+		* @brief Tokenizes a string, based on an input token separator
+		*
+		* @param separator Separator to split tokens by
+		* @return TokenizeResult containing the results of the tokenization process.
+		*/
+		TokenizeResult<Jupiter::Reference_String> tokenize(const T &separator) const;
+		TokenizeResult<Jupiter::Reference_String> tokenize(const Jupiter::Readable_String<T> &separator) const;
+		TokenizeResult<Jupiter::Reference_String> tokenize(const T *separator, size_t separator_size) const;
+
+		/**
+		* @brief Tokenizes a string, based on an input token separator
+		*
+		* @param in String to split into tokens
+		* @param separator Separator to split tokens by
+		* @return TokenizeResult containing the results of the tokenization process.
+		*/
+		static TokenizeResult<Jupiter::Reference_String> tokenize(const Jupiter::Readable_String<T> &in, const T &separator);
+		static TokenizeResult<Jupiter::Reference_String> tokenize(const Jupiter::Readable_String<T> &in, const Jupiter::Readable_String<T> &separator);
+		static TokenizeResult<Jupiter::Reference_String> tokenize(const Jupiter::Readable_String<T> &in, const T *separator, size_t separator_size);
+
 		/** Mutative operators */
 		inline Readable_String<T> &operator-=(size_t right) { this->truncate(right); return *this; };
 		inline Readable_String<T> &operator=(const Readable_String<T> &right) { this->set(right); return *this; };

Jupiter/Reference_String_Imp.h

@@ -237,6 +237,38 @@ template<typename T> Jupiter::Reference_String<T> Jupiter::Reference_String<T>::
 
 template<typename T> const Jupiter::Reference_String<T> Jupiter::Reference_String<T>::empty = Jupiter::Reference_String<T>();
 
+// tokenize
+
+template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::Reference_String> Jupiter::Reference_String<T>::tokenize(const T &separator) const
+{
+	return Jupiter::Reference_String<T>::tokenize(*this, separator);
+}
+
+template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::Reference_String> Jupiter::Reference_String<T>::tokenize(const Jupiter::Readable_String<T> &separator) const
+{
+	return Jupiter::Reference_String<T>::tokenize(*this, separator);
+}
+
+template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::Reference_String> Jupiter::Reference_String<T>::tokenize(const T *separator, size_t separator_size) const
+{
+	return Jupiter::Reference_String<T>::tokenize(*this, separator, separator_size);
+}
+
+template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::Reference_String> Jupiter::Reference_String<T>::tokenize(const Jupiter::Readable_String<T> &in, const T &token)
+{
+	return Jupiter::Readable_String<T>::tokenize<Jupiter::Reference_String>(in, token);
+}
+
+template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::Reference_String> Jupiter::Reference_String<T>::tokenize(const Jupiter::Readable_String<T> &in, const Jupiter::Readable_String<T> &separator)
+{
+	return Jupiter::Readable_String<T>::tokenize<Jupiter::Reference_String>(in, separator);
+}
+
+template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::Reference_String> Jupiter::Reference_String<T>::tokenize(const Jupiter::Readable_String<T> &in, const T *separator, size_t separator_size)
+{
+	return Jupiter::Readable_String<T>::tokenize<Jupiter::Reference_String>(in, separator, separator_size);
+}
+
 // Jupiter::DataBuffer specialization
 
 template<> struct _Jupiter_DataBuffer_partial_specialization_impl<Jupiter::Reference_String>

Jupiter/String.h

@@ -203,6 +203,27 @@ namespace Jupiter
 		static String_Strict<T> gotoToken(const Jupiter::Readable_String<T> &in, size_t pos, const T &token);
 		static String_Strict<T> gotoToken(const Jupiter::Readable_String<T> &in, size_t pos, const Jupiter::Readable_String<T> &token);
 
+		/**
+		* @brief Tokenizes a string, based on an input token separator
+		*
+		* @param separator Separator to split tokens by
+		* @return TokenizeResult containing the results of the tokenization process.
+		*/
+		TokenizeResult<Jupiter::String_Strict> tokenize(const T &separator) const;
+		TokenizeResult<Jupiter::String_Strict> tokenize(const Jupiter::Readable_String<T> &separator) const;
+		TokenizeResult<Jupiter::String_Strict> tokenize(const T *separator, size_t separator_size) const;
+
+		/**
+		* @brief Tokenizes a string, based on an input token separator
+		*
+		* @param in String to split into tokens
+		* @param separator Separator to split tokens by
+		* @return TokenizeResult containing the results of the tokenization process.
+		*/
+		static TokenizeResult<Jupiter::String_Strict> tokenize(const Jupiter::Readable_String<T> &in, const T &separator);
+		static TokenizeResult<Jupiter::String_Strict> tokenize(const Jupiter::Readable_String<T> &in, const Jupiter::Readable_String<T> &separator);
+		static TokenizeResult<Jupiter::String_Strict> tokenize(const Jupiter::Readable_String<T> &in, const T *separator, size_t separator_size);
+
 		/** Default Constructor */
 		String_Strict();
 
@@ -422,6 +443,27 @@ namespace Jupiter
 		static String_Loose<T> gotoToken(const Jupiter::Readable_String<T> &in, size_t pos, const T &token);
 		static String_Loose<T> gotoToken(const Jupiter::Readable_String<T> &in, size_t pos, const Jupiter::Readable_String<T> &token);
 
+		/**
+		* @brief Tokenizes a string, based on an input token separator
+		*
+		* @param separator Separator to split tokens by
+		* @return TokenizeResult containing the results of the tokenization process.
+		*/
+		TokenizeResult<Jupiter::String_Loose> tokenize(const T &separator);
+		TokenizeResult<Jupiter::String_Loose> tokenize(const Jupiter::Readable_String<T> &separator);
+		TokenizeResult<Jupiter::String_Loose> tokenize(const T *separator, size_t separator_size);
+
+		/**
+		* @brief Tokenizes a string, based on an input token separator
+		*
+		* @param in String to split into tokens
+		* @param separator Separator to split tokens by
+		* @return TokenizeResult containing the results of the tokenization process.
+		*/
+		static TokenizeResult<Jupiter::String_Loose> tokenize(const Jupiter::Readable_String<T> &in, const T &separator);
+		static TokenizeResult<Jupiter::String_Loose> tokenize(const Jupiter::Readable_String<T> &in, const Jupiter::Readable_String<T> &separator);
+		static TokenizeResult<Jupiter::String_Loose> tokenize(const Jupiter::Readable_String<T> &in, const T *separator, size_t separator_size);
+
 		/** Default constructor */
 		String_Loose();
 

Jupiter/String_Imp.h

@@ -315,7 +315,40 @@ template<typename T> Jupiter::String_Strict<T> Jupiter::String_Strict<T>::gotoTo
 	return Jupiter::Readable_String<T>::gotoToken<Jupiter::String_Strict>(in, pos, token);
 }
 
+// tokenize
+
+template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::String_Strict> Jupiter::String_Strict<T>::tokenize(const T &separator) const
+{
+	return Jupiter::String_Strict<T>::tokenize(*this, separator);
+}
+
+template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::String_Strict> Jupiter::String_Strict<T>::tokenize(const Jupiter::Readable_String<T> &separator) const
+{
+	return Jupiter::String_Strict<T>::tokenize(*this, separator);
+}
+
+template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::String_Strict> Jupiter::String_Strict<T>::tokenize(const T *separator, size_t separator_size) const
+{
+	return Jupiter::String_Strict<T>::tokenize(*this, separator, separator_size);
+}
+
+template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::String_Strict> Jupiter::String_Strict<T>::tokenize(const Jupiter::Readable_String<T> &in, const T &token)
+{
+	return Jupiter::Readable_String<T>::tokenize<Jupiter::String_Strict>(in, token);
+}
+
+template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::String_Strict> Jupiter::String_Strict<T>::tokenize(const Jupiter::Readable_String<T> &in, const Jupiter::Readable_String<T> &separator)
+{
+	return Jupiter::Readable_String<T>::tokenize<Jupiter::String_Strict>(in, separator);
+}
+
+template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::String_Strict> Jupiter::String_Strict<T>::tokenize(const Jupiter::Readable_String<T> &in, const T *separator, size_t separator_size)
+{
+	return Jupiter::Readable_String<T>::tokenize<Jupiter::String_Strict>(in, separator, separator_size);
+}
+
 // Operators
+
 template<typename T> inline Jupiter::String_Strict<T> Jupiter::String_Strict<T>::operator+(const Jupiter::String_Strict<T> &rhs) const
 {
 	return Jupiter::String_Strict<T>::operator+(reinterpret_cast<const Jupiter::Readable_String<T> &>(rhs));
@@ -653,6 +686,7 @@ template<typename T> Jupiter::String_Loose<T> Jupiter::String_Loose<T>::gotoToke
 }
 
 // Operators
+
 template<typename T> inline Jupiter::String_Loose<T> Jupiter::String_Loose<T>::operator+(const Jupiter::String_Loose<T> &rhs) const
 {
 	return Jupiter::String_Loose<T>::operator+(reinterpret_cast<const Jupiter::Readable_String<T> &>(rhs));
@@ -663,6 +697,38 @@ template<typename T> inline Jupiter::String_Loose<T> Jupiter::String_Loose<T>::o
 	return Jupiter::operator+(*this, rhs);
 }
 
+// tokenize
+
+template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::String_Loose> Jupiter::String_Loose<T>::tokenize(const T &separator)
+{
+	return Jupiter::String_Loose<T>::tokenize(*this, separator);
+}
+
+template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::String_Loose> Jupiter::String_Loose<T>::tokenize(const Jupiter::Readable_String<T> &separator)
+{
+	return Jupiter::String_Loose<T>::tokenize(*this, separator);
+}
+
+template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::String_Loose> Jupiter::String_Loose<T>::tokenize(const T *separator, size_t separator_size)
+{
+	return Jupiter::String_Loose<T>::tokenize(*this, separator, separator_size);
+}
+
+template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::String_Loose> Jupiter::String_Loose<T>::tokenize(const Jupiter::Readable_String<T> &in, const T &separator)
+{
+	return Jupiter::Readable_String<T>::tokenize<Jupiter::String_Loose>(in, separator);
+}
+
+template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::String_Loose> Jupiter::String_Loose<T>::tokenize(const Jupiter::Readable_String<T> &in, const Jupiter::Readable_String<T> &separator)
+{
+	return Jupiter::Readable_String<T>::tokenize<Jupiter::String_Loose>(in, separator);
+}
+
+template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::String_Loose> Jupiter::String_Loose<T>::tokenize(const Jupiter::Readable_String<T> &in, const T *separator, size_t separator_size)
+{
+	return Jupiter::Readable_String<T>::tokenize<Jupiter::String_Loose>(in, separator, separator_size);
+}
+
 #if !defined JUPITER_STRING_STRICT_OPERATOR_PLUS
 #if !defined DISABLE_DEFAULT_JUPITER_STRING_OPERATOR_PLUS
 template<typename T> static inline Jupiter::String_Loose<T> Jupiter::operator+(const Jupiter::Readable_String<T> &lhs, const Jupiter::Readable_String<T> &rhs)