Browse Source

Added tokenize() functions to Readable_String and various subclasses.

General fixes to CString.
release/0.19
JustinAJ 9 years ago
parent
commit
629ffccfea
  1. 60
      Jupiter/CString.h
  2. 130
      Jupiter/CString_Imp.h
  3. 2
      Jupiter/Jupiter.h
  4. 59
      Jupiter/Readable_String.h
  5. 193
      Jupiter/Readable_String_Imp.h
  6. 21
      Jupiter/Reference_String.h
  7. 32
      Jupiter/Reference_String_Imp.h
  8. 42
      Jupiter/String.h
  9. 66
      Jupiter/String_Imp.h
  10. BIN
      Release/Jupiter.lib

60
Jupiter/CString.h

@ -183,16 +183,38 @@ namespace Jupiter
*/ */
static CString_Type<T> gotoWord(const Jupiter::Readable_String<T> &in, size_t pos, const T *whitespace); static CString_Type<T> gotoWord(const Jupiter::Readable_String<T> &in, size_t pos, const T *whitespace);
/**
* @brief Tokenizes a string, based on an input token separator
*
* @param separator Separator to split tokens by
* @return TokenizeResult containing the results of the tokenization process.
*/
TokenizeResult<Jupiter::CString_Type> tokenize(const T &separator) const;
TokenizeResult<Jupiter::CString_Type> tokenize(const Jupiter::Readable_String<T> &separator) const;
TokenizeResult<Jupiter::CString_Type> tokenize(const T *separator, size_t separator_size) const;
/**
* @brief Tokenizes a string, based on an input token separator
*
* @param in String to split into tokens
* @param separator Separator to split tokens by
* @return TokenizeResult containing the results of the tokenization process.
*/
static TokenizeResult<Jupiter::CString_Type> tokenize(const Jupiter::Readable_String<T> &in, const T &separator);
static TokenizeResult<Jupiter::CString_Type> tokenize(const Jupiter::Readable_String<T> &in, const Jupiter::Readable_String<T> &separator);
static TokenizeResult<Jupiter::CString_Type> tokenize(const Jupiter::Readable_String<T> &in, const T *separator, size_t separator_size);
/** /**
* @brief Copies the data from the input string to the CString. * @brief Copies the data from the input string to the CString.
* *
* @param in String containing the data to be copied. * @param in String containing the data to be copied.
* @return New size of the CString. * @return New size of the CString.
*/ */
size_t set(const Readable_String<T> &in); size_t set(const T *in, size_t inSize) override;
size_t set(const std::basic_string<T> &in); size_t set(const Jupiter::Readable_String<T> &in) override;
size_t set(const T *in); size_t set(const std::basic_string<T> &in) override;
size_t set(const T in); size_t set(const T *in) override;
size_t set(const T &in) override;
/** /**
* @brief Copies the data from the input string and concatenates it to the end of CString. * @brief Copies the data from the input string and concatenates it to the end of CString.
@ -200,10 +222,11 @@ namespace Jupiter
* @param in String containing the data to be concatenated. * @param in String containing the data to be concatenated.
* @return New size of the CString. * @return New size of the CString.
*/ */
size_t concat(const Readable_String<T> &in); size_t concat(const T *in, size_t inSize) override;
size_t concat(const std::basic_string<T> &in); size_t concat(const Readable_String<T> &in) override;
size_t concat(const T *in); size_t concat(const std::basic_string<T> &in) override;
size_t concat(const T in); size_t concat(const T *in) override;
size_t concat(const T &in) override;
/** Default Constructor */ /** Default Constructor */
CString_Type(); CString_Type();
@ -380,6 +403,27 @@ namespace Jupiter
*/ */
static CString_Loose<T> gotoWord(const Jupiter::Readable_String<T> &in, size_t pos, const T *whitespace); static CString_Loose<T> gotoWord(const Jupiter::Readable_String<T> &in, size_t pos, const T *whitespace);
/**
* @brief Tokenizes a string, based on an input token separator
*
* @param separator Separator to split tokens by
* @return TokenizeResult containing the results of the tokenization process.
*/
TokenizeResult<Jupiter::CString_Loose> tokenize(const T &separator);
TokenizeResult<Jupiter::CString_Loose> tokenize(const Jupiter::Readable_String<T> &separator);
TokenizeResult<Jupiter::CString_Loose> tokenize(const T *separator, size_t separator_size);
/**
* @brief Tokenizes a string, based on an input token separator
*
* @param in String to split into tokens
* @param separator Separator to split tokens by
* @return TokenizeResult containing the results of the tokenization process.
*/
static TokenizeResult<Jupiter::CString_Loose> tokenize(const Jupiter::Readable_String<T> &in, const T &separator);
static TokenizeResult<Jupiter::CString_Loose> tokenize(const Jupiter::Readable_String<T> &in, const Jupiter::Readable_String<T> &separator);
static TokenizeResult<Jupiter::CString_Loose> tokenize(const Jupiter::Readable_String<T> &in, const T *separator, size_t separator_size);
/** Default constructor */ /** Default constructor */
CString_Loose(); CString_Loose();

130
Jupiter/CString_Imp.h

@ -264,6 +264,8 @@ template<typename T> Jupiter::CString_Type<T> Jupiter::CString_Type<T>::Format(c
return r; return r;
} }
// substring
template<typename T> Jupiter::CString_Type<T> Jupiter::CString_Type<T>::substring(size_t pos) const template<typename T> Jupiter::CString_Type<T> Jupiter::CString_Type<T>::substring(size_t pos) const
{ {
return Jupiter::CString_Type<T>::substring(*this, pos); return Jupiter::CString_Type<T>::substring(*this, pos);
@ -306,6 +308,8 @@ template<typename T> Jupiter::CString_Type<T> Jupiter::CString_Type<T>::substrin
return r; return r;
} }
// getWord
template<typename T> Jupiter::CString_Type<T> Jupiter::CString_Type<T>::getWord(size_t pos, const T *whitespace) const template<typename T> Jupiter::CString_Type<T> Jupiter::CString_Type<T>::getWord(size_t pos, const T *whitespace) const
{ {
return Jupiter::CString_Type<T>::getWord(*this, pos, whitespace); return Jupiter::CString_Type<T>::getWord(*this, pos, whitespace);
@ -321,6 +325,8 @@ template<typename T> Jupiter::CString_Type<T> Jupiter::CString_Type<T>::getWord(
return Jupiter::Readable_String<T>::getWord<Jupiter::CString_Type>(in, pos, whitespace); return Jupiter::Readable_String<T>::getWord<Jupiter::CString_Type>(in, pos, whitespace);
} }
// gotoWord
template<typename T> Jupiter::CString_Type<T> Jupiter::CString_Type<T>::gotoWord(size_t pos, const T *whitespace) const template<typename T> Jupiter::CString_Type<T> Jupiter::CString_Type<T>::gotoWord(size_t pos, const T *whitespace) const
{ {
return Jupiter::CString_Type<T>::gotoWord(*this, pos, whitespace); return Jupiter::CString_Type<T>::gotoWord(*this, pos, whitespace);
@ -331,22 +337,54 @@ template<typename T> Jupiter::CString_Type<T> Jupiter::CString_Type<T>::gotoWord
return Jupiter::Readable_String<T>::gotoWord<Jupiter::CString_Type>(in, pos, whitespace); return Jupiter::Readable_String<T>::gotoWord<Jupiter::CString_Type>(in, pos, whitespace);
} }
template<typename T> size_t Jupiter::CString_Type<T>::set(const Jupiter::Readable_String<T> &in) // tokenize
template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::CString_Type> Jupiter::CString_Type<T>::tokenize(const T &separator) const
{ {
this->setBufferSizeNoCopy(in.size()); return Jupiter::CString_Type<T>::tokenize(*this, separator);
for (Jupiter::String_Type<T>::length = 0; Jupiter::String_Type<T>::length != in.size() && in.get(Jupiter::String_Type<T>::length) != 0; Jupiter::String_Type<T>::length++) }
Jupiter::String_Type<T>::str[Jupiter::String_Type<T>::length] = in.get(Jupiter::String_Type<T>::length);
Jupiter::String_Type<T>::str[Jupiter::String_Type<T>::length] = 0; template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::CString_Type> Jupiter::CString_Type<T>::tokenize(const Jupiter::Readable_String<T> &separator) const
{
return Jupiter::CString_Type<T>::tokenize(*this, separator);
}
template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::CString_Type> Jupiter::CString_Type<T>::tokenize(const T *separator, size_t separator_size) const
{
return Jupiter::CString_Type<T>::tokenize(*this, separator, separator_size);
}
template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::CString_Type> Jupiter::CString_Type<T>::tokenize(const Jupiter::Readable_String<T> &in, const T &token)
{
return Jupiter::Readable_String<T>::tokenize<Jupiter::CString_Type>(in, token);
}
template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::CString_Type> Jupiter::CString_Type<T>::tokenize(const Jupiter::Readable_String<T> &in, const Jupiter::Readable_String<T> &separator)
{
return Jupiter::Readable_String<T>::tokenize<Jupiter::CString_Type>(in, separator);
}
template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::CString_Type> Jupiter::CString_Type<T>::tokenize(const Jupiter::Readable_String<T> &in, const T *separator, size_t separator_size)
{
return Jupiter::Readable_String<T>::tokenize<Jupiter::CString_Type>(in, separator, separator_size);
}
// set
template<typename T> size_t Jupiter::CString_Type<T>::set(const T *in, size_t in_size)
{
Jupiter::String_Type<T>::str[Jupiter::String_Type<T>::set(in, in_size)] = 0;
return Jupiter::String_Type<T>::length; return Jupiter::String_Type<T>::length;
} }
template<typename T> size_t Jupiter::CString_Type<T>::set(const Jupiter::Readable_String<T> &in)
{
return this->set(in.ptr(), in.size());
}
template<typename T> size_t Jupiter::CString_Type<T>::set(const std::basic_string<T> &in) template<typename T> size_t Jupiter::CString_Type<T>::set(const std::basic_string<T> &in)
{ {
this->setBufferSizeNoCopy(in.size()); return this->set(in.data(), in.size());
for (Jupiter::String_Type<T>::length = 0; Jupiter::String_Type<T>::length != in.size() && in.at(Jupiter::String_Type<T>::length) != 0; Jupiter::String_Type<T>::length++)
Jupiter::String_Type<T>::str[Jupiter::String_Type<T>::length] = in.at(Jupiter::String_Type<T>::length);
Jupiter::String_Type<T>::str[Jupiter::String_Type<T>::length];
return Jupiter::String_Type<T>::length;
} }
template<typename T> size_t Jupiter::CString_Type<T>::set(const T *in) template<typename T> size_t Jupiter::CString_Type<T>::set(const T *in)
@ -355,7 +393,7 @@ template<typename T> size_t Jupiter::CString_Type<T>::set(const T *in)
return Jupiter::String_Type<T>::length; return Jupiter::String_Type<T>::length;
} }
template<typename T> size_t Jupiter::CString_Type<T>::set(const T in) template<typename T> size_t Jupiter::CString_Type<T>::set(const T &in)
{ {
if (in == 0) if (in == 0)
{ {
@ -369,34 +407,22 @@ template<typename T> size_t Jupiter::CString_Type<T>::set(const T in)
return Jupiter::String_Type<T>::length = 1; return Jupiter::String_Type<T>::length = 1;
} }
template<typename T> size_t Jupiter::CString_Type<T>::concat(const Jupiter::Readable_String<T> &in) // concat
{
size_t nSize = Jupiter::String_Type<T>::length + in.size(); template<typename T> size_t Jupiter::CString_Type<T>::concat(const T *in, size_t in_size)
const T *inData = in.ptr();
this->setBufferSize(nSize);
while (Jupiter::String_Type<T>::length != nSize && *inData != 0)
{ {
Jupiter::String_Type<T>::str[Jupiter::String_Type<T>::length] = *inData; Jupiter::String_Type<T>::str[Jupiter::String_Type<T>::concat(in, in_size)] = 0;
Jupiter::String_Type<T>::length++;
inData++;
}
Jupiter::String_Type<T>::str[Jupiter::String_Type<T>::length] = 0;
return Jupiter::String_Type<T>::length; return Jupiter::String_Type<T>::length;
} }
template<typename T> size_t Jupiter::CString_Type<T>::concat(const std::basic_string<T> &in) template<typename T> size_t Jupiter::CString_Type<T>::concat(const Jupiter::Readable_String<T> &in)
{
size_t nSize = Jupiter::String_Type<T>::length + in.size();
const T *inData = in.data();
this->setBufferSize(nSize);
while (Jupiter::String_Type<T>::length != nSize && *inData != 0)
{ {
Jupiter::String_Type<T>::str[Jupiter::String_Type<T>::length] = *inData; return this->concat(in.ptr(), in.size());
Jupiter::String_Type<T>::length++;
inData++;
} }
Jupiter::String_Type<T>::str[Jupiter::String_Type<T>::length] = 0;
return Jupiter::String_Type<T>::length; template<typename T> size_t Jupiter::CString_Type<T>::concat(const std::basic_string<T> &in)
{
return this->concat(in.data(), in.size());
} }
template<typename T> size_t Jupiter::CString_Type<T>::concat(const T *in) template<typename T> size_t Jupiter::CString_Type<T>::concat(const T *in)
@ -407,15 +433,16 @@ template<typename T> size_t Jupiter::CString_Type<T>::concat(const T *in)
return Jupiter::String_Type<T>::length = nSize; return Jupiter::String_Type<T>::length = nSize;
} }
template<typename T> size_t Jupiter::CString_Type<T>::concat(const T c) template<typename T> size_t Jupiter::CString_Type<T>::concat(const T &in)
{ {
this->setBufferSize(Jupiter::String_Type<T>::length + 1); this->setBufferSize(Jupiter::String_Type<T>::length + 1);
Jupiter::String_Type<T>::str[Jupiter::String_Type<T>::length] = c; Jupiter::String_Type<T>::str[Jupiter::String_Type<T>::length] = in;
Jupiter::String_Type<T>::str[++Jupiter::String_Type<T>::length] = 0; Jupiter::String_Type<T>::str[++Jupiter::String_Type<T>::length] = 0;
return Jupiter::String_Type<T>::length; return Jupiter::String_Type<T>::length;
} }
// Operators // Operators
template<typename T> inline Jupiter::CString_Type<T> Jupiter::CString_Type<T>::operator+(const Jupiter::CString_Type<T> &rhs) const template<typename T> inline Jupiter::CString_Type<T> Jupiter::CString_Type<T>::operator+(const Jupiter::CString_Type<T> &rhs) const
{ {
return Jupiter::CString_Type<T>::operator+(reinterpret_cast<const Jupiter::Readable_String<T> &>(rhs)); return Jupiter::CString_Type<T>::operator+(reinterpret_cast<const Jupiter::Readable_String<T> &>(rhs));
@ -668,7 +695,40 @@ template<typename T> Jupiter::CString_Loose<T> Jupiter::CString_Loose<T>::gotoWo
return Jupiter::Readable_String<T>::gotoWord<Jupiter::CString_Loose>(in, pos, whitespace); return Jupiter::Readable_String<T>::gotoWord<Jupiter::CString_Loose>(in, pos, whitespace);
} }
// tokenize
template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::CString_Loose> Jupiter::CString_Loose<T>::tokenize(const T &token)
{
return Jupiter::CString_Loose<T>::tokenize(*this, token);
}
template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::CString_Loose> Jupiter::CString_Loose<T>::tokenize(const Jupiter::Readable_String<T> &separator)
{
return Jupiter::CString_Loose<T>::tokenize(*this, separator);
}
template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::CString_Loose> Jupiter::CString_Loose<T>::tokenize(const T *separator, size_t separator_size)
{
return Jupiter::CString_Loose<T>::tokenize(*this, separator, separator_size);
}
template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::CString_Loose> Jupiter::CString_Loose<T>::tokenize(const Jupiter::Readable_String<T> &in, const T &token)
{
return Jupiter::Readable_String<T>::tokenize<Jupiter::CString_Loose>(in, token);
}
template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::CString_Loose> Jupiter::CString_Loose<T>::tokenize(const Jupiter::Readable_String<T> &in, const Jupiter::Readable_String<T> &separator)
{
return Jupiter::Readable_String<T>::tokenize<Jupiter::CString_Loose>(in, separator);
}
template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::CString_Loose> Jupiter::CString_Loose<T>::tokenize(const Jupiter::Readable_String<T> &in, const T *separator, size_t separator_size)
{
return Jupiter::Readable_String<T>::tokenize<Jupiter::CString_Loose>(in, separator, separator_size);
}
// Operators // Operators
template<typename T> inline Jupiter::CString_Loose<T> Jupiter::CString_Loose<T>::operator+(const Jupiter::CString_Loose<T> &rhs) const template<typename T> inline Jupiter::CString_Loose<T> Jupiter::CString_Loose<T>::operator+(const Jupiter::CString_Loose<T> &rhs) const
{ {
return Jupiter::CString_Loose<T>::operator+(reinterpret_cast<const Jupiter::Readable_String<T> &>(rhs)); return Jupiter::CString_Loose<T>::operator+(reinterpret_cast<const Jupiter::Readable_String<T> &>(rhs));

2
Jupiter/Jupiter.h

@ -35,7 +35,7 @@
#define JUPITER_API #define JUPITER_API
#endif // _WIN32 #endif // _WIN32
#define JUPITER_VERSION "Jupiter 0.16" /** Version of this program at compile time. */ #define JUPITER_VERSION "Jupiter 0.17-dev" /** Version of this program at compile time. */
#if defined __cplusplus #if defined __cplusplus
extern "C" extern "C"

59
Jupiter/Readable_String.h

@ -315,6 +315,65 @@ namespace Jupiter
template<template<typename> class R> static R<T> gotoToken(const Jupiter::Readable_String<T> &in, size_t pos, const T &token); template<template<typename> class R> static R<T> gotoToken(const Jupiter::Readable_String<T> &in, size_t pos, const T &token);
template<template<typename> class R> static R<T> gotoToken(const Jupiter::Readable_String<T> &in, size_t pos, const Jupiter::Readable_String<T> &token); template<template<typename> class R> static R<T> gotoToken(const Jupiter::Readable_String<T> &in, size_t pos, const Jupiter::Readable_String<T> &token);
/**
* @brief Contains the results from a call to tokenize().
*/
template<template<typename> class R> struct TokenizeResult
{
/** Underlying type used to represent each token */
typedef R<T> token_type;
/** Array of tokens, with a length of 'token_count' */
R<T> *tokens;
/** Length of 'tokens' array */
size_t token_count;
/**
* @brief Default constructor for the TokenizeResult class.
* Note: 'tokens' assigned nullptr, and 'token_count' is assigned 0.
*/
TokenizeResult();
/**
* @brief Allocation constructor for the TokenizeResult class.
*/
TokenizeResult(size_t in_token_count);
/**
* @brief Capture constructor for the TokenizeResult class.
*/
TokenizeResult(R<T> *in_tokens, size_t in_token_count);
/**
* @brief Copy constructor for the TokenizeResult class.
*/
TokenizeResult(const TokenizeResult &source);
/**
* @brief Move constructor for the TokenizeResult class.
*/
TokenizeResult(const TokenizeResult &&source);
/**
* @brief Destructor for the TokenizeResult class.
*/
~TokenizeResult();
};
/**
* @brief Tokenizes a string, based on an input token separator
*
* @param R String type to use when constructing results
*
* @param in String to split into tokens
* @param separator Separator to split tokens by
* @return TokenizeResult containing the results of the tokenization process.
*/
template<template<typename> class R> static TokenizeResult<R> tokenize(const Jupiter::Readable_String<T> &in, const T &separator);
template<template<typename> class R> static TokenizeResult<R> tokenize(const Jupiter::Readable_String<T> &in, const Jupiter::Readable_String<T> &separator);
template<template<typename> class R> static TokenizeResult<R> tokenize(const Jupiter::Readable_String<T> &in, const T *separator, size_t separator_size);
/** Access operator */ /** Access operator */
inline const T &operator[](size_t index) const { return this->get(index); }; inline const T &operator[](size_t index) const { return this->get(index); };

193
Jupiter/Readable_String_Imp.h

@ -1301,6 +1301,199 @@ template<typename T> template<template<typename> class R> R<T> Jupiter::Readable
return R<T>(); return R<T>();
} }
/** tokenization */
template<typename T> template<template<typename> class R> Jupiter::Readable_String<T>::TokenizeResult<R>::TokenizeResult()
{
Jupiter::Readable_String<T>::TokenizeResult<R>::tokens = nullptr;
Jupiter::Readable_String<T>::TokenizeResult<R>::token_count = 0;
}
template<typename T> template<template<typename> class R> Jupiter::Readable_String<T>::TokenizeResult<R>::TokenizeResult(size_t in_token_count)
{
Jupiter::Readable_String<T>::TokenizeResult<R>::token_count = in_token_count;
Jupiter::Readable_String<T>::TokenizeResult<R>::tokens = new R<T>[Jupiter::Readable_String<T>::TokenizeResult<R>::token_count];
}
template<typename T> template<template<typename> class R> Jupiter::Readable_String<T>::TokenizeResult<R>::TokenizeResult(R<T> *in_tokens, size_t in_token_count)
{
Jupiter::Readable_String<T>::TokenizeResult<R>::tokens = in_tokens;
Jupiter::Readable_String<T>::TokenizeResult<R>::token_count = in_token_count;
}
template<typename T> template<template<typename> class R> Jupiter::Readable_String<T>::TokenizeResult<R>::TokenizeResult(const TokenizeResult &source)
{
Jupiter::Readable_String<T>::TokenizeResult<R>::tokens = new R<T>[source.token_count];
Jupiter::Readable_String<T>::TokenizeResult<R>::token_count = 0;
while (Jupiter::Readable_String<T>::TokenizeResult<R>::token_count != source.token_count)
{
Jupiter::Readable_String<T>::TokenizeResult<R>::tokens[Jupiter::Readable_String<T>::TokenizeResult<R>::token_count] = source.tokens[Jupiter::Readable_String<T>::TokenizeResult<R>::token_count];
++Jupiter::Readable_String<T>::TokenizeResult<R>::token_count;
}
}
template<typename T> template<template<typename> class R> Jupiter::Readable_String<T>::TokenizeResult<R>::TokenizeResult(const TokenizeResult &&source)
{
Jupiter::Readable_String<T>::TokenizeResult<R>::tokens = source.tokens;
Jupiter::Readable_String<T>::TokenizeResult<R>::token_count = source.token_count;
source.tokens = nullptr;
source.token_count = 0;
}
template<typename T> template<template<typename> class R> Jupiter::Readable_String<T>::TokenizeResult<R>::~TokenizeResult()
{
if (Jupiter::Readable_String<T>::TokenizeResult<R>::tokens != nullptr)
delete[] Jupiter::Readable_String<T>::TokenizeResult<R>::tokens;
}
// tokenize
template<typename T> template<template<typename> class R> typename Jupiter::Readable_String<T>::TokenizeResult<R> Jupiter::Readable_String<T>::tokenize(const Jupiter::Readable_String<T> &in, const T &token)
{
// special case: no input
if (in.isEmpty())
return TokenizeResult<R>();
size_t length = in.tokenCount(token);
R<T> *tokens = new R<T>[length];
// special case: only 1 token
if (length == 1)
{
*tokens = in;
return TokenizeResult<R>(tokens, length);
}
R<T> *itr = tokens;
const T *in_itr = in.ptr();
const T *in_end = in.ptr() + in.size();
const T *token_start = in_itr;
while (in_itr != in_end)
{
loop_start:
if (*in_itr == token)
{
// push token to output
itr->set(token_start, in_itr - token_start);
++itr;
// increment over separator
++in_itr;
// iterator is at end of string
if (in_itr == in_end)
return TokenizeResult<R>(tokens, length);
// start reading new token
token_start = in_itr;
goto loop_start;
}
++in_itr;
}
itr->set(token_start, in_end - token_start);
return TokenizeResult<R>(tokens, length);
}
template<typename T> template<template<typename> class R> typename Jupiter::Readable_String<T>::TokenizeResult<R> Jupiter::Readable_String<T>::tokenize(const Jupiter::Readable_String<T> &in, const Jupiter::Readable_String<T> &separator)
{
return Jupiter::Readable_String<T>::tokenize<R>(in, separator.ptr(), separator.size());
}
template<typename T> template<template<typename> class R> typename Jupiter::Readable_String<T>::TokenizeResult<R> Jupiter::Readable_String<T>::tokenize(const Jupiter::Readable_String<T> &in, const T *separator, size_t separator_size)
{
// special case: separator is a single element
if (separator_size == 1)
return Jupiter::Readable_String<T>::tokenize<R>(in, *separator);
// special case: no input
if (in.isEmpty())
return TokenizeResult<R>();
R<T> *tokens;
// special case: no separator
if (separator_size == 0)
{
tokens = new R<T>[1];
*tokens = in;
return TokenizeResult<R>(tokens, 1);
}
// special case: only enough space for 1 token, or 2 empty tokens
if (separator_size >= in.size())
{
if (in.equals(separator, separator_size))
return TokenizeResult<R>(2);
tokens = new R<T>[1];
*tokens = in;
return TokenizeResult<R>(tokens, 1);
}
size_t length = in.tokenCount(separator);
tokens = new R<T>[length];
// special case: only 1 token
if (length == 1)
{
*tokens = in;
return TokenizeResult<R>(tokens, length);
}
R<T> *itr = tokens;
const T *in_itr = in.ptr();
const T *in_end = in.ptr() + in.size();
const T *token_start = in_itr;
auto is_separator = [](const T *sep, size_t sep_size, const T *ptr)
{
while (*sep == *ptr)
{
if (--sep_size == 0)
return true;
++sep, ++ptr;
}
return false;
};
while (in_itr + separator_size - 1 != in_end)
{
loop_start:
if (is_separator(separator, separator_size, in_itr))
{
// push token to output
itr->set(token_start, in_itr - token_start);
++itr;
in_itr += separator_size;
token_start = in_itr;
// not enough room for another separator
if (in_itr + separator_size > in_end)
break;
// only enough room for 1 separator (empty token), or 1 token
if (in_itr + separator_size == in_end)
{
if (is_separator(separator, separator_size, in_itr))
return TokenizeResult<R>(tokens, length);
break;
}
goto loop_start;
}
++in_itr;
}
itr->set(token_start, in_end - token_start);
return TokenizeResult<R>(tokens, length);
}
// Jupiter::DataBuffer specialization // Jupiter::DataBuffer specialization
template<> struct _Jupiter_DataBuffer_partial_specialization_impl<Jupiter::Readable_String> template<> struct _Jupiter_DataBuffer_partial_specialization_impl<Jupiter::Readable_String>

21
Jupiter/Reference_String.h

@ -234,6 +234,27 @@ namespace Jupiter
static Reference_String<T> gotoToken(const Jupiter::Readable_String<T> &in, size_t pos, const T &token); static Reference_String<T> gotoToken(const Jupiter::Readable_String<T> &in, size_t pos, const T &token);
static Reference_String<T> gotoToken(const Jupiter::Readable_String<T> &in, size_t pos, const Jupiter::Readable_String<T> &token); static Reference_String<T> gotoToken(const Jupiter::Readable_String<T> &in, size_t pos, const Jupiter::Readable_String<T> &token);
/**
* @brief Tokenizes a string, based on an input token separator
*
* @param separator Separator to split tokens by
* @return TokenizeResult containing the results of the tokenization process.
*/
TokenizeResult<Jupiter::Reference_String> tokenize(const T &separator) const;
TokenizeResult<Jupiter::Reference_String> tokenize(const Jupiter::Readable_String<T> &separator) const;
TokenizeResult<Jupiter::Reference_String> tokenize(const T *separator, size_t separator_size) const;
/**
* @brief Tokenizes a string, based on an input token separator
*
* @param in String to split into tokens
* @param separator Separator to split tokens by
* @return TokenizeResult containing the results of the tokenization process.
*/
static TokenizeResult<Jupiter::Reference_String> tokenize(const Jupiter::Readable_String<T> &in, const T &separator);
static TokenizeResult<Jupiter::Reference_String> tokenize(const Jupiter::Readable_String<T> &in, const Jupiter::Readable_String<T> &separator);
static TokenizeResult<Jupiter::Reference_String> tokenize(const Jupiter::Readable_String<T> &in, const T *separator, size_t separator_size);
/** Mutative operators */ /** Mutative operators */
inline Readable_String<T> &operator-=(size_t right) { this->truncate(right); return *this; }; inline Readable_String<T> &operator-=(size_t right) { this->truncate(right); return *this; };
inline Readable_String<T> &operator=(const Readable_String<T> &right) { this->set(right); return *this; }; inline Readable_String<T> &operator=(const Readable_String<T> &right) { this->set(right); return *this; };

32
Jupiter/Reference_String_Imp.h

@ -237,6 +237,38 @@ template<typename T> Jupiter::Reference_String<T> Jupiter::Reference_String<T>::
template<typename T> const Jupiter::Reference_String<T> Jupiter::Reference_String<T>::empty = Jupiter::Reference_String<T>(); template<typename T> const Jupiter::Reference_String<T> Jupiter::Reference_String<T>::empty = Jupiter::Reference_String<T>();
// tokenize
template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::Reference_String> Jupiter::Reference_String<T>::tokenize(const T &separator) const
{
return Jupiter::Reference_String<T>::tokenize(*this, separator);
}
template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::Reference_String> Jupiter::Reference_String<T>::tokenize(const Jupiter::Readable_String<T> &separator) const
{
return Jupiter::Reference_String<T>::tokenize(*this, separator);
}
template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::Reference_String> Jupiter::Reference_String<T>::tokenize(const T *separator, size_t separator_size) const
{
return Jupiter::Reference_String<T>::tokenize(*this, separator, separator_size);
}
template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::Reference_String> Jupiter::Reference_String<T>::tokenize(const Jupiter::Readable_String<T> &in, const T &token)
{
return Jupiter::Readable_String<T>::tokenize<Jupiter::Reference_String>(in, token);
}
template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::Reference_String> Jupiter::Reference_String<T>::tokenize(const Jupiter::Readable_String<T> &in, const Jupiter::Readable_String<T> &separator)
{
return Jupiter::Readable_String<T>::tokenize<Jupiter::Reference_String>(in, separator);
}
template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::Reference_String> Jupiter::Reference_String<T>::tokenize(const Jupiter::Readable_String<T> &in, const T *separator, size_t separator_size)
{
return Jupiter::Readable_String<T>::tokenize<Jupiter::Reference_String>(in, separator, separator_size);
}
// Jupiter::DataBuffer specialization // Jupiter::DataBuffer specialization
template<> struct _Jupiter_DataBuffer_partial_specialization_impl<Jupiter::Reference_String> template<> struct _Jupiter_DataBuffer_partial_specialization_impl<Jupiter::Reference_String>

42
Jupiter/String.h

@ -203,6 +203,27 @@ namespace Jupiter
static String_Strict<T> gotoToken(const Jupiter::Readable_String<T> &in, size_t pos, const T &token); static String_Strict<T> gotoToken(const Jupiter::Readable_String<T> &in, size_t pos, const T &token);
static String_Strict<T> gotoToken(const Jupiter::Readable_String<T> &in, size_t pos, const Jupiter::Readable_String<T> &token); static String_Strict<T> gotoToken(const Jupiter::Readable_String<T> &in, size_t pos, const Jupiter::Readable_String<T> &token);
/**
* @brief Tokenizes a string, based on an input token separator
*
* @param separator Separator to split tokens by
* @return TokenizeResult containing the results of the tokenization process.
*/
TokenizeResult<Jupiter::String_Strict> tokenize(const T &separator) const;
TokenizeResult<Jupiter::String_Strict> tokenize(const Jupiter::Readable_String<T> &separator) const;
TokenizeResult<Jupiter::String_Strict> tokenize(const T *separator, size_t separator_size) const;
/**
* @brief Tokenizes a string, based on an input token separator
*
* @param in String to split into tokens
* @param separator Separator to split tokens by
* @return TokenizeResult containing the results of the tokenization process.
*/
static TokenizeResult<Jupiter::String_Strict> tokenize(const Jupiter::Readable_String<T> &in, const T &separator);
static TokenizeResult<Jupiter::String_Strict> tokenize(const Jupiter::Readable_String<T> &in, const Jupiter::Readable_String<T> &separator);
static TokenizeResult<Jupiter::String_Strict> tokenize(const Jupiter::Readable_String<T> &in, const T *separator, size_t separator_size);
/** Default Constructor */ /** Default Constructor */
String_Strict(); String_Strict();
@ -422,6 +443,27 @@ namespace Jupiter
static String_Loose<T> gotoToken(const Jupiter::Readable_String<T> &in, size_t pos, const T &token); static String_Loose<T> gotoToken(const Jupiter::Readable_String<T> &in, size_t pos, const T &token);
static String_Loose<T> gotoToken(const Jupiter::Readable_String<T> &in, size_t pos, const Jupiter::Readable_String<T> &token); static String_Loose<T> gotoToken(const Jupiter::Readable_String<T> &in, size_t pos, const Jupiter::Readable_String<T> &token);
/**
* @brief Tokenizes a string, based on an input token separator
*
* @param separator Separator to split tokens by
* @return TokenizeResult containing the results of the tokenization process.
*/
TokenizeResult<Jupiter::String_Loose> tokenize(const T &separator);
TokenizeResult<Jupiter::String_Loose> tokenize(const Jupiter::Readable_String<T> &separator);
TokenizeResult<Jupiter::String_Loose> tokenize(const T *separator, size_t separator_size);
/**
* @brief Tokenizes a string, based on an input token separator
*
* @param in String to split into tokens
* @param separator Separator to split tokens by
* @return TokenizeResult containing the results of the tokenization process.
*/
static TokenizeResult<Jupiter::String_Loose> tokenize(const Jupiter::Readable_String<T> &in, const T &separator);
static TokenizeResult<Jupiter::String_Loose> tokenize(const Jupiter::Readable_String<T> &in, const Jupiter::Readable_String<T> &separator);
static TokenizeResult<Jupiter::String_Loose> tokenize(const Jupiter::Readable_String<T> &in, const T *separator, size_t separator_size);
/** Default constructor */ /** Default constructor */
String_Loose(); String_Loose();

66
Jupiter/String_Imp.h

@ -315,7 +315,40 @@ template<typename T> Jupiter::String_Strict<T> Jupiter::String_Strict<T>::gotoTo
return Jupiter::Readable_String<T>::gotoToken<Jupiter::String_Strict>(in, pos, token); return Jupiter::Readable_String<T>::gotoToken<Jupiter::String_Strict>(in, pos, token);
} }
// tokenize
template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::String_Strict> Jupiter::String_Strict<T>::tokenize(const T &separator) const
{
return Jupiter::String_Strict<T>::tokenize(*this, separator);
}
template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::String_Strict> Jupiter::String_Strict<T>::tokenize(const Jupiter::Readable_String<T> &separator) const
{
return Jupiter::String_Strict<T>::tokenize(*this, separator);
}
template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::String_Strict> Jupiter::String_Strict<T>::tokenize(const T *separator, size_t separator_size) const
{
return Jupiter::String_Strict<T>::tokenize(*this, separator, separator_size);
}
template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::String_Strict> Jupiter::String_Strict<T>::tokenize(const Jupiter::Readable_String<T> &in, const T &token)
{
return Jupiter::Readable_String<T>::tokenize<Jupiter::String_Strict>(in, token);
}
template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::String_Strict> Jupiter::String_Strict<T>::tokenize(const Jupiter::Readable_String<T> &in, const Jupiter::Readable_String<T> &separator)
{
return Jupiter::Readable_String<T>::tokenize<Jupiter::String_Strict>(in, separator);
}
template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::String_Strict> Jupiter::String_Strict<T>::tokenize(const Jupiter::Readable_String<T> &in, const T *separator, size_t separator_size)
{
return Jupiter::Readable_String<T>::tokenize<Jupiter::String_Strict>(in, separator, separator_size);
}
// Operators // Operators
template<typename T> inline Jupiter::String_Strict<T> Jupiter::String_Strict<T>::operator+(const Jupiter::String_Strict<T> &rhs) const template<typename T> inline Jupiter::String_Strict<T> Jupiter::String_Strict<T>::operator+(const Jupiter::String_Strict<T> &rhs) const
{ {
return Jupiter::String_Strict<T>::operator+(reinterpret_cast<const Jupiter::Readable_String<T> &>(rhs)); return Jupiter::String_Strict<T>::operator+(reinterpret_cast<const Jupiter::Readable_String<T> &>(rhs));
@ -653,6 +686,7 @@ template<typename T> Jupiter::String_Loose<T> Jupiter::String_Loose<T>::gotoToke
} }
// Operators // Operators
template<typename T> inline Jupiter::String_Loose<T> Jupiter::String_Loose<T>::operator+(const Jupiter::String_Loose<T> &rhs) const template<typename T> inline Jupiter::String_Loose<T> Jupiter::String_Loose<T>::operator+(const Jupiter::String_Loose<T> &rhs) const
{ {
return Jupiter::String_Loose<T>::operator+(reinterpret_cast<const Jupiter::Readable_String<T> &>(rhs)); return Jupiter::String_Loose<T>::operator+(reinterpret_cast<const Jupiter::Readable_String<T> &>(rhs));
@ -663,6 +697,38 @@ template<typename T> inline Jupiter::String_Loose<T> Jupiter::String_Loose<T>::o
return Jupiter::operator+(*this, rhs); return Jupiter::operator+(*this, rhs);
} }
// tokenize
template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::String_Loose> Jupiter::String_Loose<T>::tokenize(const T &separator)
{
return Jupiter::String_Loose<T>::tokenize(*this, separator);
}
template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::String_Loose> Jupiter::String_Loose<T>::tokenize(const Jupiter::Readable_String<T> &separator)
{
return Jupiter::String_Loose<T>::tokenize(*this, separator);
}
template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::String_Loose> Jupiter::String_Loose<T>::tokenize(const T *separator, size_t separator_size)
{
return Jupiter::String_Loose<T>::tokenize(*this, separator, separator_size);
}
template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::String_Loose> Jupiter::String_Loose<T>::tokenize(const Jupiter::Readable_String<T> &in, const T &separator)
{
return Jupiter::Readable_String<T>::tokenize<Jupiter::String_Loose>(in, separator);
}
template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::String_Loose> Jupiter::String_Loose<T>::tokenize(const Jupiter::Readable_String<T> &in, const Jupiter::Readable_String<T> &separator)
{
return Jupiter::Readable_String<T>::tokenize<Jupiter::String_Loose>(in, separator);
}
template<typename T> typename Jupiter::Readable_String<T>::TokenizeResult<Jupiter::String_Loose> Jupiter::String_Loose<T>::tokenize(const Jupiter::Readable_String<T> &in, const T *separator, size_t separator_size)
{
return Jupiter::Readable_String<T>::tokenize<Jupiter::String_Loose>(in, separator, separator_size);
}
#if !defined JUPITER_STRING_STRICT_OPERATOR_PLUS #if !defined JUPITER_STRING_STRICT_OPERATOR_PLUS
#if !defined DISABLE_DEFAULT_JUPITER_STRING_OPERATOR_PLUS #if !defined DISABLE_DEFAULT_JUPITER_STRING_OPERATOR_PLUS
template<typename T> static inline Jupiter::String_Loose<T> Jupiter::operator+(const Jupiter::Readable_String<T> &lhs, const Jupiter::Readable_String<T> &rhs) template<typename T> static inline Jupiter::String_Loose<T> Jupiter::operator+(const Jupiter::Readable_String<T> &lhs, const Jupiter::Readable_String<T> &rhs)

BIN
Release/Jupiter.lib

Binary file not shown.
Loading…
Cancel
Save