MirrorHub/construct
0
0
Fork 0
mirror of https://github.com/matrix-construct/construct synced 2025-01-14 00:34:18 +01:00
Code Issues Releases Wiki Activity

ircd::tokens: Optimize character tokenizer impls.

Browse source
This commit is contained in:
Jason Volk 2022-05-01 15:40:48 -07:00
parent bf36da742b
commit 5a995f4bac
1 changed files with 87 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

131
ircd/tokens.cc
View file

@ -13,6 +13,7 @@
namespace ircd
{
struct string_separator;
struct char_separator;
};
struct [[gnu::visibility("internal")]]
@ -29,6 +30,20 @@ ircd::string_separator
~string_separator() noexcept;
};
struct [[gnu::visibility("internal")]]
ircd::char_separator
{
char delim;
template<class iterator,
class token>
bool operator()(iterator &next, iterator end, token &ret) const noexcept;
void reset() noexcept;
char_separator(const char &delim) noexcept;
~char_separator() noexcept;
};
//
// interface
//
@ -206,15 +221,9 @@ ircd::token(const string_view &str,
{
using type = string_view;
using iter = typename type::const_iterator;
using delim = boost::char_separator<char>;
using delim = char_separator;
assert(sep != '\0');
const char _sep[2]
{
sep, '\0'
};
const delim d{_sep};
const delim d{sep};
const boost::tokenizer<delim, iter, type> view
{
str, d
@ -248,15 +257,9 @@ ircd::token_exists(const string_view &str,
{
using type = string_view;
using iter = typename type::const_iterator;
using delim = boost::char_separator<char>;
using delim = char_separator;
assert(sep != '\0');
const char _sep[2]
{
sep, '\0'
};
const delim d{_sep};
const delim d{sep};
const boost::tokenizer<delim, iter, type> view
{
str, d
@ -289,15 +292,9 @@ ircd::token_count(const string_view &str,
{
using type = string_view;
using iter = typename type::const_iterator;
using delim = boost::char_separator<char>;
using delim = char_separator;
assert(sep != '\0');
const char _sep[2]
{
sep, '\0'
};
const delim d{_sep};
const delim d{sep};
const boost::tokenizer<delim, iter, type> view
{
str, d
@ -325,17 +322,22 @@ ircd::token_count(const string_view &str,
size_t
ircd::tokens(const string_view &str,
const char &sep,
const char &sep_,
const mutable_buffer &buf,
const token_view &closure)
{
assert(sep != '\0');
assert(sep_ != '\0');
const char _sep[2]
{
sep, '\0'
sep_, '\0'
};
return tokens(str, _sep, buf, closure);
const string_view sep
{
_sep
};
return tokens(str, sep, buf, closure);
}
size_t
@ -370,15 +372,9 @@ ircd::tokens(const string_view &str,
{
using type = string_view;
using iter = typename type::const_iterator;
using delim = boost::char_separator<char>;
using delim = char_separator;
assert(sep != '\0');
const char _sep[2]
{
sep, '\0'
};
const delim d{_sep};
const delim d{sep};
const boost::tokenizer<delim, iter, type> view
{
str, d
@ -423,15 +419,9 @@ ircd::tokens(const string_view &str,
{
using type = string_view;
using iter = typename type::const_iterator;
using delim = boost::char_separator<char>;
using delim = char_separator;
assert(sep != '\0');
const char _sep[2]
{
sep, '\0'
};
const delim d{_sep};
const delim d{sep};
const boost::tokenizer<delim, iter, type> view
{
str, d
@ -518,3 +508,56 @@ const noexcept
while(!ret);
return true;
}
//
// char_separator
//
ircd::char_separator::char_separator(const char &delim)
noexcept
:delim
{
delim
}
{
}
ircd::char_separator::~char_separator()
noexcept
{
}
void
ircd::char_separator::reset()
noexcept
{
//TODO: ???
}
template<class iterator,
class token>
bool
ircd::char_separator::operator()(iterator &start,
iterator stop,
token &ret)
const noexcept
{
do
{
if(start == stop)
return false;
const string_view input
{
start, stop
};
string_view remain;
std::tie(ret, remain) = ircd::split(input, delim);
start = remain?
begin(remain):
stop;
}
while(!ret);
return true;
}