From f049bbf0d07c0465823dd84aeb6343e77e60f1c5 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 24 Apr 2018 16:40:55 -0700 Subject: [PATCH] ircd: Split tokens.cc from lexical.cc --- ircd/Makefile.am | 1 + ircd/lexical.cc | 270 ------------------------------------------- ircd/tokens.cc | 293 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 294 insertions(+), 270 deletions(-) create mode 100644 ircd/tokens.cc diff --git a/ircd/Makefile.am b/ircd/Makefile.am index e3a2c4f90..e0cc30c3a 100644 --- a/ircd/Makefile.am +++ b/ircd/Makefile.am @@ -76,6 +76,7 @@ libircd_la_LIBADD = \ libircd_la_SOURCES = \ exception.cc \ lexical.cc \ + tokens.cc \ json.cc \ locale.cc \ logger.cc \ diff --git a/ircd/lexical.cc b/ircd/lexical.cc index 0cf6e9ad7..022b1cba7 100644 --- a/ircd/lexical.cc +++ b/ircd/lexical.cc @@ -16,278 +16,8 @@ /// can also be replaced. /// -#include #include -/////////////////////////////////////////////////////////////////////////////// -// -// ircd/tokens.h -// - -ircd::string_view -ircd::tokens_before(const string_view &str, - const char &sep, - const size_t &i) -{ - const char ssep[2] { sep, '\0' }; - return tokens_before(str, ssep, i); -} - -ircd::string_view -ircd::tokens_before(const string_view &str, - const char *const &sep, - const size_t &i) -{ - using type = string_view; - using iter = typename type::const_iterator; - using delim = boost::char_separator; - - const delim d(sep); - const boost::tokenizer view(str, d); - - string_view ret; - auto it(begin(view)); - for(size_t j(0); it != end(view) && j < i; ++it, j++) - ret = { begin(view)->data(), it->data() + it->size() }; - - return ret; -} - -ircd::string_view -ircd::tokens_after(const string_view &str, - const char &sep, - const size_t &i) -{ - const char ssep[2] { sep, '\0' }; - return tokens_after(str, ssep, i); -} - -ircd::string_view -ircd::tokens_after(const string_view &str, - const char *const &sep, - const size_t &i) -{ - using type = string_view; - using iter = typename type::const_iterator; - using delim = boost::char_separator; - - const delim d(sep); - const boost::tokenizer view(str, d); - - auto it(begin(view)); - for(size_t j(0); it != end(view); ++it, j++) - if(j > i) - return string_view{it->data(), str.data() + str.size()}; - - return {}; -} - -ircd::string_view -ircd::token_first(const string_view &str, - const char &sep) -{ - const char ssep[2] { sep, '\0' }; - return token(str, ssep, 0); -} - -ircd::string_view -ircd::token_first(const string_view &str, - const char *const &sep) -{ - return token(str, sep, 0); -} - -ircd::string_view -ircd::token_last(const string_view &str, - const char &sep) -{ - const char ssep[2] { sep, '\0' }; - return token_last(str, ssep); -} - -ircd::string_view -ircd::token_last(const string_view &str, - const char *const &sep) -{ - using type = string_view; - using iter = typename type::const_iterator; - using delim = boost::char_separator; - - const delim d(sep); - const boost::tokenizer view(str, d); - - auto it(begin(view)); - if(it == end(view)) - return str.empty()? str : throw std::out_of_range("token out of range"); - - string_view ret(*it); - for(++it; it != end(view); ++it) - ret = *it; - - return ret; -} - -ircd::string_view -ircd::token(const string_view &str, - const char &sep, - const size_t &i, - const string_view &def) -{ - const char ssep[2] { sep, '\0' }; - return token(str, ssep, i, def); -} - -ircd::string_view -ircd::token(const string_view &str, - const char *const &sep, - const size_t &i, - const string_view &def) -try -{ - return token(str, sep, i); -} -catch(const std::out_of_range &) -{ - return def; -} - -ircd::string_view -ircd::token(const string_view &str, - const char &sep, - const size_t &i) -{ - const char ssep[2] { sep, '\0' }; - return token(str, ssep, i); -} - -ircd::string_view -ircd::token(const string_view &str, - const char *const &sep, - const size_t &i) -{ - using type = string_view; - using iter = typename type::const_iterator; - using delim = boost::char_separator; - - const delim d(sep); - const boost::tokenizer view(str, d); - const auto it(at(begin(view), end(view), i)); - return *it; -} - -size_t -ircd::token_count(const string_view &str, - const char &sep) -{ - const char ssep[2] { sep, '\0' }; - return token_count(str, ssep); -} - -size_t -ircd::token_count(const string_view &str, - const char *const &sep) -{ - using type = string_view; - using iter = typename type::const_iterator; - using delim = boost::char_separator; - - const delim d(sep); - const boost::tokenizer view(str, d); - return std::distance(begin(view), end(view)); -} - -size_t -ircd::tokens(const string_view &str, - const char &sep, - const mutable_buffer &buf, - const token_view &closure) -{ - const char ssep[2] { sep, '\0' }; - return tokens(str, ssep, buf, closure); -} - -size_t -ircd::tokens(const string_view &str, - const char *const &sep, - const mutable_buffer &buf, - const token_view &closure) -{ - char *ptr(data(buf)); - char *const stop(data(buf) + size(buf)); - tokens(str, sep, [&closure, &ptr, &stop] - (const string_view &token) - { - const size_t terminated_size(token.size() + 1); - const size_t remaining(std::distance(ptr, stop)); - if(remaining < terminated_size) - return; - - char *const dest(ptr); - ptr += strlcpy(dest, token.data(), terminated_size); - closure(string_view(dest, token.size())); - }); - - return std::distance(data(buf), ptr); -} - -size_t -ircd::tokens(const string_view &str, - const char &sep, - const size_t &limit, - const token_view &closure) -{ - const char ssep[2] { sep, '\0' }; - return tokens(str, ssep, limit, closure); -} - -size_t -ircd::tokens(const string_view &str, - const char *const &sep, - const size_t &limit, - const token_view &closure) -{ - using type = string_view; - using iter = typename type::const_iterator; - using delim = boost::char_separator; - - const delim d(sep); - const boost::tokenizer view(str, d); - - size_t i(0); - for(auto it(begin(view)); i < limit && it != end(view); ++it, i++) - closure(*it); - - return i; -} - -void -ircd::tokens(const string_view &str, - const char &sep, - const token_view &closure) -{ - const char ssep[2] { sep, '\0' }; - tokens(str, ssep, closure); -} - -void -ircd::tokens(const string_view &str, - const char *const &sep, - const token_view &closure) -{ - using type = string_view; - using iter = typename type::const_iterator; - using delim = boost::char_separator; - - const delim d(sep); - const boost::tokenizer view(str, d); - std::for_each(begin(view), end(view), closure); -} - -/////////////////////////////////////////////////////////////////////////////// -// -// ircd/lex_cast.h -// - namespace ircd { /// The static lex_cast ring buffers are each LEX_CAST_BUFSIZE bytes; diff --git a/ircd/tokens.cc b/ircd/tokens.cc new file mode 100644 index 000000000..c6511a2f7 --- /dev/null +++ b/ircd/tokens.cc @@ -0,0 +1,293 @@ +// Matrix Construct +// +// Copyright (C) Matrix Construct Developers, Authors & Contributors +// Copyright (C) 2016-2018 Jason Volk +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice is present in all copies. The +// full license for this software is available in the LICENSE file. + +#include + +ircd::string_view +ircd::tokens_before(const string_view &str, + const char &sep, + const size_t &i) +{ + const char ssep[2] { sep, '\0' }; + return tokens_before(str, ssep, i); +} + +ircd::string_view +ircd::tokens_before(const string_view &str, + const char *const &sep, + const size_t &i) +{ + using type = string_view; + using iter = typename type::const_iterator; + using delim = boost::char_separator; + + const delim d{sep}; + const boost::tokenizer view + { + str, d + }; + + string_view ret; + auto it(begin(view)); + for(size_t j(0); it != end(view) && j < i; ++it, j++) + ret = { begin(view)->data(), it->data() + it->size() }; + + return ret; +} + +ircd::string_view +ircd::tokens_after(const string_view &str, + const char &sep, + const size_t &i) +{ + const char ssep[2] { sep, '\0' }; + return tokens_after(str, ssep, i); +} + +ircd::string_view +ircd::tokens_after(const string_view &str, + const char *const &sep, + const size_t &i) +{ + using type = string_view; + using iter = typename type::const_iterator; + using delim = boost::char_separator; + + const delim d{sep}; + const boost::tokenizer view + { + str, d + }; + + auto it(begin(view)); + for(size_t j(0); it != end(view); ++it, j++) + if(j > i) + return string_view{it->data(), str.data() + str.size()}; + + return {}; +} + +ircd::string_view +ircd::token_first(const string_view &str, + const char &sep) +{ + const char ssep[2] { sep, '\0' }; + return token(str, ssep, 0); +} + +ircd::string_view +ircd::token_first(const string_view &str, + const char *const &sep) +{ + return token(str, sep, 0); +} + +ircd::string_view +ircd::token_last(const string_view &str, + const char &sep) +{ + const char ssep[2] { sep, '\0' }; + return token_last(str, ssep); +} + +ircd::string_view +ircd::token_last(const string_view &str, + const char *const &sep) +{ + using type = string_view; + using iter = typename type::const_iterator; + using delim = boost::char_separator; + + const delim d{sep}; + const boost::tokenizer view + { + str, d + }; + + auto it(begin(view)); + if(it == end(view)) + return str.empty()? str : throw std::out_of_range("token out of range"); + + string_view ret(*it); + for(++it; it != end(view); ++it) + ret = *it; + + return ret; +} + +ircd::string_view +ircd::token(const string_view &str, + const char &sep, + const size_t &i, + const string_view &def) +{ + const char ssep[2] { sep, '\0' }; + return token(str, ssep, i, def); +} + +ircd::string_view +ircd::token(const string_view &str, + const char *const &sep, + const size_t &i, + const string_view &def) +try +{ + return token(str, sep, i); +} +catch(const std::out_of_range &) +{ + return def; +} + +ircd::string_view +ircd::token(const string_view &str, + const char &sep, + const size_t &i) +{ + const char ssep[2] { sep, '\0' }; + return token(str, ssep, i); +} + +ircd::string_view +ircd::token(const string_view &str, + const char *const &sep, + const size_t &i) +{ + using type = string_view; + using iter = typename type::const_iterator; + using delim = boost::char_separator; + + const delim d{sep}; + const boost::tokenizer view + { + str, d + }; + + return *at(begin(view), end(view), i); +} + +size_t +ircd::token_count(const string_view &str, + const char &sep) +{ + const char ssep[2] { sep, '\0' }; + return token_count(str, ssep); +} + +size_t +ircd::token_count(const string_view &str, + const char *const &sep) +{ + using type = string_view; + using iter = typename type::const_iterator; + using delim = boost::char_separator; + + const delim d{sep}; + const boost::tokenizer view + { + str, d + }; + + return std::distance(begin(view), end(view)); +} + +size_t +ircd::tokens(const string_view &str, + const char &sep, + const mutable_buffer &buf, + const token_view &closure) +{ + const char ssep[2] { sep, '\0' }; + return tokens(str, ssep, buf, closure); +} + +size_t +ircd::tokens(const string_view &str, + const char *const &sep, + const mutable_buffer &buf, + const token_view &closure) +{ + char *ptr(data(buf)); + char *const stop(data(buf) + size(buf)); + tokens(str, sep, [&closure, &ptr, &stop] + (const string_view &token) + { + const size_t terminated_size(token.size() + 1); + const size_t remaining(std::distance(ptr, stop)); + if(remaining < terminated_size) + return; + + char *const dest(ptr); + ptr += strlcpy(dest, token.data(), terminated_size); + closure(string_view(dest, token.size())); + }); + + return std::distance(data(buf), ptr); +} + +size_t +ircd::tokens(const string_view &str, + const char &sep, + const size_t &limit, + const token_view &closure) +{ + const char ssep[2] { sep, '\0' }; + return tokens(str, ssep, limit, closure); +} + +size_t +ircd::tokens(const string_view &str, + const char *const &sep, + const size_t &limit, + const token_view &closure) +{ + using type = string_view; + using iter = typename type::const_iterator; + using delim = boost::char_separator; + + const delim d{sep}; + const boost::tokenizer view + { + str, d + }; + + size_t i(0); + for(auto it(begin(view)); i < limit && it != end(view); ++it, i++) + closure(*it); + + return i; +} + +void +ircd::tokens(const string_view &str, + const char &sep, + const token_view &closure) +{ + const char ssep[2] { sep, '\0' }; + tokens(str, ssep, closure); +} + +void +ircd::tokens(const string_view &str, + const char *const &sep, + const token_view &closure) +{ + using type = string_view; + using iter = typename type::const_iterator; + using delim = boost::char_separator; + + const delim d{sep}; + const boost::tokenizer view + { + str, d + }; + + std::for_each(begin(view), end(view), closure); +}