mirror of
https://github.com/matrix-construct/construct
synced 2024-11-12 04:51:08 +01:00
79 lines
1.8 KiB
C++
79 lines
1.8 KiB
C++
// Matrix Construct
|
|
//
|
|
// Copyright (C) Matrix Construct Developers, Authors & Contributors
|
|
// Copyright (C) 2016-2021 Jason Volk <jason@zemos.net>
|
|
//
|
|
// Permission to use, copy, modify, and/or distribute this software for any
|
|
// purpose with or without fee is hereby granted, provided that the above
|
|
// copyright notice and this permission notice is present in all copies. The
|
|
// full license for this software is available in the LICENSE file.
|
|
|
|
#pragma once
|
|
#define HAVE_IRCD_GPT_TOKEN_H
|
|
|
|
namespace ircd::gpt
|
|
{
|
|
struct token;
|
|
}
|
|
|
|
/// Token is just a 16-bit index into the vocabulary. This lightweight wrapper
|
|
/// convenience constructs from a string lookup or from a u16 directly.
|
|
class ircd::gpt::token
|
|
{
|
|
uint16_t val;
|
|
|
|
public:
|
|
operator const uint16_t &() const;
|
|
operator uint16_t &();
|
|
|
|
operator string_view() const;
|
|
|
|
token(const_buffer &buf) noexcept;
|
|
token(const string_view &, const bool prefix_space = false);
|
|
token(const uint16_t &) noexcept;
|
|
};
|
|
|
|
static_assert(sizeof(ircd::gpt::token) == sizeof(uint16_t));
|
|
static_assert(std::is_standard_layout<ircd::gpt::token>::value);
|
|
|
|
/// Direct construction; no lookup
|
|
inline
|
|
ircd::gpt::token::token(const uint16_t &val)
|
|
noexcept
|
|
:val{val}
|
|
{}
|
|
|
|
/// Must resolve to one token or error thrown.
|
|
/// prefix_space=true internally prepends space for potentially better token.
|
|
inline
|
|
ircd::gpt::token::token(const string_view &str,
|
|
const bool prefix_space)
|
|
:val{vocab::tokenize(str, prefix_space)}
|
|
{}
|
|
|
|
/// Consumes input for one token off front of buf
|
|
inline
|
|
ircd::gpt::token::token(const_buffer &buf)
|
|
noexcept
|
|
:val{vocab::tokenize(buf)}
|
|
{}
|
|
|
|
inline ircd::gpt::token::operator
|
|
string_view()
|
|
const
|
|
{
|
|
return vocab::token[val];
|
|
}
|
|
|
|
inline ircd::gpt::token::operator
|
|
uint16_t &()
|
|
{
|
|
return val;
|
|
}
|
|
|
|
inline ircd::gpt::token::operator
|
|
const uint16_t &()
|
|
const
|
|
{
|
|
return val;
|
|
}
|