mirror of
https://github.com/matrix-construct/construct
synced 2024-09-26 18:38:52 +02:00
modules/console: Add gpt tokenize diagnostic cmd.
This commit is contained in:
parent
3e6fcf3a47
commit
43a02bb598
1 changed files with 25 additions and 0 deletions
|
@ -17413,6 +17413,31 @@ console_cmd__gpt__token(opt &out, const string_view &line)
|
|||
return true;
|
||||
}
|
||||
|
||||
bool
|
||||
console_cmd__gpt__tokenize(opt &out, const string_view &line)
|
||||
{
|
||||
const auto text
|
||||
{
|
||||
line
|
||||
};
|
||||
|
||||
u16 buf[1024];
|
||||
const auto tokens
|
||||
{
|
||||
gpt::vocab::tokenize(buf, text)
|
||||
};
|
||||
|
||||
uint i(0);
|
||||
char dbuf[512];
|
||||
for(const auto &token : tokens)
|
||||
out
|
||||
<< std::setw(3) << (i++) << ": "
|
||||
<< gpt::vocab::debug(dbuf, token)
|
||||
<< std::endl;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool
|
||||
console_cmd__gpt__data(opt &out, const string_view &line)
|
||||
{
|
||||
|
|
Loading…
Reference in a new issue