mirror of
https://github.com/matrix-construct/construct
synced 2024-09-26 18:38:52 +02:00
modules/console: Add gpt tokenize diagnostic cmd.
This commit is contained in:
parent
3e6fcf3a47
commit
43a02bb598
1 changed files with 25 additions and 0 deletions
|
@ -17413,6 +17413,31 @@ console_cmd__gpt__token(opt &out, const string_view &line)
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool
|
||||||
|
console_cmd__gpt__tokenize(opt &out, const string_view &line)
|
||||||
|
{
|
||||||
|
const auto text
|
||||||
|
{
|
||||||
|
line
|
||||||
|
};
|
||||||
|
|
||||||
|
u16 buf[1024];
|
||||||
|
const auto tokens
|
||||||
|
{
|
||||||
|
gpt::vocab::tokenize(buf, text)
|
||||||
|
};
|
||||||
|
|
||||||
|
uint i(0);
|
||||||
|
char dbuf[512];
|
||||||
|
for(const auto &token : tokens)
|
||||||
|
out
|
||||||
|
<< std::setw(3) << (i++) << ": "
|
||||||
|
<< gpt::vocab::debug(dbuf, token)
|
||||||
|
<< std::endl;
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
bool
|
bool
|
||||||
console_cmd__gpt__data(opt &out, const string_view &line)
|
console_cmd__gpt__data(opt &out, const string_view &line)
|
||||||
{
|
{
|
||||||
|
|
Loading…
Reference in a new issue