MirrorHub/construct
0
0
Fork 0
mirror of https://github.com/matrix-construct/construct synced 2024-06-25 05:18:23 +02:00
Code Issues Releases Wiki Activity

modules/console: Add gpt tokenize diagnostic cmd.

Browse source
This commit is contained in:
Jason Volk 2021-08-24 17:32:33 -07:00
parent 3e6fcf3a47
commit 43a02bb598
1 changed files with 25 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
modules/console.cc
View file

@ -17413,6 +17413,31 @@ console_cmd__gpt__token(opt &out, const string_view &line)
return true;
}
bool
console_cmd__gpt__tokenize(opt &out, const string_view &line)
{
const auto text
{
line
};
u16 buf[1024];
const auto tokens
{
gpt::vocab::tokenize(buf, text)
};
uint i(0);
char dbuf[512];
for(const auto &token : tokens)
out
<< std::setw(3) << (i++) << ": "
<< gpt::vocab::debug(dbuf, token)
<< std::endl;
return true;
}
bool
console_cmd__gpt__data(opt &out, const string_view &line)
{