From 782379aeb43fab6f72c008c4b9d7bef02cca0f94 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 24 Aug 2021 17:44:38 -0700 Subject: [PATCH] ircd::gpt::vocab: Simplify UTF-8 length gauge. --- ircd/gpt_vocab.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ircd/gpt_vocab.cc b/ircd/gpt_vocab.cc index 6833148c1..8bc7744a0 100644 --- a/ircd/gpt_vocab.cc +++ b/ircd/gpt_vocab.cc @@ -678,9 +678,9 @@ ircd::gpt::vocab::bpe_prepare(u8x16 (&out)[16][2], simd::strlen(in) }; - const u32x16 cplen + const u8x16 cplen ( - utf8::length(utf8::decode(in)) + utf8::length(in) ); u32x16 idx;