maxmustermann/terminal
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

wip

Browse source
This commit is contained in:
Leonard Hecker 2021-07-20 03:58:47 +02:00
parent dbf9343320
commit e7d00cab8e
7 changed files with 67 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

75
src/inc/til/u8u16convert.h
View file

@ -20,15 +20,14 @@ functions MultiByteToWideChar and WideCharToMultiByte.
namespace til // Terminal Implementation Library. Also: "Today I Learned"
{
struct u8accumulator
struct u8state
{
uint32_t buffer{};
uint32_t remaining{};
};
struct u16accumulator
{
uint32_t buffer{};
constexpr void reset() noexcept {
*this = {};
}
};
// Routine Description:
@ -61,7 +60,7 @@ namespace til // Terminal Implementation Library. Also: "Today I Learned"
// - out - reference to the resulting UTF-16 string
// - state - reference to a til::u8state class holding the status of the current partials handling
template<typename Output>
void u8u16(const std::string_view& in, Output& out, u8accumulator& state) noexcept
void u8u16(const std::string_view& in, Output& out, u8state& state) noexcept
{
auto data = in.data();
auto size = in.size();
@ -75,7 +74,7 @@ namespace til // Terminal Implementation Library. Also: "Today I Learned"
{
if (remaining > size)
{
remaining = size;
remaining = gsl::narrow_cast<uint32_t>(size);
}
state.remaining -= remaining;
@ -83,14 +82,15 @@ namespace til // Terminal Implementation Library. Also: "Today I Learned"
do
{
state.buffer <<= 6;
state.buffer |= *data++;
state.buffer |= *data++ & 0x3f;
} while (--remaining);
if (!state.remaining)
{
if (state.buffer < 0x10000)
{
out.append(static_cast<wchar_t>(state.buffer));
const auto buffer = static_cast<wchar_t>(state.buffer);
out.append(&buffer, 1);
}
else
{
@ -104,36 +104,36 @@ namespace til // Terminal Implementation Library. Also: "Today I Learned"
{
auto end = data + size;
size_t count = 1;
size_t have = 1;
// Skip UTF-8 continuation bytes in the form of 0b10xxxxxx.
while ((*--end & 0b11000000) == 0b10000000 && end != data) {
++count;
while ((*--end & 0b11000000) == 0b10000000 && end != data)
{
++have;
}
// A leading UTF-8 byte is either of:
// * 0b110xxxxx
// * 0b1110xxxx
// * 0b11110xxx
if (count != 1)
if (have != 1)
{
DWORD index;
DWORD index = 0;
if (_BitScanReverse(&index, ~*end & 0xff))
{
index -= 24;
if ((index <= 4) & (index > count))
const auto want = 7 - index;
if (want <= 4 && want > have)
{
auto ptr = end;
uint32_t buffer = *ptr++ & (0xff >> (7 - index));
uint32_t buffer = *ptr++ & ((1 << index) - 1);
for (size_t i = 0; i < count; ++i)
for (size_t i = 1; i < have; ++i)
{
buffer <<= 6;
buffer |= *ptr++;
buffer |= *ptr++ & 0x3f;
}
state.buffer = buffer;
state.remaining = index - count;
state.remaining = gsl::narrow_cast<uint32_t>(want - have);
}
}
else
@ -141,11 +141,11 @@ namespace til // Terminal Implementation Library. Also: "Today I Learned"
++end;
}
size = data - end;
size = end - data;
}
}
u16u8({ data, size }, out);
u8u16({ data, size }, out);
}
// Routine Description:
@ -188,6 +188,16 @@ namespace til // Terminal Implementation Library. Also: "Today I Learned"
THROW_LAST_ERROR_IF(lengthOut == 0);
}
struct u16state
{
wchar_t buffer{};
constexpr void reset() noexcept
{
*this = {};
}
};
// Routine Description:
// - Takes a UTF-16 string, complements and/or caches partials, and performs the conversion to UTF-8.
// Arguments:
@ -195,7 +205,7 @@ namespace til // Terminal Implementation Library. Also: "Today I Learned"
// - out - reference to the resulting UTF-8 string
// - state - reference to a til::u16state class holding the status of the current partials handling
template<typename Output>
void u16u8(const std::wstring_view& in, Output& out, u16accumulator& state) noexcept
void u16u8(const std::wstring_view& in, Output& out, u16state& state) noexcept
{
auto data = in.data();
auto size = in.size();
@ -207,14 +217,17 @@ namespace til // Terminal Implementation Library. Also: "Today I Learned"
if (state.buffer)
{
if (*data >= 0xDC00 & *data <= 0xDFFF)
if (*data >= 0xDC00 && *data <= 0xDFFF)
{
const auto buffer = (state.buffer << 10) | (*data - 0xDC00);
const uint32_t high = state.buffer - 0xD800;
const uint32_t low = *data - 0xDC00;
const auto codePoint = ((high << 10) | low) + 0x10000;
char buffer[4];
buffer[0] = 0b11110000 | ((buffer >> 18) & 0x3f);
buffer[1] = 0b10000000 | ((buffer >> 12) & 0x3f);
buffer[2] = 0b10000000 | ((buffer >> 6) & 0x3f);
buffer[3] = 0b10000000 | ((buffer >> 0) & 0x3f);
buffer[0] = 0b11110000 | ((codePoint >> 18) & 0x3f);
buffer[1] = 0b10000000 | ((codePoint >> 12) & 0x3f);
buffer[2] = 0b10000000 | ((codePoint >> 6) & 0x3f);
buffer[3] = 0b10000000 | ((codePoint >> 0) & 0x3f);
out.append(&buffer[0], 4);
++data;
@ -230,7 +243,7 @@ namespace til // Terminal Implementation Library. Also: "Today I Learned"
if (auto end = data + size - 1; *end >= 0xD800 && *end <= 0xDBFF)
{
state.buffer = *end - 0xD800;
state.buffer = *end;
--size;
if (!size)

2
src/renderer/vt/VtSequences.cpp
View file

@ -5,6 +5,8 @@
#include "vtrenderer.hpp"
#include "../../inc/conattrs.hpp"
#include <til/u8u16convert.h>
#pragma hdrstop
using namespace Microsoft::Console::Render;

6
src/renderer/vt/state.cpp
View file

@ -148,7 +148,11 @@ VtEngine::VtEngine(_In_ wil::unique_hfile pipe,
// - S_OK or suitable HRESULT error from either conversion or writing pipe.
[[nodiscard]] HRESULT VtEngine::_WriteTerminalUtf8(const std::wstring_view wstr) noexcept
{
RETURN_IF_FAILED(til::u16u8(wstr, _conversionBuffer));
try
{
til::u16u8(wstr, _conversionBuffer);
}
CATCH_RETURN();
return _Write(_conversionBuffer);
}

2
src/renderer/vt/tracing.cpp
View file

@ -150,7 +150,7 @@ void RenderTracing::TraceInvalidateScroll(const til::point scroll) const
}
void RenderTracing::TraceStartPaint(const bool quickReturn,
const til::pmr::bitmap& invalidMap,
const til::bitmap& invalidMap,
const til::rectangle lastViewport,
const til::point scrollDelt,
const bool cursorMoved,

2
src/renderer/vt/tracing.hpp
View file

@ -39,7 +39,7 @@ namespace Microsoft::Console::VirtualTerminal
void TraceTriggerCircling(const bool newFrame) const;
void TraceInvalidateScroll(const til::point scroll) const;
void TraceStartPaint(const bool quickReturn,
const til::pmr::bitmap& invalidMap,
const til::bitmap& invalidMap,
const til::rectangle lastViewport,
const til::point scrollDelta,
const bool cursorMoved,

2
src/til/ut_til/SPSCTests.cpp
View file

@ -4,6 +4,8 @@
#include "precomp.h"
#include "WexTestClass.h"
#include <til/spsc.h>
using namespace WEX::Common;
using namespace WEX::Logging;
using namespace WEX::TestExecution;

32
src/til/ut_til/u8u16convertTests.cpp
View file

@ -45,8 +45,7 @@ void Utf8Utf16ConvertTests::TestU8ToU16()
};
std::wstring u16Out{};
const HRESULT hRes{ til::u8u16(u8String, u16Out) };
VERIFY_ARE_EQUAL(S_OK, hRes);
til::u8u16(u8String, u16Out);
VERIFY_ARE_EQUAL(u16StringComp, u16Out);
}
@ -74,8 +73,7 @@ void Utf8Utf16ConvertTests::TestU16ToU8()
};
std::string u8Out{};
const HRESULT hRes{ til::u16u8(u16String, u8Out) };
VERIFY_ARE_EQUAL(S_OK, hRes);
til::u16u8(u16String, u8Out);
VERIFY_ARE_EQUAL(u8StringComp, u8Out);
}
@ -117,23 +115,19 @@ void Utf8Utf16ConvertTests::TestU8ToU16Partials()
til::u8state state{};
std::wstring u16Out1{};
const HRESULT hRes1{ til::u8u16(u8String1, u16Out1, state) };
VERIFY_ARE_EQUAL(S_OK, hRes1);
til::u8u16(u8String1, u16Out1, state);
VERIFY_ARE_EQUAL(u16StringComp1, u16Out1);
std::wstring u16Out2{};
const HRESULT hRes2{ til::u8u16(u8String2, u16Out2, state) };
VERIFY_ARE_EQUAL(S_OK, hRes2);
til::u8u16(u8String2, u16Out2, state);
VERIFY_ARE_EQUAL(u16StringComp1, u16Out2);
std::wstring u16Out3{};
const HRESULT hRes3{ til::u8u16(u8String3, u16Out3, state) };
VERIFY_ARE_EQUAL(S_OK, hRes3);
til::u8u16(u8String3, u16Out3, state);
VERIFY_ARE_EQUAL(std::wstring{}, u16Out3);
std::wstring u16Out4{};
const HRESULT hRes4{ til::u8u16(u8String4, u16Out4, state) };
VERIFY_ARE_EQUAL(S_OK, hRes4);
til::u8u16(u8String4, u16Out4, state);
VERIFY_ARE_EQUAL(u16StringComp2, u16Out4);
}
@ -159,13 +153,11 @@ void Utf8Utf16ConvertTests::TestU16ToU8Partials()
til::u16state state{};
std::string u8Out1{};
const HRESULT hRes1{ til::u16u8(u16String1, u8Out1, state) };
VERIFY_ARE_EQUAL(S_OK, hRes1);
til::u16u8(u16String1, u8Out1, state);
VERIFY_ARE_EQUAL(u8StringComp, u8Out1);
std::string u8Out2{};
const HRESULT hRes2{ til::u16u8(u16String2, u8Out2, state) };
VERIFY_ARE_EQUAL(S_OK, hRes2);
til::u16u8(u16String2, u8Out2, state);
VERIFY_ARE_EQUAL(u8StringComp, u8Out2);
}
@ -184,12 +176,12 @@ void Utf8Utf16ConvertTests::TestU8ToU16OneByOne()
til::u8state state{};
std::wstring u16Out1{};
VERIFY_SUCCEEDED(til::u8u16(u8String1_1, u16Out1, state));
til::u8u16(u8String1_1, u16Out1, state);
VERIFY_ARE_EQUAL(L"", u16Out1); // There should be no output for the first three bytes
VERIFY_SUCCEEDED(til::u8u16(u8String1_2, u16Out1, state));
til::u8u16(u8String1_2, u16Out1, state);
VERIFY_ARE_EQUAL(L"", u16Out1);
VERIFY_SUCCEEDED(til::u8u16(u8String1_3, u16Out1, state));
til::u8u16(u8String1_3, u16Out1, state);
VERIFY_ARE_EQUAL(L"", u16Out1);
VERIFY_SUCCEEDED(til::u8u16(u8String1_4, u16Out1, state));
til::u8u16(u8String1_4, u16Out1, state);
VERIFY_ARE_EQUAL(u16StringComp1, u16Out1);
}