wip

src/buffer/out/CharRow.cpp

@@ -17,8 +17,8 @@
 // Note: will through if unable to allocate char/attribute buffers
 #pragma warning(push)
 #pragma warning(disable : 26447) // small_vector's constructor says it can throw but it should not given how we use it.  This suppresses this error for the AuditMode build.
-CharRow::CharRow(size_t rowWidth, ROW* const pParent) noexcept :
+CharRow::CharRow(CharRowCell* buffer, size_t rowWidth, ROW* const pParent) noexcept :
-    _data(rowWidth, value_type()),
+    _data(buffer, rowWidth),
     _pParent{ FAIL_FAST_IF_NULL(pParent) }
 {
 }
@@ -53,38 +53,9 @@ void CharRow::Reset() noexcept
 // - resizes the width of the CharRowBase
 // Arguments:
 // - newSize - the new width of the character and attributes rows
-// Return Value:
+void CharRow::Resize(CharRowCell* buffer, const size_t newSize) noexcept
-// - S_OK on success, otherwise relevant error code
-[[nodiscard]] HRESULT CharRow::Resize(const size_t newSize) noexcept
 {
-    try
+    _data = {buffer, newSize};
-    {
-        const value_type insertVals;
-        _data.resize(newSize, insertVals);
-    }
-    CATCH_RETURN();
-
-    return S_OK;
-}
-
-typename CharRow::iterator CharRow::begin() noexcept
-{
-    return _data.begin();
-}
-
-typename CharRow::const_iterator CharRow::cbegin() const noexcept
-{
-    return _data.cbegin();
-}
-
-typename CharRow::iterator CharRow::end() noexcept
-{
-    return _data.end();
-}
-
-typename CharRow::const_iterator CharRow::cend() const noexcept
-{
-    return _data.cend();
 }
 
 // Routine Description:
@@ -95,12 +66,16 @@ typename CharRow::const_iterator CharRow::cend() const noexcept
 // - The calculated left boundary of the internal string.
 size_t CharRow::MeasureLeft() const noexcept
 {
-    const_iterator it = _data.cbegin();
+    const auto beg = _data.begin();
-    while (it != _data.cend() && it->IsSpace())
+    const auto end = _data.end();
+
+    auto it = beg;
+    while (it != end && it->IsSpace())
     {
         ++it;
     }
-    return it - _data.cbegin();
+
+    return it - beg;
 }
 
 // Routine Description:
@@ -111,17 +86,21 @@ size_t CharRow::MeasureLeft() const noexcept
 // - The calculated right boundary of the internal string.
 size_t CharRow::MeasureRight() const
 {
-    const_reverse_iterator it = _data.crbegin();
+    const auto beg = _data.rbegin();
-    while (it != _data.crend() && it->IsSpace())
+    const auto end = _data.rend();
+
+    auto it = beg;
+    while (it != end && it->IsSpace())
     {
         ++it;
     }
-    return _data.crend() - it;
+
+    return end - it;
 }
 
 void CharRow::ClearCell(const size_t column)
 {
-    _data.at(column).Reset();
+    _data[column].Reset();
 }
 
 // Routine Description:
@@ -132,7 +111,7 @@ void CharRow::ClearCell(const size_t column)
 // - True if there is valid text in this row. False otherwise.
 bool CharRow::ContainsText() const noexcept
 {
-    for (const value_type& cell : _data)
+    for (const auto& cell : _data)
     {
         if (!cell.IsSpace())
         {
@@ -151,7 +130,7 @@ bool CharRow::ContainsText() const noexcept
 // Note: will throw exception if column is out of bounds
 const DbcsAttribute& CharRow::DbcsAttrAt(const size_t column) const
 {
-    return _data.at(column).DbcsAttr();
+    return _data[column].DbcsAttr();
 }
 
 // Routine Description:
@@ -163,7 +142,7 @@ const DbcsAttribute& CharRow::DbcsAttrAt(const size_t column) const
 // Note: will throw exception if column is out of bounds
 DbcsAttribute& CharRow::DbcsAttrAt(const size_t column)
 {
-    return _data.at(column).DbcsAttr();
+    return _data[column].DbcsAttr();
 }
 
 // Routine Description:
@@ -175,7 +154,7 @@ DbcsAttribute& CharRow::DbcsAttrAt(const size_t column)
 // Note: will throw exception if column is out of bounds
 void CharRow::ClearGlyph(const size_t column)
 {
-    _data.at(column).EraseChars();
+    _data[column].EraseChars();
 }
 
 // Routine Description:

src/buffer/out/CharRow.hpp

@@ -49,15 +49,12 @@ class CharRow final
 public:
     using glyph_type = typename wchar_t;
     using value_type = typename CharRowCell;
-    using iterator = typename boost::container::small_vector_base<value_type>::iterator;
-    using const_iterator = typename boost::container::small_vector_base<value_type>::const_iterator;
-    using const_reverse_iterator = typename boost::container::small_vector_base<value_type>::const_reverse_iterator;
     using reference = typename CharRowCellReference;
 
-    CharRow(size_t rowWidth, ROW* const pParent) noexcept;
+    CharRow(CharRowCell* buffer, size_t rowWidth, ROW* const pParent) noexcept;
 
     size_t size() const noexcept;
-    [[nodiscard]] HRESULT Resize(const size_t newSize) noexcept;
+    void Resize(CharRowCell* buffer, const size_t newSize) noexcept;
     size_t MeasureLeft() const noexcept;
     size_t MeasureRight() const;
     bool ContainsText() const noexcept;
@@ -71,14 +68,25 @@ public:
     const reference GlyphAt(const size_t column) const;
     reference GlyphAt(const size_t column);
 
-    // iterators
+    auto begin() noexcept
-    iterator begin() noexcept;
+    {
-    const_iterator cbegin() const noexcept;
+        return _data.begin();
-    const_iterator begin() const noexcept { return cbegin(); }
+    }
 
-    iterator end() noexcept;
+    auto begin() const noexcept
-    const_iterator cend() const noexcept;
+    {
-    const_iterator end() const noexcept { return cend(); }
+        return _data.begin();
+    }
+
+    auto end() noexcept
+    {
+        return _data.end();
+    }
+
+    auto end() const noexcept
+    {
+        return _data.end();
+    }
 
     UnicodeStorage& GetUnicodeStorage() noexcept;
     const UnicodeStorage& GetUnicodeStorage() const noexcept;
@@ -96,20 +104,21 @@ private:
 
 protected:
     // storage for glyph data and dbcs attributes
-    boost::container::small_vector<value_type, 120> _data;
+    gsl::span<CharRowCell> _data;
 
     // ROW that this CharRow belongs to
     ROW* _pParent;
 };
 
-template<typename InputIt1, typename InputIt2>
+template<typename InputIt1, typename InputIt2, typename OutputIt>
-void OverwriteColumns(InputIt1 startChars, InputIt1 endChars, InputIt2 startAttrs, CharRow::iterator outIt)
+void OverwriteColumns(InputIt1 startChars, InputIt1 endChars, InputIt2 startAttrs, OutputIt outIt)
 {
-    std::transform(startChars,
+    std::transform(
-                   endChars,
+        startChars,
-                   startAttrs,
+        endChars,
-                   outIt,
+        startAttrs,
-                   [](const wchar_t wch, const DbcsAttribute attr) {
+        outIt,
-                       return CharRow::value_type{ wch, attr };
+        [](const wchar_t wch, const DbcsAttribute attr) {
-                   });
+            return CharRow::value_type{ wch, attr };
+        });
 }

src/buffer/out/CharRowCellReference.cpp

@@ -41,7 +41,7 @@ CharRowCellReference::operator std::wstring_view() const
 // - ref to the CharRowCell
 CharRowCell& CharRowCellReference::_cellData()
 {
-    return _parent._data.at(_index);
+    return _parent._data[_index];
 }
 
 // Routine Description:
@@ -50,7 +50,7 @@ CharRowCell& CharRowCellReference::_cellData()
 // - ref to the CharRowCell
 const CharRowCell& CharRowCellReference::_cellData() const
 {
-    return _parent._data.at(_index);
+    return _parent._data[_index];
 }
 
 // Routine Description:

src/buffer/out/Row.cpp

@@ -16,10 +16,9 @@
 // - pParent - the text buffer that this row belongs to
 // Return Value:
 // - constructed object
-ROW::ROW(const SHORT rowId, const unsigned short rowWidth, const TextAttribute fillAttribute, TextBuffer* const pParent) :
+ROW::ROW(const SHORT rowId, CharRowCell* buffer, const unsigned short rowWidth, const TextAttribute& fillAttribute, TextBuffer* const pParent) :
     _id{ rowId },
-    _rowWidth{ rowWidth },
+    _charRow{ buffer, rowWidth, this },
-    _charRow{ rowWidth, this },
     _attrRow{ rowWidth, fillAttribute },
     _lineRendition{ LineRendition::SingleWidth },
     _wrapForced{ false },
@@ -34,7 +33,7 @@ ROW::ROW(const SHORT rowId, const unsigned short rowWidth, const TextAttribute f
 // - Attr - The default attribute (color) to fill
 // Return Value:
 // - <none>
-bool ROW::Reset(const TextAttribute Attr)
+bool ROW::Reset(const TextAttribute& Attr)
 {
     _lineRendition = LineRendition::SingleWidth;
     _wrapForced = false;
@@ -52,26 +51,6 @@ bool ROW::Reset(const TextAttribute Attr)
     return true;
 }
 
-// Routine Description:
-// - resizes ROW to new width
-// Arguments:
-// - width - the new width, in cells
-// Return Value:
-// - S_OK if successful, otherwise relevant error
-[[nodiscard]] HRESULT ROW::Resize(const unsigned short width)
-{
-    RETURN_IF_FAILED(_charRow.Resize(width));
-    try
-    {
-        _attrRow.Resize(width);
-    }
-    CATCH_RETURN();
-
-    _rowWidth = width;
-
-    return S_OK;
-}
-
 // Routine Description:
 // - clears char data in column in row
 // Arguments:

src/buffer/out/Row.hpp

@@ -32,9 +32,9 @@ class TextBuffer;
 class ROW final
 {
 public:
-    ROW(const SHORT rowId, const unsigned short rowWidth, const TextAttribute fillAttribute, TextBuffer* const pParent);
+    ROW(const SHORT rowId, CharRowCell* buffer, const unsigned short rowWidth, const TextAttribute& fillAttribute, TextBuffer* const pParent);
 
-    size_t size() const noexcept { return _rowWidth; }
+    size_t size() const noexcept { return _charRow.size(); }
 
     void SetWrapForced(const bool wrap) noexcept { _wrapForced = wrap; }
     bool WasWrapForced() const noexcept { return _wrapForced; }
@@ -54,8 +54,7 @@ public:
     SHORT GetId() const noexcept { return _id; }
     void SetId(const SHORT id) noexcept { _id = id; }
 
-    bool Reset(const TextAttribute Attr);
+    bool Reset(const TextAttribute& Attr);
-    [[nodiscard]] HRESULT Resize(const unsigned short width);
 
     void ClearColumn(const size_t column);
     std::wstring GetText() const { return _charRow.GetText(); }
@@ -74,13 +73,12 @@ private:
     CharRow _charRow;
     ATTR_ROW _attrRow;
     LineRendition _lineRendition;
+    TextBuffer* _pParent; // non ownership pointer
     SHORT _id;
-    unsigned short _rowWidth;
     // Occurs when the user runs out of text in a given row and we're forced to wrap the cursor to the next line
     bool _wrapForced;
     // Occurs when the user runs out of text to support a double byte character and we're forced to the next line
     bool _doubleBytePadded;
-    TextBuffer* _pParent; // non ownership pointer
 };
 
 #ifdef UNIT_TESTING

src/buffer/out/UnicodeStorage.cpp

@@ -47,7 +47,7 @@ void UnicodeStorage::Erase(const key_type key) noexcept
 // - rowMap - A map of the old row IDs to the new row IDs.
 // - width - The width of the new row. Remove any items that are beyond the row width.
 //         - Use nullopt if we're not resizing the width of the row, just renumbering the rows.
-void UnicodeStorage::Remap(const std::unordered_map<SHORT, SHORT>& rowMap, const std::optional<SHORT> width)
+void UnicodeStorage::Remap(const std::unordered_map<SHORT, SHORT>& rowMap, SHORT width)
 {
     // Make a temporary map to hold all the new row positioning
     std::unordered_map<key_type, mapped_type> newMap;
@@ -58,18 +58,10 @@ void UnicodeStorage::Remap(const std::unordered_map<SHORT, SHORT>& rowMap, const
         // Extract the old coordinate position
         const auto oldCoord = pair.first;
 
-        // Only try to short-circuit based on width if we were told it changed
+        // If the column index is at/beyond the row width, don't bother copying it to the new map.
-        // by being given a new width value.
+        if (oldCoord.X >= width)
-        if (width.has_value())
         {
-            // Get the column ID
+            continue;
-            const auto oldColId = oldCoord.X;
-
-            // If the column index is at/beyond the row width, don't bother copying it to the new map.
-            if (oldColId >= width.value())
-            {
-                continue;
-            }
         }
 
         // Get the row ID from the position as that's what we need to remap

src/buffer/out/UnicodeStorage.hpp

@@ -55,7 +55,7 @@ public:
 
     void Erase(const key_type key) noexcept;
 
-    void Remap(const std::unordered_map<SHORT, SHORT>& rowMap, const std::optional<SHORT> width);
+    void Remap(const std::unordered_map<SHORT, SHORT>& rowMap, SHORT width);
 
 private:
     std::unordered_map<key_type, mapped_type> _map;

src/buffer/out/textBuffer.cpp

@@ -31,26 +31,32 @@ TextBuffer::TextBuffer(const COORD screenBufferSize,
                        const TextAttribute defaultAttributes,
                        const UINT cursorSize,
                        Microsoft::Console::Render::IRenderTarget& renderTarget) :
-    _firstRow{ 0 },
     _currentAttributes{ defaultAttributes },
     _cursor{ cursorSize, *this },
-    _storage{},
+    _renderTarget{ renderTarget }
-    _unicodeStorage{},
-    _renderTarget{ renderTarget },
-    _size{},
-    _currentHyperlinkId{ 1 },
-    _currentPatternId{ 0 }
 {
-    // initialize ROWs
+    const auto dx = static_cast<size_t>(screenBufferSize.X);
+    const auto dy = static_cast<size_t>(screenBufferSize.Y);
+    auto buffer = static_cast<CharRowCell*>(VirtualAlloc(nullptr, dx * dy * sizeof(CharRowCell), MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE));
+    THROW_IF_NULL_ALLOC(buffer);
+
+    _charBuffer = buffer;
+
     _storage.reserve(static_cast<size_t>(screenBufferSize.Y));
-    for (size_t i = 0; i < static_cast<size_t>(screenBufferSize.Y); ++i)
+    for (SHORT i = 0; i < screenBufferSize.Y; ++i)
     {
-        _storage.emplace_back(static_cast<SHORT>(i), screenBufferSize.X, _currentAttributes, this);
+        _storage.emplace_back(i, buffer, screenBufferSize.X, _currentAttributes, this);
+        buffer += dx;
     }
 
     _UpdateSize();
 }
 
+TextBuffer::~TextBuffer()
+{
+    VirtualFree(_charBuffer, 0, MEM_RELEASE);
+}
+
 // Routine Description:
 // - Copies properties from another text buffer into this one.
 // - This is primarily to copy properties that would otherwise not be specified during CreateInstance
@@ -83,11 +89,9 @@ UINT TextBuffer::TotalRowCount() const noexcept
 // - const reference to the requested row. Asserts if out of bounds.
 const ROW& TextBuffer::GetRowByOffset(const size_t index) const
 {
-    const size_t totalRows = TotalRowCount();
-
     // Rows are stored circularly, so the index you ask for is offset by the start position and mod the total of rows.
-    const size_t offsetIndex = (_firstRow + index) % totalRows;
+    const size_t offsetIndex = (_firstRow + index) % _storage.size();
-    return _storage.at(offsetIndex);
+    return _storage[offsetIndex];
 }
 
 // Routine Description:
@@ -99,11 +103,9 @@ const ROW& TextBuffer::GetRowByOffset(const size_t index) const
 // - reference to the requested row. Asserts if out of bounds.
 ROW& TextBuffer::GetRowByOffset(const size_t index)
 {
-    const size_t totalRows = TotalRowCount();
-
     // Rows are stored circularly, so the index you ask for is offset by the start position and mod the total of rows.
-    const size_t offsetIndex = (_firstRow + index) % totalRows;
+    const size_t offsetIndex = (_firstRow + index) % _storage.size();
-    return _storage.at(offsetIndex);
+    return _storage[offsetIndex];
 }
 
 // Routine Description:
@@ -400,7 +402,7 @@ OutputCellIterator TextBuffer::WriteLine(const OutputCellIterator givenIt,
 //Return Value:
 // - true if we successfully inserted the character
 // - false otherwise (out of memory)
-bool TextBuffer::InsertCharacter(const std::wstring_view chars,
+bool TextBuffer::InsertCharacter(const std::wstring_view& chars,
                                  const DbcsAttribute dbcsAttribute,
                                  const TextAttribute attr)
 {
@@ -779,7 +781,7 @@ void TextBuffer::ScrollRows(const SHORT firstRow, const SHORT size, const SHORT
 
     // Renumber the IDs now that we've rearranged where the rows sit within the buffer.
     // Refreshing should also delegate to the UnicodeStorage to re-key all the stored unicode sequences (where applicable).
-    _RefreshRowIDs(std::nullopt);
+    _RefreshRowIDs(_size.Width());
 }
 
 Cursor& TextBuffer::GetCursor() noexcept
@@ -897,6 +899,11 @@ void TextBuffer::Reset()
 {
     RETURN_HR_IF(E_INVALIDARG, newSize.X < 0 || newSize.Y < 0);
 
+    const auto dx = static_cast<size_t>(newSize.X);
+    const auto dy = static_cast<size_t>(newSize.Y);
+    auto buffer = static_cast<CharRowCell*>(VirtualAlloc(nullptr, dx * dy * sizeof(CharRowCell), MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE));
+    RETURN_IF_NULL_ALLOC(buffer);
+
     try
     {
         const auto currentSize = GetSize().Dimensions();
@@ -910,12 +917,7 @@ void TextBuffer::Reset()
         const SHORT TopRowIndex = (GetFirstRowIndex() + TopRow) % currentSize.Y;
 
         // rotate rows until the top row is at index 0
-        for (int i = 0; i < TopRowIndex; i++)
+        std::rotate(_storage.begin(), _storage.begin() + TopRowIndex, _storage.end());
-        {
-            _storage.emplace_back(std::move(_storage.front()));
-            _storage.erase(_storage.begin());
-        }
-
         _SetFirstRowIndex(0);
 
         // realloc in the Y direction
@@ -927,19 +929,26 @@ void TextBuffer::Reset()
         // add rows if we're growing
         while (_storage.size() < static_cast<size_t>(newSize.Y))
         {
-            _storage.emplace_back(static_cast<short>(_storage.size()), newSize.X, attributes, this);
+            _storage.emplace_back(static_cast<short>(_storage.size()), nullptr, newSize.X, attributes, this);
         }
 
         // Now that we've tampered with the row placement, refresh all the row IDs.
         // Also take advantage of the row ID refresh loop to resize the rows in the X dimension
         // and cleanup the UnicodeStorage characters that might fall outside the resized buffer.
         _RefreshRowIDs(newSize.X);
+        _RefreshRowWidth(buffer, newSize.X);
 
         // Update the cached size value
         _UpdateSize();
     }
-    CATCH_RETURN();
+    catch (...)
+    {
+        VirtualFree(buffer, 0, MEM_RELEASE);
+        RETURN_CAUGHT_EXCEPTION();
+    };
 
+    VirtualFree(_charBuffer, 0, MEM_RELEASE);
+    _charBuffer = buffer;
     return S_OK;
 }
 
@@ -962,7 +971,7 @@ UnicodeStorage& TextBuffer::GetUnicodeStorage() noexcept
 //   any high unicode (UnicodeStorage) runs while we're already looping through the rows.
 // Arguments:
 // - newRowWidth - Optional new value for the row width.
-void TextBuffer::_RefreshRowIDs(std::optional<SHORT> newRowWidth)
+void TextBuffer::_RefreshRowIDs(SHORT width)
 {
     std::unordered_map<SHORT, SHORT> rowMap;
     SHORT i = 0;
@@ -976,17 +985,19 @@ void TextBuffer::_RefreshRowIDs(std::optional<SHORT> newRowWidth)
 
         // Also update the char row parent pointers as they can get shuffled up in the rotates.
         it.GetCharRow().UpdateParent(&it);
-
-        // Resize the rows in the X dimension if we have a new width
-        if (newRowWidth.has_value())
-        {
-            // Realloc in the X direction
-            THROW_IF_FAILED(it.Resize(newRowWidth.value()));
-        }
     }
 
     // Give the new mapping to Unicode Storage
-    _unicodeStorage.Remap(rowMap, newRowWidth);
+    _unicodeStorage.Remap(rowMap, width);
+}
+
+void TextBuffer::_RefreshRowWidth(CharRowCell *data, size_t width) noexcept
+{
+    for (auto& it : _storage)
+    {
+        it.GetCharRow().Resize(data, width);
+        data += width;
+    }
 }
 
 void TextBuffer::_NotifyPaint(const Viewport& viewport) const
@@ -1045,7 +1056,7 @@ Microsoft::Console::Render::IRenderTarget& TextBuffer::GetRenderTarget() noexcep
 // - wordDelimiters: the delimiters defined as a part of the DelimiterClass::DelimiterChar
 // Return Value:
 // - the delimiter class for the given char
-const DelimiterClass TextBuffer::_GetDelimiterClassAt(const COORD pos, const std::wstring_view wordDelimiters) const
+const DelimiterClass TextBuffer::_GetDelimiterClassAt(const COORD pos, const std::wstring_view& wordDelimiters) const
 {
     return GetRowByOffset(pos.Y).GetCharRow().DelimiterClassAt(pos.X, wordDelimiters);
 }
@@ -1060,7 +1071,7 @@ const DelimiterClass TextBuffer::_GetDelimiterClassAt(const COORD pos, const std
 //                        (or a row boundary is encountered)
 // Return Value:
 // - The COORD for the first character on the "word" (inclusive)
-const COORD TextBuffer::GetWordStart(const COORD target, const std::wstring_view wordDelimiters, bool accessibilityMode) const
+const COORD TextBuffer::GetWordStart(const COORD target, const std::wstring_view& wordDelimiters, bool accessibilityMode) const
 {
     // Consider a buffer with this text in it:
     // "  word   other  "
@@ -1105,7 +1116,7 @@ const COORD TextBuffer::GetWordStart(const COORD target, const std::wstring_view
 // - wordDelimiters - what characters are we considering for the separation of words
 // Return Value:
 // - The COORD for the first character on the current/previous READABLE "word" (inclusive)
-const COORD TextBuffer::_GetWordStartForAccessibility(const COORD target, const std::wstring_view wordDelimiters) const
+const COORD TextBuffer::_GetWordStartForAccessibility(const COORD target, const std::wstring_view& wordDelimiters) const
 {
     COORD result = target;
     const auto bufferSize = GetSize();
@@ -1150,7 +1161,7 @@ const COORD TextBuffer::_GetWordStartForAccessibility(const COORD target, const
 // - wordDelimiters - what characters are we considering for the separation of words
 // Return Value:
 // - The COORD for the first character on the current word or delimiter run (stopped by the left margin)
-const COORD TextBuffer::_GetWordStartForSelection(const COORD target, const std::wstring_view wordDelimiters) const
+const COORD TextBuffer::_GetWordStartForSelection(const COORD target, const std::wstring_view& wordDelimiters) const
 {
     COORD result = target;
     const auto bufferSize = GetSize();
@@ -1182,7 +1193,7 @@ const COORD TextBuffer::_GetWordStartForSelection(const COORD target, const std:
 //                        (or a row boundary is encountered)
 // Return Value:
 // - The COORD for the last character on the "word" (inclusive)
-const COORD TextBuffer::GetWordEnd(const COORD target, const std::wstring_view wordDelimiters, bool accessibilityMode) const
+const COORD TextBuffer::GetWordEnd(const COORD target, const std::wstring_view& wordDelimiters, bool accessibilityMode) const
 {
     // Consider a buffer with this text in it:
     // "  word   other  "
@@ -1219,7 +1230,7 @@ const COORD TextBuffer::GetWordEnd(const COORD target, const std::wstring_view w
 // - lastCharPos - the position of the last nonspace character in the text buffer (to improve performance)
 // Return Value:
 // - The COORD for the first character of the next readable "word". If no next word, return one past the end of the buffer
-const COORD TextBuffer::_GetWordEndForAccessibility(const COORD target, const std::wstring_view wordDelimiters, const COORD lastCharPos) const
+const COORD TextBuffer::_GetWordEndForAccessibility(const COORD target, const std::wstring_view& wordDelimiters, const COORD lastCharPos) const
 {
     const auto bufferSize = GetSize();
     COORD result = target;
@@ -1267,7 +1278,7 @@ const COORD TextBuffer::_GetWordEndForAccessibility(const COORD target, const st
 // - wordDelimiters - what characters are we considering for the separation of words
 // Return Value:
 // - The COORD for the last character of the current word or delimiter run (stopped by right margin)
-const COORD TextBuffer::_GetWordEndForSelection(const COORD target, const std::wstring_view wordDelimiters) const
+const COORD TextBuffer::_GetWordEndForSelection(const COORD target, const std::wstring_view& wordDelimiters) const
 {
     const auto bufferSize = GetSize();
 
@@ -1350,7 +1361,7 @@ void TextBuffer::_PruneHyperlinks()
 // Return Value:
 // - true, if successfully updated pos. False, if we are unable to move (usually due to a buffer boundary)
 // - pos - The COORD for the first character on the "word" (inclusive)
-bool TextBuffer::MoveToNextWord(COORD& pos, const std::wstring_view wordDelimiters, COORD lastCharPos) const
+bool TextBuffer::MoveToNextWord(COORD& pos, const std::wstring_view& wordDelimiters, COORD lastCharPos) const
 {
     // move to the beginning of the next word
     // NOTE: _GetWordEnd...() returns the exclusive position of the "end of the word"
@@ -1374,7 +1385,7 @@ bool TextBuffer::MoveToNextWord(COORD& pos, const std::wstring_view wordDelimite
 // Return Value:
 // - true, if successfully updated pos. False, if we are unable to move (usually due to a buffer boundary)
 // - pos - The COORD for the first character on the "word" (inclusive)
-bool TextBuffer::MoveToPreviousWord(COORD& pos, std::wstring_view wordDelimiters) const
+bool TextBuffer::MoveToPreviousWord(COORD& pos, const std::wstring_view& wordDelimiters) const
 {
     // move to the beginning of the current word
     auto copy{ GetWordStart(pos, wordDelimiters, true) };
@@ -1732,7 +1743,7 @@ const TextBuffer::TextAndColor TextBuffer::GetText(const bool includeCRLF,
 // - string containing the generated HTML
 std::string TextBuffer::GenHTML(const TextAndColor& rows,
                                 const int fontHeightPoints,
-                                const std::wstring_view fontFaceName,
+                                const std::wstring_view& fontFaceName,
                                 const COLORREF backgroundColor)
 {
     try
@@ -1795,7 +1806,7 @@ std::string TextBuffer::GenHTML(const TextAndColor& rows,
                 const auto writeAccumulatedChars = [&](bool includeCurrent) {
                     if (col >= startOffset)
                     {
-                        const auto unescapedText = ConvertToA(CP_UTF8, std::wstring_view(rows.text.at(row)).substr(startOffset, col - startOffset + includeCurrent));
+                        const auto unescapedText = ConvertToA(CP_UTF8, rows.text.at(row).substr(startOffset, col - startOffset + includeCurrent));
                         for (const auto c : unescapedText)
                         {
                             switch (c)
@@ -1921,7 +1932,7 @@ std::string TextBuffer::GenHTML(const TextAndColor& rows,
 // - htmlTitle - value used in title tag of html header. Used to name the application
 // Return Value:
 // - string containing the generated RTF
-std::string TextBuffer::GenRTF(const TextAndColor& rows, const int fontHeightPoints, const std::wstring_view fontFaceName, const COLORREF backgroundColor)
+std::string TextBuffer::GenRTF(const TextAndColor& rows, const int fontHeightPoints, const std::wstring_view& fontFaceName, const COLORREF backgroundColor)
 {
     try
     {
@@ -1983,7 +1994,7 @@ std::string TextBuffer::GenRTF(const TextAndColor& rows, const int fontHeightPoi
                 const auto writeAccumulatedChars = [&](bool includeCurrent) {
                     if (col >= startOffset)
                     {
-                        const auto unescapedText = ConvertToA(CP_UTF8, std::wstring_view(rows.text.at(row)).substr(startOffset, col - startOffset + includeCurrent));
+                        const auto unescapedText = ConvertToA(CP_UTF8, rows.text.at(row).substr(startOffset, col - startOffset + includeCurrent));
                         for (const auto c : unescapedText)
                         {
                             switch (c)
@@ -2361,9 +2372,9 @@ HRESULT TextBuffer::Reflow(TextBuffer& oldBuffer,
 // - Adds or updates a hyperlink in our hyperlink table
 // Arguments:
 // - The hyperlink URI, the hyperlink id (could be new or old)
-void TextBuffer::AddHyperlinkToMap(std::wstring_view uri, uint16_t id)
+void TextBuffer::AddHyperlinkToMap(const std::wstring_view& uri, uint16_t id)
 {
-    _hyperlinkMap[id] = uri;
+    _hyperlinkMap.emplace(id, uri);
 }
 
 // Method Description:
@@ -2383,28 +2394,31 @@ std::wstring TextBuffer::GetHyperlinkUriFromId(uint16_t id) const
 // - The user-defined id
 // Return value:
 // - The internal hyperlink ID
-uint16_t TextBuffer::GetHyperlinkId(std::wstring_view uri, std::wstring_view id)
+uint16_t TextBuffer::GetHyperlinkId(const std::wstring_view& uri, const std::wstring_view& id)
 {
     uint16_t numericId = 0;
     if (id.empty())
     {
         // no custom id specified, return our internal count
-        numericId = _currentHyperlinkId;
+        numericId = _currentHyperlinkId++;
-        ++_currentHyperlinkId;
     }
     else
     {
-        // assign _currentHyperlinkId if the custom id does not already exist
+        // We need to use both uri and id for hashing. See GH#7698
-        std::wstring newId{ id };
+        std::wstring key;
-        // hash the URL and add it to the custom ID - GH#7698
+        key.reserve(uri.size() + id.size());
-        newId += L"%" + std::to_wstring(std::hash<std::wstring_view>{}(uri));
+        key.append(uri);
-        const auto result = _hyperlinkCustomIdMap.emplace(newId, _currentHyperlinkId);
+        key.append(id);
+
+        const auto result = _hyperlinkCustomIdMap.emplace(key, _currentHyperlinkId);
+        numericId = result.first->second;
+
         if (result.second)
         {
             // the custom id did not already exist
+            _hyperlinkCustomIdMapReverse.insert_or_assign(_currentHyperlinkId, key);
             ++_currentHyperlinkId;
         }
-        numericId = (*(result.first)).second;
     }
     // _currentHyperlinkId could overflow, make sure its not 0
     if (_currentHyperlinkId == 0)
@@ -2422,13 +2436,11 @@ uint16_t TextBuffer::GetHyperlinkId(std::wstring_view uri, std::wstring_view id)
 void TextBuffer::RemoveHyperlinkFromMap(uint16_t id) noexcept
 {
     _hyperlinkMap.erase(id);
-    for (const auto& customIdPair : _hyperlinkCustomIdMap)
+
+    if (auto it = _hyperlinkCustomIdMapReverse.find(id); it != _hyperlinkCustomIdMapReverse.end())
     {
-        if (customIdPair.second == id)
+        _hyperlinkCustomIdMap.erase(it->second);
-        {
+        _hyperlinkCustomIdMapReverse.erase(it);
-            _hyperlinkCustomIdMap.erase(customIdPair.first);
-            break;
-        }
     }
 }
 
@@ -2441,12 +2453,9 @@ void TextBuffer::RemoveHyperlinkFromMap(uint16_t id) noexcept
 // - The custom ID if there was one, empty string otherwise
 std::wstring TextBuffer::GetCustomIdFromId(uint16_t id) const
 {
-    for (auto customIdPair : _hyperlinkCustomIdMap)
+    if (auto it = _hyperlinkCustomIdMapReverse.find(id); it != _hyperlinkCustomIdMapReverse.end())
     {
-        if (customIdPair.second == id)
+        return it->second;
-        {
-            return customIdPair.first;
-        }
     }
     return {};
 }
@@ -2460,6 +2469,7 @@ void TextBuffer::CopyHyperlinkMaps(const TextBuffer& other)
 {
     _hyperlinkMap = other._hyperlinkMap;
     _hyperlinkCustomIdMap = other._hyperlinkCustomIdMap;
+    _hyperlinkCustomIdMapReverse = other._hyperlinkCustomIdMapReverse;
     _currentHyperlinkId = other._currentHyperlinkId;
 }
 
@@ -2470,7 +2480,7 @@ void TextBuffer::CopyHyperlinkMaps(const TextBuffer& other)
 // - The regex pattern
 // Return value:
 // - An ID that the caller should associate with the given pattern
-const size_t TextBuffer::AddPatternRecognizer(const std::wstring_view regexString)
+const size_t TextBuffer::AddPatternRecognizer(const std::wstring_view& regexString)
 {
     ++_currentPatternId;
     _idsAndPatterns.emplace(std::make_pair(_currentPatternId, regexString));

src/buffer/out/textBuffer.hpp

@@ -69,6 +69,9 @@ public:
                const TextAttribute defaultAttributes,
                const UINT cursorSize,
                Microsoft::Console::Render::IRenderTarget& renderTarget);
+
+    ~TextBuffer();
+
     TextBuffer(const TextBuffer& a) = delete;
 
     // Used for duplicating properties to another text buffer
@@ -98,7 +101,7 @@ public:
                                  const std::optional<size_t> limitRight = std::nullopt);
 
     bool InsertCharacter(const wchar_t wch, const DbcsAttribute dbcsAttribute, const TextAttribute attr);
-    bool InsertCharacter(const std::wstring_view chars, const DbcsAttribute dbcsAttribute, const TextAttribute attr);
+    bool InsertCharacter(const std::wstring_view& chars, const DbcsAttribute dbcsAttribute, const TextAttribute attr);
     bool IncrementCursor();
     bool NewlineCursor();
 
@@ -141,10 +144,10 @@ public:
 
     Microsoft::Console::Render::IRenderTarget& GetRenderTarget() noexcept;
 
-    const COORD GetWordStart(const COORD target, const std::wstring_view wordDelimiters, bool accessibilityMode = false) const;
+    const COORD GetWordStart(const COORD target, const std::wstring_view& wordDelimiters, bool accessibilityMode = false) const;
-    const COORD GetWordEnd(const COORD target, const std::wstring_view wordDelimiters, bool accessibilityMode = false) const;
+    const COORD GetWordEnd(const COORD target, const std::wstring_view& wordDelimiters, bool accessibilityMode = false) const;
-    bool MoveToNextWord(COORD& pos, const std::wstring_view wordDelimiters, COORD lastCharPos) const;
+    bool MoveToNextWord(COORD& pos, const std::wstring_view& wordDelimiters, COORD lastCharPos) const;
-    bool MoveToPreviousWord(COORD& pos, const std::wstring_view wordDelimiters) const;
+    bool MoveToPreviousWord(COORD& pos, const std::wstring_view& wordDelimiters) const;
 
     const til::point GetGlyphStart(const til::point pos) const;
     const til::point GetGlyphEnd(const til::point pos) const;
@@ -153,9 +156,9 @@ public:
 
     const std::vector<SMALL_RECT> GetTextRects(COORD start, COORD end, bool blockSelection, bool bufferCoordinates) const;
 
-    void AddHyperlinkToMap(std::wstring_view uri, uint16_t id);
+    void AddHyperlinkToMap(const std::wstring_view& uri, uint16_t id);
     std::wstring GetHyperlinkUriFromId(uint16_t id) const;
-    uint16_t GetHyperlinkId(std::wstring_view uri, std::wstring_view id);
+    uint16_t GetHyperlinkId(const std::wstring_view& uri, const std::wstring_view& id);
     void RemoveHyperlinkFromMap(uint16_t id) noexcept;
     std::wstring GetCustomIdFromId(uint16_t id) const;
     void CopyHyperlinkMaps(const TextBuffer& OtherBuffer);
@@ -176,12 +179,12 @@ public:
 
     static std::string GenHTML(const TextAndColor& rows,
                                const int fontHeightPoints,
-                               const std::wstring_view fontFaceName,
+                               const std::wstring_view& fontFaceName,
                                const COLORREF backgroundColor);
 
     static std::string GenRTF(const TextAndColor& rows,
                               const int fontHeightPoints,
-                              const std::wstring_view fontFaceName,
+                              const std::wstring_view& fontFaceName,
                               const COLORREF backgroundColor);
 
     struct PositionInformation
@@ -195,60 +198,47 @@ public:
                           const std::optional<Microsoft::Console::Types::Viewport> lastCharacterViewport,
                           std::optional<std::reference_wrapper<PositionInformation>> positionInfo);
 
-    const size_t AddPatternRecognizer(const std::wstring_view regexString);
+    const size_t AddPatternRecognizer(const std::wstring_view& regexString);
     void ClearPatternRecognizers() noexcept;
     void CopyPatterns(const TextBuffer& OtherBuffer);
     interval_tree::IntervalTree<til::point, size_t> GetPatterns(const size_t firstRow, const size_t lastRow) const;
 
 private:
     void _UpdateSize();
-    Microsoft::Console::Types::Viewport _size;
+    void _RefreshRowIDs(SHORT width);
-    std::vector<ROW> _storage;
+    void _RefreshRowWidth(CharRowCell* data, size_t width) noexcept;
-    Cursor _cursor;
-
-    SHORT _firstRow; // indexes top row (not necessarily 0)
-
-    TextAttribute _currentAttributes;
-
-    // storage location for glyphs that can't fit into the buffer normally
-    UnicodeStorage _unicodeStorage;
-
-    std::unordered_map<uint16_t, std::wstring> _hyperlinkMap;
-    std::unordered_map<std::wstring, uint16_t> _hyperlinkCustomIdMap;
-    uint16_t _currentHyperlinkId;
-
-    void _RefreshRowIDs(std::optional<SHORT> newRowWidth);
-
-    Microsoft::Console::Render::IRenderTarget& _renderTarget;
-
     void _SetFirstRowIndex(const SHORT FirstRowIndex) noexcept;
-
     COORD _GetPreviousFromCursor() const;
-
     void _SetWrapOnCurrentRow();
     void _AdjustWrapOnCurrentRow(const bool fSet);
-
     void _NotifyPaint(const Microsoft::Console::Types::Viewport& viewport) const;
-
     // Assist with maintaining proper buffer state for Double Byte character sequences
     bool _PrepareForDoubleByteSequence(const DbcsAttribute dbcsAttribute);
     bool _AssertValidDoubleByteSequence(const DbcsAttribute dbcsAttribute);
-
     ROW& _GetFirstRow();
     ROW& _GetPrevRowNoWrap(const ROW& row);
-
     void _ExpandTextRow(SMALL_RECT& selectionRow) const;
-
+    const DelimiterClass _GetDelimiterClassAt(const COORD pos, const std::wstring_view& wordDelimiters) const;
-    const DelimiterClass _GetDelimiterClassAt(const COORD pos, const std::wstring_view wordDelimiters) const;
+    const COORD _GetWordStartForAccessibility(const COORD target, const std::wstring_view& wordDelimiters) const;
-    const COORD _GetWordStartForAccessibility(const COORD target, const std::wstring_view wordDelimiters) const;
+    const COORD _GetWordStartForSelection(const COORD target, const std::wstring_view& wordDelimiters) const;
-    const COORD _GetWordStartForSelection(const COORD target, const std::wstring_view wordDelimiters) const;
+    const COORD _GetWordEndForAccessibility(const COORD target, const std::wstring_view& wordDelimiters, const COORD lastCharPos) const;
-    const COORD _GetWordEndForAccessibility(const COORD target, const std::wstring_view wordDelimiters, const COORD lastCharPos) const;
+    const COORD _GetWordEndForSelection(const COORD target, const std::wstring_view& wordDelimiters) const;
-    const COORD _GetWordEndForSelection(const COORD target, const std::wstring_view wordDelimiters) const;
-
     void _PruneHyperlinks();
 
+    Microsoft::Console::Render::IRenderTarget& _renderTarget;
+    std::vector<ROW> _storage;
+    std::unordered_map<uint16_t, std::wstring> _hyperlinkMap;
+    std::unordered_map<std::wstring, uint16_t> _hyperlinkCustomIdMap;
+    std::unordered_map<uint16_t, std::wstring> _hyperlinkCustomIdMapReverse;
     std::unordered_map<size_t, std::wstring> _idsAndPatterns;
-    size_t _currentPatternId;
+    TextAttribute _currentAttributes;
+    UnicodeStorage _unicodeStorage;
+    Cursor _cursor;
+    void* _charBuffer;
+    Microsoft::Console::Types::Viewport _size;
+    uint16_t _currentHyperlinkId{ 1 };
+    size_t _currentPatternId{ 0 };
+    SHORT _firstRow{ 0 }; // indexes top row (not necessarily 0)
 
 #ifdef UNIT_TESTING
     friend class TextBufferTests;

src/cascadia/LocalTests_SettingsModel/TerminalSettingsTests.cpp

@@ -51,7 +51,6 @@ namespace SettingsModelLocalTests
     void TerminalSettingsTests::TryCreateWinRTType()
     {
         TerminalSettings settings;
-        VERIFY_IS_NOT_NULL(settings);
         auto oldFontSize = settings.FontSize();
         settings.FontSize(oldFontSize + 5);
         auto newFontSize = settings.FontSize();
@@ -60,7 +59,10 @@ namespace SettingsModelLocalTests
 
     void TerminalSettingsTests::TestTerminalArgsForBinding()
     {
-        const std::string settingsJson{ R"(
+        const winrt::guid guid0{ ::Microsoft::Console::Utils::GuidFromString(L"{6239a42c-0000-49a3-80bd-e8fdd045185c}") };
+        const winrt::guid guid1{ ::Microsoft::Console::Utils::GuidFromString(L"{6239a42c-1111-49a3-80bd-e8fdd045185c}") };
+
+        CascadiaSettings settings{ LR"(
         {
             "defaultProfile": "{6239a42c-0000-49a3-80bd-e8fdd045185c}",
             "profiles": [
@@ -98,11 +100,6 @@ namespace SettingsModelLocalTests
             ]
         })" };
 
-        const winrt::guid guid0{ ::Microsoft::Console::Utils::GuidFromString(L"{6239a42c-0000-49a3-80bd-e8fdd045185c}") };
-        const winrt::guid guid1{ ::Microsoft::Console::Utils::GuidFromString(L"{6239a42c-1111-49a3-80bd-e8fdd045185c}") };
-
-        CascadiaSettings settings{ til::u8u16(settingsJson) };
-
         auto actionMap = settings.GlobalSettings().ActionMap();
         VERIFY_ARE_EQUAL(3u, settings.ActiveProfiles().Size());
 
@@ -385,7 +382,7 @@ namespace SettingsModelLocalTests
     void TerminalSettingsTests::MakeSettingsForProfileThatDoesntExist()
     {
         // Test that making settings throws when the GUID doesn't exist
-        const std::string settingsString{ R"(
+        CascadiaSettings settings{ LR"(
         {
             "defaultProfile": "{6239a42c-1111-49a3-80bd-e8fdd045185c}",
             "profiles": [
@@ -401,7 +398,6 @@ namespace SettingsModelLocalTests
                 }
             ]
         })" };
-        CascadiaSettings settings{ til::u8u16(settingsString) };
 
         const auto guid1 = ::Microsoft::Console::Utils::GuidFromString(L"{6239a42c-1111-49a3-80bd-e8fdd045185c}");
         const auto guid2 = ::Microsoft::Console::Utils::GuidFromString(L"{6239a42c-2222-49a3-80bd-e8fdd045185c}");
@@ -449,7 +445,7 @@ namespace SettingsModelLocalTests
         // defaultProfile that's not in the list, we validate the settings, and
         // then call MakeSettings(nullopt). The validation should ensure that
         // the default profile is something reasonable
-        const std::string settingsString{ R"(
+        CascadiaSettings settings{ LR"(
         {
             "defaultProfile": "{6239a42c-3333-49a3-80bd-e8fdd045185c}",
             "profiles": [
@@ -465,7 +461,6 @@ namespace SettingsModelLocalTests
                 }
             ]
         })" };
-        CascadiaSettings settings{ til::u8u16(settingsString) };
 
         VERIFY_ARE_EQUAL(2u, settings.Warnings().Size());
         VERIFY_ARE_EQUAL(2u, settings.ActiveProfiles().Size());
@@ -487,7 +482,7 @@ namespace SettingsModelLocalTests
         Log::Comment(NoThrowString().Format(
             L"Ensure that setting (or not) a property in the profile that should override a property of the color scheme works correctly."));
 
-        const std::string settings0String{ R"(
+        CascadiaSettings settings{ LR"(
         {
             "defaultProfile": "profile5",
             "profiles": [
@@ -528,8 +523,6 @@ namespace SettingsModelLocalTests
             ]
         })" };
 
-        CascadiaSettings settings{ til::u8u16(settings0String) };
-
         VERIFY_ARE_EQUAL(6u, settings.ActiveProfiles().Size());
         VERIFY_ARE_EQUAL(2u, settings.GlobalSettings().ColorSchemes().Size());
 

src/cascadia/LocalTests_TerminalApp/SettingsTests.cpp

@@ -79,7 +79,10 @@ namespace TerminalAppLocalTests
         // containing a ${profile.name} to replace. When we expand it, it should
         // have created one command for each profile.
 
-        const std::string settingsJson{ R"(
+        const auto guid0 = ::Microsoft::Console::Utils::GuidFromString(L"{6239a42c-0000-49a3-80bd-e8fdd045185c}");
+        const auto guid1 = ::Microsoft::Console::Utils::GuidFromString(L"{6239a42c-1111-49a3-80bd-e8fdd045185c}");
+
+        CascadiaSettings settings{ LR"(
         {
             "defaultProfile": "{6239a42c-0000-49a3-80bd-e8fdd045185c}",
             "profiles": [
@@ -111,11 +114,6 @@ namespace TerminalAppLocalTests
             "schemes": [ { "name": "Campbell" } ] // This is included here to prevent settings validation errors.
         })" };
 
-        const auto guid0 = ::Microsoft::Console::Utils::GuidFromString(L"{6239a42c-0000-49a3-80bd-e8fdd045185c}");
-        const auto guid1 = ::Microsoft::Console::Utils::GuidFromString(L"{6239a42c-1111-49a3-80bd-e8fdd045185c}");
-
-        CascadiaSettings settings{ til::u8u16(settingsJson) };
-
         VERIFY_ARE_EQUAL(0u, settings.Warnings().Size());
 
         VERIFY_ARE_EQUAL(3u, settings.ActiveProfiles().Size());
@@ -207,7 +205,10 @@ namespace TerminalAppLocalTests
         // For this test, put an iterable command without a given `name` to
         // replace. When we expand it, it should still work.
 
-        const std::string settingsJson{ R"(
+        const auto guid0 = ::Microsoft::Console::Utils::GuidFromString(L"{6239a42c-0000-49a3-80bd-e8fdd045185c}");
+        const auto guid1 = ::Microsoft::Console::Utils::GuidFromString(L"{6239a42c-1111-49a3-80bd-e8fdd045185c}");
+
+        CascadiaSettings settings{ LR"(
         {
             "defaultProfile": "{6239a42c-0000-49a3-80bd-e8fdd045185c}",
             "profiles": [
@@ -238,11 +239,6 @@ namespace TerminalAppLocalTests
             "schemes": [ { "name": "Campbell" } ] // This is included here to prevent settings validation errors.
         })" };
 
-        const auto guid0 = ::Microsoft::Console::Utils::GuidFromString(L"{6239a42c-0000-49a3-80bd-e8fdd045185c}");
-        const auto guid1 = ::Microsoft::Console::Utils::GuidFromString(L"{6239a42c-1111-49a3-80bd-e8fdd045185c}");
-
-        CascadiaSettings settings{ til::u8u16(settingsJson) };
-
         VERIFY_ARE_EQUAL(0u, settings.Warnings().Size());
 
         VERIFY_ARE_EQUAL(3u, settings.ActiveProfiles().Size());
@@ -335,7 +331,10 @@ namespace TerminalAppLocalTests
         // cause bad json to be filled in. Something like a profile with a name
         // of "Foo\"", so the trailing '"' might break the json parsing.
 
-        const std::string settingsJson{ R"(
+        const auto guid0 = ::Microsoft::Console::Utils::GuidFromString(L"{6239a42c-0000-49a3-80bd-e8fdd045185c}");
+        const auto guid1 = ::Microsoft::Console::Utils::GuidFromString(L"{6239a42c-1111-49a3-80bd-e8fdd045185c}");
+
+        CascadiaSettings settings{ LR"(
         {
             "defaultProfile": "{6239a42c-0000-49a3-80bd-e8fdd045185c}",
             "profiles": [
@@ -367,11 +366,6 @@ namespace TerminalAppLocalTests
             "schemes": [ { "name": "Campbell" } ] // This is included here to prevent settings validation errors.
         })" };
 
-        const auto guid0 = ::Microsoft::Console::Utils::GuidFromString(L"{6239a42c-0000-49a3-80bd-e8fdd045185c}");
-        const auto guid1 = ::Microsoft::Console::Utils::GuidFromString(L"{6239a42c-1111-49a3-80bd-e8fdd045185c}");
-
-        CascadiaSettings settings{ til::u8u16(settingsJson) };
-
         VERIFY_ARE_EQUAL(0u, settings.Warnings().Size());
 
         VERIFY_ARE_EQUAL(3u, settings.ActiveProfiles().Size());
@@ -468,7 +462,7 @@ namespace TerminalAppLocalTests
         //    ├─ first.com
         //    └─ second.com
 
-        const std::string settingsJson{ R"(
+        CascadiaSettings settings{ LR"(
         {
             "defaultProfile": "{6239a42c-0000-49a3-80bd-e8fdd045185c}",
             "profiles": [
@@ -508,8 +502,6 @@ namespace TerminalAppLocalTests
             "schemes": [ { "name": "Campbell" } ] // This is included here to prevent settings validation errors.
         })" };
 
-        CascadiaSettings settings{ til::u8u16(settingsJson) };
-
         VERIFY_ARE_EQUAL(0u, settings.Warnings().Size());
         VERIFY_ARE_EQUAL(3u, settings.ActiveProfiles().Size());
 
@@ -558,7 +550,7 @@ namespace TerminalAppLocalTests
         //       ├─ child1
         //       └─ child2
 
-        const std::string settingsJson{ R"(
+        CascadiaSettings settings{ LR"(
         {
             "defaultProfile": "{6239a42c-0000-49a3-80bd-e8fdd045185c}",
             "profiles": [
@@ -603,8 +595,6 @@ namespace TerminalAppLocalTests
             "schemes": [ { "name": "Campbell" } ] // This is included here to prevent settings validation errors.
         })" };
 
-        CascadiaSettings settings{ til::u8u16(settingsJson) };
-
         VERIFY_ARE_EQUAL(0u, settings.Warnings().Size());
         VERIFY_ARE_EQUAL(3u, settings.ActiveProfiles().Size());
 
@@ -691,7 +681,7 @@ namespace TerminalAppLocalTests
         //     ├─ Split pane, direction: vertical, profile: profile2
         //     └─ Split pane, direction: horizontal, profile: profile2
 
-        const std::string settingsJson{ R"(
+        CascadiaSettings settings{ LR"(
         {
             "defaultProfile": "{6239a42c-0000-49a3-80bd-e8fdd045185c}",
             "profiles": [
@@ -727,8 +717,6 @@ namespace TerminalAppLocalTests
             "schemes": [ { "name": "Campbell" } ] // This is included here to prevent settings validation errors.
         })" };
 
-        CascadiaSettings settings{ til::u8u16(settingsJson) };
-
         VERIFY_ARE_EQUAL(0u, settings.Warnings().Size());
         VERIFY_ARE_EQUAL(3u, settings.ActiveProfiles().Size());
 
@@ -828,7 +816,7 @@ namespace TerminalAppLocalTests
         //    ├─ Profile 2
         //    └─ Profile 3
 
-        const std::string settingsJson{ R"(
+        CascadiaSettings settings{ LR"(
         {
             "defaultProfile": "{6239a42c-0000-49a3-80bd-e8fdd045185c}",
             "profiles": [
@@ -864,8 +852,6 @@ namespace TerminalAppLocalTests
             "schemes": [ { "name": "Campbell" } ] // This is included here to prevent settings validation errors.
         })" };
 
-        CascadiaSettings settings{ til::u8u16(settingsJson) };
-
         VERIFY_ARE_EQUAL(0u, settings.Warnings().Size());
         VERIFY_ARE_EQUAL(3u, settings.ActiveProfiles().Size());
 
@@ -926,7 +912,7 @@ namespace TerminalAppLocalTests
         //       ├─ Split vertically
         //       └─ Split horizontally
 
-        const std::string settingsJson{ R"(
+        CascadiaSettings settings{ LR"(
         {
             "defaultProfile": "{6239a42c-0000-49a3-80bd-e8fdd045185c}",
             "profiles": [
@@ -967,8 +953,6 @@ namespace TerminalAppLocalTests
             "schemes": [ { "name": "Campbell" } ] // This is included here to prevent settings validation errors.
         })" };
 
-        CascadiaSettings settings{ til::u8u16(settingsJson) };
-
         VERIFY_ARE_EQUAL(0u, settings.Warnings().Size());
         VERIFY_ARE_EQUAL(3u, settings.ActiveProfiles().Size());
 
@@ -1071,7 +1055,7 @@ namespace TerminalAppLocalTests
         // containing a ${profile.name} to replace. When we expand it, it should
         // have created one command for each profile.
 
-        const std::string settingsJson{ R"(
+        CascadiaSettings settings{ LR"(
         {
             "defaultProfile": "{6239a42c-0000-49a3-80bd-e8fdd045185c}",
             "profiles": [
@@ -1107,8 +1091,6 @@ namespace TerminalAppLocalTests
             ]
         })" };
 
-        CascadiaSettings settings{ til::u8u16(settingsJson) };
-
         // Since at least one profile does not reference a color scheme,
         // we add a warning saying "the color scheme is unknown"
         VERIFY_ARE_EQUAL(1u, settings.Warnings().Size());

src/cascadia/LocalTests_TerminalApp/TabTests.cpp

@@ -129,7 +129,6 @@ namespace TerminalAppLocalTests
         // Verify we can create a WinRT type we authored
         // Just creating it is enough to know that everything is working.
         TerminalSettings settings;
-        VERIFY_IS_NOT_NULL(settings);
         auto oldFontSize = settings.FontSize();
         settings.FontSize(oldFontSize + 5);
         auto newFontSize = settings.FontSize();
@@ -307,7 +306,7 @@ namespace TerminalAppLocalTests
         // TerminalPage and not only create them successfully, but also create a
         // tab using those settings successfully.
 
-        const std::string settingsJson0{ R"(
+        CascadiaSettings settings0{ LR"(
         {
             "defaultProfile": "{6239a42c-1111-49a3-80bd-e8fdd045185c}",
             "profiles": [
@@ -324,9 +323,6 @@ namespace TerminalAppLocalTests
             ]
         })" };
 
-        CascadiaSettings settings0{ til::u8u16(settingsJson0) };
-        VERIFY_IS_NOT_NULL(settings0);
-
         // This is super wacky, but we can't just initialize the
         // com_ptr<impl::TerminalPage> in the lambda and assign it back out of
         // the lambda. We'll crash trying to get a weak_ref to the TerminalPage
@@ -353,7 +349,7 @@ namespace TerminalAppLocalTests
         //
         // Created to test GH#2455
 
-        const std::string settingsJson0{ R"(
+        CascadiaSettings settings0{ LR"(
         {
             "defaultProfile": "{6239a42c-1111-49a3-80bd-e8fdd045185c}",
             "profiles": [
@@ -368,9 +364,9 @@ namespace TerminalAppLocalTests
                     "historySize": 2
                 }
             ]
-        })" };
+        })") };
 
-        const std::string settingsJson1{ R"(
+        CascadiaSettings settings1{ LR"(
         {
             "defaultProfile": "{6239a42c-1111-49a3-80bd-e8fdd045185c}",
             "profiles": [
@@ -382,12 +378,6 @@ namespace TerminalAppLocalTests
             ]
         })" };
 
-        CascadiaSettings settings0{ til::u8u16(settingsJson0) };
-        VERIFY_IS_NOT_NULL(settings0);
-
-        CascadiaSettings settings1{ til::u8u16(settingsJson1) };
-        VERIFY_IS_NOT_NULL(settings1);
-
         const auto guid1 = Microsoft::Console::Utils::GuidFromString(L"{6239a42c-1111-49a3-80bd-e8fdd045185c}");
         const auto guid2 = Microsoft::Console::Utils::GuidFromString(L"{6239a42c-2222-49a3-80bd-e8fdd045185c}");
         const auto guid3 = Microsoft::Console::Utils::GuidFromString(L"{6239a42c-3333-49a3-80bd-e8fdd045185c}");
@@ -440,7 +430,7 @@ namespace TerminalAppLocalTests
         //
         // Created to test GH#2455
 
-        const std::string settingsJson0{ R"(
+        CascadiaSettings settings0{ LR"(
         {
             "defaultProfile": "{6239a42c-1111-49a3-80bd-e8fdd045185c}",
             "profiles": [
@@ -457,7 +447,7 @@ namespace TerminalAppLocalTests
             ]
         })" };
 
-        const std::string settingsJson1{ R"(
+        CascadiaSettings settings1{ LR"(
         {
             "defaultProfile": "{6239a42c-1111-49a3-80bd-e8fdd045185c}",
             "profiles": [
@@ -469,12 +459,6 @@ namespace TerminalAppLocalTests
             ]
         })" };
 
-        CascadiaSettings settings0{ til::u8u16(settingsJson0) };
-        VERIFY_IS_NOT_NULL(settings0);
-
-        CascadiaSettings settings1{ til::u8u16(settingsJson1) };
-        VERIFY_IS_NOT_NULL(settings1);
-
         const auto guid1 = Microsoft::Console::Utils::GuidFromString(L"{6239a42c-1111-49a3-80bd-e8fdd045185c}");
         const auto guid2 = Microsoft::Console::Utils::GuidFromString(L"{6239a42c-2222-49a3-80bd-e8fdd045185c}");
         const auto guid3 = Microsoft::Console::Utils::GuidFromString(L"{6239a42c-3333-49a3-80bd-e8fdd045185c}");
@@ -554,7 +538,7 @@ namespace TerminalAppLocalTests
     // - The initialized TerminalPage, ready to use.
     winrt::com_ptr<winrt::TerminalApp::implementation::TerminalPage> TabTests::_commonSetup()
     {
-        const std::string settingsJson0{ R"(
+        CascadiaSettings settings0{ LR"(
         {
             "defaultProfile": "{6239a42c-1111-49a3-80bd-e8fdd045185c}",
             "showTabsInTitlebar": false,
@@ -655,9 +639,6 @@ namespace TerminalAppLocalTests
             ]
         })" };
 
-        CascadiaSettings settings0{ til::u8u16(settingsJson0) };
-        VERIFY_IS_NOT_NULL(settings0);
-
         const auto guid1 = Microsoft::Console::Utils::GuidFromString(L"{6239a42c-1111-49a3-80bd-e8fdd045185c}");
         const auto guid2 = Microsoft::Console::Utils::GuidFromString(L"{6239a42c-2222-49a3-80bd-e8fdd045185c}");
 

src/cascadia/TerminalApp/AppLogic.cpp

@@ -1335,8 +1335,7 @@ namespace winrt::TerminalApp::implementation
                 {
                     // The string they provided wasn't an int, it wasn't "new"
                     // or "last", so whatever it is, that's the name they get.
-                    winrt::hstring winrtName{ til::u8u16(parsedTarget) };
+                    return winrt::make<FindTargetWindowResult>(WindowingBehaviorUseName, til::u8u16(parsedTarget));
-                    return winrt::make<FindTargetWindowResult>(WindowingBehaviorUseName, winrtName);
                 }
             }
         }

src/cascadia/TerminalConnection/ConptyConnection.cpp

@@ -456,8 +456,11 @@ namespace winrt::Microsoft::Terminal::TerminalConnection::implementation
                 // else we call convertUTF8ChunkToUTF16 with an empty string_view to convert possible remaining partials to U+FFFD
             }
 
-            const HRESULT result{ til::u8u16(std::string_view{ _buffer.data(), read }, _u16Str, _u8State) };
+            try
-            if (FAILED(result))
+            {
+                til::u8u16(std::string_view{ _buffer.data(), read }, _u16Str, _u8State);
+            }
+            catch (...)
             {
                 if (_isStateAtOrBeyond(ConnectionState::Closing))
                 {

src/cascadia/TerminalSettingsModel/CascadiaSettings.cpp

@@ -5,6 +5,7 @@
 #include "CascadiaSettings.h"
 #include "CascadiaSettings.g.cpp"
 
+#include <til/u8u16convert.h>
 #include <LibraryResources.h>
 
 #include "AzureCloudShellGenerator.h"

src/cascadia/TerminalSettingsModel/CascadiaSettingsSerialization.cpp

@@ -5,6 +5,7 @@
 #include "CascadiaSettings.h"
 
 #include <fmt/chrono.h>
+#include <til/u8u16convert.h>
 #include <shlobj.h>
 
 // defaults.h is a file containing the default json settings in a std::string_view

src/cascadia/TerminalSettingsModel/Command.cpp

@@ -5,9 +5,11 @@
 #include "Command.h"
 #include "Command.g.cpp"
 
+#include <til/u8u16convert.h>
+#include <LibraryResources.h>
+
 #include "ActionAndArgs.h"
 #include "KeyChordSerialization.h"
-#include <LibraryResources.h>
 #include "TerminalSettingsSerializationHelpers.h"
 
 using namespace winrt::Microsoft::Terminal::Settings::Model;

src/cascadia/TerminalSettingsModel/KeyChordSerialization.cpp

@@ -6,6 +6,7 @@
 #include "KeyChordSerialization.g.cpp"
 
 #include <til/static_map.h>
+#include <til/u8u16convert.h>
 
 using namespace winrt::Microsoft::Terminal::Control;
 using namespace winrt::Microsoft::Terminal::Settings::Model::implementation;

src/host/ApiRoutines.h

@@ -375,4 +375,7 @@ public:
                                                       const CONSOLE_FONT_INFOEX& consoleFontInfoEx) noexcept override;
 
 #pragma endregion
+
+private:
+    til::u8state _u8State;
 };

src/host/VtInputThread.cpp

@@ -70,20 +70,22 @@ VtInputThread::VtInputThread(_In_ wil::unique_hfile hPipe,
     LockConsole();
     auto Unlock = wil::scope_exit([&] { UnlockConsole(); });
 
+    std::wstring wstr;
+    try
+    {
+        til::u8u16(u8Str, wstr, _u8State);
+    }
+    catch (...)
+    {
+        return S_FALSE;
+    }
+
     try
     {
-        std::wstring wstr{};
-        auto hr = til::u8u16(u8Str, wstr, _u8State);
-        // If we hit a parsing error, eat it. It's bad utf-8, we can't do anything with it.
-        if (FAILED(hr))
-        {
-            return S_FALSE;
-        }
         _pInputStateMachine->ProcessString(wstr);
+        return S_OK;
     }
     CATCH_RETURN();
-
-    return S_OK;
 }
 
 // Function Description:

src/host/VtInputThread.hpp

@@ -15,6 +15,7 @@ Author(s):
 #pragma once
 
 #include "../terminal/parser/StateMachine.hpp"
+#include <til/u8u16convert.h>
 
 namespace Microsoft::Console
 {

src/host/_stream.cpp

@@ -1137,19 +1137,22 @@ constexpr unsigned int LOCAL_BUFFER_SIZE = 100;
         auto leadByteCaptured{ false };
         auto leadByteConsumed{ false };
         std::wstring wstr{};
-        static til::u8state u8State{};
 
         // Convert our input parameters to Unicode
         if (codepage == CP_UTF8)
         {
-            RETURN_IF_FAILED(til::u8u16(buffer, wstr, u8State));
+            try
+            {
+                til::u8u16(buffer, wstr, _u8State);
+            }
+            CATCH_RETURN()
             read = buffer.size();
         }
         else
         {
             // In case the codepage changes from UTF-8 to another,
             // we discard partials that might still be cached.
-            u8State.reset();
+            _u8State.reset();
 
             int mbPtrLength{};
             RETURN_IF_FAILED(SizeTToInt(buffer.size(), &mbPtrLength));

src/host/exe/Host.EXE.vcxproj

@@ -43,9 +43,6 @@
     <ProjectReference Include="..\..\renderer\base\lib\base.vcxproj">
       <Project>{af0a096a-8b3a-4949-81ef-7df8f0fee91f}</Project>
     </ProjectReference>
-    <ProjectReference Include="..\..\renderer\dx\lib\dx.vcxproj">
-      <Project>{48d21369-3d7b-4431-9967-24e81292cf62}</Project>
-    </ProjectReference>
     <ProjectReference Include="..\..\renderer\gdi\lib\gdi.vcxproj">
       <Project>{1c959542-bac2-4e55-9a6d-13251914cbb9}</Project>
     </ProjectReference>

src/inc/til.h

@@ -15,8 +15,6 @@
 #include "til/rectangle.h"
 #include "til/rle.h"
 #include "til/bitmap.h"
-#include "til/u8u16convert.h"
-#include "til/spsc.h"
 #include "til/coalesce.h"
 #include "til/replace.h"
 #include "til/string.h"

src/inc/til/u8u16convert.h

@@ -14,288 +14,43 @@ could overcome disadvantages of syscalls. Test results can be read up
 in PR #4093 and the test algorithms are available in src\tools\U8U16Test.
 Based on the results the decision was made to keep using the platform
 functions MultiByteToWideChar and WideCharToMultiByte.
-
-Author(s):
-- Steffen Illhardt (german-one) 2020
 --*/
 
 #pragma once
 
 namespace til // Terminal Implementation Library. Also: "Today I Learned"
 {
-    template<class charT>
+    struct u8state
-    class u8u16state final
     {
-    public:
+        uint32_t buffer{};
-        u8u16state() noexcept :
+        uint32_t remaining{};
-            _buffer{},
+
-            _utfPartials{}
+        constexpr void reset() noexcept {
-        {
+            *this = {};
         }
-
-        // Method Description:
-        // - Takes a UTF-8 string and populates it with *complete* UTF-8 codepoints.
-        //   If it receives an incomplete codepoint, it will cache it until it can be completed.
-        // Arguments:
-        // - in - UTF-8 string_view potentially containing partial code points
-        // - out - on return, populated with complete codepoints at the string end
-        // Return Value:
-        // - S_OK          - the resulting string doesn't end with a partial
-        // - S_FALSE       - the resulting string contains the previously cached partials only
-        // - E_OUTOFMEMORY - the method failed to allocate memory for the resulting string
-        // - E_ABORT       - the resulting string length would exceed the max_size and thus, the processing was aborted
-        // - E_UNEXPECTED  - an unexpected error occurred
-        template<class T = charT>
-        [[nodiscard]] typename std::enable_if<std::is_same<T, char>::value, HRESULT>::type
-        operator()(const std::basic_string_view<T> in, std::basic_string_view<T>& out) noexcept
-        {
-            try
-            {
-                size_t capacity{};
-                RETURN_HR_IF(E_ABORT, !base::CheckAdd(in.length(), _partialsLen).AssignIfValid(&capacity));
-
-                _buffer.clear();
-                _buffer.reserve(capacity);
-
-                // copy UTF-8 code units that were remaining from the previous call (if any)
-                if (_partialsLen != 0u)
-                {
-                    _buffer.assign(_utfPartials.cbegin(), _utfPartials.cbegin() + _partialsLen);
-                    _partialsLen = 0u;
-                }
-
-                if (in.empty())
-                {
-                    out = _buffer;
-                    if (_buffer.empty())
-                    {
-                        return S_OK;
-                    }
-
-                    return S_FALSE; // the partial is populated
-                }
-
-                _buffer.append(in);
-                size_t remainingLength{ _buffer.length() };
-
-                auto backIter = _buffer.end();
-                // If the last byte in the string was a byte belonging to a UTF-8 multi-byte character
-                if ((*(backIter - 1) & _Utf8BitMasks::MaskAsciiByte) > _Utf8BitMasks::IsAsciiByte)
-                {
-                    // Check only up to 3 last bytes, if no Lead Byte was found then the byte before must be the Lead Byte and no partials are in the string
-                    const size_t stopLen{ std::min(_buffer.length(), gsl::narrow_cast<size_t>(3u)) };
-                    for (size_t sequenceLen{ 1u }; sequenceLen <= stopLen; ++sequenceLen)
-                    {
-                        --backIter;
-                        // If Lead Byte found
-                        if ((*backIter & _Utf8BitMasks::MaskContinuationByte) > _Utf8BitMasks::IsContinuationByte)
-                        {
-                            // If the Lead Byte indicates that the last bytes in the string is a partial UTF-8 code point then cache them:
-                            //  Use the bitmask at index `sequenceLen`. Compare the result with the operand having the same index. If they
-                            //  are not equal then the sequence has to be cached because it is a partial code point. Otherwise the
-                            //  sequence is a complete UTF-8 code point and the whole string is ready for the conversion into a UTF-16 string.
-                            if ((*backIter & _cmpMasks.at(sequenceLen)) != _cmpOperands.at(sequenceLen))
-                            {
-                                std::move(backIter, _buffer.end(), _utfPartials.begin());
-                                remainingLength -= sequenceLen;
-                                _partialsLen = sequenceLen;
-                            }
-
-                            break;
-                        }
-                    }
-                }
-
-                // populate the part of the string that contains complete code points only
-                out = { _buffer.data(), remainingLength };
-
-                return S_OK;
-            }
-            catch (std::length_error&)
-            {
-                return E_ABORT;
-            }
-            catch (std::bad_alloc&)
-            {
-                return E_OUTOFMEMORY;
-            }
-            catch (...)
-            {
-                return E_UNEXPECTED;
-            }
-        }
-
-        // Method Description:
-        // - Takes a UTF-16 string and populates it with *complete* UTF-16 codepoints.
-        //   If it receives an incomplete codepoint, it will cache it until it can be completed.
-        // Arguments:
-        // - in - UTF-16 string_view potentially containing partial code points
-        // - out - on return, populated with complete codepoints at the string end
-        // Return Value:
-        // - S_OK          - the resulting string doesn't end with a partial
-        // - S_FALSE       - the resulting string contains the previously cached partials only
-        // - E_OUTOFMEMORY - the method failed to allocate memory for the resulting string
-        // - E_ABORT       - the resulting string length would exceed the max_size and thus, the processing was aborted
-        // - E_UNEXPECTED  - an unexpected error occurred
-        template<class T = charT>
-        [[nodiscard]] typename std::enable_if<std::is_same<T, wchar_t>::value, HRESULT>::type
-        operator()(const std::basic_string_view<T> in, std::basic_string_view<T>& out) noexcept
-        {
-            try
-            {
-                size_t remainingLength{ in.length() };
-                size_t capacity{};
-
-                RETURN_HR_IF(E_ABORT, !base::CheckAdd(remainingLength, _partialsLen).AssignIfValid(&capacity));
-
-                _buffer.clear();
-                _buffer.reserve(capacity);
-
-                // copy UTF-8 code units that were remaining from the previous call (if any)
-                if (_partialsLen != 0u)
-                {
-                    _buffer.push_back(_utfPartials.front());
-                    _partialsLen = 0u;
-                }
-
-                if (in.empty())
-                {
-                    out = _buffer;
-                    if (_buffer.empty())
-                    {
-                        return S_OK;
-                    }
-
-                    return S_FALSE; // the high surrogate is populated
-                }
-
-                // cache the last value in the string if it is in the range of high surrogates
-                if (in.back() >= 0xD800u && in.back() <= 0xDBFFu)
-                {
-                    _utfPartials.front() = in.back();
-                    --remainingLength;
-                    _partialsLen = 1u;
-                }
-                else
-                {
-                    _partialsLen = 0u;
-                }
-
-                // populate the part of the string that contains complete code points only
-                _buffer.append(in, 0u, remainingLength);
-                out = _buffer;
-
-                return S_OK;
-            }
-            catch (std::length_error&)
-            {
-                return E_ABORT;
-            }
-            catch (std::bad_alloc&)
-            {
-                return E_OUTOFMEMORY;
-            }
-            catch (...)
-            {
-                return E_UNEXPECTED;
-            }
-        }
-
-        // Method Description:
-        // - Discard cached partials.
-        // Arguments:
-        // - none
-        // Return Value:
-        // - void
-        void reset() noexcept
-        {
-            _partialsLen = 0u;
-        }
-
-    private:
-        enum _Utf8BitMasks : BYTE
-        {
-            IsAsciiByte = 0b0'0000000, // Any byte representing an ASCII character has the MSB set to 0
-            MaskAsciiByte = 0b1'0000000, // Bit mask to be used in a bitwise AND operation to find out whether or not a byte match the IsAsciiByte pattern
-            IsContinuationByte = 0b10'000000, // Continuation bytes of any UTF-8 non-ASCII character have the MSB set to 1 and the adjacent bit set to 0
-            MaskContinuationByte = 0b11'000000, // Bit mask to be used in a bitwise AND operation to find out whether or not a byte match the IsContinuationByte pattern
-            IsLeadByteTwoByteSequence = 0b110'00000, // A lead byte that indicates a UTF-8 non-ASCII character consisting of two bytes has the two highest bits set to 1 and the adjacent bit set to 0
-            MaskLeadByteTwoByteSequence = 0b111'00000, // Bit mask to be used in a bitwise AND operation to find out whether or not a lead byte match the IsLeadByteTwoByteSequence pattern
-            IsLeadByteThreeByteSequence = 0b1110'0000, // A lead byte that indicates a UTF-8 non-ASCII character consisting of three bytes has the three highest bits set to 1 and the adjacent bit set to 0
-            MaskLeadByteThreeByteSequence = 0b1111'0000, // Bit mask to be used in a bitwise AND operation to find out whether or not a lead byte match the IsLeadByteThreeByteSequence pattern
-            IsLeadByteFourByteSequence = 0b11110'000, // A lead byte that indicates a UTF-8 non-ASCII character consisting of four bytes has the four highest bits set to 1 and the adjacent bit set to 0
-            MaskLeadByteFourByteSequence = 0b11111'000 // Bit mask to be used in a bitwise AND operation to find out whether or not a lead byte match the IsLeadByteFourByteSequence pattern
-        };
-
-        // array of bitmasks
-        constexpr static std::array<BYTE, 4> _cmpMasks{
-            0, // unused
-            _Utf8BitMasks::MaskContinuationByte,
-            _Utf8BitMasks::MaskLeadByteTwoByteSequence,
-            _Utf8BitMasks::MaskLeadByteThreeByteSequence,
-        };
-
-        // array of values for the comparisons
-        constexpr static std::array<BYTE, 4> _cmpOperands{
-            0, // unused
-            _Utf8BitMasks::IsAsciiByte, // intentionally conflicts with MaskContinuationByte
-            _Utf8BitMasks::IsLeadByteTwoByteSequence,
-            _Utf8BitMasks::IsLeadByteThreeByteSequence,
-        };
-
-        std::basic_string<charT> _buffer; // buffer to which the populated string_view refers
-        std::array<charT, 4> _utfPartials; // buffer for code units of a partial code point that have to be cached
-        size_t _partialsLen{}; // number of cached code units
     };
 
-    // make clear what incoming string type the state is for
-    typedef u8u16state<char> u8state;
-    typedef u8u16state<wchar_t> u16state;
-
     // Routine Description:
     // - Takes a UTF-8 string and performs the conversion to UTF-16. NOTE: The function relies on getting complete UTF-8 characters at the string boundaries.
     // Arguments:
     // - in - UTF-8 string to be converted
     // - out - reference to the resulting UTF-16 string
-    // Return Value:
+    template<typename Output>
-    // - S_OK          - the conversion succeeded
+    void u8u16(const std::string_view& in, Output& out)
-    // - E_OUTOFMEMORY - the function failed to allocate memory for the resulting string
-    // - E_ABORT       - the resulting string length would exceed the upper boundary of an int and thus, the conversion was aborted before the conversion has been completed
-    // - E_UNEXPECTED  - an unexpected error occurred
-    template<class inT, class outT>
-    [[nodiscard]] typename std::enable_if<std::is_same<typename inT::value_type, char>::value && std::is_same<typename outT::value_type, wchar_t>::value, HRESULT>::type
-    u8u16(const inT in, outT& out) noexcept
     {
-        try
+        out.clear();
-        {
-            out.clear();
 
-            if (in.empty())
+        if (in.empty())
-            {
+        {
-                return S_OK;
+            return;
-            }
+        }
 
-            int lengthRequired{};
+        // The worst ratio of UTF-8 code units to UTF-16 code units is 1 to 1 if UTF-8 consists of ASCII only.
-            // The worst ratio of UTF-8 code units to UTF-16 code units is 1 to 1 if UTF-8 consists of ASCII only.
+        const auto lengthRequired = gsl::narrow<int>(in.length());
-            RETURN_HR_IF(E_ABORT, !base::MakeCheckedNum(in.length()).AssignIfValid(&lengthRequired));
+        out.resize(in.length());
-            out.resize(in.length()); // avoid to call MultiByteToWideChar twice only to get the required size
+        const int lengthOut = MultiByteToWideChar(CP_UTF8, 0, in.data(), lengthRequired, out.data(), lengthRequired);
-            const int lengthOut = MultiByteToWideChar(gsl::narrow_cast<UINT>(CP_UTF8), 0ul, in.data(), lengthRequired, out.data(), lengthRequired);
+        out.resize(gsl::narrow_cast<size_t>(lengthOut));
-            out.resize(gsl::narrow_cast<size_t>(lengthOut));
+        THROW_LAST_ERROR_IF(lengthOut == 0);
-
-            return lengthOut == 0 ? E_UNEXPECTED : S_OK;
-        }
-        catch (std::length_error&)
-        {
-            return E_ABORT;
-        }
-        catch (std::bad_alloc&)
-        {
-            return E_OUTOFMEMORY;
-        }
-        catch (...)
-        {
-            return E_UNEXPECTED;
-        }
     }
 
     // Routine Description:
@@ -304,87 +59,93 @@ namespace til // Terminal Implementation Library. Also: "Today I Learned"
     // - in - UTF-8 string to be converted
     // - out - reference to the resulting UTF-16 string
     // - state - reference to a til::u8state class holding the status of the current partials handling
-    // Return Value:
+    template<typename Output>
-    // - S_OK          - the conversion succeeded
+    void u8u16(const std::string_view& in, Output& out, u8state& state) noexcept
-    // - E_OUTOFMEMORY - the function failed to allocate memory for the resulting string
-    // - E_ABORT       - the resulting string length would exceed the upper boundary of an int and thus, the conversion was aborted before the conversion has been completed
-    // - E_UNEXPECTED  - an unexpected error occurred
-    template<class inT, class outT>
-    [[nodiscard]] typename std::enable_if<std::is_same<typename inT::value_type, char>::value && std::is_same<typename outT::value_type, wchar_t>::value, HRESULT>::type
-    u8u16(const inT in, outT& out, u8state& state) noexcept
     {
-        std::string_view sv{};
+        auto data = in.data();
-        RETURN_IF_FAILED(state(std::string_view{ in }, sv));
+        auto size = in.size();
-        return til::u8u16(sv, out);
-    }
 
-    // Routine Description:
+        if (!size)
-    // - Takes a UTF-16 string and performs the conversion to UTF-8. NOTE: The function relies on getting complete UTF-16 characters at the string boundaries.
-    // Arguments:
-    // - in - UTF-16 string to be converted
-    // - out - reference to the resulting UTF-8 string
-    // Return Value:
-    // - S_OK          - the conversion succeeded
-    // - E_OUTOFMEMORY - the function failed to allocate memory for the resulting string
-    // - E_ABORT       - the resulting string length would exceed the upper boundary of an int and thus, the conversion was aborted before the conversion has been completed
-    // - E_UNEXPECTED  - an unexpected error occurred
-    template<class inT, class outT>
-    [[nodiscard]] typename std::enable_if<std::is_same<typename inT::value_type, wchar_t>::value && std::is_same<typename outT::value_type, char>::value, HRESULT>::type
-    u16u8(const inT in, outT& out) noexcept
-    {
-        try
         {
-            out.clear();
+            return;
+        }
 
-            if (in.empty())
+        if (auto remaining = state.remaining)
+        {
+            if (remaining > size)
             {
-                return S_OK;
+                remaining = gsl::narrow_cast<uint32_t>(size);
             }
 
-            int lengthIn{};
+            state.remaining -= remaining;
-            int lengthRequired{};
-            // Code Point U+0000..U+FFFF: 1 UTF-16 code unit --> 1..3 UTF-8 code units.
-            // Code Points >U+FFFF: 2 UTF-16 code units --> 4 UTF-8 code units.
-            // Thus, the worst ratio of UTF-16 code units to UTF-8 code units is 1 to 3.
-            RETURN_HR_IF(E_ABORT, !base::MakeCheckedNum(in.length()).AssignIfValid(&lengthIn) || !base::CheckMul(lengthIn, 3).AssignIfValid(&lengthRequired));
-            out.resize(gsl::narrow_cast<size_t>(lengthRequired)); // avoid to call WideCharToMultiByte twice only to get the required size
-            const int lengthOut = WideCharToMultiByte(gsl::narrow_cast<UINT>(CP_UTF8), 0ul, in.data(), lengthIn, out.data(), lengthRequired, nullptr, nullptr);
-            out.resize(gsl::narrow_cast<size_t>(lengthOut));
 
-            return lengthOut == 0 ? E_UNEXPECTED : S_OK;
+            do
-        }
+            {
-        catch (std::length_error&)
+                state.buffer <<= 6;
-        {
+                state.buffer |= *data++ & 0x3f;
-            return E_ABORT;
+            } while (--remaining);
-        }
-        catch (std::bad_alloc&)
-        {
-            return E_OUTOFMEMORY;
-        }
-        catch (...)
-        {
-            return E_UNEXPECTED;
-        }
-    }
 
-    // Routine Description:
+            if (!state.remaining)
-    // - Takes a UTF-16 string, complements and/or caches partials, and performs the conversion to UTF-8.
+            {
-    // Arguments:
+                if (state.buffer < 0x10000)
-    // - in - UTF-16 string to be converted
+                {
-    // - out - reference to the resulting UTF-8 string
+                    const auto buffer = static_cast<wchar_t>(state.buffer);
-    // - state - reference to a til::u16state class holding the status of the current partials handling
+                    out.append(&buffer, 1);
-    // Return Value:
+                }
-    // - S_OK          - the conversion succeeded without any change of the represented code points
+                else
-    // - E_OUTOFMEMORY - the function failed to allocate memory for the resulting string
+                {
-    // - E_ABORT       - the resulting string length would exceed the upper boundary of an int and thus, the conversion was aborted before the conversion has been completed
+                    wchar_t buffer[2];
-    // - E_UNEXPECTED  - an unexpected error occurred
+                    buffer[0] = ((state.buffer >> 10) & 0x3FF) + 0xD800;
-    template<class inT, class outT>
+                    buffer[1] = (state.buffer & 0x3FF) + 0xDC00;
-    [[nodiscard]] typename std::enable_if<std::is_same<typename inT::value_type, wchar_t>::value && std::is_same<typename outT::value_type, char>::value, HRESULT>::type
+                    out.append(&buffer[0], 2);
-    u16u8(const inT in, outT& out, u16state& state) noexcept
+                }
-    {
+            }
-        std::wstring_view sv{};
+        }
-        RETURN_IF_FAILED(state(std::wstring_view{ in }, sv));
+
-        return u16u8(sv, out);
+        {
+            auto end = data + size;
+            size_t have = 1;
+            // Skip UTF-8 continuation bytes in the form of 0b10xxxxxx.
+            while ((*--end & 0b11000000) == 0b10000000 && end != data)
+            {
+                ++have;
+            }
+
+            // A leading UTF-8 byte is either of:
+            // * 0b110xxxxx
+            // * 0b1110xxxx
+            // * 0b11110xxx
+            if (have != 1)
+            {
+                DWORD index = 0;
+                if (_BitScanReverse(&index, ~*end & 0xff))
+                {
+                    const auto want = 7 - index;
+                    if (want <= 4 && want > have)
+                    {
+                        auto ptr = end;
+                        uint32_t buffer = *ptr++ & ((1 << index) - 1);
+
+                        for (size_t i = 1; i < have; ++i)
+                        {
+                            buffer <<= 6;
+                            buffer |= *ptr++ & 0x3f;
+                        }
+
+                        state.buffer = buffer;
+                        state.remaining = gsl::narrow_cast<uint32_t>(want - have);
+                    }
+                }
+                else
+                {
+                    ++end;
+                }
+
+                size = end - data;
+            }
+        }
+
+        u8u16({ data, size }, out);
     }
 
     // Routine Description:
@@ -394,30 +155,104 @@ namespace til // Terminal Implementation Library. Also: "Today I Learned"
     // Return Value:
     // - the resulting UTF-16 string
     // - NOTE: Throws HRESULT errors that the non-throwing sibling returns
-    template<class inT>
+    _TIL_INLINEPREFIX std::wstring u8u16(const std::string_view& in)
-    typename std::enable_if<std::is_same<typename inT::value_type, char>::value, std::wstring>::type
-    u8u16(const inT in)
     {
-        std::wstring out{};
+        std::wstring out;
-        THROW_IF_FAILED(u8u16(std::string_view{ in }, out));
+        u8u16(in, out);
         return out;
     }
 
     // Routine Description:
-    // Takes a UTF-8 string, complements and/or caches partials, and performs the conversion to UTF-16.
+    // - Takes a UTF-16 string and performs the conversion to UTF-8. NOTE: The function relies on getting complete UTF-16 characters at the string boundaries.
     // Arguments:
-    // - in - UTF-8 string to be converted
+    // - in - UTF-16 string to be converted
-    // - state - reference to a til::u8state class holding the status of the current partials handling
+    // - out - reference to the resulting UTF-8 string
-    // Return Value:
+    template<typename Output>
-    // - the resulting UTF-16 string
+    void u16u8(const std::wstring_view& in, Output& out) noexcept
-    // - NOTE: Throws HRESULT errors that the non-throwing sibling returns
-    template<class inT>
-    typename std::enable_if<std::is_same<typename inT::value_type, char>::value, std::wstring>::type
-    u8u16(const inT in, u8state& state)
     {
-        std::wstring out{};
+        out.clear();
-        THROW_IF_FAILED(u8u16(std::string_view{ in }, out, state));
+
-        return out;
+        if (in.empty())
+        {
+            return;
+        }
+
+        // Code Point U+0000..U+FFFF: 1 UTF-16 code unit --> 1..3 UTF-8 code units.
+        // Code Points >U+FFFF: 2 UTF-16 code units --> 4 UTF-8 code units.
+        // Thus, the worst ratio of UTF-16 code units to UTF-8 code units is 1 to 3.
+        const size_t lengthIn = in.length();
+        const size_t lengthRequired = base::CheckMul(lengthIn, 3).ValueOrDie();
+        out.resize(lengthRequired);
+        const int lengthOut = WideCharToMultiByte(gsl::narrow_cast<UINT>(CP_UTF8), 0ul, in.data(), gsl::narrow<int>(lengthIn), out.data(), gsl::narrow<int>(lengthRequired), nullptr, nullptr);
+        out.resize(lengthOut);
+        THROW_LAST_ERROR_IF(lengthOut == 0);
+    }
+
+    struct u16state
+    {
+        wchar_t buffer{};
+
+        constexpr void reset() noexcept
+        {
+            *this = {};
+        }
+    };
+
+    // Routine Description:
+    // - Takes a UTF-16 string, complements and/or caches partials, and performs the conversion to UTF-8.
+    // Arguments:
+    // - in - UTF-16 string to be converted
+    // - out - reference to the resulting UTF-8 string
+    // - state - reference to a til::u16state class holding the status of the current partials handling
+    template<typename Output>
+    void u16u8(const std::wstring_view& in, Output& out, u16state& state) noexcept
+    {
+        auto data = in.data();
+        auto size = in.size();
+
+        if (!size)
+        {
+            return;
+        }
+
+        if (state.buffer)
+        {
+            if (*data >= 0xDC00 && *data <= 0xDFFF)
+            {
+                const uint32_t high = state.buffer - 0xD800;
+                const uint32_t low = *data - 0xDC00;
+                const auto codePoint = ((high << 10) | low) + 0x10000;
+
+                char buffer[4];
+                buffer[0] = 0b11110000 | ((codePoint >> 18) & 0x3f);
+                buffer[1] = 0b10000000 | ((codePoint >> 12) & 0x3f);
+                buffer[2] = 0b10000000 | ((codePoint >> 6) & 0x3f);
+                buffer[3] = 0b10000000 | ((codePoint >> 0) & 0x3f);
+                out.append(&buffer[0], 4);
+
+                ++data;
+                --size;
+                if (!size)
+                {
+                    return;
+                }
+            }
+
+            state = {};
+        }
+
+        if (auto end = data + size - 1; *end >= 0xD800 && *end <= 0xDBFF)
+        {
+            state.buffer = *end;
+
+            --size;
+            if (!size)
+            {
+                return;
+            }
+        }
+
+        u16u8({ data, size }, out);
     }
 
     // Routine Description:
@@ -427,29 +262,10 @@ namespace til // Terminal Implementation Library. Also: "Today I Learned"
     // Return Value:
     // - the resulting UTF-8 string
     // - NOTE: Throws HRESULT errors that the non-throwing sibling returns
-    template<class inT>
+    _TIL_INLINEPREFIX std::string u16u8(const std::wstring_view& in)
-    typename std::enable_if<std::is_same<typename inT::value_type, wchar_t>::value, std::string>::type
-    u16u8(const inT in)
     {
         std::string out{};
-        THROW_IF_FAILED(u16u8(std::wstring_view{ in }, out));
+        u16u8(in, out);
-        return out;
-    }
-
-    // Routine Description:
-    // Takes a UTF-16 string, complements and/or caches partials, and performs the conversion to UTF-8.
-    // Arguments:
-    // - in - UTF-16 string to be converted
-    // - state - reference to a til::u16state class holding the status of the current partials handling
-    // Return Value:
-    // - the resulting UTF-8 string
-    // - NOTE: Throws HRESULT errors that the non-throwing sibling returns
-    template<class inT>
-    typename std::enable_if<std::is_same<typename inT::value_type, wchar_t>::value, std::string>::type
-    u16u8(const inT in, u16state& state)
-    {
-        std::string out{};
-        THROW_IF_FAILED(u16u8(std::wstring_view{ in }, out, state));
         return out;
     }
 }

src/renderer/vt/VtSequences.cpp

@@ -5,6 +5,8 @@
 #include "vtrenderer.hpp"
 #include "../../inc/conattrs.hpp"
 
+#include <til/u8u16convert.h>
+
 #pragma hdrstop
 using namespace Microsoft::Console::Render;
 

src/renderer/vt/state.cpp

@@ -5,6 +5,7 @@
 #include "vtrenderer.hpp"
 #include "../../inc/conattrs.hpp"
 #include "../../types/inc/convert.hpp"
+#include <til/u8u16convert.h>
 
 // For _vcprintf
 #include <conio.h>
@@ -31,8 +32,7 @@ VtEngine::VtEngine(_In_ wil::unique_hfile pipe,
     _hFile(std::move(pipe)),
     _lastTextAttributes(INVALID_COLOR, INVALID_COLOR),
     _lastViewport(initialViewport),
-    _pool(til::pmr::get_default_resource()),
+    _invalidMap(initialViewport.Dimensions(), false),
-    _invalidMap(initialViewport.Dimensions(), false, &_pool),
     _lastText({ 0 }),
     _scrollDelta({ 0, 0 }),
     _quickReturn(false),
@@ -148,7 +148,11 @@ VtEngine::VtEngine(_In_ wil::unique_hfile pipe,
 // - S_OK or suitable HRESULT error from either conversion or writing pipe.
 [[nodiscard]] HRESULT VtEngine::_WriteTerminalUtf8(const std::wstring_view wstr) noexcept
 {
-    RETURN_IF_FAILED(til::u16u8(wstr, _conversionBuffer));
+    try
+    {
+        til::u16u8(wstr, _conversionBuffer);
+    }
+    CATCH_RETURN();
     return _Write(_conversionBuffer);
 }
 

src/renderer/vt/tracing.cpp

@@ -150,7 +150,7 @@ void RenderTracing::TraceInvalidateScroll(const til::point scroll) const
 }
 
 void RenderTracing::TraceStartPaint(const bool quickReturn,
-                                    const til::pmr::bitmap& invalidMap,
+                                    const til::bitmap& invalidMap,
                                     const til::rectangle lastViewport,
                                     const til::point scrollDelt,
                                     const bool cursorMoved,

src/renderer/vt/tracing.hpp

@@ -39,7 +39,7 @@ namespace Microsoft::Console::VirtualTerminal
         void TraceTriggerCircling(const bool newFrame) const;
         void TraceInvalidateScroll(const til::point scroll) const;
         void TraceStartPaint(const bool quickReturn,
-                             const til::pmr::bitmap& invalidMap,
+                             const til::bitmap& invalidMap,
                              const til::rectangle lastViewport,
                              const til::point scrollDelta,
                              const bool cursorMoved,

src/renderer/vt/vtrenderer.hpp

@@ -119,8 +119,7 @@ namespace Microsoft::Console::Render
 
         Microsoft::Console::Types::Viewport _lastViewport;
 
-        std::pmr::unsynchronized_pool_resource _pool;
+        til::bitmap _invalidMap;
-        til::pmr::bitmap _invalidMap;
 
         COORD _lastText;
         til::point _scrollDelta;

src/server/ApiMessage.cpp

@@ -104,8 +104,9 @@ try
     {
         RETURN_HR_IF(E_FAIL, State.ReadOffset > Descriptor.InputSize);
 
-        ULONG const cbReadSize = Descriptor.InputSize - State.ReadOffset;
+        // We need to limit the read buffer to something reasonable (here: 16MiB) unless we want to
-
+        // consume the user's entire system memory when someone calls WriteFile() with a huge buffer.
+        const ULONG cbReadSize = std::min(16777216ul, Descriptor.InputSize - State.ReadOffset);
         _inputBuffer.resize(cbReadSize);
 
         RETURN_IF_FAILED(ReadMessageInput(0, _inputBuffer.data(), cbReadSize));

src/terminal/parser/OutputStateMachineEngine.cpp

@@ -1081,22 +1081,27 @@ bool OutputStateMachineEngine::_GetOscSetClipboard(const std::wstring_view strin
                                                    std::wstring& content,
                                                    bool& queryClipboard) const noexcept
 {
-    const size_t pos = string.find(';');
+    const auto pos = string.find(L';');
-    if (pos != std::wstring_view::npos)
+    if (pos == std::wstring_view::npos)
     {
-        const std::wstring_view substr = string.substr(pos + 1);
+        return false;
-        if (substr == L"?")
-        {
-            queryClipboard = true;
-            return true;
-        }
-        else
-        {
-            return Base64::s_Decode(substr, content);
-        }
     }
 
-    return false;
+    const auto substr = string.substr(pos + 1);
+    if (substr == L"?")
+    {
+        queryClipboard = true;
+        return true;
+    }
+
+    try {
+        Base64::s_Decode(substr, content);
+        return true;
+    }
+    catch (...)
+    {
+        return false;
+    }
 }
 
 // Method Description:

src/terminal/parser/base64.cpp

@@ -4,66 +4,22 @@
 #include "precomp.h"
 #include "base64.hpp"
 
+#include <til/u8u16convert.h>
+
 using namespace Microsoft::Console::VirtualTerminal;
 
-static const char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+// clang-format off
-static const char padChar = '=';
+static constexpr uint8_t decodeTable[128] = {
-
+	255 /* NUL */, 255 /* SOH */, 255 /* STX */, 255 /* ETX */, 255 /* EOT */, 255 /* ENQ */, 255 /* ACK */, 255 /* BEL */, 255 /* BS  */, 255 /* HT  */, 64  /* LF  */, 255 /* VT  */, 255 /* FF  */, 64  /* CR  */, 255 /* SO  */, 255 /* SI  */,
-#pragma warning(disable : 26446 26447 26482 26485 26493 26494)
+	255 /* DLE */, 255 /* DC1 */, 255 /* DC2 */, 255 /* DC3 */, 255 /* DC4 */, 255 /* NAK */, 255 /* SYN */, 255 /* ETB */, 255 /* CAN */, 255 /* EM  */, 255 /* SUB */, 255 /* ESC */, 255 /* FS  */, 255 /* GS  */, 255 /* RS  */, 255 /* US  */,
-
+	255 /* SP  */, 255 /* !   */, 255 /* "   */, 255 /* #   */, 255 /* $   */, 255 /* %   */, 255 /* &   */, 255 /* '   */, 255 /* (   */, 255 /* )   */, 255 /* *   */, 62  /* +   */, 255 /* ,   */, 62  /* -   */, 255 /* .   */, 63  /* /   */,
-// Routine Description:
+	52  /* 0   */, 53  /* 1   */, 54  /* 2   */, 55  /* 3   */, 56  /* 4   */, 57  /* 5   */, 58  /* 6   */, 59  /* 7   */, 60  /* 8   */, 61  /* 9   */, 255 /* :   */, 255 /* ;   */, 255 /* <   */, 255 /* =   */, 255 /* >   */, 255 /* ?   */,
-// - Encode a string using base64. When there are not enough characters
+	255 /* @   */, 0   /* A   */, 1   /* B   */, 2   /* C   */, 3   /* D   */, 4   /* E   */, 5   /* F   */, 6   /* G   */, 7   /* H   */, 8   /* I   */, 9   /* J   */, 10  /* K   */, 11  /* L   */, 12  /* M   */, 13  /* N   */, 14  /* O   */,
-//      for one quantum, paddings are added.
+	15  /* P   */, 16  /* Q   */, 17  /* R   */, 18  /* S   */, 19  /* T   */, 20  /* U   */, 21  /* V   */, 22  /* W   */, 23  /* X   */, 24  /* Y   */, 25  /* Z   */, 255 /* [   */, 255 /* \   */, 255 /* ]   */, 255 /* ^   */, 63  /* _   */,
-// Arguments:
+	255 /* `   */, 26  /* a   */, 27  /* b   */, 28  /* c   */, 29  /* d   */, 30  /* e   */, 31  /* f   */, 32  /* g   */, 33  /* h   */, 34  /* i   */, 35  /* j   */, 36  /* k   */, 37  /* l   */, 38  /* m   */, 39  /* n   */, 40  /* o   */,
-// - src - String to base64 encode.
+	41  /* p   */, 42  /* q   */, 43  /* r   */, 44  /* s   */, 45  /* t   */, 46  /* u   */, 47  /* v   */, 48  /* w   */, 49  /* x   */, 50  /* y   */, 51  /* z   */, 255 /* {   */, 255 /* |   */, 255 /* }   */, 255 /* ~   */, 255 /* DEL */,
-// Return Value:
+};
-// - the encoded string.
+// clang-format on
-std::wstring Base64::s_Encode(const std::wstring_view src) noexcept
-{
-    std::wstring dst;
-    wchar_t input[3];
-
-    const auto len = (src.size() + 2) / 3 * 4;
-    if (len == 0)
-    {
-        return dst;
-    }
-    dst.reserve(len);
-
-    auto iter = src.cbegin();
-    // Encode each three chars into one quantum (four chars).
-    while (iter < src.cend() - 2)
-    {
-        input[0] = *iter++;
-        input[1] = *iter++;
-        input[2] = *iter++;
-        dst.push_back(base64Chars[input[0] >> 2]);
-        dst.push_back(base64Chars[(input[0] & 0x03) << 4 | input[1] >> 4]);
-        dst.push_back(base64Chars[(input[1] & 0x0f) << 2 | input[2] >> 6]);
-        dst.push_back(base64Chars[(input[2] & 0x3f)]);
-    }
-
-    // Here only zero, or one, or two chars are left. We may need to add paddings.
-    if (iter < src.cend())
-    {
-        input[0] = *iter++;
-        dst.push_back(base64Chars[input[0] >> 2]);
-        if (iter < src.cend()) // Two chars left.
-        {
-            input[1] = *iter++;
-            dst.push_back(base64Chars[(input[0] & 0x03) << 4 | input[1] >> 4]);
-            dst.push_back(base64Chars[(input[1] & 0x0f) << 2]);
-        }
-        else // Only one char left.
-        {
-            dst.push_back(base64Chars[(input[0] & 0x03) << 4]);
-            dst.push_back(padChar);
-        }
-        dst.push_back(padChar);
-    }
-
-    return dst;
-}
 
 // Routine Description:
 // - Decode a base64 string. This requires the base64 string is properly padded.
@@ -73,121 +29,98 @@ std::wstring Base64::s_Encode(const std::wstring_view src) noexcept
 // - dst - Destination to decode into.
 // Return Value:
 // - true if decoding successfully, otherwise false.
-bool Base64::s_Decode(const std::wstring_view src, std::wstring& dst) noexcept
+void Base64::s_Decode(const std::wstring_view src, std::wstring& dst)
 {
-    std::string mbStr;
+    std::string result;
-    int state = 0;
+    result.resize((src.size() / 4) * 3);
-    char tmp;
 
-    const auto len = src.size() / 4 * 3;
+    auto in = src.data();
-    if (len == 0)
+    const auto inEnd = in + src.size();
+    const auto inEndBatched = inEnd - 3;
+    const auto outBeg = reinterpret_cast<uint8_t*>(result.data());
+    auto out = outBeg;
+    uint_fast32_t r = 0;
+    uint_fast8_t ri = 0;
+    uint_fast16_t error = 0;
+
+#define accumulate(ch)                         \
+    do                                         \
+    {                                          \
+        const auto n = decodeTable[ch & 0x7f]; \
+                                               \
+        error |= (ch | n) & 0xff80;            \
+                                               \
+        if ((n & 0b01000000) == 0)             \
+        {                                      \
+            r = r << 6 | n;                    \
+            ri++;                              \
+        }                                      \
+    } while (0)
+
+    while (in < inEndBatched)
     {
-        return false;
+        const auto a = in[0];
-    }
+        const auto b = in[1];
-    mbStr.reserve(len);
+        const auto c = in[2];
+        const auto d = in[3];
 
-    auto iter = src.cbegin();
+        accumulate(a);
-    while (iter < src.cend())
+        accumulate(b);
-    {
+        accumulate(c);
-        if (s_IsSpace(*iter)) // Skip whitespace anywhere.
+        accumulate(d);
-        {
-            iter++;
-            continue;
-        }
 
-        if (*iter == padChar)
+        switch (ri)
         {
-            break;
-        }
-
-        auto pos = strchr(base64Chars, *iter);
-        if (!pos) // A non-base64 character found.
-        {
-            return false;
-        }
-
-        switch (state)
-        {
-        case 0:
-            tmp = (char)(pos - base64Chars) << 2;
-            state = 1;
-            break;
-        case 1:
-            tmp |= (char)(pos - base64Chars) >> 4;
-            mbStr += tmp;
-            tmp = (char)((pos - base64Chars) & 0x0f) << 4;
-            state = 2;
-            break;
         case 2:
-            tmp |= (char)(pos - base64Chars) >> 2;
+            out[0] = uint8_t(r >> 4);
-            mbStr += tmp;
+            out += 1;
-            tmp = (char)((pos - base64Chars) & 0x03) << 6;
+            ri = 1;
-            state = 3;
             break;
         case 3:
-            tmp |= pos - base64Chars;
+            out[0] = uint8_t(r >> 10);
-            mbStr += tmp;
+            out[1] = uint8_t(r >> 2);
-            state = 0;
+            out += 2;
+            ri = 1;
             break;
-        default:
+        case 4:
+            out[0] = uint8_t(r >> 16);
+            out[1] = uint8_t(r >> 8);
+            out[2] = uint8_t(r >> 0);
+            out += 3;
+            ri = 0;
             break;
         }
 
-        iter++;
+        in += 4;
     }
 
-    if (iter < src.cend()) // Padding char is met.
+    for (size_t i = 0, remaining = inEnd - in; i < remaining; i++)
     {
-        iter++;
+        const auto ch = in[i];
-        switch (state)
+        accumulate(ch);
-        {
-        // Invalid when state is 0 or 1.
-        case 0:
-        case 1:
-            return false;
-        case 2:
-            // Skip any number of spaces.
-            while (iter < src.cend() && s_IsSpace(*iter))
-            {
-                iter++;
-            }
-            // Make sure there is another trailing padding character.
-            if (iter == src.cend() || *iter != padChar)
-            {
-                return false;
-            }
-            iter++; // Skip the padding character and fallthrough to "single trailing padding character" case.
-            [[fallthrough]];
-        case 3:
-            while (iter < src.cend())
-            {
-                if (!s_IsSpace(*iter))
-                {
-                    return false;
-                }
-                iter++;
-            }
-            break;
-        default:
-            break;
-        }
     }
-    else if (state != 0) // When no padding, we must be in state 0.
+
+    switch (ri)
     {
-        return false;
+    case 2:
+        out[0] = uint8_t(r >> 4);
+        break;
+    case 3:
+        out[0] = uint8_t(r >> 10);
+        out[1] = uint8_t(r >> 2);
+        break;
+    case 4:
+        out[0] = uint8_t(r >> 16);
+        out[1] = uint8_t(r >> 8);
+        out[2] = uint8_t(r >> 0);
+        break;
     }
 
-    return SUCCEEDED(til::u8u16(mbStr, dst));
+    if (error)
-}
+    {
+        throw std::runtime_error("invalid base64");
+    }
 
-// Routine Description:
+    result.resize(out - outBeg);
-// - Check if parameter is a base64 whitespace. Only carriage return or line feed
+    til::u8u16(result, dst);
-//      is valid whitespace.
-// Arguments:
-// - ch - Character to check.
-// Return Value:
-// - true iff ch is a carriage return or line feed.
-constexpr bool Base64::s_IsSpace(const wchar_t ch) noexcept
-{
-    return ch == L'\r' || ch == L'\n';
 }

src/terminal/parser/base64.hpp

@@ -16,10 +16,6 @@ namespace Microsoft::Console::VirtualTerminal
     class Base64
     {
     public:
-        static std::wstring s_Encode(const std::wstring_view src) noexcept;
+        static void s_Decode(const std::wstring_view src, std::wstring& dst);
-        static bool s_Decode(const std::wstring_view src, std::wstring& dst) noexcept;
-
-    private:
-        static constexpr bool s_IsSpace(const wchar_t ch) noexcept;
     };
 }

src/terminal/parser/ut_parser/Base64Test.cpp

@@ -28,15 +28,6 @@ class Microsoft::Console::VirtualTerminal::Base64Test
 {
     TEST_CLASS(Base64Test);
 
-    TEST_METHOD(TestBase64Encode)
-    {
-        VERIFY_ARE_EQUAL(L"Zm9v", Base64::s_Encode(L"foo"));
-        VERIFY_ARE_EQUAL(L"Zm9vYg==", Base64::s_Encode(L"foob"));
-        VERIFY_ARE_EQUAL(L"Zm9vYmE=", Base64::s_Encode(L"fooba"));
-        VERIFY_ARE_EQUAL(L"Zm9vYmFy", Base64::s_Encode(L"foobar"));
-        VERIFY_ARE_EQUAL(L"Zm9vYmFyDQo=", Base64::s_Encode(L"foobar\r\n"));
-    }
-
     TEST_METHOD(TestBase64Decode)
     {
         std::wstring result;

src/til/ut_til/SPSCTests.cpp

@@ -4,6 +4,8 @@
 #include "precomp.h"
 #include "WexTestClass.h"
 
+#include <til/spsc.h>
+
 using namespace WEX::Common;
 using namespace WEX::Logging;
 using namespace WEX::TestExecution;

src/til/ut_til/u8u16convertTests.cpp

@@ -4,6 +4,8 @@
 #include "precomp.h"
 #include "WexTestClass.h"
 
+#include <til/u8u16convert.h>
+
 using namespace WEX::Common;
 using namespace WEX::Logging;
 using namespace WEX::TestExecution;
@@ -43,8 +45,7 @@ void Utf8Utf16ConvertTests::TestU8ToU16()
     };
 
     std::wstring u16Out{};
-    const HRESULT hRes{ til::u8u16(u8String, u16Out) };
+    til::u8u16(u8String, u16Out);
-    VERIFY_ARE_EQUAL(S_OK, hRes);
     VERIFY_ARE_EQUAL(u16StringComp, u16Out);
 }
 
@@ -72,8 +73,7 @@ void Utf8Utf16ConvertTests::TestU16ToU8()
     };
 
     std::string u8Out{};
-    const HRESULT hRes{ til::u16u8(u16String, u8Out) };
+    til::u16u8(u16String, u8Out);
-    VERIFY_ARE_EQUAL(S_OK, hRes);
     VERIFY_ARE_EQUAL(u8StringComp, u8Out);
 }
 
@@ -115,23 +115,19 @@ void Utf8Utf16ConvertTests::TestU8ToU16Partials()
     til::u8state state{};
 
     std::wstring u16Out1{};
-    const HRESULT hRes1{ til::u8u16(u8String1, u16Out1, state) };
+    til::u8u16(u8String1, u16Out1, state);
-    VERIFY_ARE_EQUAL(S_OK, hRes1);
     VERIFY_ARE_EQUAL(u16StringComp1, u16Out1);
 
     std::wstring u16Out2{};
-    const HRESULT hRes2{ til::u8u16(u8String2, u16Out2, state) };
+    til::u8u16(u8String2, u16Out2, state);
-    VERIFY_ARE_EQUAL(S_OK, hRes2);
     VERIFY_ARE_EQUAL(u16StringComp1, u16Out2);
 
     std::wstring u16Out3{};
-    const HRESULT hRes3{ til::u8u16(u8String3, u16Out3, state) };
+    til::u8u16(u8String3, u16Out3, state);
-    VERIFY_ARE_EQUAL(S_OK, hRes3);
     VERIFY_ARE_EQUAL(std::wstring{}, u16Out3);
 
     std::wstring u16Out4{};
-    const HRESULT hRes4{ til::u8u16(u8String4, u16Out4, state) };
+    til::u8u16(u8String4, u16Out4, state);
-    VERIFY_ARE_EQUAL(S_OK, hRes4);
     VERIFY_ARE_EQUAL(u16StringComp2, u16Out4);
 }
 
@@ -157,13 +153,11 @@ void Utf8Utf16ConvertTests::TestU16ToU8Partials()
     til::u16state state{};
 
     std::string u8Out1{};
-    const HRESULT hRes1{ til::u16u8(u16String1, u8Out1, state) };
+    til::u16u8(u16String1, u8Out1, state);
-    VERIFY_ARE_EQUAL(S_OK, hRes1);
     VERIFY_ARE_EQUAL(u8StringComp, u8Out1);
 
     std::string u8Out2{};
-    const HRESULT hRes2{ til::u16u8(u16String2, u8Out2, state) };
+    til::u16u8(u16String2, u8Out2, state);
-    VERIFY_ARE_EQUAL(S_OK, hRes2);
     VERIFY_ARE_EQUAL(u8StringComp, u8Out2);
 }
 
@@ -182,12 +176,12 @@ void Utf8Utf16ConvertTests::TestU8ToU16OneByOne()
     til::u8state state{};
 
     std::wstring u16Out1{};
-    VERIFY_SUCCEEDED(til::u8u16(u8String1_1, u16Out1, state));
+    til::u8u16(u8String1_1, u16Out1, state);
     VERIFY_ARE_EQUAL(L"", u16Out1); // There should be no output for the first three bytes
-    VERIFY_SUCCEEDED(til::u8u16(u8String1_2, u16Out1, state));
+    til::u8u16(u8String1_2, u16Out1, state);
     VERIFY_ARE_EQUAL(L"", u16Out1);
-    VERIFY_SUCCEEDED(til::u8u16(u8String1_3, u16Out1, state));
+    til::u8u16(u8String1_3, u16Out1, state);
     VERIFY_ARE_EQUAL(L"", u16Out1);
-    VERIFY_SUCCEEDED(til::u8u16(u8String1_4, u16Out1, state));
+    til::u8u16(u8String1_4, u16Out1, state);
     VERIFY_ARE_EQUAL(u16StringComp1, u16Out1);
 }