From 680577f55c6927ae4f1a63ae396aff6b9b0fae66 Mon Sep 17 00:00:00 2001 From: Michael Niksa Date: Tue, 24 Mar 2020 19:41:10 -0700 Subject: [PATCH] Update til::bitmap to use dynamic_bitset<> + libpopcnt (#5092) This commit replaces `std::vector` with `dynamic_bitset<>` by @pinam45 (https://github.com/pinam45/dynamic_bitset) and with `libpopcnt` for high-performance bit counting by @kimwalisch (https://github.com/kimwalisch/libpopcnt). * [x] In support of performance, incremental rendering, and Terminal "not speed enough" as well as my sanity relative to `std::vector` * [x] Tests updated and passed. * [x] `LICENSE`, `NOTICE`, and provenance files updated. * [x] I'm a core contributor. I discussed it with @DHowett-MSFT and cleared the licensing checks before pulling this in. ## Details `std::vector` provided by the Microsoft VC Runtime is incapable of a great many things. Many of the methods you come to expect off of `std::vector` that are dutifully presented through the `bool` variant will spontaneously fail at some future date because it decides you allocated, resized, or manipulated the `vector` specialization in an unsupported manner. Half of the methods will straight up not work for filling/resizing in bulk. And you will tear your hair out as it will somehow magically forget the assignment of half the bits you gave it part way through an iteration then assert out and die. As such, to preserve my sanity, I searched for an alternative. I came across the self-contained header-only library `dynamic_bitset` by @pinam45 which appears to do as much of `boost::dynamic_bitset` as I wanted, but without including 400kg of boost libraries. It also has a nifty optional dependency on `libpopcnt` by @kimwalisch that will use processor-specific extensions for rapidly counting bits. @DHowett-MSFT and I briefly discussed how nice `popcnt` would have been on `std::vector` last week... and now we can have it. (To be fair, I don't believe I'm using it yet... but we'll be able to easily dial in `til::bitmap` soon and not worry about a performance hit if we do have to walk bits and count them thanks to `libpopcnt`.) This PR specifically focuses on swapping the dependencies out and ingesting the new libraries. We'll further tune `til::bitmap` in future pulls as necessary. ## Validation * [x] Ran the automated tests for bitmap. * [x] Ran the terminal manually and it looks fine still. --- NOTICE.md | 69 +- dep/dynamic_bitset/LICENSE | 21 + dep/dynamic_bitset/MAINTAINER_README.md | 17 + dep/dynamic_bitset/cgmanifest.json | 13 + dep/dynamic_bitset/dynamic_bitset.hpp | 1944 +++++++++++++++++ dep/libpopcnt/LICENSE | 26 + dep/libpopcnt/MAINTAINER_README.md | 17 + dep/libpopcnt/cgmanifest.json | 13 + dep/libpopcnt/libpopcnt.h | 841 +++++++ .../WindowsTerminalUniversal.vcxproj | 2 +- src/common.build.pre.props | 2 +- src/inc/LibraryIncludes.h | 7 + src/inc/til/bitmap.h | 33 +- src/til/ut_til/BitmapTests.cpp | 17 +- 14 files changed, 2988 insertions(+), 34 deletions(-) create mode 100644 dep/dynamic_bitset/LICENSE create mode 100644 dep/dynamic_bitset/MAINTAINER_README.md create mode 100644 dep/dynamic_bitset/cgmanifest.json create mode 100644 dep/dynamic_bitset/dynamic_bitset.hpp create mode 100644 dep/libpopcnt/LICENSE create mode 100644 dep/libpopcnt/MAINTAINER_README.md create mode 100644 dep/libpopcnt/cgmanifest.json create mode 100644 dep/libpopcnt/libpopcnt.h diff --git a/NOTICE.md b/NOTICE.md index 0f50008fb..9582b76cb 100644 --- a/NOTICE.md +++ b/NOTICE.md @@ -80,7 +80,7 @@ SOFTWARE. ## chromium/base/numerics -**Source**: +**Source**: https://github.com/chromium/chromium/tree/master/base/numerics ### License @@ -112,4 +112,71 @@ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` + +## kimwalisch/libpopcnt + +**Source**: https://github.com/kimwalisch/libpopcnt + +### License + +``` +BSD 2-Clause License + +Copyright (c) 2016 - 2019, Kim Walisch +Copyright (c) 2016 - 2019, Wojciech Muła + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +``` + +## dynamic_bitset + +**Source**: https://github.com/pinam45/dynamic_bitset + +### License + +``` +MIT License + +Copyright (c) 2019 Maxime Pinard + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + ``` \ No newline at end of file diff --git a/dep/dynamic_bitset/LICENSE b/dep/dynamic_bitset/LICENSE new file mode 100644 index 000000000..ee9bc7066 --- /dev/null +++ b/dep/dynamic_bitset/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 Maxime Pinard + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/dep/dynamic_bitset/MAINTAINER_README.md b/dep/dynamic_bitset/MAINTAINER_README.md new file mode 100644 index 000000000..4782372cd --- /dev/null +++ b/dep/dynamic_bitset/MAINTAINER_README.md @@ -0,0 +1,17 @@ +### Notes for Future Maintainers + +This was originally imported by @miniksa in March 2020. + +The provenance information (where it came from and which commit) is stored in the file `cgmanifest.json` in the same directory as this readme. +Please update the provenance information in that file when ingesting an updated version of the dependent library. +That provenance file is automatically read and inventoried by Microsoft systems to ensure compliance with appropiate governance standards. + +## What should be done to update this in the future? + +1. Go to pinam45/dynamic_bitset repository on GitHub. +2. Take the entire contents of the include directory wholesale and drop it in the root directory here. +3. Don't change anything about it. +4. Validate that the license in the root of the repository didn't change and update it if so. It is sitting in the same directory as this readme. + If it changed dramatically, ensure that it is still compatible with our license scheme. Also update the NOTICE file in the root of our repository to declare the third-party usage. +5. Submit the pull. + diff --git a/dep/dynamic_bitset/cgmanifest.json b/dep/dynamic_bitset/cgmanifest.json new file mode 100644 index 000000000..1585faeb5 --- /dev/null +++ b/dep/dynamic_bitset/cgmanifest.json @@ -0,0 +1,13 @@ +{"Registrations":[ + { + "component": { + "type": "git", + "git": { + "repositoryUrl": "https://github.com/pinam45/dynamic_bitset", + "commitHash": "00f2d066ce9deebf28b006636150e5a882beb83f" + } + } + } + ], + "Version": 1 +} \ No newline at end of file diff --git a/dep/dynamic_bitset/dynamic_bitset.hpp b/dep/dynamic_bitset/dynamic_bitset.hpp new file mode 100644 index 000000000..8a0e385fb --- /dev/null +++ b/dep/dynamic_bitset/dynamic_bitset.hpp @@ -0,0 +1,1944 @@ +// +// Copyright (c) 2019 Maxime Pinard +// +// Distributed under the MIT license +// See accompanying file LICENSE or copy at +// https://opensource.org/licenses/MIT +// +#ifndef DYNAMIC_BITSET_DYNAMIC_BITSET_HPP +#define DYNAMIC_BITSET_DYNAMIC_BITSET_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef DYNAMIC_BITSET_NO_LIBPOPCNT +# if __has_include() +# include +# define DYNAMIC_BITSET_USE_LIBPOPCNT +# endif +#endif + +#if defined(__clang__) +# define DYNAMIC_BITSET_CLANG +# ifdef __has_builtin +# if __has_builtin(__builtin_popcount) && __has_builtin(__builtin_popcountl) \ + && __has_builtin(__builtin_popcountll) +# define DYNAMIC_BITSET_CLANG_builtin_popcount +# endif +# if __has_builtin(__builtin_ctz) && __has_builtin(__builtin_ctzl) \ + && __has_builtin(__builtin_ctzll) +# define DYNAMIC_BITSET_CLANG_builtin_ctz +# endif +# endif +#elif defined(__GNUC__) +# define DYNAMIC_BITSET_GCC +#elif defined(_MSC_VER) +# define DYNAMIC_BITSET_MSVC +# include +# pragma intrinsic(_BitScanForward) +# if defined(_M_X64) || defined(_M_ARM64) +# define DYNAMIC_BITSET_MSVC_64 +# pragma intrinsic(_BitScanForward64) +# else +# endif +#endif + +template> +class dynamic_bitset +{ + static_assert(std::is_unsigned::value, "Block is not an unsigned integral type"); + +public: + typedef size_t size_type; + typedef Block block_type; + typedef Allocator allocator_type; + + static constexpr size_type bits_per_block = CHAR_BIT * sizeof(block_type); + static constexpr size_type npos = std::numeric_limits::max(); + + class reference + { + public: + constexpr reference(dynamic_bitset& bitset, size_type bit_pos); + constexpr reference(const reference&) noexcept = default; + constexpr reference(reference&&) noexcept = default; + ~reference() noexcept = default; + + constexpr reference& operator=(bool v); + constexpr reference& operator=(const reference& rhs); + constexpr reference& operator=(reference&& rhs) noexcept; + + constexpr reference& operator&=(bool v); + constexpr reference& operator|=(bool v); + constexpr reference& operator^=(bool v); + constexpr reference& operator-=(bool v); + + [[nodiscard]] constexpr bool operator~() const; + [[nodiscard]] constexpr operator bool() const; + constexpr void operator&() = delete; + + constexpr reference& set(); + constexpr reference& reset(); + constexpr reference& flip(); + constexpr reference& assign(bool v); + + private: + block_type& m_block; + block_type m_mask; + }; + typedef bool const_reference; + + // copy/move constructors = default + constexpr dynamic_bitset(const dynamic_bitset& other) = default; + constexpr dynamic_bitset(dynamic_bitset&& other) noexcept = default; + constexpr dynamic_bitset& operator=( + const dynamic_bitset& other) = default; + constexpr dynamic_bitset& operator=( + dynamic_bitset&& other) noexcept = default; + + // other constructors + constexpr explicit dynamic_bitset(const allocator_type& allocator = allocator_type()); + constexpr explicit dynamic_bitset(size_type nbits, + unsigned long long init_val = 0, + const allocator_type& allocator = allocator_type()); + constexpr dynamic_bitset(std::initializer_list init_vals, + const allocator_type& allocator = allocator_type()); + + // string constructors + template + constexpr explicit dynamic_bitset( + std::basic_string_view<_CharT, _Traits> str, + typename std::basic_string_view<_CharT, _Traits>::size_type pos = 0, + typename std::basic_string_view<_CharT, _Traits>::size_type n = + std::basic_string_view<_CharT, _Traits>::npos, + _CharT zero = _CharT('0'), + _CharT one = _CharT('1'), + const allocator_type& allocator = allocator_type()); + + template + constexpr explicit dynamic_bitset( + const std::basic_string<_CharT, _Traits, _Alloc>& str, + typename std::basic_string<_CharT, _Traits, _Alloc>::size_type pos = 0, + typename std::basic_string<_CharT, _Traits, _Alloc>::size_type n = + std::basic_string<_CharT, _Traits, _Alloc>::npos, + _CharT zero = _CharT('0'), + _CharT one = _CharT('1'), + const allocator_type& allocator = allocator_type()); + + template + constexpr explicit dynamic_bitset( + const _CharT* str, + typename std::basic_string<_CharT>::size_type pos = 0, + typename std::basic_string<_CharT>::size_type n = std::basic_string<_CharT>::npos, + _CharT zero = _CharT('0'), + _CharT one = _CharT('1'), + const allocator_type& allocator = allocator_type()); + + // destructor + ~dynamic_bitset() noexcept = default; + + // size changing operations + constexpr void resize(size_type nbits, bool value = false); + constexpr void clear(); + constexpr void push_back(bool value); + constexpr void pop_back(); + constexpr void append(block_type block); + constexpr void append(std::initializer_list blocks); + template + constexpr void append(BlockInputIterator first, BlockInputIterator last); + + // bitset operations + constexpr dynamic_bitset& operator&=( + const dynamic_bitset& rhs); + constexpr dynamic_bitset& operator|=( + const dynamic_bitset& rhs); + constexpr dynamic_bitset& operator^=( + const dynamic_bitset& rhs); + constexpr dynamic_bitset& operator-=( + const dynamic_bitset& rhs); + constexpr dynamic_bitset& operator<<=(size_type shift); + constexpr dynamic_bitset& operator>>=(size_type shift); + [[nodiscard]] constexpr dynamic_bitset operator<<(size_type shift) const; + [[nodiscard]] constexpr dynamic_bitset operator>>(size_type shift) const; + [[nodiscard]] constexpr dynamic_bitset operator~() const; + + // bit operations + constexpr dynamic_bitset& set(size_type pos, size_type len, bool value); + constexpr dynamic_bitset& set(size_type pos, bool value = true); + constexpr dynamic_bitset& set(); + constexpr dynamic_bitset& reset(size_type pos, size_type len); + constexpr dynamic_bitset& reset(size_type pos); + constexpr dynamic_bitset& reset(); + constexpr dynamic_bitset& flip(size_type pos, size_type len); + constexpr dynamic_bitset& flip(size_type pos); + constexpr dynamic_bitset& flip(); + [[nodiscard]] constexpr bool test(size_type pos) const; + [[nodiscard]] constexpr bool test_set(size_type pos, bool value = true); + [[nodiscard]] constexpr bool all() const; + [[nodiscard]] constexpr bool any() const; + [[nodiscard]] constexpr bool none() const; + [[nodiscard]] constexpr size_type count() const noexcept; + + // subscript operators + [[nodiscard]] constexpr reference operator[](size_type pos); + [[nodiscard]] constexpr const_reference operator[](size_type pos) const; + + //container-like functions + [[nodiscard]] constexpr size_type size() const noexcept; + [[nodiscard]] constexpr size_type num_blocks() const noexcept; + [[nodiscard]] constexpr bool empty() const noexcept; + [[nodiscard]] constexpr size_type capacity() const noexcept; + constexpr void reserve(size_type num_bits); + constexpr void shrink_to_fit(); + + // subsets + [[nodiscard]] constexpr bool is_subset_of(const dynamic_bitset& bitset) const; + [[nodiscard]] constexpr bool is_proper_subset_of( + const dynamic_bitset& bitset) const; + [[nodiscard]] constexpr bool intersects(const dynamic_bitset& bitset) const; + + // find functions + [[nodiscard]] constexpr size_type find_first() const; + [[nodiscard]] constexpr size_type find_next(size_type prev) const; + + // utils + constexpr void swap(dynamic_bitset& other); + [[nodiscard]] constexpr allocator_type get_allocator() const; + template, + typename _Alloc = std::allocator<_CharT>> + [[nodiscard]] constexpr std::basic_string<_CharT, _Traits, _Alloc> to_string( + _CharT zero = _CharT('0'), + _CharT one = _CharT('1')) const; + template + constexpr void iterate_bits_on(Function&& function, Parameters&&... parameters) const; + + // friend external binary operators + template + friend constexpr bool operator==(const dynamic_bitset& lhs, + const dynamic_bitset& rhs); + template + friend constexpr bool operator<(const dynamic_bitset& lhs, + const dynamic_bitset& rhs); + +private: + template + struct dependent_false : public std::false_type + { + }; + + std::vector m_blocks; + size_type m_bits_number; + + static constexpr block_type zero_block = block_type(0); + static constexpr block_type one_block = block_type(~zero_block); + static constexpr size_type block_last_bit_index = bits_per_block - 1; + + static constexpr size_type blocks_required(size_type nbits) noexcept; + + static constexpr size_type block_index(size_type pos) noexcept; + static constexpr size_type bit_index(size_type pos) noexcept; + + static constexpr block_type bit_mask(size_type pos) noexcept; + static constexpr block_type bit_mask(size_type first, size_type last) noexcept; + + static constexpr void set_block_bits(block_type& block, + size_type first, + size_type last, + bool val = true) noexcept; + static constexpr void flip_block_bits(block_type& block, + size_type first, + size_type last) noexcept; + + static constexpr size_type block_count(const block_type& block) noexcept; + static constexpr size_type block_count(const block_type& block, size_type nbits) noexcept; + + static constexpr size_type first_on(const block_type& block) noexcept; + + template + constexpr void init_from_string(std::basic_string_view<_CharT, _Traits> str, + typename std::basic_string_view<_CharT, _Traits>::size_type pos, + typename std::basic_string_view<_CharT, _Traits>::size_type n, + _CharT zero, + _CharT one); + + constexpr block_type& get_block(size_type pos); + constexpr const block_type& get_block(size_type pos) const; + constexpr block_type& last_block(); + constexpr block_type last_block() const; + + // used bits in the last block + constexpr size_type extra_bits_number() const noexcept; + // unused bits in the last block + constexpr size_type unused_bits_number() const noexcept; + + template + constexpr void apply(const dynamic_bitset& other, BinaryOperation binary_op); + template + constexpr void apply(UnaryOperation unary_op); + constexpr void apply_left_shift(size_type shift); + constexpr void apply_right_shift(size_type shift); + + // reset unused bits to 0 + constexpr void sanitize(); + + // check functions used in asserts + constexpr bool check_unused_bits() const noexcept; + constexpr bool check_size() const noexcept; + constexpr bool check_consistency() const noexcept; +}; + +// Deduction guideline for expressions like "dynamic_bitset a(32);" with an integral type as parameter +// to use the constructor with the initial size instead of the constructor with the allocator. +template>> +dynamic_bitset(integral_type)->dynamic_bitset<>; + +//================================================================================================= +// dynamic_bitset external functions declarations +//================================================================================================= + +template +constexpr bool operator!=(const dynamic_bitset& lhs, + const dynamic_bitset& rhs); +template +constexpr bool operator<=(const dynamic_bitset& lhs, + const dynamic_bitset& rhs); +template +constexpr bool operator>(const dynamic_bitset& lhs, + const dynamic_bitset& rhs); +template +constexpr bool operator>=(const dynamic_bitset& lhs, + const dynamic_bitset& rhs); + +template +constexpr dynamic_bitset operator&(const dynamic_bitset& lhs, + const dynamic_bitset& rhs); +template +constexpr dynamic_bitset operator|(const dynamic_bitset& lhs, + const dynamic_bitset& rhs); +template +constexpr dynamic_bitset operator^(const dynamic_bitset& lhs, + const dynamic_bitset& rhs); +template +constexpr dynamic_bitset operator-(const dynamic_bitset& lhs, + const dynamic_bitset& rhs); + +template +constexpr std::basic_ostream<_CharT, _Traits>& operator<<( + std::basic_ostream<_CharT, _Traits>& os, + const dynamic_bitset& bitset); + +template +constexpr std::basic_istream<_CharT, _Traits>& operator>>(std::basic_istream<_CharT, _Traits>& is, + dynamic_bitset& bitset); + +template +constexpr void swap(dynamic_bitset& bitset1, + dynamic_bitset& bitset2); + +//================================================================================================= +// dynamic_bitset::reference functions implementations +//================================================================================================= + +template +constexpr dynamic_bitset::reference::reference( + dynamic_bitset& bitset, + size_type bit_pos) + : m_block(bitset.get_block(bit_pos)), m_mask(dynamic_bitset::bit_mask(bit_pos)) +{ +} + +template +constexpr typename dynamic_bitset::reference& dynamic_bitset:: + reference::operator=(bool v) +{ + assign(v); + return *this; +} + +template +constexpr typename dynamic_bitset::reference& dynamic_bitset:: + reference::operator=(const dynamic_bitset::reference& rhs) +{ + assign(rhs); + return *this; +} + +template +constexpr typename dynamic_bitset::reference& dynamic_bitset:: + reference::operator=(dynamic_bitset::reference&& rhs) noexcept +{ + assign(rhs); + return *this; +} + +template +constexpr typename dynamic_bitset::reference& dynamic_bitset:: + reference::operator&=(bool v) +{ + if(!v) + { + reset(); + } + return *this; +} + +template +constexpr typename dynamic_bitset::reference& dynamic_bitset:: + reference::operator|=(bool v) +{ + if(v) + { + set(); + } + return *this; +} + +template +constexpr typename dynamic_bitset::reference& dynamic_bitset:: + reference::operator^=(bool v) +{ + if(v) + { + flip(); + } + return *this; +} + +template +constexpr typename dynamic_bitset::reference& dynamic_bitset:: + reference::operator-=(bool v) +{ + if(v) + { + reset(); + } + return *this; +} + +template +constexpr bool dynamic_bitset::reference::operator~() const +{ + return (m_block & m_mask) == zero_block; +} + +template +constexpr dynamic_bitset::reference::operator bool() const +{ + return (m_block & m_mask) != zero_block; +} + +template +constexpr typename dynamic_bitset::reference& dynamic_bitset:: + reference::set() +{ + m_block |= m_mask; + return *this; +} + +template +constexpr typename dynamic_bitset::reference& dynamic_bitset:: + reference::reset() +{ + m_block &= ~m_mask; + return *this; +} + +template +constexpr typename dynamic_bitset::reference& dynamic_bitset:: + reference::flip() +{ + m_block ^= m_mask; + return *this; +} + +template +constexpr typename dynamic_bitset::reference& dynamic_bitset:: + reference::assign(bool v) +{ + if(v) + { + set(); + } + else + { + reset(); + } + return *this; +} + +//================================================================================================= +// dynamic_bitset public functions implementations +//================================================================================================= + +template +constexpr dynamic_bitset::dynamic_bitset(const allocator_type& allocator) + : m_blocks(allocator), m_bits_number(0) +{ +} + +template +constexpr dynamic_bitset::dynamic_bitset(size_type nbits, + unsigned long long init_val, + const allocator_type& allocator) + : m_blocks(blocks_required(nbits), allocator), m_bits_number(nbits) +{ + if(nbits == 0 || init_val == 0) + { + return; + } + + constexpr size_type init_val_required_blocks = sizeof(unsigned long long) / sizeof(block_type); + if constexpr(init_val_required_blocks == 1) + { + m_blocks[0] = init_val; + } + else + { + const unsigned long long block_mask = static_cast(one_block); + const size_type blocks_to_init = std::min(m_blocks.size(), init_val_required_blocks); + for(size_type i = 0; i < blocks_to_init; ++i) + { + m_blocks[i] = block_type((init_val >> (i * bits_per_block) & block_mask)); + } + } + sanitize(); +} + +template +constexpr dynamic_bitset::dynamic_bitset( + std::initializer_list init_vals, + const allocator_type& allocator) + : m_blocks(allocator), m_bits_number(0) +{ + append(init_vals); +} + +template +template +constexpr dynamic_bitset::dynamic_bitset( + std::basic_string_view<_CharT, _Traits> str, + typename std::basic_string_view<_CharT, _Traits>::size_type pos, + typename std::basic_string_view<_CharT, _Traits>::size_type n, + _CharT zero, + _CharT one, + const allocator_type& allocator) + : m_blocks(allocator), m_bits_number(0) +{ + assert(pos < str.size()); + init_from_string(str, pos, n, zero, one); +} + +template +template +constexpr dynamic_bitset::dynamic_bitset( + const std::basic_string<_CharT, _Traits, _Alloc>& str, + typename std::basic_string<_CharT, _Traits, _Alloc>::size_type pos, + typename std::basic_string<_CharT, _Traits, _Alloc>::size_type n, + _CharT zero, + _CharT one, + const allocator_type& allocator) + : m_blocks(allocator), m_bits_number(0) +{ + assert(pos < str.size()); + init_from_string(std::basic_string_view<_CharT, _Traits>(str), pos, n, zero, one); +} + +template +template +constexpr dynamic_bitset::dynamic_bitset( + const _CharT* str, + typename std::basic_string<_CharT>::size_type pos, + typename std::basic_string<_CharT>::size_type n, + _CharT zero, + _CharT one, + const allocator_type& allocator) + : m_blocks(allocator), m_bits_number(0) +{ + init_from_string(std::basic_string_view<_CharT>(str), pos, n, zero, one); +} + +template +constexpr void dynamic_bitset::resize(size_type nbits, bool value) +{ + if(nbits == m_bits_number) + { + return; + } + + const size_type old_num_blocks = num_blocks(); + const size_type new_num_blocks = blocks_required(nbits); + + const block_type init_value = value ? one_block : zero_block; + if(new_num_blocks != old_num_blocks) + { + m_blocks.resize(new_num_blocks, init_value); + } + + if(value && nbits > m_bits_number && old_num_blocks > 0) + { + // set value of the new bits in the old last block + const size_type extra_bits = extra_bits_number(); + if(extra_bits > 0) + { + m_blocks[old_num_blocks - 1] |= (init_value << extra_bits); + } + } + + m_bits_number = nbits; + sanitize(); + assert(check_consistency()); +} + +template +constexpr void dynamic_bitset::clear() +{ + m_blocks.clear(); + m_bits_number = 0; +} + +template +constexpr void dynamic_bitset::push_back(bool value) +{ + const size_type new_last_bit = m_bits_number++; + if(m_bits_number <= m_blocks.size() * bits_per_block) + { + if(value) + { + set(new_last_bit, value); + } + } + else + { + m_blocks.push_back(block_type(value)); + } + assert(operator[](new_last_bit) == value); + assert(check_consistency()); +} + +template +constexpr void dynamic_bitset::pop_back() +{ + if(empty()) + { + return; + } + + --m_bits_number; + if(m_blocks.size() > blocks_required(m_bits_number)) + { + m_blocks.pop_back(); + // no extra bits: sanitize not required + assert(extra_bits_number() == 0); + } + else + { + sanitize(); + } + assert(check_consistency()); +} + +template +constexpr void dynamic_bitset::append(block_type block) +{ + const size_type extra_bits = extra_bits_number(); + if(extra_bits == 0) + { + m_blocks.push_back(block); + } + else + { + last_block() |= (block << extra_bits); + m_blocks.push_back(block_type(block >> (bits_per_block - extra_bits))); + } + + m_bits_number += bits_per_block; + assert(check_consistency()); +} + +template +constexpr void dynamic_bitset::append(std::initializer_list blocks) +{ + if(blocks.size() == 0) + { + return; + } + + append(std::cbegin(blocks), std::cend(blocks)); +} + +template +template +constexpr void dynamic_bitset::append(BlockInputIterator first, + BlockInputIterator last) +{ + if(first == last) + { + return; + } + + // if random access iterators, std::distance complexity is constant + if constexpr(std::is_same_v< + typename std::iterator_traits::iterator_category, + std::random_access_iterator_tag>) + { + assert(std::distance(first, last) > 0); + m_blocks.reserve(m_blocks.size() + static_cast(std::distance(first, last))); + } + + const size_type extra_bits = extra_bits_number(); + const size_type unused_bits = unused_bits_number(); + if(extra_bits == 0) + { + auto pos = m_blocks.insert(std::end(m_blocks), first, last); + assert(std::distance(pos, std::end(m_blocks)) > 0); + m_bits_number += + static_cast(std::distance(pos, std::end(m_blocks))) * bits_per_block; + } + else + { + last_block() |= (*first << extra_bits); + block_type block = block_type(*first >> unused_bits); + ++first; + while(first != last) + { + block |= (*first << extra_bits); + m_blocks.push_back(block); + m_bits_number += bits_per_block; + block = block_type(*first >> unused_bits); + ++first; + } + m_blocks.push_back(block); + m_bits_number += bits_per_block; + } + + assert(check_consistency()); +} +template +constexpr dynamic_bitset& dynamic_bitset::operator&=( + const dynamic_bitset& rhs) +{ + assert(size() == rhs.size()); + //apply(rhs, std::bit_and()); + for(size_type i = 0; i < m_blocks.size(); ++i) + { + m_blocks[i] &= rhs.m_blocks[i]; + } + return *this; +} + +template +constexpr dynamic_bitset& dynamic_bitset::operator|=( + const dynamic_bitset& rhs) +{ + assert(size() == rhs.size()); + //apply(rhs, std::bit_or()); + for(size_type i = 0; i < m_blocks.size(); ++i) + { + m_blocks[i] |= rhs.m_blocks[i]; + } + return *this; +} + +template +constexpr dynamic_bitset& dynamic_bitset::operator^=( + const dynamic_bitset& rhs) +{ + assert(size() == rhs.size()); + //apply(rhs, std::bit_xor()); + for(size_type i = 0; i < m_blocks.size(); ++i) + { + m_blocks[i] ^= rhs.m_blocks[i]; + } + return *this; +} + +template +constexpr dynamic_bitset& dynamic_bitset::operator-=( + const dynamic_bitset& rhs) +{ + assert(size() == rhs.size()); + //apply(rhs, [](const block_type& x, const block_type& y) { return (x & ~y); }); + for(size_type i = 0; i < m_blocks.size(); ++i) + { + m_blocks[i] &= ~rhs.m_blocks[i]; + } + return *this; +} + +template +constexpr dynamic_bitset& dynamic_bitset::operator<<=( + size_type shift) +{ + if(shift != 0) + { + if(shift >= m_bits_number) + { + reset(); + } + else + { + apply_left_shift(shift); + sanitize(); // unused bits can have changed, reset them to 0 + } + } + return *this; +} + +template +constexpr dynamic_bitset& dynamic_bitset::operator>>=( + size_type shift) +{ + if(shift != 0) + { + if(shift >= m_bits_number) + { + reset(); + } + else + { + apply_right_shift(shift); + } + } + return *this; +} + +template +constexpr dynamic_bitset dynamic_bitset::operator<<( + size_type shift) const +{ + return dynamic_bitset(*this) <<= shift; +} + +template +constexpr dynamic_bitset dynamic_bitset::operator>>( + size_type shift) const +{ + return dynamic_bitset(*this) >>= shift; +} + +template +constexpr dynamic_bitset dynamic_bitset::operator~() const +{ + dynamic_bitset bitset(*this); + bitset.flip(); + return bitset; +} + +template +constexpr dynamic_bitset& dynamic_bitset::set(size_type pos, + size_type len, + bool value) +{ + assert(pos < size()); + assert(pos + len - 1 < size()); + if(len == 0) + { + return *this; + } + + const size_type first_block = block_index(pos); + const size_type last_block = block_index(pos + len - 1); + const size_type first_bit_index = bit_index(pos); + const size_type last_bit_index = bit_index(pos + len - 1); + + if(first_block == last_block) + { + set_block_bits(m_blocks[first_block], first_bit_index, last_bit_index, value); + } + else + { + size_type first_full_block = first_block; + size_type last_full_block = last_block; + + if(first_bit_index != 0) + { + ++first_full_block; // first block is not full + set_block_bits(m_blocks[first_block], first_bit_index, block_last_bit_index, value); + } + + if(last_bit_index != block_last_bit_index) + { + --last_full_block; // last block is not full + set_block_bits(m_blocks[last_block], 0, last_bit_index, value); + } + + const block_type full_block = value ? one_block : zero_block; + for(size_type i = first_full_block; i <= last_full_block; ++i) + { + m_blocks[i] = full_block; + } + } + + return *this; +} + +template +constexpr dynamic_bitset& dynamic_bitset::set(size_type pos, + bool value) +{ + assert(pos < size()); + + if(value) + { + m_blocks[block_index(pos)] |= bit_mask(pos); + } + else + { + m_blocks[block_index(pos)] &= ~bit_mask(pos); + } + + return *this; +} + +template +constexpr dynamic_bitset& dynamic_bitset::set() +{ + std::fill(std::begin(m_blocks), std::end(m_blocks), one_block); + sanitize(); + return *this; +} + +template +constexpr dynamic_bitset& dynamic_bitset::reset(size_type pos, + size_type len) +{ + return set(pos, len, false); +} + +template +constexpr dynamic_bitset& dynamic_bitset::reset(size_type pos) +{ + return set(pos, false); +} + +template +constexpr dynamic_bitset& dynamic_bitset::reset() +{ + std::fill(std::begin(m_blocks), std::end(m_blocks), zero_block); + return *this; +} + +template +constexpr dynamic_bitset& dynamic_bitset::flip(size_type pos, + size_type len) +{ + assert(pos < size()); + assert(pos + len - 1 < size()); + if(len == 0) + { + return *this; + } + + const size_type first_block = block_index(pos); + const size_type last_block = block_index(pos + len - 1); + const size_type first_bit_index = bit_index(pos); + const size_type last_bit_index = bit_index(pos + len - 1); + + if(first_block == last_block) + { + flip_block_bits(m_blocks[first_block], first_bit_index, last_bit_index); + } + else + { + size_type first_full_block = first_block; + size_type last_full_block = last_block; + + if(first_bit_index != 0) + { + ++first_full_block; // first block is not full + flip_block_bits(m_blocks[first_block], first_bit_index, block_last_bit_index); + } + + if(last_bit_index != block_last_bit_index) + { + --last_full_block; // last block is not full + flip_block_bits(m_blocks[last_block], 0, last_bit_index); + } + + for(size_type i = first_full_block; i <= last_full_block; ++i) + { + m_blocks[i] = block_type(~m_blocks[i]); + } + } + + return *this; +} + +template +constexpr dynamic_bitset& dynamic_bitset::flip(size_type pos) +{ + assert(pos < size()); + m_blocks[block_index(pos)] ^= bit_mask(pos); + return *this; +} + +template +constexpr dynamic_bitset& dynamic_bitset::flip() +{ + std::transform( + std::cbegin(m_blocks), std::cend(m_blocks), std::begin(m_blocks), std::bit_not()); + sanitize(); + return *this; +} + +template +constexpr bool dynamic_bitset::test(size_type pos) const +{ + assert(pos < size()); + return (m_blocks[block_index(pos)] & bit_mask(pos)) != zero_block; +} + +template +constexpr bool dynamic_bitset::test_set(size_type pos, bool value) +{ + bool const result = test(pos); + if(result != value) + { + set(pos, value); + } + return result; +} + +template +constexpr bool dynamic_bitset::all() const +{ + if(empty()) + { + return true; + } + + const block_type full_block = one_block; + if(extra_bits_number() == 0) + { + for(const block_type& block: m_blocks) + { + if(block != full_block) + { + return false; + } + } + } + else + { + for(size_type i = 0; i < m_blocks.size() - 1; ++i) + { + if(m_blocks[i] != full_block) + { + return false; + } + } + if(last_block() != (full_block >> unused_bits_number())) + { + return false; + } + } + return true; +} + +template +constexpr bool dynamic_bitset::any() const +{ + for(const block_type& block: m_blocks) + { + if(block != zero_block) + { + return true; + } + } + return false; +} + +template +constexpr bool dynamic_bitset::none() const +{ + return !any(); +} + +template +constexpr typename dynamic_bitset::size_type dynamic_bitset::count() + const noexcept +{ + if(empty()) + { + return 0; + } + +#ifdef DYNAMIC_BITSET_USE_LIBPOPCNT + const size_type count = + static_cast(popcnt(m_blocks.data(), m_blocks.size() * sizeof(block_type))); +#else + size_type count = 0; + + // full blocks + for(size_type i = 0; i < m_blocks.size() - 1; ++i) + { + count += block_count(m_blocks[i]); + } + + // last block + const block_type& block = last_block(); + if(block != zero_block) + { + const size_t extra_bits = extra_bits_number(); + if(extra_bits == 0) + { + count += block_count(block); + } + else + { + count += block_count(block, extra_bits); + } + } +#endif + return count; +} + +template +constexpr + typename dynamic_bitset::reference dynamic_bitset::operator[]( + size_type pos) +{ + assert(pos < size()); + return dynamic_bitset::reference(*this, pos); +} + +template +constexpr typename dynamic_bitset:: + const_reference dynamic_bitset::operator[](size_type pos) const +{ + return test(pos); +} + +template +constexpr typename dynamic_bitset::size_type dynamic_bitset::size() + const noexcept +{ + return m_bits_number; +} + +template +constexpr typename dynamic_bitset::size_type dynamic_bitset:: + num_blocks() const noexcept +{ + return m_blocks.size(); +} + +template +constexpr bool dynamic_bitset::empty() const noexcept +{ + return size() == 0; +} + +template +constexpr typename dynamic_bitset::size_type dynamic_bitset::capacity() + const noexcept +{ + return m_blocks.capacity() * bits_per_block; +} + +template +constexpr void dynamic_bitset::reserve(size_type num_bits) +{ + m_blocks.reserve(blocks_required(num_bits)); +} + +template +constexpr void dynamic_bitset::shrink_to_fit() +{ + m_blocks.shrink_to_fit(); +} + +template +constexpr bool dynamic_bitset::is_subset_of( + const dynamic_bitset& bitset) const +{ + assert(size() == bitset.size()); + for(size_type i = 0; i < m_blocks.size(); ++i) + { + if((m_blocks[i] & ~bitset.m_blocks[i]) != zero_block) + { + return false; + } + } + return true; +} + +template +constexpr bool dynamic_bitset::is_proper_subset_of( + const dynamic_bitset& bitset) const +{ + assert(size() == bitset.size()); + bool is_proper = false; + for(size_type i = 0; i < m_blocks.size(); ++i) + { + const block_type& self_block = m_blocks[i]; + const block_type& other_block = bitset.m_blocks[i]; + + if((self_block & ~other_block) != zero_block) + { + return false; + } + if((~self_block & other_block) != zero_block) + { + is_proper = true; + } + } + return is_proper; +} + +template +constexpr bool dynamic_bitset::intersects( + const dynamic_bitset& bitset) const +{ + const size_type min_blocks_number = std::min(m_blocks.size(), bitset.m_blocks.size()); + for(size_type i = 0; i < min_blocks_number; ++i) + { + if((m_blocks[i] & bitset.m_blocks[i]) != zero_block) + { + return true; + } + } + return false; +} + +template +constexpr typename dynamic_bitset::size_type dynamic_bitset:: + find_first() const +{ + for(size_type i = 0; i < m_blocks.size(); ++i) + { + if(m_blocks[i] != zero_block) + { + return i * bits_per_block + first_on(m_blocks[i]); + } + } + return npos; +} + +template +constexpr typename dynamic_bitset::size_type dynamic_bitset:: + find_next(size_type prev) const +{ + if(empty() || prev >= (size() - 1)) + { + return npos; + } + + const size_type first_bit = prev + 1; + const size_type first_block = block_index(first_bit); + const size_type first_bit_index = bit_index(first_bit); + const block_type first_block_shifted = block_type(m_blocks[first_block] >> first_bit_index); + + if(first_block_shifted != zero_block) + { + return first_bit + first_on(first_block_shifted); + } + else + { + for(size_type i = first_block + 1; i < m_blocks.size(); ++i) + { + if(m_blocks[i] != zero_block) + { + return i * bits_per_block + first_on(m_blocks[i]); + } + } + } + return npos; +} + +template +constexpr void dynamic_bitset::swap(dynamic_bitset& other) +{ + std::swap(m_blocks, other.m_blocks); + std::swap(m_bits_number, other.m_bits_number); +} + +template +constexpr typename dynamic_bitset::allocator_type dynamic_bitset< + Block, + Allocator>::get_allocator() const +{ + return m_blocks.get_allocator(); +} + +template +template +constexpr std::basic_string<_CharT, _Traits, _Alloc> dynamic_bitset::to_string( + _CharT zero, + _CharT one) const +{ + const size_type len = size(); + std::basic_string<_CharT, _Traits, _Alloc> str(len, zero); + for(size_type i_block = 0; i_block < m_blocks.size(); ++i_block) + { + if(m_blocks[i_block] == zero_block) + { + continue; + } + block_type mask = block_type(1); + const size_type limit = + i_block * bits_per_block < len ? len - i_block * bits_per_block : bits_per_block; + for(size_type i_bit = 0; i_bit < limit; ++i_bit) + { + if((m_blocks[i_block] & mask) != zero_block) + { + _Traits::assign(str[len - (i_block * bits_per_block + i_bit + 1)], one); + } + mask <<= 1; + } + } + return str; +} + +template +template +constexpr void dynamic_bitset::iterate_bits_on(Function&& function, + Parameters&&... parameters) const +{ + if constexpr(!std::is_invocable_v) + { + static_assert(dependent_false::value, "Function take invalid arguments"); + // function should take (size_t, parameters...) as arguments + } + + if constexpr(std::is_same_v, void>) + { + size_t i_bit = find_first(); + while(i_bit != npos) + { + std::invoke( + std::forward(function), i_bit, std::forward(parameters)...); + i_bit = find_next(i_bit); + } + } + else if constexpr(std::is_convertible_v, + bool>) + { + size_t i_bit = find_first(); + while(i_bit != npos) + { + if(!std::invoke( + std::forward(function), i_bit, std::forward(parameters)...)) + { + break; + } + i_bit = find_next(i_bit); + } + } + else + { + static_assert(dependent_false::value, "Function have invalid return type"); + // return type should be void, or convertible to bool + } +} + +template +[[nodiscard]] constexpr bool operator==(const dynamic_bitset& lhs, + const dynamic_bitset& rhs) +{ + return (lhs.m_bits_number == rhs.m_bits_number) && (lhs.m_blocks == rhs.m_blocks); +} + +template +[[nodiscard]] constexpr bool operator<(const dynamic_bitset& lhs, + const dynamic_bitset& rhs) +{ + using size_type = typename dynamic_bitset::size_type; + using block_type = typename dynamic_bitset::block_type; + const size_type lhs_size = lhs.size(); + const size_type rhs_size = rhs.size(); + const size_type lhs_blocks_size = lhs.m_blocks.size(); + const size_type rhs_blocks_size = rhs.m_blocks.size(); + + if(lhs_size == rhs_size) + { + // if comparison of two empty bitsets + if(lhs_size == 0) + { + return false; + } + + for(size_type i = lhs_blocks_size - 1; i > 0; --i) + { + if(lhs.m_blocks[i] != rhs.m_blocks[i]) + { + return lhs.m_blocks[i] < rhs.m_blocks[i]; + } + } + return lhs.m_blocks[0] < rhs.m_blocks[0]; + } + + // empty bitset inferior to 0-only bitset + if(lhs_size == 0) + { + return true; + } + if(rhs_size == 0) + { + return false; + } + + const bool rhs_longer = rhs_size > lhs_size; + const dynamic_bitset& longest_bitset = rhs_longer ? rhs : lhs; + const size_type longest_blocks_size = std::max(lhs_blocks_size, rhs_blocks_size); + const size_type shortest_blocks_size = std::min(lhs_blocks_size, rhs_blocks_size); + for(size_type i = longest_blocks_size - 1; i >= shortest_blocks_size; --i) + { + if(longest_bitset.m_blocks[i] != block_type(0)) + { + return rhs_longer; + } + } + + for(size_type i = shortest_blocks_size - 1; i > 0; --i) + { + if(lhs.m_blocks[i] != rhs.m_blocks[i]) + { + return lhs.m_blocks[i] < rhs.m_blocks[i]; + } + } + if(lhs.m_blocks[0] != rhs.m_blocks[0]) + { + return lhs.m_blocks[0] < rhs.m_blocks[0]; + } + return lhs_size < rhs_size; +} + +//================================================================================================= +// dynamic_bitset private functions implementations +//================================================================================================= + +template +constexpr typename dynamic_bitset::size_type dynamic_bitset:: + blocks_required(size_type nbits) noexcept +{ + return nbits / bits_per_block + static_cast(nbits % bits_per_block > 0); +} + +template +constexpr typename dynamic_bitset::size_type dynamic_bitset:: + block_index(size_type pos) noexcept +{ + return pos / bits_per_block; +} + +template +constexpr typename dynamic_bitset::size_type dynamic_bitset:: + bit_index(size_type pos) noexcept +{ + return pos % bits_per_block; +} + +template +constexpr typename dynamic_bitset::block_type dynamic_bitset:: + bit_mask(size_type pos) noexcept +{ + return block_type(block_type(1) << bit_index(pos)); +} + +template +constexpr typename dynamic_bitset::block_type dynamic_bitset:: + bit_mask(size_type first, size_type last) noexcept +{ + first = bit_index(first); + last = bit_index(last); + if(last == (block_last_bit_index)) + { + return block_type(one_block << first); + } + else + { + return block_type(((block_type(1) << (last + 1)) - 1) ^ ((block_type(1) << first) - 1)); + } +} + +template +constexpr void dynamic_bitset::set_block_bits(block_type& block, + size_type first, + size_type last, + bool val) noexcept +{ + if(val) + { + block |= bit_mask(first, last); + } + else + { + block &= ~bit_mask(first, last); + } +} + +template +constexpr void dynamic_bitset::flip_block_bits(block_type& block, + size_type first, + size_type last) noexcept +{ + block ^= bit_mask(first, last); +} + +template +constexpr typename dynamic_bitset::size_type dynamic_bitset:: + block_count(const block_type& block) noexcept +{ + if(block == zero_block) + { + return 0; + } + +#if defined(DYNAMIC_BITSET_GCC) \ + || (defined(DYNAMIC_BITSET_CLANG) && defined(DYNAMIC_BITSET_CLANG_builtin_popcount)) + if constexpr(std::is_same_v) + { + return static_cast(__builtin_popcountll(block)); + } + if constexpr(std::is_same_v) + { + return static_cast(__builtin_popcountl(block)); + } + if constexpr(sizeof(block_type) <= sizeof(unsigned int)) + { + return static_cast(__builtin_popcount(static_cast(block))); + } +#endif + + size_type count = 0; + block_type mask = 1; + for(size_type bit_index = 0; bit_index < bits_per_block; ++bit_index) + { + count += ((block & mask) != zero_block); + mask <<= 1; + } + return count; +} + +template +constexpr typename dynamic_bitset::size_type dynamic_bitset:: + block_count(const block_type& block, size_type nbits) noexcept +{ + assert(nbits <= bits_per_block); + if(block == zero_block) + { + return 0; + } + +#if defined(DYNAMIC_BITSET_GCC) \ + || (defined(DYNAMIC_BITSET_CLANG) && defined(DYNAMIC_BITSET_CLANG_builtin_popcount)) + const block_type shifted_block = block_type(block << (bits_per_block - nbits)); + if constexpr(std::is_same_v) + { + return static_cast(__builtin_popcountll(shifted_block)); + } + if constexpr(std::is_same_v) + { + return static_cast(__builtin_popcountl(shifted_block)); + } + if constexpr(sizeof(block_type) <= sizeof(unsigned int)) + { + return static_cast(__builtin_popcount(static_cast(shifted_block))); + } +#endif + + size_type count = 0; + block_type mask = 1; + for(size_type bit_index = 0; bit_index < nbits; ++bit_index) + { + count += ((block & mask) != zero_block); + mask <<= 1; + } + + return count; +} + +template +constexpr typename dynamic_bitset::size_type dynamic_bitset:: + first_on(const block_type& block) noexcept +{ + assert(block != zero_block); + +#if defined(DYNAMIC_BITSET_GCC) \ + || (defined(DYNAMIC_BITSET_CLANG) && defined(DYNAMIC_BITSET_CLANG_builtin_ctz)) + if constexpr(std::is_same_v) + { + return static_cast(__builtin_ctzll(block)); + } + if constexpr(std::is_same_v) + { + return static_cast(__builtin_ctzl(block)); + } + if constexpr(sizeof(block_type) <= sizeof(unsigned int)) + { + return static_cast(__builtin_ctz(static_cast(block))); + } +#elif defined(DYNAMIC_BITSET_MSVC) +# if defined(DYNAMIC_BITSET_MSVC_64) + if constexpr(std::is_same_v) + { + unsigned long index = std::numeric_limits::max(); + _BitScanForward64(&index, block); + return static_cast(index); + } +# endif + if constexpr(std::is_same_v) + { + unsigned long index = std::numeric_limits::max(); + _BitScanForward(&index, block); + return static_cast(index); + } + if constexpr(sizeof(block_type) <= sizeof(unsigned long)) + { + unsigned long index = std::numeric_limits::max(); + _BitScanForward(&index, static_cast(block)); + return static_cast(index); + } +#endif + + block_type mask = block_type(1); + for(size_type i = 0; i < bits_per_block; ++i) + { + if((block & mask) != zero_block) + { + return i; + } + mask <<= 1; + } + return npos; +} + +template +template +constexpr void dynamic_bitset::init_from_string( + std::basic_string_view<_CharT, _Traits> str, + typename std::basic_string_view<_CharT, _Traits>::size_type pos, + typename std::basic_string_view<_CharT, _Traits>::size_type n, + [[maybe_unused]] _CharT zero, + _CharT one) +{ + assert(pos < str.size()); + + const size_type size = std::min(n, str.size() - pos); + m_bits_number = size; + + m_blocks.clear(); + m_blocks.resize(blocks_required(size)); + for(size_t i = 0; i < size; ++i) + { + const _CharT c = str[(pos + size - 1) - i]; + assert(c == zero || c == one); + if(c == one) + { + set(i); + } + } +} + +template +constexpr typename dynamic_bitset::block_type& dynamic_bitset:: + get_block(size_type pos) +{ + return m_blocks[block_index(pos)]; +} + +template +constexpr const typename dynamic_bitset::block_type& dynamic_bitset< + Block, + Allocator>::get_block(size_type pos) const +{ + return m_blocks[block_index(pos)]; +} + +template +constexpr typename dynamic_bitset::block_type& dynamic_bitset:: + last_block() +{ + return m_blocks[m_blocks.size() - 1]; +} + +template +constexpr typename dynamic_bitset::block_type dynamic_bitset:: + last_block() const +{ + return m_blocks[m_blocks.size() - 1]; +} + +template +constexpr typename dynamic_bitset::size_type dynamic_bitset:: + extra_bits_number() const noexcept +{ + return bit_index(m_bits_number); +} + +template +constexpr typename dynamic_bitset::size_type dynamic_bitset:: + unused_bits_number() const noexcept +{ + return bits_per_block - extra_bits_number(); +} + +template +template +constexpr void dynamic_bitset::apply( + const dynamic_bitset& other, + BinaryOperation binary_op) +{ + assert(num_blocks() == other.num_blocks()); + std::transform(std::cbegin(m_blocks), + std::cend(m_blocks), + std::cbegin(other.m_blocks), + std::begin(m_blocks), + binary_op); +} + +template +template +constexpr void dynamic_bitset::apply(UnaryOperation unary_op) +{ + std::transform(std::cbegin(m_blocks), std::cend(m_blocks), std::begin(m_blocks), unary_op); +} + +template +constexpr void dynamic_bitset::apply_left_shift(size_type shift) +{ + assert(shift > 0); + assert(shift < capacity()); + + const size_type blocks_shift = shift / bits_per_block; + const size_type bits_offset = shift % bits_per_block; + + if(bits_offset == 0) + { + for(size_type i = m_blocks.size() - 1; i >= blocks_shift; --i) + { + m_blocks[i] = m_blocks[i - blocks_shift]; + } + } + else + { + const size_type reverse_bits_offset = bits_per_block - bits_offset; + for(size_type i = m_blocks.size() - 1; i > blocks_shift; --i) + { + m_blocks[i] = + block_type((m_blocks[i - blocks_shift] << bits_offset) + | block_type(m_blocks[i - blocks_shift - 1] >> reverse_bits_offset)); + } + m_blocks[blocks_shift] = block_type(m_blocks[0] << bits_offset); + } + + // set bit that came at the right to 0 in unmodified blocks + std::fill(std::begin(m_blocks), + std::begin(m_blocks) + + static_cast(blocks_shift), + zero_block); +} + +template +constexpr void dynamic_bitset::apply_right_shift(size_type shift) +{ + assert(shift > 0); + assert(shift < capacity()); + + const size_type blocks_shift = shift / bits_per_block; + const size_type bits_offset = shift % bits_per_block; + const size_type last_block_to_shift = m_blocks.size() - blocks_shift - 1; + + if(bits_offset == 0) + { + for(size_type i = 0; i <= last_block_to_shift; ++i) + { + m_blocks[i] = m_blocks[i + blocks_shift]; + } + } + else + { + const size_type reverse_bits_offset = bits_per_block - bits_offset; + for(size_type i = 0; i < last_block_to_shift; ++i) + { + m_blocks[i] = + block_type((m_blocks[i + blocks_shift] >> bits_offset) + | block_type(m_blocks[i + blocks_shift + 1] << reverse_bits_offset)); + } + m_blocks[last_block_to_shift] = block_type(m_blocks[m_blocks.size() - 1] >> bits_offset); + } + + // set bit that came at the left to 0 in unmodified blocks + std::fill( + std::begin(m_blocks) + + static_cast(last_block_to_shift + 1), + std::end(m_blocks), + zero_block); +} + +template +constexpr void dynamic_bitset::sanitize() +{ + size_type shift = m_bits_number % bits_per_block; + if(shift > 0) + { + last_block() &= ~(one_block << shift); + } +} + +template +constexpr bool dynamic_bitset::check_unused_bits() const noexcept +{ + const size_type extra_bits = extra_bits_number(); + if(extra_bits > 0) + { + return (last_block() & (one_block << extra_bits)) == zero_block; + } + return true; +} + +template +constexpr bool dynamic_bitset::check_size() const noexcept +{ + return blocks_required(size()) == m_blocks.size(); +} + +template +constexpr bool dynamic_bitset::check_consistency() const noexcept +{ + return check_unused_bits() && check_size(); +} + +//================================================================================================= +// dynamic_bitset external functions implementations +//================================================================================================= + +template +constexpr bool operator!=(const dynamic_bitset& lhs, + const dynamic_bitset& rhs) +{ + return !(lhs == rhs); +} + +template +constexpr bool operator<=(const dynamic_bitset& lhs, + const dynamic_bitset& rhs) +{ + return !(rhs < lhs); +} + +template +constexpr bool operator>(const dynamic_bitset& lhs, + const dynamic_bitset& rhs) +{ + return rhs < lhs; +} + +template +constexpr bool operator>=(const dynamic_bitset& lhs, + const dynamic_bitset& rhs) +{ + return !(lhs < rhs); +} + +template +constexpr dynamic_bitset operator&(const dynamic_bitset& lhs, + const dynamic_bitset& rhs) +{ + dynamic_bitset result(lhs); + return result &= rhs; +} + +template +constexpr dynamic_bitset operator|(const dynamic_bitset& lhs, + const dynamic_bitset& rhs) +{ + dynamic_bitset result(lhs); + return result |= rhs; +} + +template +constexpr dynamic_bitset operator^(const dynamic_bitset& lhs, + const dynamic_bitset& rhs) +{ + dynamic_bitset result(lhs); + return result ^= rhs; +} + +template +constexpr dynamic_bitset operator-(const dynamic_bitset& lhs, + const dynamic_bitset& rhs) +{ + dynamic_bitset result(lhs); + return result -= rhs; +} + +template +constexpr std::basic_ostream<_CharT, _Traits>& operator<<( + std::basic_ostream<_CharT, _Traits>& os, + const dynamic_bitset& bitset) +{ + // A better implementation is possible + return os << bitset.template to_string<_CharT, _Traits>(); +} + +template +constexpr std::basic_istream<_CharT, _Traits>& operator>>(std::basic_istream<_CharT, _Traits>& is, + dynamic_bitset& bitset) +{ + // A better implementation is possible + constexpr _CharT zero = _CharT('0'); + constexpr _CharT one = _CharT('1'); + typename std::basic_istream<_CharT, _Traits>::sentry s(is); + if(!s) + { + return is; + } + + dynamic_bitset reverse_bitset; + _CharT val; + is.get(val); + while(is.good()) + { + if(val == one) + { + reverse_bitset.push_back(true); + } + else if(val == zero) + { + reverse_bitset.push_back(false); + } + else + { + is.unget(); + break; + } + is.get(val); + } + + bitset.clear(); + if(!reverse_bitset.empty()) + { + for(typename dynamic_bitset::size_type i = reverse_bitset.size() - 1; + i > 0; + --i) + { + bitset.push_back(reverse_bitset.test(i)); + } + bitset.push_back(reverse_bitset.test(0)); + } + + return is; +} + +template +constexpr void swap(dynamic_bitset& bitset1, + dynamic_bitset& bitset2) +{ + bitset1.swap(bitset2); +} + +#endif //DYNAMIC_BITSET_DYNAMIC_BITSET_HPP diff --git a/dep/libpopcnt/LICENSE b/dep/libpopcnt/LICENSE new file mode 100644 index 000000000..15a721096 --- /dev/null +++ b/dep/libpopcnt/LICENSE @@ -0,0 +1,26 @@ +BSD 2-Clause License + +Copyright (c) 2016 - 2019, Kim Walisch +Copyright (c) 2016 - 2019, Wojciech Muła + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/dep/libpopcnt/MAINTAINER_README.md b/dep/libpopcnt/MAINTAINER_README.md new file mode 100644 index 000000000..45048fce5 --- /dev/null +++ b/dep/libpopcnt/MAINTAINER_README.md @@ -0,0 +1,17 @@ +### Notes for Future Maintainers + +This was originally imported by @miniksa in March 2020. + +The provenance information (where it came from and which commit) is stored in the file `cgmanifest.json` in the same directory as this readme. +Please update the provenance information in that file when ingesting an updated version of the dependent library. +That provenance file is automatically read and inventoried by Microsoft systems to ensure compliance with appropiate governance standards. + +## What should be done to update this in the future? + +1. Go to kimwalisch/libpopcnt repository on GitHub. +2. Take the `libpopcnt.h` file. +3. Don't change anything about it. +4. Validate that the `LICENSE` in the root of the repository didn't change and update it if so. It is sitting in the same directory as this readme. + If it changed dramatically, ensure that it is still compatible with our license scheme. Also update the NOTICE file in the root of our repository to declare the third-party usage. +5. Submit the pull. + diff --git a/dep/libpopcnt/cgmanifest.json b/dep/libpopcnt/cgmanifest.json new file mode 100644 index 000000000..90bfc49cc --- /dev/null +++ b/dep/libpopcnt/cgmanifest.json @@ -0,0 +1,13 @@ +{"Registrations":[ + { + "component": { + "type": "git", + "git": { + "repositoryUrl": "https://github.com/kimwalisch/libpopcnt", + "commitHash": "043a99fba31121a70bcb2f589faa17f534ae6085" + } + } + } + ], + "Version": 1 +} \ No newline at end of file diff --git a/dep/libpopcnt/libpopcnt.h b/dep/libpopcnt/libpopcnt.h new file mode 100644 index 000000000..e23b8e408 --- /dev/null +++ b/dep/libpopcnt/libpopcnt.h @@ -0,0 +1,841 @@ +/* + * libpopcnt.h - C/C++ library for counting the number of 1 bits (bit + * population count) in an array as quickly as possible using + * specialized CPU instructions i.e. POPCNT, AVX2, AVX512, NEON. + * + * Copyright (c) 2016 - 2019, Kim Walisch + * Copyright (c) 2016 - 2018, Wojciech Muła + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef LIBPOPCNT_H +#define LIBPOPCNT_H + +#include + +#ifndef __has_builtin + #define __has_builtin(x) 0 +#endif + +#ifndef __has_attribute + #define __has_attribute(x) 0 +#endif + +#ifdef __GNUC__ + #define GNUC_PREREQ(x, y) \ + (__GNUC__ > x || (__GNUC__ == x && __GNUC_MINOR__ >= y)) +#else + #define GNUC_PREREQ(x, y) 0 +#endif + +#ifdef __clang__ + #define CLANG_PREREQ(x, y) \ + (__clang_major__ > x || (__clang_major__ == x && __clang_minor__ >= y)) +#else + #define CLANG_PREREQ(x, y) 0 +#endif + +#if (_MSC_VER < 1900) && \ + !defined(__cplusplus) + #define inline __inline +#endif + +#if (defined(__i386__) || \ + defined(__x86_64__) || \ + defined(_M_IX86) || \ + defined(_M_X64)) + #define X86_OR_X64 +#endif + +#if defined(X86_OR_X64) && \ + (defined(__cplusplus) || \ + defined(_MSC_VER) || \ + (GNUC_PREREQ(4, 2) || \ + __has_builtin(__sync_val_compare_and_swap))) + #define HAVE_CPUID +#endif + +#if GNUC_PREREQ(4, 2) || \ + __has_builtin(__builtin_popcount) + #define HAVE_BUILTIN_POPCOUNT +#endif + +#if GNUC_PREREQ(4, 2) || \ + CLANG_PREREQ(3, 0) + #define HAVE_ASM_POPCNT +#endif + +#if defined(HAVE_CPUID) && \ + (defined(HAVE_ASM_POPCNT) || \ + defined(_MSC_VER)) + #define HAVE_POPCNT +#endif + +#if defined(HAVE_CPUID) && \ + GNUC_PREREQ(4, 9) + #define HAVE_AVX2 +#endif + +#if defined(HAVE_CPUID) && \ + GNUC_PREREQ(5, 0) + #define HAVE_AVX512 +#endif + +#if defined(HAVE_CPUID) && \ + defined(_MSC_VER) && \ + defined(__AVX2__) + #define HAVE_AVX2 +#endif + +#if defined(HAVE_CPUID) && \ + defined(_MSC_VER) && \ + defined(__AVX512__) + #define HAVE_AVX512 +#endif + +#if defined(HAVE_CPUID) && \ + CLANG_PREREQ(3, 8) && \ + __has_attribute(target) && \ + (!defined(_MSC_VER) || defined(__AVX2__)) && \ + (!defined(__apple_build_version__) || __apple_build_version__ >= 8000000) + #define HAVE_AVX2 + #define HAVE_AVX512 +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * This uses fewer arithmetic operations than any other known + * implementation on machines with fast multiplication. + * It uses 12 arithmetic operations, one of which is a multiply. + * http://en.wikipedia.org/wiki/Hamming_weight#Efficient_implementation + */ +static inline uint64_t popcount64(uint64_t x) +{ + uint64_t m1 = 0x5555555555555555ll; + uint64_t m2 = 0x3333333333333333ll; + uint64_t m4 = 0x0F0F0F0F0F0F0F0Fll; + uint64_t h01 = 0x0101010101010101ll; + + x -= (x >> 1) & m1; + x = (x & m2) + ((x >> 2) & m2); + x = (x + (x >> 4)) & m4; + + return (x * h01) >> 56; +} + +#if defined(HAVE_ASM_POPCNT) && \ + defined(__x86_64__) + +static inline uint64_t popcnt64(uint64_t x) +{ + __asm__ ("popcnt %1, %0" : "=r" (x) : "0" (x)); + return x; +} + +#elif defined(HAVE_ASM_POPCNT) && \ + defined(__i386__) + +static inline uint32_t popcnt32(uint32_t x) +{ + __asm__ ("popcnt %1, %0" : "=r" (x) : "0" (x)); + return x; +} + +static inline uint64_t popcnt64(uint64_t x) +{ + return popcnt32((uint32_t) x) + + popcnt32((uint32_t)(x >> 32)); +} + +#elif defined(_MSC_VER) && \ + defined(_M_X64) + +#include + +static inline uint64_t popcnt64(uint64_t x) +{ + return _mm_popcnt_u64(x); +} + +#elif defined(_MSC_VER) && \ + defined(_M_IX86) + +#include + +static inline uint64_t popcnt64(uint64_t x) +{ + return _mm_popcnt_u32((uint32_t) x) + + _mm_popcnt_u32((uint32_t)(x >> 32)); +} + +/* non x86 CPUs */ +#elif defined(HAVE_BUILTIN_POPCOUNT) + +static inline uint64_t popcnt64(uint64_t x) +{ + return __builtin_popcountll(x); +} + +/* no hardware POPCNT, + * use pure integer algorithm */ +#else + +static inline uint64_t popcnt64(uint64_t x) +{ + return popcount64(x); +} + +#endif + +static inline uint64_t popcnt64_unrolled(const uint64_t* data, uint64_t size) +{ + uint64_t i = 0; + uint64_t limit = size - size % 4; + uint64_t cnt = 0; + + for (; i < limit; i += 4) + { + cnt += popcnt64(data[i+0]); + cnt += popcnt64(data[i+1]); + cnt += popcnt64(data[i+2]); + cnt += popcnt64(data[i+3]); + } + + for (; i < size; i++) + cnt += popcnt64(data[i]); + + return cnt; +} + +#if defined(HAVE_CPUID) + +#if defined(_MSC_VER) + #include + #include +#endif + +/* %ecx bit flags */ +#define bit_POPCNT (1 << 23) + +/* %ebx bit flags */ +#define bit_AVX2 (1 << 5) +#define bit_AVX512 (1 << 30) + +/* xgetbv bit flags */ +#define XSTATE_SSE (1 << 1) +#define XSTATE_YMM (1 << 2) +#define XSTATE_ZMM (7 << 5) + +static inline void run_cpuid(int eax, int ecx, int* abcd) +{ +#if defined(_MSC_VER) + __cpuidex(abcd, eax, ecx); +#else + int ebx = 0; + int edx = 0; + + #if defined(__i386__) && \ + defined(__PIC__) + /* in case of PIC under 32-bit EBX cannot be clobbered */ + __asm__ ("movl %%ebx, %%edi;" + "cpuid;" + "xchgl %%ebx, %%edi;" + : "=D" (ebx), + "+a" (eax), + "+c" (ecx), + "=d" (edx)); + #else + __asm__ ("cpuid;" + : "+b" (ebx), + "+a" (eax), + "+c" (ecx), + "=d" (edx)); + #endif + + abcd[0] = eax; + abcd[1] = ebx; + abcd[2] = ecx; + abcd[3] = edx; +#endif +} + +#if defined(HAVE_AVX2) || \ + defined(HAVE_AVX512) + +static inline int get_xcr0() +{ + int xcr0; + +#if defined(_MSC_VER) + xcr0 = (int) _xgetbv(0); +#else + __asm__ ("xgetbv" : "=a" (xcr0) : "c" (0) : "%edx" ); +#endif + + return xcr0; +} + +#endif + +static inline int get_cpuid() +{ + int flags = 0; + int abcd[4]; + + run_cpuid(1, 0, abcd); + + if ((abcd[2] & bit_POPCNT) == bit_POPCNT) + flags |= bit_POPCNT; + +#if defined(HAVE_AVX2) || \ + defined(HAVE_AVX512) + + int osxsave_mask = (1 << 27); + + /* ensure OS supports extended processor state management */ + if ((abcd[2] & osxsave_mask) != osxsave_mask) + return 0; + + int ymm_mask = XSTATE_SSE | XSTATE_YMM; + int zmm_mask = XSTATE_SSE | XSTATE_YMM | XSTATE_ZMM; + + int xcr0 = get_xcr0(); + + if ((xcr0 & ymm_mask) == ymm_mask) + { + run_cpuid(7, 0, abcd); + + if ((abcd[1] & bit_AVX2) == bit_AVX2) + flags |= bit_AVX2; + + if ((xcr0 & zmm_mask) == zmm_mask) + { + if ((abcd[1] & bit_AVX512) == bit_AVX512) + flags |= bit_AVX512; + } + } + +#endif + + return flags; +} + +#endif /* cpuid */ + +#if defined(HAVE_AVX2) + +#include + +#if !defined(_MSC_VER) + __attribute__ ((target ("avx2"))) +#endif +static inline void CSA256(__m256i* h, __m256i* l, __m256i a, __m256i b, __m256i c) +{ + __m256i u = _mm256_xor_si256(a, b); + *h = _mm256_or_si256(_mm256_and_si256(a, b), _mm256_and_si256(u, c)); + *l = _mm256_xor_si256(u, c); +} + +#if !defined(_MSC_VER) + __attribute__ ((target ("avx2"))) +#endif +static inline __m256i popcnt256(__m256i v) +{ + __m256i lookup1 = _mm256_setr_epi8( + 4, 5, 5, 6, 5, 6, 6, 7, + 5, 6, 6, 7, 6, 7, 7, 8, + 4, 5, 5, 6, 5, 6, 6, 7, + 5, 6, 6, 7, 6, 7, 7, 8 + ); + + __m256i lookup2 = _mm256_setr_epi8( + 4, 3, 3, 2, 3, 2, 2, 1, + 3, 2, 2, 1, 2, 1, 1, 0, + 4, 3, 3, 2, 3, 2, 2, 1, + 3, 2, 2, 1, 2, 1, 1, 0 + ); + + __m256i low_mask = _mm256_set1_epi8(0x0f); + __m256i lo = _mm256_and_si256(v, low_mask); + __m256i hi = _mm256_and_si256(_mm256_srli_epi16(v, 4), low_mask); + __m256i popcnt1 = _mm256_shuffle_epi8(lookup1, lo); + __m256i popcnt2 = _mm256_shuffle_epi8(lookup2, hi); + + return _mm256_sad_epu8(popcnt1, popcnt2); +} + +/* + * AVX2 Harley-Seal popcount (4th iteration). + * The algorithm is based on the paper "Faster Population Counts + * using AVX2 Instructions" by Daniel Lemire, Nathan Kurz and + * Wojciech Mula (23 Nov 2016). + * @see https://arxiv.org/abs/1611.07612 + */ +#if !defined(_MSC_VER) + __attribute__ ((target ("avx2"))) +#endif +static inline uint64_t popcnt_avx2(const __m256i* data, uint64_t size) +{ + __m256i cnt = _mm256_setzero_si256(); + __m256i ones = _mm256_setzero_si256(); + __m256i twos = _mm256_setzero_si256(); + __m256i fours = _mm256_setzero_si256(); + __m256i eights = _mm256_setzero_si256(); + __m256i sixteens = _mm256_setzero_si256(); + __m256i twosA, twosB, foursA, foursB, eightsA, eightsB; + + uint64_t i = 0; + uint64_t limit = size - size % 16; + uint64_t* cnt64; + + for(; i < limit; i += 16) + { + CSA256(&twosA, &ones, ones, data[i+0], data[i+1]); + CSA256(&twosB, &ones, ones, data[i+2], data[i+3]); + CSA256(&foursA, &twos, twos, twosA, twosB); + CSA256(&twosA, &ones, ones, data[i+4], data[i+5]); + CSA256(&twosB, &ones, ones, data[i+6], data[i+7]); + CSA256(&foursB, &twos, twos, twosA, twosB); + CSA256(&eightsA, &fours, fours, foursA, foursB); + CSA256(&twosA, &ones, ones, data[i+8], data[i+9]); + CSA256(&twosB, &ones, ones, data[i+10], data[i+11]); + CSA256(&foursA, &twos, twos, twosA, twosB); + CSA256(&twosA, &ones, ones, data[i+12], data[i+13]); + CSA256(&twosB, &ones, ones, data[i+14], data[i+15]); + CSA256(&foursB, &twos, twos, twosA, twosB); + CSA256(&eightsB, &fours, fours, foursA, foursB); + CSA256(&sixteens, &eights, eights, eightsA, eightsB); + + cnt = _mm256_add_epi64(cnt, popcnt256(sixteens)); + } + + cnt = _mm256_slli_epi64(cnt, 4); + cnt = _mm256_add_epi64(cnt, _mm256_slli_epi64(popcnt256(eights), 3)); + cnt = _mm256_add_epi64(cnt, _mm256_slli_epi64(popcnt256(fours), 2)); + cnt = _mm256_add_epi64(cnt, _mm256_slli_epi64(popcnt256(twos), 1)); + cnt = _mm256_add_epi64(cnt, popcnt256(ones)); + + for(; i < size; i++) + cnt = _mm256_add_epi64(cnt, popcnt256(data[i])); + + cnt64 = (uint64_t*) &cnt; + + return cnt64[0] + + cnt64[1] + + cnt64[2] + + cnt64[3]; +} + +/* Align memory to 32 bytes boundary */ +static inline void align_avx2(const uint8_t** p, uint64_t* size, uint64_t* cnt) +{ + for (; (uintptr_t) *p % 8; (*p)++) + { + *cnt += popcnt64(**p); + *size -= 1; + } + for (; (uintptr_t) *p % 32; (*p) += 8) + { + *cnt += popcnt64( + *(const uint64_t*) *p); + *size -= 8; + } +} + +#endif + +#if defined(HAVE_AVX512) + +#include + +#if !defined(_MSC_VER) + __attribute__ ((target ("avx512bw"))) +#endif +static inline __m512i popcnt512(__m512i v) +{ + __m512i m1 = _mm512_set1_epi8(0x55); + __m512i m2 = _mm512_set1_epi8(0x33); + __m512i m4 = _mm512_set1_epi8(0x0F); + __m512i t1 = _mm512_sub_epi8(v, (_mm512_srli_epi16(v, 1) & m1)); + __m512i t2 = _mm512_add_epi8(t1 & m2, (_mm512_srli_epi16(t1, 2) & m2)); + __m512i t3 = _mm512_add_epi8(t2, _mm512_srli_epi16(t2, 4)) & m4; + + return _mm512_sad_epu8(t3, _mm512_setzero_si512()); +} + +#if !defined(_MSC_VER) + __attribute__ ((target ("avx512bw"))) +#endif +static inline void CSA512(__m512i* h, __m512i* l, __m512i a, __m512i b, __m512i c) +{ + *l = _mm512_ternarylogic_epi32(c, b, a, 0x96); + *h = _mm512_ternarylogic_epi32(c, b, a, 0xe8); +} + +/* + * AVX512 Harley-Seal popcount (4th iteration). + * The algorithm is based on the paper "Faster Population Counts + * using AVX2 Instructions" by Daniel Lemire, Nathan Kurz and + * Wojciech Mula (23 Nov 2016). + * @see https://arxiv.org/abs/1611.07612 + */ +#if !defined(_MSC_VER) + __attribute__ ((target ("avx512bw"))) +#endif +static inline uint64_t popcnt_avx512(const __m512i* data, const uint64_t size) +{ + __m512i cnt = _mm512_setzero_si512(); + __m512i ones = _mm512_setzero_si512(); + __m512i twos = _mm512_setzero_si512(); + __m512i fours = _mm512_setzero_si512(); + __m512i eights = _mm512_setzero_si512(); + __m512i sixteens = _mm512_setzero_si512(); + __m512i twosA, twosB, foursA, foursB, eightsA, eightsB; + + uint64_t i = 0; + uint64_t limit = size - size % 16; + uint64_t* cnt64; + + for(; i < limit; i += 16) + { + CSA512(&twosA, &ones, ones, data[i+0], data[i+1]); + CSA512(&twosB, &ones, ones, data[i+2], data[i+3]); + CSA512(&foursA, &twos, twos, twosA, twosB); + CSA512(&twosA, &ones, ones, data[i+4], data[i+5]); + CSA512(&twosB, &ones, ones, data[i+6], data[i+7]); + CSA512(&foursB, &twos, twos, twosA, twosB); + CSA512(&eightsA, &fours, fours, foursA, foursB); + CSA512(&twosA, &ones, ones, data[i+8], data[i+9]); + CSA512(&twosB, &ones, ones, data[i+10], data[i+11]); + CSA512(&foursA, &twos, twos, twosA, twosB); + CSA512(&twosA, &ones, ones, data[i+12], data[i+13]); + CSA512(&twosB, &ones, ones, data[i+14], data[i+15]); + CSA512(&foursB, &twos, twos, twosA, twosB); + CSA512(&eightsB, &fours, fours, foursA, foursB); + CSA512(&sixteens, &eights, eights, eightsA, eightsB); + + cnt = _mm512_add_epi64(cnt, popcnt512(sixteens)); + } + + cnt = _mm512_slli_epi64(cnt, 4); + cnt = _mm512_add_epi64(cnt, _mm512_slli_epi64(popcnt512(eights), 3)); + cnt = _mm512_add_epi64(cnt, _mm512_slli_epi64(popcnt512(fours), 2)); + cnt = _mm512_add_epi64(cnt, _mm512_slli_epi64(popcnt512(twos), 1)); + cnt = _mm512_add_epi64(cnt, popcnt512(ones)); + + for(; i < size; i++) + cnt = _mm512_add_epi64(cnt, popcnt512(data[i])); + + cnt64 = (uint64_t*) &cnt; + + return cnt64[0] + + cnt64[1] + + cnt64[2] + + cnt64[3] + + cnt64[4] + + cnt64[5] + + cnt64[6] + + cnt64[7]; +} + +/* Align memory to 64 bytes boundary */ +static inline void align_avx512(const uint8_t** p, uint64_t* size, uint64_t* cnt) +{ + for (; (uintptr_t) *p % 8; (*p)++) + { + *cnt += popcnt64(**p); + *size -= 1; + } + for (; (uintptr_t) *p % 64; (*p) += 8) + { + *cnt += popcnt64( + *(const uint64_t*) *p); + *size -= 8; + } +} + +#endif + +/* x86 CPUs */ +#if defined(X86_OR_X64) + +/* Align memory to 8 bytes boundary */ +static inline void align_8(const uint8_t** p, uint64_t* size, uint64_t* cnt) +{ + for (; *size > 0 && (uintptr_t) *p % 8; (*p)++) + { + *cnt += popcount64(**p); + *size -= 1; + } +} + +static inline uint64_t popcount64_unrolled(const uint64_t* data, uint64_t size) +{ + uint64_t i = 0; + uint64_t limit = size - size % 4; + uint64_t cnt = 0; + + for (; i < limit; i += 4) + { + cnt += popcount64(data[i+0]); + cnt += popcount64(data[i+1]); + cnt += popcount64(data[i+2]); + cnt += popcount64(data[i+3]); + } + + for (; i < size; i++) + cnt += popcount64(data[i]); + + return cnt; +} + +/* + * Count the number of 1 bits in the data array + * @data: An array + * @size: Size of data in bytes + */ +static inline uint64_t popcnt(const void* data, uint64_t size) +{ + const uint8_t* ptr = (const uint8_t*) data; + uint64_t cnt = 0; + uint64_t i; + +#if defined(HAVE_CPUID) + #if defined(__cplusplus) + /* C++11 thread-safe singleton */ + static const int cpuid = get_cpuid(); + #else + static int cpuid_ = -1; + int cpuid = cpuid_; + if (cpuid == -1) + { + cpuid = get_cpuid(); + + #if defined(_MSC_VER) + _InterlockedCompareExchange(&cpuid_, cpuid, -1); + #else + __sync_val_compare_and_swap(&cpuid_, -1, cpuid); + #endif + } + #endif +#endif + +#if defined(HAVE_AVX512) + + /* AVX512 requires arrays >= 1024 bytes */ + if ((cpuid & bit_AVX512) && + size >= 1024) + { + align_avx512(&ptr, &size, &cnt); + cnt += popcnt_avx512((const __m512i*) ptr, size / 64); + ptr += size - size % 64; + size = size % 64; + } + +#endif + +#if defined(HAVE_AVX2) + + /* AVX2 requires arrays >= 512 bytes */ + if ((cpuid & bit_AVX2) && + size >= 512) + { + align_avx2(&ptr, &size, &cnt); + cnt += popcnt_avx2((const __m256i*) ptr, size / 32); + ptr += size - size % 32; + size = size % 32; + } + +#endif + +#if defined(HAVE_POPCNT) + + if (cpuid & bit_POPCNT) + { + cnt += popcnt64_unrolled((const uint64_t*) ptr, size / 8); + ptr += size - size % 8; + size = size % 8; + for (i = 0; i < size; i++) + cnt += popcnt64(ptr[i]); + + return cnt; + } + +#endif + + /* pure integer popcount algorithm */ + if (size >= 8) + { + align_8(&ptr, &size, &cnt); + cnt += popcount64_unrolled((const uint64_t*) ptr, size / 8); + ptr += size - size % 8; + size = size % 8; + } + + /* pure integer popcount algorithm */ + for (i = 0; i < size; i++) + cnt += popcount64(ptr[i]); + + return cnt; +} + +#elif defined(__ARM_NEON) || \ + defined(__aarch64__) + +#include + +/* Align memory to 8 bytes boundary */ +static inline void align_8(const uint8_t** p, uint64_t* size, uint64_t* cnt) +{ + for (; *size > 0 && (uintptr_t) *p % 8; (*p)++) + { + *cnt += popcnt64(**p); + *size -= 1; + } +} + +static inline uint64x2_t vpadalq(uint64x2_t sum, uint8x16_t t) +{ + return vpadalq_u32(sum, vpaddlq_u16(vpaddlq_u8(t))); +} + +/* + * Count the number of 1 bits in the data array + * @data: An array + * @size: Size of data in bytes + */ +static inline uint64_t popcnt(const void* data, uint64_t size) +{ + uint64_t cnt = 0; + uint64_t chunk_size = 64; + const uint8_t* ptr = (const uint8_t*) data; + + if (size >= chunk_size) + { + uint64_t i = 0; + uint64_t iters = size / chunk_size; + uint64x2_t sum = vcombine_u64(vcreate_u64(0), vcreate_u64(0)); + uint8x16_t zero = vcombine_u8(vcreate_u8(0), vcreate_u8(0)); + + do + { + uint8x16_t t0 = zero; + uint8x16_t t1 = zero; + uint8x16_t t2 = zero; + uint8x16_t t3 = zero; + + /* + * After every 31 iterations we need to add the + * temporary sums (t0, t1, t2, t3) to the total sum. + * We must ensure that the temporary sums <= 255 + * and 31 * 8 bits = 248 which is OK. + */ + uint64_t limit = (i + 31 < iters) ? i + 31 : iters; + + /* Each iteration processes 64 bytes */ + for (; i < limit; i++) + { + uint8x16x4_t input = vld4q_u8(ptr); + ptr += chunk_size; + + t0 = vaddq_u8(t0, vcntq_u8(input.val[0])); + t1 = vaddq_u8(t1, vcntq_u8(input.val[1])); + t2 = vaddq_u8(t2, vcntq_u8(input.val[2])); + t3 = vaddq_u8(t3, vcntq_u8(input.val[3])); + } + + sum = vpadalq(sum, t0); + sum = vpadalq(sum, t1); + sum = vpadalq(sum, t2); + sum = vpadalq(sum, t3); + } + while (i < iters); + + uint64_t tmp[2]; + vst1q_u64(tmp, sum); + cnt += tmp[0]; + cnt += tmp[1]; + } + + size %= chunk_size; + align_8(&ptr, &size, &cnt); + const uint64_t* ptr64 = (const uint64_t*) ptr; + uint64_t iters = size / 8; + + for (uint64_t i = 0; i < iters; i++) + cnt += popcnt64(ptr64[i]); + + ptr += size - size % 8; + size = size % 8; + + for (uint64_t i = 0; i < size; i++) + cnt += popcnt64(ptr[i]); + + return cnt; +} + +/* all other CPUs */ +#else + +/* Align memory to 8 bytes boundary */ +static inline void align_8(const uint8_t** p, uint64_t* size, uint64_t* cnt) +{ + for (; *size > 0 && (uintptr_t) *p % 8; (*p)++) + { + *cnt += popcnt64(**p); + *size -= 1; + } +} + +/* + * Count the number of 1 bits in the data array + * @data: An array + * @size: Size of data in bytes + */ +static inline uint64_t popcnt(const void* data, uint64_t size) +{ + const uint8_t* ptr = (const uint8_t*) data; + uint64_t cnt = 0; + uint64_t i; + + align_8(&ptr, &size, &cnt); + cnt += popcnt64_unrolled((const uint64_t*) ptr, size / 8); + ptr += size - size % 8; + size = size % 8; + for (i = 0; i < size; i++) + cnt += popcnt64(ptr[i]); + + return cnt; +} + +#endif + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* LIBPOPCNT_H */ diff --git a/src/cascadia/WindowsTerminalUniversal/WindowsTerminalUniversal.vcxproj b/src/cascadia/WindowsTerminalUniversal/WindowsTerminalUniversal.vcxproj index c178c5bb6..3ae1e552d 100644 --- a/src/cascadia/WindowsTerminalUniversal/WindowsTerminalUniversal.vcxproj +++ b/src/cascadia/WindowsTerminalUniversal/WindowsTerminalUniversal.vcxproj @@ -86,7 +86,7 @@ - $(OpenConsoleDir)\src\inc;$(OpenConsoleDir)\dep;$(OpenConsoleDir)\dep\Console;$(OpenConsoleDir)\dep\chromium;$(OpenConsoleDir)\dep\Win32K;$(OpenConsoleDir)\dep\gsl\include;$(OpenConsoleDir)\dep\wil\include;%(AdditionalIncludeDirectories); + $(OpenConsoleDir)\src\inc;$(OpenConsoleDir)\dep;$(OpenConsoleDir)\dep\Console;$(OpenConsoleDir)\dep\chromium;$(SolutionDir)\dep\dynamic_bitset;$(SolutionDir)\dep\libpopcnt;$(OpenConsoleDir)\dep\Win32K;$(OpenConsoleDir)\dep\gsl\include;$(OpenConsoleDir)\dep\wil\include;%(AdditionalIncludeDirectories);