mirror of
https://github.com/matrix-construct/construct
synced 2024-11-18 07:50:57 +01:00
ircd::simd: Split bit manipulations suites to headers; renames.
This commit is contained in:
Jason Volk
parent
291038a014
commit
3a70943b94
5 changed files with 208 additions and 140 deletions
66
include/ircd/simd/lzcnt.h
Normal file
66
include/ircd/simd/lzcnt.h
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
// The Construct
|
||||
//
|
||||
// Copyright (C) The Construct Developers, Authors & Contributors
|
||||
// Copyright (C) 2016-2020 Jason Volk <jason@zemos.net>
|
||||
//
|
||||
// Permission to use, copy, modify, and/or distribute this software for any
|
||||
// purpose with or without fee is hereby granted, provided that the above
|
||||
// copyright notice and this permission notice is present in all copies. The
|
||||
// full license for this software is available in the LICENSE file.
|
||||
|
||||
#pragma once
|
||||
#define HAVE_IRCD_SIMD_LZCNT_H
|
||||
|
||||
namespace ircd::simd
|
||||
{
|
||||
template<class T> uint lzcnt(const T) noexcept;
|
||||
}
|
||||
|
||||
/// Convenience template. Unfortunately this drops to scalar until specific
|
||||
/// targets and specializations are created.
|
||||
template<class T>
|
||||
inline uint
|
||||
__attribute__((target("lzcnt")))
|
||||
ircd::simd::lzcnt(const T a)
|
||||
noexcept
|
||||
{
|
||||
// The behavior of lzcnt can differ among platforms; when true we expect
|
||||
// lzcnt to fall back to bsr-like behavior.
|
||||
constexpr auto bitscan
|
||||
{
|
||||
#ifdef __LZCNT__
|
||||
false
|
||||
#else
|
||||
true
|
||||
#endif
|
||||
};
|
||||
|
||||
uint ret(0), i(0); do
|
||||
{
|
||||
const auto mask
|
||||
{
|
||||
boolmask(uint(ret == sizeof_lane<T>() * 8 * i))
|
||||
};
|
||||
|
||||
if constexpr(bitscan && sizeof_lane<T>() <= sizeof(u16))
|
||||
ret += (15 - __lzcnt16(__builtin_bswap16(a[i++]))) & mask;
|
||||
|
||||
else if constexpr(sizeof_lane<T>() <= sizeof(u16))
|
||||
ret += __lzcnt16(__builtin_bswap16(a[i++])) & mask;
|
||||
|
||||
else if constexpr(bitscan && sizeof_lane<T>() <= sizeof(u32))
|
||||
ret += (31 - __lzcnt32(__builtin_bswap32(a[i++]))) & mask;
|
||||
|
||||
else if constexpr(sizeof_lane<T>() <= sizeof(u32))
|
||||
ret += __lzcnt32(__builtin_bswap32(a[i++])) & mask;
|
||||
|
||||
else if constexpr(bitscan)
|
||||
ret += (63 - __lzcnt64(__builtin_bswap64(a[i++]))) & mask;
|
||||
|
||||
else
|
||||
ret += __lzcnt64(__builtin_bswap64(a[i++])) & mask;
|
||||
}
|
||||
while(i < lanes<T>());
|
||||
|
||||
return ret;
|
||||
}
|
||||
59
include/ircd/simd/popcnt.h
Normal file
59
include/ircd/simd/popcnt.h
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
// The Construct
|
||||
//
|
||||
// Copyright (C) The Construct Developers, Authors & Contributors
|
||||
// Copyright (C) 2016-2020 Jason Volk <jason@zemos.net>
|
||||
//
|
||||
// Permission to use, copy, modify, and/or distribute this software for any
|
||||
// purpose with or without fee is hereby granted, provided that the above
|
||||
// copyright notice and this permission notice is present in all copies. The
|
||||
// full license for this software is available in the LICENSE file.
|
||||
|
||||
#pragma once
|
||||
#define HAVE_IRCD_SIMD_POPCNT_H
|
||||
|
||||
namespace ircd::simd
|
||||
{
|
||||
template<class T> T popmask(const T) noexcept;
|
||||
template<class T> T boolmask(const T) noexcept;
|
||||
template<class T> uint popcnt(const T) noexcept;
|
||||
}
|
||||
|
||||
/// Convenience template. Unfortunately this drops to scalar until specific
|
||||
/// targets and specializations are created.
|
||||
template<class T>
|
||||
inline uint
|
||||
ircd::simd::popcnt(const T a)
|
||||
noexcept
|
||||
{
|
||||
uint ret(0), i(0);
|
||||
for(; i < lanes<T>(); ++i)
|
||||
if constexpr(sizeof_lane<T>() <= sizeof(int))
|
||||
ret += __builtin_popcount(a[i]);
|
||||
else if constexpr(sizeof_lane<T>() <= sizeof(long))
|
||||
ret += __builtin_popcountl(a[i]);
|
||||
else
|
||||
ret += __builtin_popcountll(a[i]);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
/// Convenience template. Extends a bool value where the lsb is 1 or 0 into a
|
||||
/// mask value like the result of vector comparisons.
|
||||
template<class T>
|
||||
inline T
|
||||
ircd::simd::boolmask(const T a)
|
||||
noexcept
|
||||
{
|
||||
return ~(popmask(a) - 1);
|
||||
}
|
||||
|
||||
/// Convenience template. Vector compare instructions yield 0xff on equal;
|
||||
/// sometimes one might need an actual value of 1 for accumulators or maybe
|
||||
/// some bool-type reason...
|
||||
template<class T>
|
||||
inline T
|
||||
ircd::simd::popmask(const T a)
|
||||
noexcept
|
||||
{
|
||||
return a & 1;
|
||||
}
|
||||
|
|
@ -16,144 +16,22 @@
|
|||
#include "lane_cast.h"
|
||||
#include "broad_cast.h"
|
||||
#include "shift.h"
|
||||
#include "popcnt.h"
|
||||
#include "lzcnt.h"
|
||||
#include "tzcnt.h"
|
||||
#include "print.h"
|
||||
|
||||
namespace ircd::simd
|
||||
{
|
||||
template<class T> T popmask(const T) noexcept;
|
||||
template<class T> T boolmask(const T) noexcept;
|
||||
|
||||
template<class T> uint popcount(const T) noexcept;
|
||||
template<class T> uint clz(const T) noexcept;
|
||||
template<class T> uint ctz(const T) noexcept;
|
||||
}
|
||||
|
||||
namespace ircd
|
||||
{
|
||||
using simd::lane_cast;
|
||||
|
||||
using simd::shl;
|
||||
using simd::shr;
|
||||
}
|
||||
|
||||
/// Convenience template. Unfortunately this drops to scalar until specific
|
||||
/// targets and specializations are created.
|
||||
template<class T>
|
||||
inline uint
|
||||
__attribute__((target("lzcnt")))
|
||||
ircd::simd::clz(const T a)
|
||||
noexcept
|
||||
{
|
||||
constexpr auto uses_bsr
|
||||
{
|
||||
#ifndef __LZCNT__
|
||||
true
|
||||
#else
|
||||
false
|
||||
#endif
|
||||
};
|
||||
|
||||
uint ret(0), i(0); do
|
||||
{
|
||||
const auto mask
|
||||
{
|
||||
boolmask(uint(ret == sizeof_lane<T>() * 8 * i))
|
||||
};
|
||||
|
||||
if constexpr(sizeof_lane<T>() <= sizeof(u16) && uses_bsr)
|
||||
ret += (15 - __lzcnt16(__builtin_bswap16(a[i++]))) & mask;
|
||||
else if constexpr(sizeof_lane<T>() <= sizeof(u16))
|
||||
ret += __lzcnt16(__builtin_bswap16(a[i++])) & mask;
|
||||
else if constexpr(sizeof_lane<T>() <= sizeof(u32) && uses_bsr)
|
||||
ret += (31 - __lzcnt32(__builtin_bswap32(a[i++]))) & mask;
|
||||
else if constexpr(sizeof_lane<T>() <= sizeof(u32))
|
||||
ret += __lzcnt32(__builtin_bswap32(a[i++])) & mask;
|
||||
else if constexpr(uses_bsr)
|
||||
ret += (63 - __lzcnt64(__builtin_bswap64(a[i++]))) & mask;
|
||||
else
|
||||
ret += __lzcnt64(__builtin_bswap64(a[i++])) & mask;
|
||||
}
|
||||
while(i < lanes<T>());
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
/// Convenience template. Unfortunately this drops to scalar until specific
|
||||
/// targets and specializations are created.
|
||||
template<class T>
|
||||
inline uint
|
||||
__attribute__((target("lzcnt")))
|
||||
ircd::simd::ctz(const T a)
|
||||
noexcept
|
||||
{
|
||||
constexpr auto uses_bsr
|
||||
{
|
||||
#ifndef __LZCNT__
|
||||
true
|
||||
#else
|
||||
false
|
||||
#endif
|
||||
};
|
||||
|
||||
uint ret(0), i(lanes<T>()), mask(-1U); do
|
||||
{
|
||||
if constexpr(sizeof_lane<T>() <= sizeof(u16) && uses_bsr)
|
||||
ret += (15 - __lzcnt16(a[--i])) & mask;
|
||||
else if constexpr(sizeof_lane<T>() <= sizeof(u16))
|
||||
ret += __lzcnt16(a[--i]) & mask;
|
||||
else if constexpr(sizeof_lane<T>() <= sizeof(u32) && uses_bsr)
|
||||
ret += (31 - __lzcnt32(a[--i])) & mask;
|
||||
else if constexpr(sizeof_lane<T>() <= sizeof(u32))
|
||||
ret += __lzcnt32(a[--i]) & mask;
|
||||
else if constexpr(uses_bsr)
|
||||
ret += (63 - __lzcnt64(a[--i])) & mask;
|
||||
else
|
||||
ret += __lzcnt64(a[--i]) & mask;
|
||||
|
||||
static const auto lane_bits(sizeof_lane<T>() * 8);
|
||||
mask &= boolmask(uint(ret % lane_bits == 0));
|
||||
mask &= boolmask(uint(ret != 0));
|
||||
}
|
||||
while(i);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
/// Convenience template. Unfortunately this drops to scalar until specific
|
||||
/// targets and specializations are created.
|
||||
template<class T>
|
||||
inline uint
|
||||
ircd::simd::popcount(const T a)
|
||||
noexcept
|
||||
{
|
||||
uint ret(0), i(0);
|
||||
for(; i < lanes<T>(); ++i)
|
||||
if constexpr(sizeof_lane<T>() <= sizeof(int))
|
||||
ret += __builtin_popcount(a[i]);
|
||||
else if constexpr(sizeof_lane<T>() <= sizeof(long))
|
||||
ret += __builtin_popcountl(a[i]);
|
||||
else
|
||||
ret += __builtin_popcountll(a[i]);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
/// Convenience template. Extends a bool value where the lsb is 1 or 0 into a
|
||||
/// mask value like the result of vector comparisons.
|
||||
template<class T>
|
||||
inline T
|
||||
ircd::simd::boolmask(const T a)
|
||||
noexcept
|
||||
{
|
||||
return ~(popmask(a) - 1);
|
||||
}
|
||||
|
||||
/// Convenience template. Vector compare instructions yield 0xff on equal;
|
||||
/// sometimes one might need an actual value of 1 for accumulators or maybe
|
||||
/// some bool-type reason...
|
||||
template<class T>
|
||||
inline T
|
||||
ircd::simd::popmask(const T a)
|
||||
noexcept
|
||||
{
|
||||
return a & 1;
|
||||
|
||||
using simd::lzcnt;
|
||||
using simd::tzcnt;
|
||||
|
||||
using simd::popcnt;
|
||||
using simd::popmask;
|
||||
using simd::boolmask;
|
||||
}
|
||||
|
|
|
|||
65
include/ircd/simd/tzcnt.h
Normal file
65
include/ircd/simd/tzcnt.h
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
// The Construct
|
||||
//
|
||||
// Copyright (C) The Construct Developers, Authors & Contributors
|
||||
// Copyright (C) 2016-2020 Jason Volk <jason@zemos.net>
|
||||
//
|
||||
// Permission to use, copy, modify, and/or distribute this software for any
|
||||
// purpose with or without fee is hereby granted, provided that the above
|
||||
// copyright notice and this permission notice is present in all copies. The
|
||||
// full license for this software is available in the LICENSE file.
|
||||
|
||||
#pragma once
|
||||
#define HAVE_IRCD_SIMD_TZCNT_H
|
||||
|
||||
namespace ircd::simd
|
||||
{
|
||||
template<class T> uint tzcnt(const T) noexcept;
|
||||
}
|
||||
|
||||
/// Convenience template. Unfortunately this drops to scalar until specific
|
||||
/// targets and specializations are created.
|
||||
template<class T>
|
||||
inline uint
|
||||
__attribute__((target("lzcnt")))
|
||||
ircd::simd::tzcnt(const T a)
|
||||
noexcept
|
||||
{
|
||||
// The behavior of lzcnt/tzcnt can differ among platforms; when false we
|
||||
// lzcnt/tzcnt to fall back to bsr/bsf-like behavior.
|
||||
constexpr auto bitscan
|
||||
{
|
||||
#ifdef __LZCNT__
|
||||
false
|
||||
#else
|
||||
true
|
||||
#endif
|
||||
};
|
||||
|
||||
uint ret(0), i(lanes<T>()), mask(-1U); do
|
||||
{
|
||||
if constexpr(bitscan && sizeof_lane<T>() <= sizeof(u16))
|
||||
ret += (15 - __lzcnt16(a[--i])) & mask;
|
||||
|
||||
else if constexpr(sizeof_lane<T>() <= sizeof(u16))
|
||||
ret += __lzcnt16(a[--i]) & mask;
|
||||
|
||||
else if constexpr(bitscan && sizeof_lane<T>() <= sizeof(u32))
|
||||
ret += (31 - __lzcnt32(a[--i])) & mask;
|
||||
|
||||
else if constexpr(sizeof_lane<T>() <= sizeof(u32))
|
||||
ret += __lzcnt32(a[--i]) & mask;
|
||||
|
||||
else if constexpr(bitscan)
|
||||
ret += (63 - __lzcnt64(a[--i])) & mask;
|
||||
|
||||
else
|
||||
ret += __lzcnt64(a[--i]) & mask;
|
||||
|
||||
static const auto lane_bits(sizeof_lane<T>() * 8);
|
||||
mask &= boolmask(uint(ret % lane_bits == 0));
|
||||
mask &= boolmask(uint(ret != 0));
|
||||
}
|
||||
while(i);
|
||||
|
||||
return ret;
|
||||
}
|
||||
14
ircd/json.cc
14
ircd/json.cc
|
|
@ -3405,7 +3405,7 @@ ircd::json::string_stringify(u8x16 &__restrict__ block,
|
|||
// counting past the end of the string.
|
||||
const u64 regular_prefix_count
|
||||
{
|
||||
simd::clz(u64x2(is_special | ~block_mask)) / 8
|
||||
simd::lzcnt(u64x2(is_special | ~block_mask)) / 8
|
||||
};
|
||||
|
||||
// Fast-path; backward branch to count and consume uninteresting characters
|
||||
|
|
@ -3460,7 +3460,7 @@ ircd::json::string_stringify(u8x16 &__restrict__ block,
|
|||
|
||||
const u64 match_depth
|
||||
{
|
||||
simd::clz(u64x2(match)) / 8
|
||||
simd::lzcnt(u64x2(match)) / 8
|
||||
};
|
||||
|
||||
// Legitimately escaped single char
|
||||
|
|
@ -3568,7 +3568,7 @@ ircd::json::string_stringify_utf16(u8x16 &__restrict__ block,
|
|||
|
||||
const auto surrogates
|
||||
{
|
||||
simd::popcount(u64x2(simd::popmask(u8x16(is_surrogate))))
|
||||
simd::popcnt(u64x2(simd::popmask(u8x16(is_surrogate))))
|
||||
};
|
||||
|
||||
assert(di == length[0] + length[1]);
|
||||
|
|
@ -3647,7 +3647,7 @@ ircd::json::string_serialized(const u8x16 block,
|
|||
|
||||
const u64 regular_prefix_count
|
||||
{
|
||||
simd::clz(u64x2(is_special | ~block_mask)) / 8
|
||||
simd::lzcnt(u64x2(is_special | ~block_mask)) / 8
|
||||
};
|
||||
|
||||
// Fast-path; backward branch to count and consume uninteresting characters
|
||||
|
|
@ -3691,7 +3691,7 @@ ircd::json::string_serialized(const u8x16 block,
|
|||
|
||||
const u64 match_depth
|
||||
{
|
||||
simd::clz(u64x2(match)) / 8
|
||||
simd::lzcnt(u64x2(match)) / 8
|
||||
};
|
||||
|
||||
// Legitimately escaped single char
|
||||
|
|
@ -3779,7 +3779,7 @@ ircd::json::string_serialized_utf16(const u8x16 block,
|
|||
|
||||
const auto surrogates
|
||||
{
|
||||
simd::popcount(u64x2(simd::popmask(u8x16(is_surrogate))))
|
||||
simd::popcnt(u64x2(simd::popmask(u8x16(is_surrogate))))
|
||||
};
|
||||
|
||||
return u64x2
|
||||
|
|
@ -3801,7 +3801,7 @@ ircd::json::string_serialized_ctrl(const u8x16 block,
|
|||
|
||||
const u64 ctrl_prefix_count
|
||||
{
|
||||
simd::clz(u64x2(~is_ctrl | ~block_mask)) / 8
|
||||
simd::lzcnt(u64x2(~is_ctrl | ~block_mask)) / 8
|
||||
};
|
||||
|
||||
u64 ret(0);
|
||||
|
|
|
|||
Loading…
Reference in a new issue