ircd::simd: Split bit manipulations suites to headers; renames.

include/ircd/simd/lzcnt.h

@@ -0,0 +1,66 @@
+// The Construct
+//
+// Copyright (C) The Construct Developers, Authors & Contributors
+// Copyright (C) 2016-2020 Jason Volk <jason@zemos.net>
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice is present in all copies. The
+// full license for this software is available in the LICENSE file.
+
+#pragma once
+#define HAVE_IRCD_SIMD_LZCNT_H
+
+namespace ircd::simd
+{
+	template<class T> uint lzcnt(const T) noexcept;
+}
+
+/// Convenience template. Unfortunately this drops to scalar until specific
+/// targets and specializations are created.
+template<class T>
+inline uint
+__attribute__((target("lzcnt")))
+ircd::simd::lzcnt(const T a)
+noexcept
+{
+	// The behavior of lzcnt can differ among platforms; when true we expect
+	// lzcnt to fall back to bsr-like behavior.
+	constexpr auto bitscan
+	{
+		#ifdef __LZCNT__
+			false
+		#else
+			true
+		#endif
+	};
+
+	uint ret(0), i(0); do
+	{
+		const auto mask
+		{
+			boolmask(uint(ret == sizeof_lane<T>() * 8 * i))
+		};
+
+		if constexpr(bitscan && sizeof_lane<T>() <= sizeof(u16))
+			ret += (15 - __lzcnt16(__builtin_bswap16(a[i++]))) & mask;
+
+		else if constexpr(sizeof_lane<T>() <= sizeof(u16))
+			ret += __lzcnt16(__builtin_bswap16(a[i++])) & mask;
+
+		else if constexpr(bitscan && sizeof_lane<T>() <= sizeof(u32))
+			ret += (31 - __lzcnt32(__builtin_bswap32(a[i++]))) & mask;
+
+		else if constexpr(sizeof_lane<T>() <= sizeof(u32))
+			ret += __lzcnt32(__builtin_bswap32(a[i++])) & mask;
+
+		else if constexpr(bitscan)
+			ret += (63 - __lzcnt64(__builtin_bswap64(a[i++]))) & mask;
+
+		else
+			ret += __lzcnt64(__builtin_bswap64(a[i++])) & mask;
+	}
+	while(i < lanes<T>());
+
+	return ret;
+}

include/ircd/simd/popcnt.h

@@ -0,0 +1,59 @@
+// The Construct
+//
+// Copyright (C) The Construct Developers, Authors & Contributors
+// Copyright (C) 2016-2020 Jason Volk <jason@zemos.net>
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice is present in all copies. The
+// full license for this software is available in the LICENSE file.
+
+#pragma once
+#define HAVE_IRCD_SIMD_POPCNT_H
+
+namespace ircd::simd
+{
+	template<class T> T popmask(const T) noexcept;
+	template<class T> T boolmask(const T) noexcept;
+	template<class T> uint popcnt(const T) noexcept;
+}
+
+/// Convenience template. Unfortunately this drops to scalar until specific
+/// targets and specializations are created.
+template<class T>
+inline uint
+ircd::simd::popcnt(const T a)
+noexcept
+{
+	uint ret(0), i(0);
+	for(; i < lanes<T>(); ++i)
+		if constexpr(sizeof_lane<T>() <= sizeof(int))
+			ret += __builtin_popcount(a[i]);
+		else if constexpr(sizeof_lane<T>() <= sizeof(long))
+			ret += __builtin_popcountl(a[i]);
+		else
+			ret += __builtin_popcountll(a[i]);
+
+	return ret;
+}
+
+/// Convenience template. Extends a bool value where the lsb is 1 or 0 into a
+/// mask value like the result of vector comparisons.
+template<class T>
+inline T
+ircd::simd::boolmask(const T a)
+noexcept
+{
+	return ~(popmask(a) - 1);
+}
+
+/// Convenience template. Vector compare instructions yield 0xff on equal;
+/// sometimes one might need an actual value of 1 for accumulators or maybe
+/// some bool-type reason...
+template<class T>
+inline T
+ircd::simd::popmask(const T a)
+noexcept
+{
+	return a & 1;
+}

include/ircd/simd/simd.h

@@ -16,144 +16,22 @@
 #include "lane_cast.h"
 #include "broad_cast.h"
 #include "shift.h"
+#include "popcnt.h"
+#include "lzcnt.h"
+#include "tzcnt.h"
 #include "print.h"
 
-namespace ircd::simd
-{
-	template<class T> T popmask(const T) noexcept;
-	template<class T> T boolmask(const T) noexcept;
-
-	template<class T> uint popcount(const T) noexcept;
-	template<class T> uint clz(const T) noexcept;
-	template<class T> uint ctz(const T) noexcept;
-}
-
 namespace ircd
 {
 	using simd::lane_cast;
+
 	using simd::shl;
 	using simd::shr;
-}
+
-
+	using simd::lzcnt;
-/// Convenience template. Unfortunately this drops to scalar until specific
+	using simd::tzcnt;
-/// targets and specializations are created.
+
-template<class T>
+	using simd::popcnt;
-inline uint
+	using simd::popmask;
-__attribute__((target("lzcnt")))
+	using simd::boolmask;
-ircd::simd::clz(const T a)
-noexcept
-{
-	constexpr auto uses_bsr
-	{
-		#ifndef __LZCNT__
-			true
-		#else
-			false
-		#endif
-	};
-
-	uint ret(0), i(0); do
-	{
-		const auto mask
-		{
-			boolmask(uint(ret == sizeof_lane<T>() * 8 * i))
-		};
-
-		if constexpr(sizeof_lane<T>() <= sizeof(u16) && uses_bsr)
-			ret += (15 - __lzcnt16(__builtin_bswap16(a[i++]))) & mask;
-		else if constexpr(sizeof_lane<T>() <= sizeof(u16))
-			ret += __lzcnt16(__builtin_bswap16(a[i++])) & mask;
-		else if constexpr(sizeof_lane<T>() <= sizeof(u32) && uses_bsr)
-			ret += (31 - __lzcnt32(__builtin_bswap32(a[i++]))) & mask;
-		else if constexpr(sizeof_lane<T>() <= sizeof(u32))
-			ret += __lzcnt32(__builtin_bswap32(a[i++])) & mask;
-		else if constexpr(uses_bsr)
-			ret += (63 - __lzcnt64(__builtin_bswap64(a[i++]))) & mask;
-		else
-			ret += __lzcnt64(__builtin_bswap64(a[i++])) & mask;
-	}
-	while(i < lanes<T>());
-
-	return ret;
-}
-
-/// Convenience template. Unfortunately this drops to scalar until specific
-/// targets and specializations are created.
-template<class T>
-inline uint
-__attribute__((target("lzcnt")))
-ircd::simd::ctz(const T a)
-noexcept
-{
-	constexpr auto uses_bsr
-	{
-		#ifndef __LZCNT__
-			true
-		#else
-			false
-		#endif
-	};
-
-	uint ret(0), i(lanes<T>()), mask(-1U); do
-	{
-		if constexpr(sizeof_lane<T>() <= sizeof(u16) && uses_bsr)
-			ret += (15 - __lzcnt16(a[--i])) & mask;
-		else if constexpr(sizeof_lane<T>() <= sizeof(u16))
-			ret += __lzcnt16(a[--i]) & mask;
-		else if constexpr(sizeof_lane<T>() <= sizeof(u32) && uses_bsr)
-			ret += (31 - __lzcnt32(a[--i])) & mask;
-		else if constexpr(sizeof_lane<T>() <= sizeof(u32))
-			ret += __lzcnt32(a[--i]) & mask;
-		else if constexpr(uses_bsr)
-			ret += (63 - __lzcnt64(a[--i])) & mask;
-		else
-			ret += __lzcnt64(a[--i]) & mask;
-
-		static const auto lane_bits(sizeof_lane<T>() * 8);
-		mask &= boolmask(uint(ret % lane_bits == 0));
-		mask &= boolmask(uint(ret != 0));
-	}
-	while(i);
-
-	return ret;
-}
-
-/// Convenience template. Unfortunately this drops to scalar until specific
-/// targets and specializations are created.
-template<class T>
-inline uint
-ircd::simd::popcount(const T a)
-noexcept
-{
-	uint ret(0), i(0);
-	for(; i < lanes<T>(); ++i)
-		if constexpr(sizeof_lane<T>() <= sizeof(int))
-			ret += __builtin_popcount(a[i]);
-		else if constexpr(sizeof_lane<T>() <= sizeof(long))
-			ret += __builtin_popcountl(a[i]);
-		else
-			ret += __builtin_popcountll(a[i]);
-
-	return ret;
-}
-
-/// Convenience template. Extends a bool value where the lsb is 1 or 0 into a
-/// mask value like the result of vector comparisons.
-template<class T>
-inline T
-ircd::simd::boolmask(const T a)
-noexcept
-{
-	return ~(popmask(a) - 1);
-}
-
-/// Convenience template. Vector compare instructions yield 0xff on equal;
-/// sometimes one might need an actual value of 1 for accumulators or maybe
-/// some bool-type reason...
-template<class T>
-inline T
-ircd::simd::popmask(const T a)
-noexcept
-{
-	return a & 1;
 }

include/ircd/simd/tzcnt.h

@@ -0,0 +1,65 @@
+// The Construct
+//
+// Copyright (C) The Construct Developers, Authors & Contributors
+// Copyright (C) 2016-2020 Jason Volk <jason@zemos.net>
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice is present in all copies. The
+// full license for this software is available in the LICENSE file.
+
+#pragma once
+#define HAVE_IRCD_SIMD_TZCNT_H
+
+namespace ircd::simd
+{
+	template<class T> uint tzcnt(const T) noexcept;
+}
+
+/// Convenience template. Unfortunately this drops to scalar until specific
+/// targets and specializations are created.
+template<class T>
+inline uint
+__attribute__((target("lzcnt")))
+ircd::simd::tzcnt(const T a)
+noexcept
+{
+    // The behavior of lzcnt/tzcnt can differ among platforms; when false we
+    // lzcnt/tzcnt to fall back to bsr/bsf-like behavior.
+	constexpr auto bitscan
+	{
+		#ifdef __LZCNT__
+			false
+		#else
+			true
+		#endif
+	};
+
+	uint ret(0), i(lanes<T>()), mask(-1U); do
+	{
+		if constexpr(bitscan && sizeof_lane<T>() <= sizeof(u16))
+			ret += (15 - __lzcnt16(a[--i])) & mask;
+
+		else if constexpr(sizeof_lane<T>() <= sizeof(u16))
+			ret += __lzcnt16(a[--i]) & mask;
+
+		else if constexpr(bitscan && sizeof_lane<T>() <= sizeof(u32))
+			ret += (31 - __lzcnt32(a[--i])) & mask;
+
+		else if constexpr(sizeof_lane<T>() <= sizeof(u32))
+			ret += __lzcnt32(a[--i]) & mask;
+
+		else if constexpr(bitscan)
+			ret += (63 - __lzcnt64(a[--i])) & mask;
+
+		else
+			ret += __lzcnt64(a[--i]) & mask;
+
+		static const auto lane_bits(sizeof_lane<T>() * 8);
+		mask &= boolmask(uint(ret % lane_bits == 0));
+		mask &= boolmask(uint(ret != 0));
+	}
+	while(i);
+
+	return ret;
+}

ircd/json.cc

@@ -3405,7 +3405,7 @@ ircd::json::string_stringify(u8x16 &__restrict__ block,
 	// counting past the end of the string.
 	const u64 regular_prefix_count
 	{
-		simd::clz(u64x2(is_special | ~block_mask)) / 8
+		simd::lzcnt(u64x2(is_special | ~block_mask)) / 8
 	};
 
 	// Fast-path; backward branch to count and consume uninteresting characters
@@ -3460,7 +3460,7 @@ ircd::json::string_stringify(u8x16 &__restrict__ block,
 
 	const u64 match_depth
 	{
-		simd::clz(u64x2(match)) / 8
+		simd::lzcnt(u64x2(match)) / 8
 	};
 
 	// Legitimately escaped single char
@@ -3568,7 +3568,7 @@ ircd::json::string_stringify_utf16(u8x16 &__restrict__ block,
 
 	const auto surrogates
 	{
-		simd::popcount(u64x2(simd::popmask(u8x16(is_surrogate))))
+		simd::popcnt(u64x2(simd::popmask(u8x16(is_surrogate))))
 	};
 
 	assert(di == length[0] + length[1]);
@@ -3647,7 +3647,7 @@ ircd::json::string_serialized(const u8x16 block,
 
 	const u64 regular_prefix_count
 	{
-		simd::clz(u64x2(is_special | ~block_mask)) / 8
+		simd::lzcnt(u64x2(is_special | ~block_mask)) / 8
 	};
 
 	// Fast-path; backward branch to count and consume uninteresting characters
@@ -3691,7 +3691,7 @@ ircd::json::string_serialized(const u8x16 block,
 
 	const u64 match_depth
 	{
-		simd::clz(u64x2(match)) / 8
+		simd::lzcnt(u64x2(match)) / 8
 	};
 
 	// Legitimately escaped single char
@@ -3779,7 +3779,7 @@ ircd::json::string_serialized_utf16(const u8x16 block,
 
 	const auto surrogates
 	{
-		simd::popcount(u64x2(simd::popmask(u8x16(is_surrogate))))
+		simd::popcnt(u64x2(simd::popmask(u8x16(is_surrogate))))
 	};
 
 	return u64x2
@@ -3801,7 +3801,7 @@ ircd::json::string_serialized_ctrl(const u8x16 block,
 
 	const u64 ctrl_prefix_count
 	{
-		simd::clz(u64x2(~is_ctrl | ~block_mask)) / 8
+		simd::lzcnt(u64x2(~is_ctrl | ~block_mask)) / 8
 	};
 
 	u64 ret(0);