ircd:🆑 Stop propagation of platform and device initialization errors.

.appveyor.yml

@@ -5,6 +5,10 @@ skip_tags: true
 services:
   - docker
 
+branches:
+  only:
+    - appveyor
+
 environment:
   COMMAND: if [[ "${APPVEYOR_REPO_COMMIT_MESSAGE@Q}" == *"[ci debug]"* ]]; then export DEBUG_FLAGS="--enable-debug"; fi; if [[ "${APPVEYOR_REPO_COMMIT_MESSAGE@Q}" == *"[ci gdb]"* ]]; then apt-get update && apt-get -y install gdb && export CONSTRUCT="gdb --batch -ex r -ex bt --return-child-result --args construct"; else export CONSTRUCT="construct"; fi && rmdir -v deps/rocksdb && ln -sv /usr/src/rocksdb deps && ./autogen.sh && ./configure --enable-assert ${DEBUG_FLAGS} && make install && ${CONSTRUCT} -smoketest -debug localhost
 

.github/workflows/docker.yml

@@ -0,0 +1,83 @@
+name: Docker Images
+
+env:
+  ctor_id: ${{ vars.DOCKER_ID }}
+  ctor_url: https://github.com/${{github.repository}}
+
+on:
+  push:
+    branches: [master]
+
+  workflow_dispatch:
+    inputs:
+
+concurrency:
+  group: ${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  base:
+    uses: ./.github/workflows/docker_prime.yml
+    with:
+      id: ${{github.env.ctor_id}}
+      url: ${{github.env.ctor_url}}
+      features: '["base"]'
+      distros: ${{vars.DOCKER_DISTROS}}
+      machines: ${{vars.DOCKER_MACHINES}}
+      test: ${{contains(github.events.push.commits[0].message, '[ci test]')}}
+
+  full:
+    uses: ./.github/workflows/docker_prime.yml
+    needs: [base]
+    with:
+      id: ${{github.env.ctor_id}}
+      url: ${{github.env.ctor_url}}
+      features: '["full"]'
+      distros: ${{vars.DOCKER_DISTROS}}
+      machines: ${{vars.DOCKER_MACHINES}}
+      test: ${{contains(github.events.push.commits[0].message, '[ci test]')}}
+
+  built:
+    needs: [base, full]
+    runs-on: ${{matrix.machine}}
+    strategy:
+      fail-fast: false
+      matrix:
+        feature: ${{fromJSON(vars.DOCKER_FEATURES)}}
+        distro: ${{fromJSON(vars.DOCKER_DISTROS)}}
+        machine: ${{fromJSON(vars.DOCKER_MACHINES)}}
+        toolchain: ${{fromJSON(vars.DOCKER_TOOLCHAINS)}}
+        exclude:
+          - distro: alpine-3.17
+            toolchain: gcc-10      # n/a on distro version
+          - distro: alpine-3.17
+            toolchain: gcc-11      # n/a on distro version
+          - distro: alpine-3.17
+            toolchain: clang-14    # n/a on distro version
+          - distro: ubuntu-22.04
+            toolchain: clang-15    # n/a on distro version
+          - machine: arm64
+            toolchain: gcc-12      # build hangs
+          - machine: arm64
+            toolchain: gcc-11      # build hangs
+          - machine: arm64
+            toolchain: gcc-10      # build hangs
+          - machine: arm64
+            distro: alpine-3.17
+            toolchain: clang-14    # smoketest crash
+          - machine: arm64
+            distro: alpine-3.17
+            toolchain: clang-15    # smoketest crash
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: build
+      env:
+        ctor_features: ${{matrix.feature}}
+        ctor_distros: ${{matrix.distro}}
+        ctor_machines: ${{matrix.machine}}
+        ctor_toolchains: ${{matrix.toolchain}}
+        ctor_test: ${{contains(github.events.push.commits[0].message, '[ci test]')}}
+
+      run: |
+        docker/build-and-push-images.sh

.github/workflows/docker_prime.yml

@@ -0,0 +1,50 @@
+name: Docker Images Prime
+
+on:
+  workflow_call:
+    inputs:
+      id:
+        type: string
+      url:
+        type: string
+      features:
+        type: string
+      distros:
+        type: string
+      machines:
+        type: string
+      test:
+        type: boolean
+        default: false
+        required: false
+
+env:
+  ctor_id: ${{inputs.id}}
+  ctor_url: ${{inputs.url}}
+
+concurrency:
+  group: ${{github.run_id}}
+  cancel-in-progress: true
+
+jobs:
+  prime:
+    runs-on: ${{matrix.machine}}
+    strategy:
+      fail-fast: false
+      matrix:
+        feature: ${{fromJSON(inputs.features)}}
+        distro: ${{fromJSON(inputs.distros)}}
+        machine: ${{fromJSON(inputs.machines)}}
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: build
+      env:
+        ctor_features: ${{matrix.feature}}
+        ctor_distros: ${{matrix.distro}}
+        ctor_machines: ${{matrix.machine}}
+        ctor_toolchains: false
+        ctor_test: ${{inputs.test}}
+
+      run: |
+        docker/build-and-push-images.sh ${{matrix.feature}}

.gitignore

@@ -19,6 +19,7 @@ Makefile
 *.gch
 *.cache
 *.tmp
+*.save
 *.profdata
 .deps
 .dirstamp

configure.ac

@@ -83,6 +83,16 @@ dnl
 dnl Platform
 dnl
 
+dnl
+dnl Hardwares
+dnl
+
+AM_CONDITIONAL([AMD64],    [[[[ $host_cpu = *x86_64* ]]]])
+AM_CONDITIONAL([X86_64],   [[[[ $host_cpu = *x86_64* ]]]])
+
+AM_CONDITIONAL([ARM64],    [[[[ $host_cpu = *aarch64* ]]]])
+AM_CONDITIONAL([AARCH64],  [[[[ $host_cpu = *aarch64* ]]]])
+
 dnl
 dnl Compiler brand
 dnl
@@ -769,16 +779,20 @@ AM_COND_IF([GCC],
 [
 	RB_VAR_PREPEND([CXXFLAGS], ["-fstack-protector-explicit"])
 
-	dnl These flags should not be used on Intel-CET capable platforms
-	dnl TODO: XXX
-	AS_IF([test "$CXX_EPOCH" -ge "9"],
+	AS_CASE([$host_cpu],
+	[x86_64],
 	[
-		AM_COND_IF([GENERIC],
+		dnl These flags should not be used on Intel-CET capable platforms
+		dnl TODO: XXX
+		AS_IF([test "$CXX_EPOCH" -ge "9"],
 		[
-			RB_VAR_PREPEND([CXXFLAGS], ["-fcf-protection=full"])
-		], [
-			RB_VAR_PREPEND([CXXFLAGS], ["-fcf-protection=none"])
-			RB_VAR_PREPEND([CXXFLAGS], ["-mmanual-endbr"])
+			AM_COND_IF([GENERIC],
+			[
+				RB_VAR_PREPEND([CXXFLAGS], ["-fcf-protection=return"])
+			], [
+				RB_VAR_PREPEND([CXXFLAGS], ["-fcf-protection=none"])
+				RB_VAR_PREPEND([CXXFLAGS], ["-mmanual-endbr"])
+			])
 		])
 	])
 ])
@@ -797,10 +811,22 @@ MACHINE_FLAGS=""
 dnl Initial machine
 machine_arch="native"
 machine_tune="native"
+
+AM_COND_IF([CLANG],
+[
+	AS_IF([test "$CXX_EPOCH" -lt "15"],
+	[
+		AS_IF([test "$host_cpu" = "aarch64"],
+		[
+			machine_arch=""
+		])
+	])
+])
+
 AM_COND_IF([GENERIC],
 [
 	machine_tune="generic"
-	AS_CASE([$target_cpu],
+	AS_CASE([$host_cpu],
 	[x86_64],
 	[
 		machine_arch="x86-64"
@@ -816,7 +842,7 @@ for feature in $machine; do
 		machine_tune=$(echo $feature | cut -d"=" -f2)
 	else
 		machine_feat="$feature $machine_feat"
-		RB_VAR_PREPEND([MACHINE_FLAGS], ["-m${feature}"])
+		RB_VAR_PREPEND([MACHINE_FLAGS], [-m${feature}])
 	fi
 done
 
@@ -834,18 +860,23 @@ AM_COND_IF_NOT([GENERIC],
 ])
 
 dnl Specific extension underrides
-AS_CASE([$target_cpu],
+AS_CASE([$host_cpu],
 [x86_64],
 [
 	dnl AMD K10's SSE4a doesn't work with valgrind
-	RB_VAR_PREPEND([MACHINE_FLAGS], ["-mno-sse4a"])
+	RB_VAR_PREPEND([MACHINE_FLAGS], [-mno-sse4a])
 ])
 
-RB_VAR_PREPEND([MACHINE_FLAGS], ["-mtune=$machine_tune"])
-RB_VAR_PREPEND([MACHINE_FLAGS], ["-march=$machine_arch"])
+if [[ ! -z "$machine_tune" ]]; then
+	RB_VAR_PREPEND([MACHINE_FLAGS], [-mtune=$machine_tune])
+fi
 
-machine_tuning="${target_cpu} arch=$machine_arch tune=$machine_tune :$machine_feat"
-RB_VAR_PREPEND([CXXFLAGS], ["$MACHINE_FLAGS"])
+if [[ ! -z "$machine_arch" ]]; then
+	RB_VAR_PREPEND([MACHINE_FLAGS], [-march=$machine_arch])
+fi
+
+machine_tuning="${host_cpu} arch=$machine_arch tune=$machine_tune :$machine_feat"
+RB_VAR_PREPEND([CXXFLAGS], [$MACHINE_FLAGS])
 
 dnl Get the target features for this build from gcc into a readable string
 AM_COND_IF([GCC],

docker/alpine/built/Dockerfile

@@ -10,12 +10,11 @@ FROM ${acct}/${repo}:${dist_name}-${dist_version}-${feature}-${machine}
 ARG cc
 ARG cxx
 ARG extra_packages_dev
-ARG extra_packages_dev1
-ARG extra_packages_dev2
 ARG rocksdb_version 7.4.3
 ARG rocksdb_url
 ARG ctor_url https://github.com/matrix-construct/construct
 ARG machine_spec
+ARG nprocs
 
 ENV CC ${cc}
 ENV CXX ${cxx}
@@ -25,6 +24,7 @@ ENV rocksdb_version ${rocksdb_version}
 ENV rocksdb_url https://codeload.github.com/facebook/rocksdb/tar.gz/refs/tags/v${rocksdb_version}
 ENV ctor_url ${ctor_url}
 ENV machine_spec ${machine_spec}
+ENV nprocs ${nprocs}
 
 ENV packages_dev="\
 ${packages_dev} \
@@ -38,8 +38,6 @@ curl \
 git \
 libtool \
 ${extra_packages_dev} \
-${extra_packages_dev1} \
-${extra_packages_dev2} \
 "
 
 WORKDIR /usr/src
@@ -55,7 +53,7 @@ RUN true \
 && cd /usr/src/construct \
 && ./autogen.sh \
 && (./configure --disable-iou --enable-generic --with-machine="${machine_spec}" || (cat config.log; exit 1)) \
-&& make -j `nproc` EXTRA_LDFLAGS="-Wl,--strip-all" install \
+&& make -j ${nprocs} EXTRA_LDFLAGS="-Wl,--strip-all" install \
 && rm -rf /usr/src/rocksdb \
 && rm -rf /usr/src/construct \
 && rm -rf /usr/include/ircd \

docker/build-and-push-images.sh

@@ -1,32 +1,72 @@
 #!/bin/sh
 
+# Sundry configuration
+
 BASEDIR=$(dirname "$0")
-ACCT=jevolk
-REPO=construct
-CTOR_URL="https://github.com/matrix-construct/construct"
+stage=$1
+mode=$2
 
 export DOCKER_BUILDKIT=1
-#export BUILDKIT_PROGRESS=plain
+if test "$CI" = true; then
+	export BUILDKIT_PROGRESS="plain"
+	echo "plain"
+fi
 
-features="base full"
-distros="ubuntu-22.04 ubuntu-22.10 alpine-3.16 alpine-3.17"
-machines="arm64 amd64 amd64-avx amd64-avx2 amd64-avx512"
-toolchains="gcc-10 gcc-11 gcc-12 clang-14 clang-15"
-stages="built test"
+# The stage argument can be "base" "full" "built" "test" or "push"
+default_stage="push"
+stage=${stage:=$default_stage}
+
+# The mode argument can be "real" or "test"
+default_mode="real"
+mode=${mode:=$default_mode}
+
+if test "$ctor_test" = true; then
+	mode="test"
+fi
+
+# Account configuration environment.
+#
+# Override these for yourself.
+
+default_ctor_url="https://github.com/jevolk/charybdis"
+default_ctor_id="jevolk/construct"
+
+ctor_url=${ctor_url:=$default_ctor_url}
+ctor_id=${ctor_id:=$default_ctor_id}
+ctor_acct=${ctor_acct:=$(echo $ctor_id | cut -d"/" -f1)}
+ctor_repo=${ctor_repo:=$(echo $ctor_id | cut -d"/" -f2)}
+
+# Job matrix configuration environment
+#
+# All combinations of these arrays will be run by this script. Overriding
+# these with one element for each variable allows this script to do one thing
+# at a time.
+
+default_ctor_features="base full"
+default_ctor_distros="ubuntu-22.04 ubuntu-22.10 alpine-3.17"
+default_ctor_machines="amd64 amd64-avx amd64-avx512"
+default_ctor_toolchains="gcc-10 gcc-11 gcc-12 clang-14"
+
+ctor_features=${ctor_features:=$default_ctor_features}
+ctor_distros=${ctor_distros:=$default_ctor_distros}
+ctor_machines=${ctor_machines:=$default_ctor_machines}
+ctor_toolchains=${ctor_toolchains:=$default_ctor_toolchains}
+
+###############################################################################
 
 matrix()
 {
-	for feature_ in $features; do
-		for distro_ in $distros; do
-			for machine_ in $machines; do
-				for toolchain_ in $toolchains; do
-					for stage_ in $stages; do
-						build $feature_ $distro_ $machine_ $toolchain_ $stage_
-					done
+	for ctor_feature in $ctor_features; do
+		for ctor_distro in $ctor_distros; do
+			for ctor_machine in $ctor_machines; do
+				for ctor_toolchain in $ctor_toolchains; do
+					build $ctor_feature $ctor_distro $ctor_machine $ctor_toolchain
+					if test $? -ne 0; then return 1; fi
 				done
 			done
 		done
 	done
+	return 0
 }
 
 build()
@@ -35,42 +75,77 @@ build()
 	distro=$2
 	machine=$3
 	toolchain=$4
-	stage=$5
-
 	dist_name=$(echo $distro | cut -d"-" -f1)
 	dist_version=$(echo $distro | cut -d"-" -f2)
+	runner_name=$(echo $RUNNER_NAME | cut -d"." -f1)
+	runner_num=$(echo $RUNNER_NAME | cut -d"." -f2)
 
-	args=""
+	args="$ctor_docker_build_args"
 	args="$args --compress=true"
-	args="$args --build-arg acct=$ACCT"
-	args="$args --build-arg repo=$REPO"
-	args="$args --build-arg ctor_url=$CTOR_URL"
+
+	if test ! -z "$runner_num"; then
+		cpu_num=$(expr $runner_num % $(nproc))
+		args="$args --cpuset-cpus=${cpu_num}"
+		args="$args --build-arg nprocs=1"
+		# https://github.com/moby/buildkit/issues/1276
+	else
+		nprocs=$(nproc)
+		args="$args --build-arg nprocs=${nprocs}"
+	fi
+
+	args="$args --build-arg acct=${ctor_acct}"
+	args="$args --build-arg repo=${ctor_repo}"
+	args="$args --build-arg ctor_url=${ctor_url}"
 	args="$args --build-arg dist_name=${dist_name}"
 	args="$args --build-arg dist_version=${dist_version}"
-	args="$args --build-arg feature=${feature}"
 	args="$args --build-arg machine=${machine}"
+	args="$args --build-arg feature=${feature}"
 
 	args_dist $dist_name $dist_version
 	if test $? -ne 0; then return 1; fi
 
-	args_toolchain $toolchain $dist_name $dist_version
-	if test $? -ne 0; then return 1; fi
-
 	args_machine $machine
 	if test $? -ne 0; then return 1; fi
 
 	args_platform $machine
 	if test $? -ne 0; then return 1; fi
 
-	# Intermediate stage build; usually cached from prior iteration.
-	tag="$ACCT/$REPO:${distro}-${feature}-${machine}"
-	cmd="$args -t $tag $BASEDIR/${dist_name}/${feature}"
-	docker build $cmd
+	if test $toolchain != "false"; then
+		args_toolchain $toolchain $dist_name $dist_version
+		if test $? -ne 0; then return 1; fi
+	fi
+
+	if test $mode = "test"; then
+		cmd=$(which echo)
+	else
+		cmd=$(which docker)
+	fi
+
+	# Intermediate stage build; usually cached from prior iteration.
+	tag="$ctor_acct/$ctor_repo:${distro}-${feature}-${machine}"
+	arg="$args -t $tag $BASEDIR/${dist_name}/${feature}"
+	eval "$cmd build $arg"
+	if test $? -ne 0; then return 1; fi
+	if test $stage = $feature; then return 0; fi
+	if test $toolchain = "false"; then return 0; fi
+
+	# Leaf build; unique to each iteration.
+	tag="$ctor_acct/$ctor_repo:${distro}-${feature}-built-${toolchain}-${machine}"
+	arg="$args -t $tag $BASEDIR/${dist_name}/built"
+	eval "$cmd build $arg"
+	if test $? -ne 0; then return 1; fi
+	if test $stage = "built"; then return 0; fi
+
+	# Test built;
+	arg="$args $BASEDIR/${dist_name}/test"
+	eval "$cmd build $arg"
+	if test $? -ne 0; then return 1; fi
+	if test $stage = "test"; then return 0; fi
+
+	# Push built
+	eval "$cmd push $tag"
+	if test $? -ne 0; then return 1; fi
 
-	# Leaf stage build; unique to each iteration.
-	tag="$ACCT/$REPO:${distro}-${feature}-${stage}-${toolchain}-${machine}"
-	cmd="$args -t $tag $BASEDIR/${dist_name}/${stage}"
-	docker build $cmd
 	return 0
 }
 
@@ -112,22 +187,24 @@ args_dist()
 args_toolchain()
 {
 	_name=$(echo $1 | cut -d"-" -f1)
-	_version=$(echo $1 | cut -d"-" -f2)
+	_epoch=$(echo $1 | cut -d"-" -f2)
 
 	case $2 in
 	alpine)
 		toolchain=$_name
 		case $1 in
 		gcc*)
-			args="$args --build-arg extra_packages_dev=gcc"
-			args="$args --build-arg extra_packages_dev1=g++"
+			extra_dev="gcc"
+			extra_dev="${extra_dev} g++"
+			args="$args --build-arg extra_packages_dev=\"${extra_dev}\""
 			args="$args --build-arg cc=gcc --build-arg cxx=g++"
 			return 0
 			;;
 		clang*)
-			args="$args --build-arg extra_packages_dev=clang"
-			args="$args --build-arg extra_packages_dev1=llvm"
-			args="$args --build-arg extra_packages_dev2=llvm-dev"
+			extra_dev="clang"
+			extra_dev="${extra_dev} llvm"
+			extra_dev="${extra_dev} llvm-dev"
+			args="$args --build-arg extra_packages_dev=\"${extra_dev}\""
 			args="$args --build-arg cc=clang --build-arg cxx=clang++"
 			test $3 != "3.16"
 			return $?
@@ -137,16 +214,25 @@ args_toolchain()
 	ubuntu)
 		case $1 in
 		gcc*)
-			args="$args --build-arg extra_packages_dev=gcc-${_version}"
-			args="$args --build-arg extra_packages_dev1=g++-${_version}"
-			args="$args --build-arg cc=gcc-${_version} --build-arg cxx=g++-${_version}"
+			extra_dev="gcc-${_epoch}"
+			extra_dev="${extra_dev} g++-${_epoch}"
+			args="$args --build-arg extra_packages_dev=\"${extra_dev}\""
+			args="$args --build-arg cc=gcc-${_epoch} --build-arg cxx=g++-${_epoch}"
 			return 0
 			;;
 		clang*)
-			args="$args --build-arg extra_packages_dev=clang-${_version}"
-			args="$args --build-arg extra_packages_dev1=llvm-${_version}-dev"
-			args="$args --build-arg extra_packages_dev2=llvm-spirv-${_version}"
-			args="$args --build-arg cc=clang-${_version} --build-arg cxx=clang++-${_version}"
+			if test "$_epoch" -ge 15; then
+				extra="mesa-opencl-icd"
+				extra="${extra} ocl-icd-opencl-dev"
+				extra="${extra} libclc-${_epoch}-dev"
+				args="$args --build-arg extra_packages=\"${extra}\""
+			fi
+			extra_dev="clang-${_epoch}"
+			extra_dev="${extra_dev} llvm-${_epoch}"
+			extra_dev="${extra_dev} llvm-spirv-${_epoch}"
+			extra_dev="${extra_dev} libclc-${_epoch}-dev"
+			args="$args --build-arg extra_packages_dev=\"${extra_dev}\""
+			args="$args --build-arg cc=clang-${_epoch} --build-arg cxx=clang++-${_epoch}"
 			return 0
 			;;
 		esac

docker/ubuntu/base/Dockerfile

@@ -10,6 +10,7 @@ ARG rocksdb_avx 0
 ARG rocksdb_avx2 0
 ARG boost_version 1.74
 ARG icu_version 70
+ARG nprocs 1
 
 ENV rocksdb_version ${rocksdb_version}
 ENV rocksdb_url https://codeload.github.com/facebook/rocksdb/tar.gz/refs/tags/v${rocksdb_version}
@@ -17,6 +18,7 @@ ENV rocksdb_avx ${rocksdb_avx}
 ENV rocksdb_avx2 ${rocksdb_avx2}
 ENV boost_version ${boost_version}
 ENV icu_version ${icu_version}
+ENV nprocs ${nprocs}
 
 ENV packages="\
 ca-certificates \
@@ -71,7 +73,9 @@ ENV rocksdb_cmake="\
 -DWITH_JNI=0 \
 -DWITH_TESTS=0 \
 -DWITH_BENCHMARK_TOOLS=0 \
+-DWITH_TRACE_TOOLS=0 \
 -DWITH_CORE_TOOLS=0 \
+-DWITH_TOOLS=0 \
 -DWITH_GFLAGS=0 \
 -DWITH_LIBURING=0 \
 -DWITH_JEMALLOC=1 \
@@ -99,7 +103,7 @@ ENV do_fetch_rocksdb true \
 && eval ${do_install} curl \
 && cd /usr/src \
 && curl -sL ${rocksdb_url} -o rocksdb-${rocksdb_version}.tar.gz \
-&& tar xfvz rocksdb-${rocksdb_version}.tar.gz \
+&& tar xfz rocksdb-${rocksdb_version}.tar.gz \
 && rm rocksdb-${rocksdb_version}.tar.gz \
 && mv /usr/src/rocksdb-${rocksdb_version} /usr/src/rocksdb \
 && cd - \
@@ -125,7 +129,7 @@ fi \
 CFLAGS="-g0 -ftls-model=initial-exec" \
 LDFLAGS="-Wl,--strip-all" \
 cmake -H. -Bbuild ${rocksdb_cmake} \
-&& cmake --build build --target install --parallel `nproc` \
+&& cmake --build build --target install --parallel ${nprocs} \
 && rm -rf /usr/lib/$(uname -m)-linux-gnu/librocksdb.a \
 && rm -rf /usr/src/rocksdb \
 && eval ${do_purge} ${packages_rocksdb_dev} \

docker/ubuntu/base/Dockerfile.save

@@ -1,130 +0,0 @@
-ARG distribution
-
-FROM --platform=$TARGETPLATFORM $distribution
-
-ARG skiprocks
-ARG rocksdb_version 7.4.5
-ARG rocksdb_url
-ARG rocksdb_portable 1
-ARG rocksdb_avx 0
-ARG rocksdb_avx2 0
-
-ENV rocksdb_version ${rocksdb_version}
-ENV rocksdb_url https://codeload.github.com/facebook/rocksdb/tar.gz/refs/tags/v${rocksdb_version}
-ENV rocksdb_portable ${rocksdb_portable}
-ENV rocksdb_avx ${rocksdb_avx}
-ENV rocksdb_avx2 ${rocksdb_avx2}
-
-ENV packages="\
-ca-certificates \
-libatomic1 \
-libjemalloc2 \
-libboost-chrono1.74.0 \
-libboost-context1.74.0 \
-libboost-coroutine1.74.0 \
-libboost-system1.74.0 \
-libboost-thread1.74.0 \
-libicu70 \
-libnss-db \
-libsodium23 \
-libssl3 \
-liblz4-1 \
-libmagic1 \
-libzstd1 \
-"
-
-ENV packages_rocksdb_dev="\
-build-essential \
-cmake \
-libjemalloc-dev \
-liblz4-dev \
-libzstd-dev \
-xz-utils \
-"
-
-ENV packages_dev="\
-${packages_rocksdb_dev} \
-libboost-chrono1.74-dev \
-libboost-context1.74-dev \
-libboost-coroutine1.74-dev \
-libboost-system1.74-dev \
-libboost-thread1.74-dev \
-libicu-dev \
-libmagic-dev \
-libsodium-dev \
-libssl-dev \
-"
-
-ENV rocksdb_cmake="\
--DCMAKE_RULE_MESSAGES:BOOL=OFF \
--DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \
--DCMAKE_BUILD_TYPE=Release \
--DBUILD_SHARED_LIBS=1 \
--DFAIL_ON_WARNINGS=0 \
--DUSE_RTTI=1 \
--DPORTABLE=${rocksdb_portable} \
--DFORCE_AVX=${rocksdb_avx} \
--DFORCE_AVX2=${rocksdb_avx2} \
--DWITH_JNI=0 \
--DWITH_TESTS=0 \
--DWITH_BENCHMARK_TOOLS=0 \
--DWITH_CORE_TOOLS=0 \
--DWITH_GFLAGS=0 \
--DWITH_LIBURING=0 \
--DWITH_JEMALLOC=1 \
--DWITH_LZ4=1 \
--DWITH_ZSTD=1 \
-"
-
-ENV do_install true \
-&& export DEBIAN_FRONTEND=noninteractive \
-&& apt-get update \
-&& apt-get install --no-install-recommends -y -f
-
-ENV do_purge true \
-&& export DEBIAN_FRONTEND=noninteractive \
-&& apt-get purge -y
-
-ENV do_clean true \
-&& export DEBIAN_FRONTEND=noninteractive \
-&& apt-get clean \
-&& apt-get autoremove --purge -y \
-&& rm -rf /var/lib/apt/lists/* \
-&& true
-
-ENV do_fetch_rocksdb true \
-&& eval ${do_install} curl \
-&& cd /usr/src \
-&& curl -sL ${rocksdb_url} -o rocksdb-${rocksdb_version}.tar.gz \
-&& tar xfvz rocksdb-${rocksdb_version}.tar.gz \
-&& rm rocksdb-${rocksdb_version}.tar.gz \
-&& mv /usr/src/rocksdb-${rocksdb_version} /usr/src/rocksdb \
-&& cd - \
-&& eval ${do_purge} curl \
-&& true
-
-RUN true \
-&& eval ${do_install} ${packages} \
-&& update-ca-certificates \
-&& eval ${do_clean} \
-&& mkdir /build \
-&& true
-
-WORKDIR /build
-RUN true \
-&& if test -n "$skiprocks"; then \
-	exit 0; \
-fi \
-&& eval ${do_install} ${packages_rocksdb_dev} \
-&& eval ${do_fetch_rocksdb} \
-&& cd /usr/src/rocksdb \
-&& \
-CFLAGS="-g0 -ftls-model=initial-exec" \
-LDFLAGS="-Wl,--strip-all" \
-cmake -H. -Bbuild ${rocksdb_cmake} \
-&& cmake --build build --target install --parallel `nproc` \
-&& rm -rf /usr/lib/$(uname -m)-linux-gnu/librocksdb.a \
-&& rm -rf /usr/src/rocksdb \
-&& eval ${do_purge} ${packages_rocksdb_dev} \
-&& eval ${do_clean} \
-&& true

docker/ubuntu/built/Dockerfile

@@ -10,15 +10,15 @@ FROM ${acct}/${repo}:${dist_name}-${dist_version}-${feature}-${machine}
 ARG cc
 ARG cxx
 ARG extra_packages_dev
-ARG extra_packages_dev1
-ARG extra_packages_dev2
 ARG ctor_url https://github.com/matrix-construct/construct
 ARG machine_spec
+ARG nprocs
 
 ENV CC ${cc}
 ENV CXX ${cxx}
 ENV ctor_url ${ctor_url}
 ENV machine_spec ${machine_spec}
+ENV nprocs ${nprocs}
 
 ENV packages_dev="\
 ${packages_dev} \
@@ -31,8 +31,6 @@ git \
 libtool \
 shtool \
 ${extra_packages_dev} \
-${extra_packages_dev1} \
-${extra_packages_dev2} \
 "
 
 RUN true \
@@ -43,8 +41,8 @@ RUN true \
 && rmdir -v deps/rocksdb \
 && ln -sv /usr/src/rocksdb deps \
 && ./autogen.sh \
-&& ./configure --enable-generic --with-machine="${machine_spec}" \
-&& make -j `nproc` \
+&& (./configure --enable-generic --with-machine="${machine_spec}" || (cat config.log; exit 1)) \
+&& make -j ${nprocs} \
 && make install \
 && cd .. \
 && rm -rf construct \

docker/ubuntu/full/Dockerfile

@@ -6,11 +6,12 @@ ARG machine
 
 FROM ${acct}/${repo}:${dist_name}-${dist_version}-base-${machine}
 
+ARG extra_packages
+
 ENV packages="\
 libgraphicsmagick-q16-3 \
 libpng16-16 \
-mesa-opencl-icd \
-ocl-icd-opencl-dev \
+${extra_packages} \
 "
 
 ENV packages_dev="\

include/ircd/buffer/unique_buffer.h

@@ -32,6 +32,7 @@ struct ircd::buffer::unique_buffer
 	unique_buffer() = default;
 	unique_buffer(const size_t &size, const size_t &align = 0);
 	explicit unique_buffer(const const_buffer &);
+	template<class T> unique_buffer(unique_buffer<T> &&) noexcept;
 	unique_buffer(unique_buffer &&) noexcept;
 	unique_buffer(const unique_buffer &) = delete;
 	unique_buffer &operator=(unique_buffer &&) & noexcept;
@@ -71,6 +72,19 @@ ircd::buffer::unique_buffer<buffer_type>::unique_buffer(const size_t &size,
 }
 {}
 
+template<class buffer_type>
+template<class other_type>
+inline
+ircd::buffer::unique_buffer<buffer_type>::unique_buffer(unique_buffer<other_type> &&other)
+noexcept
+:buffer_type
+{
+	other.release()
+}
+{
+	assert(std::get<0>(other) == nullptr);
+}
+
 template<class buffer_type>
 inline
 ircd::buffer::unique_buffer<buffer_type>::unique_buffer(unique_buffer &&other)

include/ircd/db/domain.h

@@ -102,3 +102,11 @@ const
 {
 	return &this->operator*();
 }
+
+inline bool
+ircd::db::seek(domain::const_iterator_base &it,
+               const string_view &p)
+{
+	it.opts.prefix = true;
+	return seek(static_cast<column::const_iterator_base &>(it), p);
+}

include/ircd/ircd.h

@@ -107,6 +107,7 @@
 #include "mods/mods.h"
 #include "net/net.h"
 #include "server/server.h"
+#include "rest.h"
 #include "png.h"
 #include "beep.h"
 #include "magick.h"

include/ircd/rest.h

@@ -0,0 +1,197 @@
+// The Construct
+//
+// Copyright (C) The Construct Developers, Authors & Contributors
+// Copyright (C) 2016-2023 Jason Volk <jason@zemos.net>
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice is present in all copies. The
+// full license for this software is available in the LICENSE file.
+
+#pragma once
+#define HAVE_IRCD_REST_H
+
+/// Simple highlevel interface for web/http requests.
+///
+/// Prior to this it was too difficult to orchestrate all the objects and
+/// buffers and low-level non-ergonomic procedures split between ircd::http
+/// and ircd::server. This should instead have some familiarity to the
+/// browser-js environment which developers can easily commit to their memory.
+namespace ircd::rest
+{
+	struct opts;
+	struct request;
+
+	struct get;
+	struct put;
+	struct post;
+}
+
+struct ircd::rest::request
+:returns<string_view>
+{
+	unique_const_buffer out;
+
+	request(const mutable_buffer &out, const rfc3986::uri &, opts);
+	request(const rfc3986::uri &, opts);
+};
+
+struct ircd::rest::get
+:request
+{
+	get(const mutable_buffer &out, const rfc3986::uri &, opts);
+	get(const rfc3986::uri &, opts);
+};
+
+struct ircd::rest::put
+:request
+{
+	put(const mutable_buffer &out, const rfc3986::uri &, const string_view &content, opts);
+	put(const rfc3986::uri &, const string_view &content, opts);
+};
+
+struct ircd::rest::post
+:request
+{
+	post(const mutable_buffer &out, const rfc3986::uri &, const string_view &content, opts);
+	post(const mutable_buffer &out, const rfc3986::uri &, opts);
+	post(const rfc3986::uri &, const string_view &content, opts);
+	post(const rfc3986::uri &, opts);
+};
+
+struct ircd::rest::opts
+{
+	/// The HTTP method to use. This is overridden and should not be set unless
+	/// using the generic rest::request() call where it must be set.
+	string_view method;
+
+	/// The HTTP request body. This is overridden and should not be set unless
+	/// using the generic rest::request() call where it's set as needed.
+	string_view content;
+
+	/// The HTTP request body content-type. It is a good idea to set this when
+	/// there is request body content.
+	string_view content_type;
+
+	/// Additional request headers to send. These are pairs of string_views.
+	vector_view<const http::header> headers;
+
+	/// This is set automatically from the URI argument's domain and scheme
+	/// (service) by default. Setting it here will override.
+	net::hostport remote;
+
+	/// Managed internally by default and passed to server::request. Setting
+	/// things here will override.
+	server::out sout;
+
+	/// Managed internally by default and passed to server::request. Setting
+	/// things here will override.
+	server::in sin;
+
+	/// Passed to server::request. The http_exceptions option is useful here
+	/// to prevent this suite from throwing on non-2xx codes.
+	server::request::opts sopts;
+
+	/// Allows the HTTP response code to be returned to the caller. This may
+	/// not be initialized if the call throws any error first.
+	http::code *code {nullptr};
+
+	/// Allows the user to override the request::out with their own for
+	/// receiving dynamic content. Supply an empty unique_buffer instance.
+	unique_const_buffer *out {nullptr};
+
+	/// Timeout for the yielding/synchronous calls of this interface.
+	seconds timeout {20s};
+
+	/// Internal use
+	opts &&set(const string_view &method, const string_view &content = {});
+};
+
+inline
+ircd::rest::post::post(const rfc3986::uri &uri,
+                       opts opts)
+:request
+{
+	uri, opts.set("POST")
+}
+{}
+
+inline
+ircd::rest::post::post(const rfc3986::uri &uri,
+                       const string_view &content,
+                       opts opts)
+:request
+{
+	uri, opts.set("POST", content)
+}
+{}
+
+inline
+ircd::rest::post::post(const mutable_buffer &out,
+                       const rfc3986::uri &uri,
+                       opts opts)
+:request
+{
+	out, uri, opts.set("POST")
+}
+{}
+
+inline
+ircd::rest::post::post(const mutable_buffer &out,
+                       const rfc3986::uri &uri,
+                       const string_view &content,
+                       opts opts)
+:request
+{
+	out, uri, opts.set("POST", content)
+}
+{}
+
+inline
+ircd::rest::put::put(const rfc3986::uri &uri,
+                     const string_view &content,
+                     opts opts)
+:request
+{
+	uri, opts.set("PUT", content)
+}
+{}
+
+inline
+ircd::rest::put::put(const mutable_buffer &out,
+                     const rfc3986::uri &uri,
+                     const string_view &content,
+                     opts opts)
+:request
+{
+	out, uri, opts.set("PUT", content)
+}
+{}
+
+inline
+ircd::rest::get::get(const rfc3986::uri &uri,
+                     opts opts)
+:request
+{
+	uri, opts.set("GET")
+}
+{}
+
+inline
+ircd::rest::get::get(const mutable_buffer &out,
+                     const rfc3986::uri &uri,
+                     opts opts)
+:request
+{
+	out, uri, opts.set("GET")
+}
+{}
+
+inline ircd::rest::opts &&
+ircd::rest::opts::set(const string_view &method,
+                      const string_view &content)
+{
+	this->method = method;
+	this->content = content?: this->content;
+	return std::move(*this);
+}

include/ircd/rfc3986.h

@@ -70,6 +70,7 @@ struct ircd::rfc3986::uri
 	string_view path;
 	string_view query;
 	string_view fragment;
+	string_view resource() const; // path and query string as one
 
 	uri(const string_view &);
 	uri() = default;
@@ -179,3 +180,12 @@ namespace ircd::rfc3986
 	void valid_remote(const string_view &);
 	bool valid_remote(std::nothrow_t, const string_view &);
 }
+
+inline ircd::string_view
+ircd::rfc3986::uri::resource()
+const
+{
+	return query?
+		string_view(begin(path), end(query)):
+		path;
+}

include/ircd/simd/unaligned.h

@@ -24,7 +24,7 @@ namespace ircd::simd
 /// T = inner aligned type
 template<class T>
 struct
-[[using clang: internal_linkage, nodebug]]
+[[using clang: internal_linkage]]
 [[using gnu: packed, aligned(1), visibility("internal")]]
 ircd::simd::unaligned
 {

include/ircd/util/bitset.h

@@ -107,6 +107,7 @@ constexpr void
 ircd::util::bitset<N>::set(const size_t pos,
                            const bool val)
 {
+	reset(pos);
 	buf[byte(pos)] |= word(val) << bit(pos);
 }
 

ircd/Makefile.am

@@ -237,6 +237,7 @@ endif
 libircd_la_SOURCES += server.cc
 libircd_la_SOURCES += client.cc
 libircd_la_SOURCES += resource.cc
+libircd_la_SOURCES += rest.cc
 if JS
 libircd_la_SOURCES += js.cc
 endif
@@ -435,6 +436,8 @@ gpt_gpu.spv.cc: gpt_gpu.spv
 # GCN-HSA
 #
 
+if AMD64
+
 GCN_HSA_TARGET = amdgcn--amdhsa
 
 GCN_HSA_CPPFLAGS = $(GPU_CPPFLAGS)
@@ -469,10 +472,14 @@ libircd_la_SOURCES += gpt_gpu.gcn_hsa.bc.cc
 gpt_gpu.gcn_hsa.bc.cc: gpt_gpu.gcn_hsa.bc
 	xxd -i $^ $@
 
+endif # AMD64
+
 #
 # R600
 #
 
+if AMD64
+
 R600_TARGET = r600--
 
 #
@@ -510,6 +517,8 @@ libircd_la_SOURCES += gpt_gpu.r600_barts.bc.cc
 gpt_gpu.r600_barts.bc.cc: gpt_gpu.r600_barts.bc
 	xxd -i $^ $@
 
+endif # AMD64
+
 #
 #
 #

ircd/cl.cc

@@ -355,6 +355,7 @@ ircd::cl::init::fini_libs()
 
 size_t
 ircd::cl::init::init_platforms()
+try
 {
 	// OpenCL sez platform=null is implementation defined.
 	constexpr auto ignore(CL_INVALID_PLATFORM);
@@ -365,9 +366,21 @@ ircd::cl::init::init_platforms()
 
 	return platforms;
 }
+catch(const std::exception &e)
+{
+	log::logf
+	{
+		log, log::level::DERROR,
+		"OpenCL platforms initialization :%s",
+		e.what(),
+	};
+
+	return 0;
+}
 
 size_t
 ircd::cl::init::init_devices()
+try
 {
 	// Get the devices.
 	size_t devices_total(0);
@@ -414,6 +427,16 @@ ircd::cl::init::init_devices()
 
 	return devices_total;
 }
+catch(const std::exception &e)
+{
+	log::error
+	{
+		log, "OpenCL devices initialization :%s",
+		e.what(),
+	};
+
+	return 0;
+}
 
 size_t
 ircd::cl::init::init_ctxs()

ircd/db.cc

@@ -2377,14 +2377,6 @@ ircd::db::seek(domain::const_iterator_base &it,
 	return seek(static_cast<column::const_iterator_base &>(it), p);
 }
 
-bool
-ircd::db::seek(domain::const_iterator_base &it,
-               const string_view &p)
-{
-	it.opts.prefix = true;
-	return seek(static_cast<column::const_iterator_base &>(it), p);
-}
-
 ircd::db::domain::const_iterator
 ircd::db::domain::begin(const string_view &key,
                         gopts opts)

ircd/http.cc

@@ -151,7 +151,7 @@ namespace ircd::http::parser
 	const rule<string_view> reason     { str                                             ,"reason" };
 
 	const rule<string_view> head_key   { raw[+(char_ - (illegal | ws | colon))]        ,"head key" };
-	const rule<string_view> head_val   { str                                         ,"head value" };
+	const rule<string_view> head_val   { -str                                        ,"head value" };
 	const rule<http::header> header    { head_key >> *ws >> colon >> *ws >> head_val     ,"header" };
 	const rule<> headers               { header % (*ws >> CRLF)                         ,"headers" };
 

ircd/rest.cc

@@ -0,0 +1,138 @@
+// The Construct
+//
+// Copyright (C) The Construct Developers, Authors & Contributors
+// Copyright (C) 2016-2023 Jason Volk <jason@zemos.net>
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice is present in all copies. The
+// full license for this software is available in the LICENSE file.
+
+ircd::rest::request::request(const rfc3986::uri &uri,
+                             opts opts)
+{
+	if(!opts.remote)
+		opts.remote = net::hostport{uri};
+
+	const unique_mutable_buffer buf
+	{
+		empty(opts.sout.head) || empty(opts.sin.head)?
+			16_KiB: 0_KiB
+	};
+
+	window_buffer window{buf};
+	if(empty(opts.sout.head))
+	{
+		assert(opts.remote);
+		assert(opts.method);
+		http::request
+		{
+			window,
+			host(opts.remote),
+			opts.method,
+			uri.resource(),
+			size(opts.content),
+			opts.content_type,
+			opts.headers
+		};
+
+		opts.sout.head = window.completed();
+	}
+
+	if(empty(opts.sout.content))
+		opts.sout.content = opts.content;
+
+	if(empty(opts.sin.head))
+		opts.sin.head = mutable_buffer{window};
+
+	// server::request will allocate dynamic content
+	opts.sin.content = mutable_buffer{};
+
+	assert(opts.remote);
+	server::request request
+	{
+		opts.remote,
+		std::move(opts.sout),
+		std::move(opts.sin),
+		&opts.sopts,
+	};
+
+	const auto code
+	{
+		request.get(opts.timeout)
+	};
+
+	if(opts.code)
+		*opts.code = code;
+
+	if(opts.out)
+		*opts.out = std::move(request.in.dynamic);
+	else
+		this->out = std::move(request.in.dynamic);
+
+	ret = request.in.content;
+}
+
+ircd::rest::request::request(const mutable_buffer &out,
+                             const rfc3986::uri &uri,
+                             opts opts)
+{
+	if(!opts.remote)
+		opts.remote = net::hostport{uri};
+
+	const unique_mutable_buffer buf
+	{
+		empty(opts.sout.head) || empty(opts.sin.head)?
+			16_KiB: 0_KiB
+	};
+
+	window_buffer window{buf};
+	if(empty(opts.sout.head))
+	{
+		assert(opts.remote);
+		assert(opts.method);
+		http::request
+		{
+			window,
+			host(opts.remote),
+			opts.method,
+			uri.resource(),
+			size(opts.content),
+			opts.content_type,
+			opts.headers
+		};
+
+		opts.sout.head = window.completed();
+	}
+
+	if(empty(opts.sout.content))
+		opts.sout.content = opts.content;
+
+	if(empty(opts.sin.head))
+		opts.sin.head = mutable_buffer{window};
+
+	if(empty(opts.sin.content))
+		opts.sin.content = out;
+
+	if(empty(opts.sin.content))
+		opts.sin.content = opts.sin.head;
+
+	assert(opts.remote);
+	server::request request
+	{
+		opts.remote,
+		std::move(opts.sout),
+		std::move(opts.sin),
+		&opts.sopts,
+	};
+
+	const auto code
+	{
+		request.get(opts.timeout)
+	};
+
+	if(opts.code)
+		*opts.code = code;
+
+	ret = request.in.content;
+}

matrix/Makefile.am

@@ -41,7 +41,7 @@ endif
 endif
 
 if LTO
-if CLANG
+if CLANG15 # trouble with ld.gold <= llvm-14
 AM_CXXFLAGS += -flto=thin
 AM_CXXFLAGS += -fstrict-vtable-pointers
 AM_CXXFLAGS += -fwhole-program-vtables

modules/console.cc

@@ -4596,7 +4596,7 @@ static void
 _print_sst_info_header(opt &out)
 {
 	out << std::left << std::setfill(' ')
-	    << std::setw(3) << "chkp"
+	    << std::setw(3) << "cp"
 	    << "  " << std::setw(12) << "name"
 	    << "  " << std::setw(32) << "creation"
 	    << "  " << std::setw(3) << "flt"
@@ -18067,6 +18067,96 @@ console_cmd__redact(opt &out, const string_view &line)
 	return true;
 }
 
+bool
+console_cmd__redact__last(opt &out, const string_view &line)
+{
+	const params param{line, " ",
+	{
+		"room_id", "sender", "redactor", "count", "type", "reason"
+	}};
+
+	const m::room::id::buf room_id
+	{
+		m::room_id(param["room_id"])
+	};
+
+	const m::room room
+	{
+		room_id
+	};
+
+	const m::user::id sender
+	{
+		param["sender"]
+	};
+
+	const m::user::id redactor
+	{
+		param["redactor"]
+	};
+
+	const auto type
+	{
+		param["type"]
+	};
+
+	auto count
+	{
+		param.at<uint>("count", 1)
+	};
+
+	m::room::iterate events
+	{
+		room, "sender"
+	};
+
+	size_t limit(10000);
+	events.for_each([&]
+	(const auto &_sender, const auto &depth, const auto &event_idx)
+	{
+		if(--limit <= 0)
+			return false;
+
+		if(_sender != sender)
+			return true;
+
+		const auto event_id
+		{
+			m::event_id(std::nothrow, event_idx)
+		};
+
+		if(unlikely(!event_id))
+			return true;
+
+		const auto type_equal{[&type]
+		(const auto &t)
+		{
+			return type == t;
+		}};
+
+		if(type && !m::query(event_idx, "type", type_equal))
+			return true;
+
+		const auto redact_id
+		{
+			m::redact(room, redactor, event_id, param["reason"])
+		};
+
+		out
+		<< std::setw(3) << std::right << count
+		<< ' '
+		<< redact_id
+		<< " redacted "
+		<< event_id
+		;
+
+		return --count > 0;
+
+	});
+
+	return true;
+}
+
 //
 // well-known
 //