MirrorHub/nixpkgs
0
0
Fork 0
mirror of https://github.com/NixOS/nixpkgs.git synced 2024-11-15 14:26:33 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions
nixpkgs/maintainers/scripts/bootstrap-files/refresh-tarballs.bash

313 lines
10 KiB
Bash
Executable file
Raw Blame History

#!/usr/bin/env nix-shell
#! nix-shell --pure
#! nix-shell -i bash
#! nix-shell -p curl cacert
#! nix-shell -p git
#! nix-shell -p nix
#! nix-shell -p jq
set -o pipefail
# How the refresher works:
#
# For a given list of <targets>:
# 1. fetch latest successful '.build` job
# 2. fetch oldest evaluation that contained that '.build', extract nixpkgs commit
# 3. fetch all the `.build` artifacts from '$out/on-server/' directory
# 4. calculate hashes and craft the commit message with the details on
# how to upload the result to 'tarballs.nixos.org'
scratch_dir=$(mktemp -d)
trap 'rm -rf -- "${scratch_dir}"' EXIT
usage() {
cat >&2 <<EOF
Usage:
$0 [ --commit ] --targets=<target>[,<target>,...]
The tool must be ran from the root directory of 'nixpkgs' repository.
Synopsis:
'refresh-tarballs.bash' script fetches latest bootstrapFiles built
by hydra, registers them in 'nixpkgs' and provides commands to
upload seed files to 'tarballs.nixos.org'.
This is usually done in the following cases:
1. Single target fix: current bootstrap files for a single target
are problematic for some reason (target-specific bug). In this
case we can refresh just that target as:
\$ $0 --commit --targets=i686-unknown-linux-gnu
2. Routine refresh: all bootstrap files should be refreshed to avoid
debugging problems that only occur on very old binaries.
\$ $0 --commit --all-targets
To get help on uploading refreshed binaries to 'tarballs.nixos.org'
please have a look at <maintainers/scripts/bootstrap-files/README.md>.
EOF
exit 1
}
# log helpers
die() {
echo "ERROR: $*" >&2
exit 1
}
info() {
echo "INFO: $*" >&2
}
[[ ${#@} -eq 0 ]] && usage
# known targets
NATIVE_TARGETS=(
aarch64-unknown-linux-gnu
aarch64-unknown-linux-musl
i686-unknown-linux-gnu
x86_64-unknown-linux-gnu
x86_64-unknown-linux-musl
aarch64-apple-darwin
x86_64-apple-darwin
)
is_native() {
local t target=$1
for t in "${NATIVE_TARGETS[@]}"; do
[[ $t == $target ]] && return 0
done
return 1
}
CROSS_TARGETS=(
armv5tel-unknown-linux-gnueabi
armv6l-unknown-linux-gnueabihf
armv6l-unknown-linux-musleabihf
armv7l-unknown-linux-gnueabihf
mips64el-unknown-linux-gnuabi64
mips64el-unknown-linux-gnuabin32
mipsel-unknown-linux-gnu
powerpc64-unknown-linux-gnuabielfv2
powerpc64le-unknown-linux-gnu
riscv64-unknown-linux-gnu
s390x-unknown-linux-gnu
x86_64-unknown-freebsd
)
is_cross() {
local t target=$1
for t in "${CROSS_TARGETS[@]}"; do
[[ $t == $target ]] && return 0
done
return 1
}
nar_sri_get() {
local restore_path store_path
((${#@} != 2)) && die "nar_sri_get /path/to/name.nar.xz name"
restore_path="${scratch_dir}/$2"
xz -d < "$1" | nix-store --restore "${restore_path}"
[[ $? -ne 0 ]] && die "Failed to unpack '$1'"
store_path=$(nix-store --add "${restore_path}")
[[ $? -ne 0 ]] && die "Failed to add '$restore_path' to store"
rm -rf -- "${restore_path}"
nix-hash --to-sri "$(nix-store --query --hash "${store_path}")"
}
# collect passed options
targets=()
commit=no
for arg in "$@"; do
case "$arg" in
--all-targets)
targets+=(
${CROSS_TARGETS[@]}
${NATIVE_TARGETS[@]}
)
;;
--targets=*)
# Convert "--targets=a,b,c" to targets=(a b c) bash array.
comma_targets=${arg#--targets=}
targets+=(${comma_targets//,/ })
;;
--commit)
commit=yes
;;
*)
usage
;;
esac
done
for target in "${targets[@]}"; do
# Native and cross jobsets differ a bit. We'll have to pick the
# one based on target name:
if is_native $target; then
jobset=nixpkgs/trunk
job="stdenvBootstrapTools.${target}.build"
elif is_cross $target; then
jobset=nixpkgs/cross-trunk
job="bootstrapTools.${target}.build"
else
die "'$target' is not present in either of 'NATIVE_TARGETS' or 'CROSS_TARGETS'. Please add one."
fi
# 'nixpkgs' prefix where we will write new tarball hashes
case "$target" in
*linux*) nixpkgs_prefix="pkgs/stdenv/linux" ;;
*darwin*) nixpkgs_prefix="pkgs/stdenv/darwin" ;;
*freebsd*) nixpkgs_prefix="pkgs/stdenv/freebsd" ;;
*) die "don't know where to put '$target'" ;;
esac
# We enforce s3 prefix for all targets here. This slightly differs
# from manual uploads targets where names were chosen inconsistently.
s3_prefix="stdenv/$target"
# resolve 'latest' build to the build 'id', construct the link.
latest_build_uri="https://hydra.nixos.org/job/$jobset/$job/latest"
latest_build="$target.latest-build"
info "Fetching latest successful build from '${latest_build_uri}'"
curl -s -H "Content-Type: application/json" -L "$latest_build_uri" > "$latest_build"
[[ $? -ne 0 ]] && die "Failed to fetch latest successful build"
latest_build_id=$(jq '.id' < "$latest_build")
[[ $? -ne 0 ]] && die "Did not find 'id' in latest build"
build_uri="https://hydra.nixos.org/build/${latest_build_id}"
# We pick oldest jobset evaluation and extract the 'nicpkgs' commit.
#
# We use oldest instead of latest to make the result more stable
# across unrelated 'nixpkgs' updates. Ideally two subsequent runs of
# this refresher should produce the same output (provided there are
# no bootstrapTools updates committed between the two runs).
oldest_eval_id=$(jq '.jobsetevals|min' < "$latest_build")
[[ $? -ne 0 ]] && die "Did not find 'jobsetevals' in latest build"
eval_uri="https://hydra.nixos.org/eval/${oldest_eval_id}"
eval_meta="$target.eval-meta"
info "Fetching oldest eval details from '${eval_uri}' (can take a minute)"
curl -s -H "Content-Type: application/json" -L "${eval_uri}" > "$eval_meta"
[[ $? -ne 0 ]] && die "Failed to fetch eval metadata"
nixpkgs_revision=$(jq --raw-output ".jobsetevalinputs.nixpkgs.revision" < "$eval_meta")
[[ $? -ne 0 ]] && die "Failed to fetch revision"
# Extract the build paths out of the build metadata
drvpath=$(jq --raw-output '.drvpath' < "${latest_build}")
[[ $? -ne 0 ]] && die "Did not find 'drvpath' in latest build"
outpath=$(jq --raw-output '.buildoutputs.out.path' < "${latest_build}")
[[ $? -ne 0 ]] && die "Did not find 'buildoutputs' in latest build"
build_timestamp=$(jq --raw-output '.timestamp' < "${latest_build}")
[[ $? -ne 0 ]] && die "Did not find 'timestamp' in latest build"
build_time=$(TZ=UTC LANG=C date --date="@${build_timestamp}" --rfc-email)
[[ $? -ne 0 ]] && die "Failed to format timestamp"
info "Fetching bootstrap tools to calculate hashes from '${outpath}'"
nix-store --realize "$outpath"
[[ $? -ne 0 ]] && die "Failed to fetch '${outpath}' from hydra"
fnames=()
target_file="${nixpkgs_prefix}/bootstrap-files/${target}.nix"
info "Writing '${target_file}'"
{
# header
cat <<EOF
# Autogenerated by maintainers/scripts/bootstrap-files/refresh-tarballs.bash as:
# $ ./refresh-tarballs.bash --targets=${target}
#
# Metadata:
# - nixpkgs revision: ${nixpkgs_revision}
# - hydra build: ${latest_build_uri}
# - resolved hydra build: ${build_uri}
# - instantiated derivation: ${drvpath}
# - output directory: ${outpath}
# - build time: ${build_time}
{
EOF
for p in "${outpath}/on-server"/*; do
fname=$(basename "$p")
fnames+=("$fname")
case "$fname" in
bootstrap-tools.tar.xz) attr=bootstrapTools ;;
busybox) attr=$fname ;;
unpack.nar.xz) attr=unpack ;;
*) die "Don't know how to map '$fname' to attribute name. Please update me."
esac
executable_arg=
executable_nix=
if [[ -x "$p" ]]; then
executable_arg="--executable"
executable_nix="executable = true;"
fi
unpack_nix=
name_nix=
if [[ $fname = *.nar.xz ]]; then
unpack_nix="unpack = true;"
name_nix="name = \"${fname%.nar.xz}\";"
sri=$(nar_sri_get "$p" "${fname%.nar.xz}")
[[ $? -ne 0 ]] && die "Failed to get hash of '$p'"
else
sha256=$(nix-prefetch-url $executable_arg --name "$fname" "file://$p")
[[ $? -ne 0 ]] && die "Failed to get the hash for '$p'"
sri=$(nix-hash --to-sri "sha256:$sha256")
[[ $? -ne 0 ]] && die "Failed to convert '$sha256' hash to an SRI form"
fi
# individual file entries
cat <<EOF
$attr = import <nix/fetchurl.nix> {
url = "http://tarballs.nixos.org/${s3_prefix}/${nixpkgs_revision}/$fname";
hash = "${sri}";$(
[[ -n ${executable_nix} ]] && printf "\n %s" "${executable_nix}"
[[ -n ${name_nix} ]] && printf "\n %s" "${name_nix}"
[[ -n ${unpack_nix} ]] && printf "\n %s" "${unpack_nix}"
)
};
EOF
done
# footer
cat <<EOF
}
EOF
} > "${target_file}"
target_file_commit_msg=${target}.commit_message
cat > "$target_file_commit_msg" <<EOF
${nixpkgs_prefix}: update ${target} bootstrap-files
sha256sum of files to be uploaded:
$(
echo "$ sha256sum ${outpath}/on-server/*"
sha256sum ${outpath}/on-server/*
)
Suggested commands to upload files to 'tarballs.nixos.org':
$ nix-store --realize ${outpath}
$ aws s3 cp --recursive --acl public-read ${outpath}/on-server/ s3://nixpkgs-tarballs/${s3_prefix}/${nixpkgs_revision}
$ aws s3 cp --recursive s3://nixpkgs-tarballs/${s3_prefix}/${nixpkgs_revision} ./
$ sha256sum ${fnames[*]}
$ sha256sum ${outpath}/on-server/*
EOF
cat "$target_file_commit_msg"
if [[ $commit == yes ]]; then
git commit "${target_file}" -F "$target_file_commit_msg"
else
info "DRY RUN: git commit ${target_file} -F $target_file_commit_msg"
fi
rm -- "$target_file_commit_msg"
# delete temp files
rm -- "$latest_build" "$eval_meta"
done
Reference in a new issue View git blame Copy permalink