Merge pull request #75563 from andir/cleanup-buildRustCrate

Cleanup buildRustCrate expression
This commit is contained in:
Andreas Rammhold 2020-01-02 13:42:33 +01:00 committed by GitHub
commit 9f03cb8562
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 261 additions and 210 deletions

View file

@ -1,151 +1,76 @@
{ lib, stdenv, echo_build_heading, noisily, makeDeps, rust }:
{ lib, stdenv, echo_build_heading, noisily, mkRustcDepArgs, rust }:
{ crateName,
dependencies,
crateFeatures, crateRenames, libName, release, libPath,
crateType, metadata, crateBin, hasCrateBin,
extraRustcOpts, verbose, colors }:
extraRustcOpts, verbose, colors,
}:
let
deps = makeDeps dependencies crateRenames;
rustcOpts =
lib.lists.foldl' (opts: opt: opts + " " + opt)
(if release then "-C opt-level=3" else "-C debuginfo=2")
(["-C codegen-units=$NIX_BUILD_CORES"] ++ extraRustcOpts);
baseRustcOpts =
[(if release then "-C opt-level=3" else "-C debuginfo=2")]
++ ["-C codegen-units=$NIX_BUILD_CORES"]
++ [(mkRustcDepArgs dependencies crateRenames)]
++ [crateFeatures]
++ extraRustcOpts
++ lib.optional (stdenv.hostPlatform != stdenv.buildPlatform) "--target ${rust.toRustTarget stdenv.hostPlatform} -C linker=${stdenv.hostPlatform.config}-gcc"
;
rustcMeta = "-C metadata=${metadata} -C extra-filename=-${metadata}";
# build the final rustc arguments that can be different between different
# crates
libRustcOpts = lib.concatStringsSep " " (
baseRustcOpts
++ [rustcMeta]
++ (map (x: "--crate-type ${x}") crateType)
);
binRustcOpts = lib.concatStringsSep " " (
baseRustcOpts
);
in ''
runHook preBuild
norm=""
bold=""
green=""
boldgreen=""
if [[ "${colors}" == "always" ]]; then
norm="$(printf '\033[0m')" #returns to "normal"
bold="$(printf '\033[0;1m')" #set bold
green="$(printf '\033[0;32m')" #set green
boldgreen="$(printf '\033[0;1;32m')" #set bold, and set green.
fi
${echo_build_heading colors}
${noisily colors verbose}
build_lib() {
lib_src=$1
echo_build_heading $lib_src ${libName}
# configure & source common build functions
LIB_RUSTC_OPTS="${libRustcOpts}"
BIN_RUSTC_OPTS="${binRustcOpts}"
LIB_EXT="${stdenv.hostPlatform.extensions.sharedLibrary}"
LIB_PATH="${libPath}"
LIB_NAME="${libName}"
source ${./lib.sh}
noisily rustc --crate-name $CRATE_NAME $lib_src \
${lib.strings.concatStrings (map (x: " --crate-type ${x}") crateType)} \
${rustcOpts} ${rustcMeta} ${crateFeatures} --out-dir target/lib \
--emit=dep-info,link -L dependency=target/deps ${deps} --cap-lints allow \
$BUILD_OUT_DIR $EXTRA_BUILD $EXTRA_FEATURES --color ${colors}
CRATE_NAME='${lib.replaceStrings ["-"] ["_"] libName}'
EXTRA_LIB=" --extern $CRATE_NAME=target/lib/lib$CRATE_NAME-${metadata}.rlib"
if [ -e target/deps/lib$CRATE_NAME-${metadata}${stdenv.hostPlatform.extensions.sharedLibrary} ]; then
EXTRA_LIB="$EXTRA_LIB --extern $CRATE_NAME=target/lib/lib$CRATE_NAME-${metadata}${stdenv.hostPlatform.extensions.sharedLibrary}"
fi
}
setup_link_paths
build_bin() {
crate_name=$1
crate_name_=$(echo $crate_name | sed -e "s/-/_/g")
main_file=""
if [[ ! -z $2 ]]; then
main_file=$2
fi
echo_build_heading $@
noisily rustc --crate-name $crate_name_ $main_file --crate-type bin ${rustcOpts}\
${crateFeatures} --out-dir target/bin --emit=dep-info,link -L dependency=target/deps \
$LINK ${deps}$EXTRA_LIB --cap-lints allow \
$BUILD_OUT_DIR $EXTRA_BUILD $EXTRA_FEATURES --color ${colors} \
${if stdenv.hostPlatform != stdenv.buildPlatform then "--target ${rust.toRustTarget stdenv.hostPlatform} -C linker=${stdenv.hostPlatform.config}-gcc" else ""}
if [ "$crate_name_" != "$crate_name" ]; then
mv target/bin/$crate_name_ target/bin/$crate_name
fi
}
EXTRA_LIB=""
CRATE_NAME=$(echo ${libName} | sed -e "s/-/_/g")
if [[ -e target/link_ ]]; then
EXTRA_BUILD="$(cat target/link_) $EXTRA_BUILD"
fi
if [[ -e "${libPath}" ]]; then
build_lib ${libPath}
if [[ -e "$LIB_PATH" ]]; then
build_lib $LIB_PATH
elif [[ -e src/lib.rs ]]; then
build_lib src/lib.rs
elif [[ -e src/${libName}.rs ]]; then
build_lib src/${libName}.rs
elif [[ -e "src/$LIB_NAME.rs" ]]; then
build_lib src/$LIB_NAME.rs
fi
echo "$EXTRA_LINK_SEARCH" | while read i; do
if [[ ! -z "$i" ]]; then
for library in $i; do
echo "-L $library" >> target/link
L=$(echo $library | sed -e "s#$(pwd)/target/build#$lib/lib#")
echo "-L $L" >> target/link.final
done
fi
done
echo "$EXTRA_LINK" | while read i; do
if [[ ! -z "$i" ]]; then
for library in $i; do
echo "-l $library" >> target/link
echo "-l $library" >> target/link.final
done
fi
done
if [[ -e target/link ]]; then
sort -u target/link.final > target/link.final.sorted
mv target/link.final.sorted target/link.final
sort -u target/link > target/link.sorted
mv target/link.sorted target/link
tr '\n' ' ' < target/link > target/link_
LINK=$(cat target/link_)
fi
${lib.optionalString (crateBin != "") ''
printf "%s\n" "${crateBin}" | head -n1 | tr -s ',' '\n' | while read -r BIN_NAME BIN_PATH; do
${lib.optionalString (lib.length crateBin > 0) (lib.concatMapStringsSep "\n" (bin: ''
mkdir -p target/bin
# filter empty entries / empty "lines"
if [[ -z "$BIN_NAME" ]]; then
continue
fi
if [[ -z "$BIN_PATH" ]]; then
# heuristic to "guess" the correct source file as found in cargo:
# https://github.com/rust-lang/cargo/blob/90fc9f620190d5fa3c80b0c8c65a1e1361e6b8ae/src/cargo/util/toml/targets.rs#L308-L325
# the first two cases are the "new" default IIRC
BIN_NAME_=$(echo $BIN_NAME | sed -e 's/-/_/g')
FILES=( "src/bin/$BIN_NAME.rs" "src/bin/$BIN_NAME/main.rs" "src/bin/$BIN_NAME_.rs" "src/bin/$BIN_NAME_/main.rs" "src/bin/main.rs" "src/main.rs" )
if ! [ -e "${libPath}" -o -e src/lib.rs -o -e "src/${libName}.rs" ]; then
# if this is not a library the following path is also valid
FILES=( "src/$BIN_NAME.rs" "src/$BIN_NAME_.rs" "''${FILES[@]}" )
fi
for file in "''${FILES[@]}";
do
echo "checking file $file"
# first file that exists wins
if [[ -e "$file" ]]; then
BIN_PATH="$file"
break
fi
done
if [[ -z "$BIN_PATH" ]]; then
echo "failed to find file for binary target: $BIN_NAME" >&2
exit 1
fi
fi
BIN_NAME='${bin.name or crateName}'
${if !bin ? path then ''
BIN_PATH=""
search_for_bin_path "$BIN_NAME"
'' else ''
BIN_PATH='${bin.path}'
''}
build_bin "$BIN_NAME" "$BIN_PATH"
done
''}
'') crateBin)}
${lib.optionalString (crateBin == "" && !hasCrateBin) ''
# If crateBin is empty and hasCrateBin is not set then we must try to
# detect some kind of bin target based on some files that might exist.
${lib.optionalString (lib.length crateBin == 0 && !hasCrateBin) ''
if [[ -e src/main.rs ]]; then
mkdir -p target/bin
build_bin ${crateName} src/main.rs

View file

@ -1,4 +1,4 @@
{ lib, stdenv, echo_build_heading, noisily, makeDeps }:
{ lib, stdenv, echo_build_heading, noisily, mkRustcDepArgs }:
{ build
, buildDependencies
, colors
@ -20,12 +20,12 @@
, verbose
, workspace_member }:
let version_ = lib.splitString "-" crateVersion;
versionPre = if lib.tail version_ == [] then "" else builtins.elemAt version_ 1;
versionPre = if lib.tail version_ == [] then "" else lib.elemAt version_ 1;
version = lib.splitVersion (lib.head version_);
rustcOpts = lib.lists.foldl' (opts: opt: opts + " " + opt)
rustcOpts = lib.foldl' (opts: opt: opts + " " + opt)
(if release then "-C opt-level=3" else "-C debuginfo=2")
(["-C codegen-units=$NIX_BUILD_CORES"] ++ extraRustcOpts);
buildDeps = makeDeps buildDependencies crateRenames;
buildDeps = mkRustcDepArgs buildDependencies crateRenames;
authors = lib.concatStringsSep ":" crateAuthors;
optLevel = if release then 3 else 0;
completeDepsDir = lib.concatStringsSep " " completeDeps;
@ -90,9 +90,9 @@ in ''
export HOST="${stdenv.hostPlatform.config}"
export PROFILE=${if release then "release" else "debug"}
export OUT_DIR=$(pwd)/target/build/${crateName}.out
export CARGO_PKG_VERSION_MAJOR=${builtins.elemAt version 0}
export CARGO_PKG_VERSION_MINOR=${builtins.elemAt version 1}
export CARGO_PKG_VERSION_PATCH=${builtins.elemAt version 2}
export CARGO_PKG_VERSION_MAJOR=${lib.elemAt version 0}
export CARGO_PKG_VERSION_MINOR=${lib.elemAt version 1}
export CARGO_PKG_VERSION_PATCH=${lib.elemAt version 2}
export CARGO_PKG_VERSION_PRE="${versionPre}"
export CARGO_PKG_HOMEPAGE="${crateHomepage}"
export NUM_JOBS=1

View file

@ -13,56 +13,34 @@ let
then "macos"
else stdenv.hostPlatform.parsed.kernel.name;
makeDeps = dependencies: crateRenames:
(lib.concatMapStringsSep " " (dep:
# Create rustc arguments to link against the given list of dependencies and
# renames
mkRustcDepArgs = dependencies: crateRenames:
lib.concatMapStringsSep " " (dep:
let
extern = lib.strings.replaceStrings ["-"] ["_"] dep.libName;
name = if builtins.hasAttr dep.crateName crateRenames then
extern = lib.replaceStrings ["-"] ["_"] dep.libName;
name = if lib.hasAttr dep.crateName crateRenames then
lib.strings.replaceStrings ["-"] ["_"] crateRenames.${dep.crateName}
else
extern;
in (if lib.lists.any (x: x == "lib") dep.crateType then
in (if lib.any (x: x == "lib") dep.crateType then
" --extern ${name}=${dep.lib}/lib/lib${extern}-${dep.metadata}.rlib"
else
" --extern ${name}=${dep.lib}/lib/lib${extern}-${dep.metadata}${stdenv.hostPlatform.extensions.sharedLibrary}")
) dependencies);
) dependencies;
echo_build_heading = colors: ''
echo_build_heading() {
start=""
end=""
if [[ "${colors}" == "always" ]]; then
start="$(printf '\033[0;1;32m')" #set bold, and set green.
end="$(printf '\033[0m')" #returns to "normal"
fi
if (( $# == 1 )); then
echo "$start""Building $1""$end"
else
echo "$start""Building $1 ($2)""$end"
fi
}
'';
noisily = colors: verbose: ''
noisily() {
start=""
end=""
if [[ "${colors}" == "always" ]]; then
start="$(printf '\033[0;1;32m')" #set bold, and set green.
end="$(printf '\033[0m')" #returns to "normal"
fi
${lib.optionalString verbose ''
echo -n "$start"Running "$end"
echo $@
''}
$@
}
'';
inherit (import ./log.nix { inherit lib; }) noisily echo_build_heading;
configureCrate = import ./configure-crate.nix { inherit lib stdenv echo_build_heading noisily makeDeps; };
buildCrate = import ./build-crate.nix { inherit lib stdenv echo_build_heading noisily makeDeps rust; };
installCrate = import ./install-crate.nix;
configureCrate = import ./configure-crate.nix {
inherit lib stdenv echo_build_heading noisily mkRustcDepArgs;
};
in
buildCrate = import ./build-crate.nix {
inherit lib stdenv echo_build_heading noisily mkRustcDepArgs rust;
};
installCrate = import ./install-crate.nix;
in
crate_: lib.makeOverridable ({ rust, release, verbose, features, buildInputs, crateOverrides,
dependencies, buildDependencies, crateRenames,
@ -81,6 +59,15 @@ let crate = crate_ // (lib.attrByPath [ crate_.crateName ] (attr: {}) crateOverr
extraDerivationAttrs = lib.filterAttrs (n: v: ! lib.elem n processedAttrs) crate;
buildInputs_ = buildInputs;
extraRustcOpts_ = extraRustcOpts;
# take a list of crates that we depend on and override them to fit our overrides, rustc, release, …
makeDependencies = map (dep: lib.getLib (dep.override { inherit release verbose crateOverrides; }));
# crate2nix has a hack for the old bash based build script that did split
# entries at `,`. No we have to work around that hack.
# https://github.com/kolloch/crate2nix/blame/5b19c1b14e1b0e5522c3e44e300d0b332dc939e7/crate2nix/templates/build.nix.tera#L89
crateBin = lib.filter (bin: !(bin ? name && bin.name == ",")) (crate.crateBin or []);
hasCrateBin = crate ? crateBin;
in
stdenv.mkDerivation (rec {
@ -94,42 +81,28 @@ stdenv.mkDerivation (rec {
name = "rust_${crate.crateName}-${crate.version}";
depsBuildBuild = [ rust stdenv.cc ];
buildInputs = (crate.buildInputs or []) ++ buildInputs_;
dependencies =
builtins.map
(dep: lib.getLib (dep.override { rust = rust; release = release; verbose = verbose; crateOverrides = crateOverrides; }))
dependencies_;
dependencies = makeDependencies dependencies_;
buildDependencies = makeDependencies buildDependencies_;
buildDependencies =
builtins.map
(dep: lib.getLib (dep.override { rust = rust; release = release; verbose = verbose; crateOverrides = crateOverrides; }))
buildDependencies_;
completeDeps = lib.lists.unique (dependencies ++ lib.lists.concatMap (dep: dep.completeDeps) dependencies);
completeBuildDeps = lib.lists.unique (
completeDeps = lib.unique (dependencies ++ lib.concatMap (dep: dep.completeDeps) dependencies);
completeBuildDeps = lib.unique (
buildDependencies
++ lib.lists.concatMap (dep: dep.completeBuildDeps ++ dep.completeDeps) buildDependencies
++ lib.concatMap (dep: dep.completeBuildDeps ++ dep.completeDeps) buildDependencies
);
crateFeatures = if crate ? features then
lib.concatMapStringsSep " " (f: "--cfg feature=\\\"${f}\\\"") (crate.features ++ features) #"
else "";
crateFeatures = lib.optionalString (crate ? features)
(lib.concatMapStringsSep " " (f: "--cfg feature=\\\"${f}\\\"") (crate.features ++ features));
libName = if crate ? libName then crate.libName else crate.crateName;
libPath = if crate ? libPath then crate.libPath else "";
depsMetadata = builtins.foldl' (str: dep: str + dep.metadata) "" (dependencies ++ buildDependencies);
metadata = builtins.substring 0 10 (builtins.hashString "sha256" (crateName + "-" + crateVersion + "___" + toString crateFeatures + "___" + depsMetadata ));
crateBin = if crate ? crateBin then
builtins.foldl' (bins: bin: let
name = (if bin ? name then bin.name else crateName);
path = if bin ? path then bin.path else "";
in
bins + (if bin == "" then "" else ",") + "${name} ${path}"
) "" crate.crateBin
else "";
hasCrateBin = crate ? crateBin;
# Seed the symbol hashes with something unique every time.
# https://doc.rust-lang.org/1.0.0/rustc/metadata/loader/index.html#frobbing-symbols
metadata = let
depsMetadata = lib.foldl' (str: dep: str + dep.metadata) "" (dependencies ++ buildDependencies);
hashedMetadata = builtins.hashString "sha256"
(crateName + "-" + crateVersion + "___" + toString crateFeatures + "___" + depsMetadata);
in lib.substring 0 10 hashedMetadata;
build = crate.build or "";
workspace_member = crate.workspace_member or ".";
@ -142,9 +115,12 @@ stdenv.mkDerivation (rec {
if lib.attrByPath ["plugin"] false crate then ["dylib"] else
(crate.type or ["lib"]);
colors = lib.attrByPath [ "colors" ] "always" crate;
extraLinkFlags = builtins.concatStringsSep " " (crate.extraLinkFlags or []);
extraLinkFlags = lib.concatStringsSep " " (crate.extraLinkFlags or []);
edition = crate.edition or null;
extraRustcOpts = (if crate ? extraRustcOpts then crate.extraRustcOpts else []) ++ extraRustcOpts_ ++ (lib.optional (edition != null) "--edition ${edition}");
extraRustcOpts =
lib.optionals (crate ? extraRustcOpts) crate.extraRustcOpts
++ extraRustcOpts_
++ (lib.optional (edition != null) "--edition ${edition}");
configurePhase = configureCrate {
inherit crateName buildDependencies completeDeps completeBuildDeps crateDescription
@ -155,7 +131,7 @@ stdenv.mkDerivation (rec {
buildPhase = buildCrate {
inherit crateName dependencies
crateFeatures crateRenames libName release libPath crateType
metadata crateBin hasCrateBin verbose colors
metadata hasCrateBin crateBin verbose colors
extraRustcOpts;
};
installPhase = installCrate crateName metadata;

View file

@ -3,23 +3,23 @@
kernel = stdenv.hostPlatform.parsed.kernel.name;
abi = stdenv.hostPlatform.parsed.abi.name;
cpu = stdenv.hostPlatform.parsed.cpu.name;
updateFeatures = f: up: functions: builtins.deepSeq f (lib.lists.foldl' (features: fun: fun features) (lib.attrsets.recursiveUpdate f up) functions);
updateFeatures = f: up: functions: lib.deepSeq f (lib.foldl' (features: fun: fun features) (lib.attrsets.recursiveUpdate f up) functions);
mapFeatures = features: map (fun: fun { features = features; });
mkFeatures = feat: lib.lists.foldl (features: featureName:
mkFeatures = feat: lib.foldl (features: featureName:
if feat.${featureName} or false then
[ featureName ] ++ features
else
features
) [] (builtins.attrNames feat);
include = includedFiles: src: builtins.filterSource (path: type:
lib.lists.any (f:
) [] (lib.attrNames feat);
include = includedFiles: src: lib.filterSource (path: type:
lib.any (f:
let p = toString (src + ("/" + f));
in
p == path || (lib.strings.hasPrefix (p + "/") path)
) includedFiles
) src;
exclude = excludedFiles: src: builtins.filterSource (path: type:
lib.lists.all (f:
exclude = excludedFiles: src: lib.filterSource (path: type:
lib.all (f:
!lib.strings.hasPrefix (toString (src + ("/" + f))) path
) excludedFiles
) src;

View file

@ -0,0 +1,117 @@
build_lib() {
lib_src=$1
echo_build_heading $lib_src ${libName}
noisily rustc \
--crate-name $CRATE_NAME \
$lib_src \
--out-dir target/lib \
--emit=dep-info,link \
-L dependency=target/deps \
--cap-lints allow \
$LIB_RUSTC_OPTS \
$BUILD_OUT_DIR \
$EXTRA_BUILD \
$EXTRA_FEATURES \
--color $colors
EXTRA_LIB=" --extern $CRATE_NAME=target/lib/lib$CRATE_NAME-$metadata.rlib"
if [ -e target/deps/lib$CRATE_NAME-$metadata$LIB_EXT ]; then
EXTRA_LIB="$EXTRA_LIB --extern $CRATE_NAME=target/lib/lib$CRATE_NAME-$metadata$LIB_EXT"
fi
}
build_bin() {
crate_name=$1
crate_name_=$(echo $crate_name | tr '-' '_')
main_file=""
if [[ ! -z $2 ]]; then
main_file=$2
fi
echo_build_heading $@
noisily rustc \
--crate-name $crate_name_ \
$main_file \
--crate-type bin \
$BIN_RUSTC_OPTS \
--out-dir target/bin \
--emit=dep-info,link \
-L dependency=target/deps \
$LINK \
$EXTRA_LIB \
--cap-lints allow \
$BUILD_OUT_DIR \
$EXTRA_BUILD \
$EXTRA_FEATURES \
--color ${colors} \
if [ "$crate_name_" != "$crate_name" ]; then
mv target/bin/$crate_name_ target/bin/$crate_name
fi
}
setup_link_paths() {
EXTRA_LIB=""
if [[ -e target/link_ ]]; then
EXTRA_BUILD="$(cat target/link_) $EXTRA_BUILD"
fi
echo "$EXTRA_LINK_SEARCH" | while read i; do
if [[ ! -z "$i" ]]; then
for library in $i; do
echo "-L $library" >> target/link
L=$(echo $library | sed -e "s#$(pwd)/target/build#$lib/lib#")
echo "-L $L" >> target/link.final
done
fi
done
echo "$EXTRA_LINK" | while read i; do
if [[ ! -z "$i" ]]; then
for library in $i; do
echo "-l $library" >> target/link
echo "-l $library" >> target/link.final
done
fi
done
if [[ -e target/link ]]; then
sort -u target/link.final > target/link.final.sorted
mv target/link.final.sorted target/link.final
sort -u target/link > target/link.sorted
mv target/link.sorted target/link
tr '\n' ' ' < target/link > target/link_
LINK=$(cat target/link_)
fi
}
search_for_bin_path() {
# heuristic to "guess" the correct source file as found in cargo:
# https://github.com/rust-lang/cargo/blob/90fc9f620190d5fa3c80b0c8c65a1e1361e6b8ae/src/cargo/util/toml/targets.rs#L308-L325
BIN_NAME=$1
BIN_NAME_=$(echo $BIN_NAME | tr '-' '_')
# the first two cases are the "new" default IIRC
FILES=( "src/bin/$BIN_NAME.rs" "src/bin/$BIN_NAME/main.rs" "src/bin/$BIN_NAME_.rs" "src/bin/$BIN_NAME_/main.rs" "src/bin/main.rs" "src/main.rs" )
if ! [ -e "$LIB_PATH" -o -e src/lib.rs -o -e "src/$LIB_NAME.rs" ]; then
# if this is not a library the following path is also valid
FILES=( "src/$BIN_NAME.rs" "src/$BIN_NAME_.rs" "${FILES[@]}" )
fi
for file in "${FILES[@]}";
do
echo "checking file $file"
# first file that exists wins
if [[ -e "$file" ]]; then
BIN_PATH="$file"
break
fi
done
if [[ -z "$BIN_PATH" ]]; then
echo "failed to find file for binary target: $BIN_NAME" >&2
exit 1
fi
}

View file

@ -0,0 +1,33 @@
{ lib }:
{
echo_build_heading = colors: ''
echo_build_heading() {
start=""
end=""
${lib.optionalString (colors == "always") ''
start="$(printf '\033[0;1;32m')" #set bold, and set green.
end="$(printf '\033[0m')" #returns to "normal"
''}
if (( $# == 1 )); then
echo "$start""Building $1""$end"
else
echo "$start""Building $1 ($2)""$end"
fi
}
'';
noisily = colors: verbose: ''
noisily() {
start=""
end=""
${lib.optionalString (colors == "always") ''
start="$(printf '\033[0;1;32m')" #set bold, and set green.
end="$(printf '\033[0m')" #returns to "normal"
''}
${lib.optionalString verbose ''
echo -n "$start"Running "$end"
echo $@
''}
$@
}
'';
}