Merge pull request #75911 from Simspace/parallel-docker-buildlayeredimage

when building a layered docker image, ignore it if tar encounters cha…

pkgs/build-support/docker/default.nix

@@ -325,7 +325,6 @@ rec {
         | jshon -d config \
         | jshon -s "1970-01-01T00:00:01Z" -i created > generic.json
 
-
       # WARNING!
       # The following code is fiddly w.r.t. ensuring every layer is
       # created, and that no paths are missed. If you change the
@@ -625,7 +624,22 @@ rec {
           -i "$imageName" > image/repositories
 
         echo "Cooking the image..."
-        tar -C image --dereference --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0  --mode=a-w --xform s:'^./':: -c . | pigz -nT > $out
+        # tar exits with an exit code of 1 if files changed while it was
+        # reading them. it considers a change in the number of hard links
+        # to be a "change", which can cause this to fail if images are being
+        # built concurrently and auto-optimise-store is turned on. since
+        # know the contents of these files will not change, we can reasonably
+        # ignore this exit code
+        set +e
+        tar -C image --dereference --hard-dereference --sort=name \
+          --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0  \
+          --mode=a-w --xform s:'^./':: --use-compress-program='pigz -nT' \
+          --warning=no-file-changed -cf $out .
+        RET=$?
+        if [ $RET -ne 0 ] && [ $RET -ne 1 ]; then
+          exit $RET
+        fi
+        set -e
 
         echo "Finished."
       '';

pkgs/build-support/docker/store-path-to-layer.sh

@@ -5,16 +5,43 @@ set -eu
 layerNumber=$1
 shift
 
+storePath="$1"
+shift
+
 layerPath="./layers/$layerNumber"
-echo "Creating layer #$layerNumber for $@"
+echo "Creating layer #$layerNumber for $storePath"
 
 mkdir -p "$layerPath"
-tar --no-recursion -rf "$layerPath/layer.tar" \
+
+# make sure /nix and /nix/store appear first in the archive.
+# we create the directories here and use them because
+# when there are other things being added to the
+# nix store, tar could fail, saying,
+# "tar: /nix/store: file changed as we read it"
+mkdir -p nix/store
+tar -cf "$layerPath/layer.tar"  \
     --mtime="@$SOURCE_DATE_EPOCH" \
-    --owner=0 --group=0 /nix /nix/store
-tar -rpf "$layerPath/layer.tar" --hard-dereference --sort=name \
+    --owner=0 --group=0 \
+    --transform='s,nix,/nix,' \
+    nix
+
+# we change into the /nix/store in order to avoid a similar
+# "file changed as we read it" error as above. Namely,
+# if we use the absolute path of /nix/store/123-pkg
+# and something new it added to the nix store while tar
+# is running, it will detect a change to /nix/store and
+# fail. Instead, if we cd into the nix store and copy
+# the relative nix store path, tar will ignore changes
+# to /nix/store. In order to create the correct structure
+# in the tar file, we transform the relative nix store
+# path to the absolute store path
+n=$(basename "$storePath")
+tar -C /nix/store -rpf "$layerPath/layer.tar" \
+    --hard-dereference --sort=name \
     --mtime="@$SOURCE_DATE_EPOCH" \
-    --owner=0 --group=0 "$@"
+    --owner=0 --group=0 \
+    --transform="s,$n,/nix/store/$n," \
+    $n
 
 # Compute a checksum of the tarball.
 tarhash=$(tarsum < $layerPath/layer.tar)