nixos/stage-1: add mechanism which lustrates all impurities from / (#17784)

lustrate /ˈlʌstreɪt/ verb.
  purify by expiatory sacrifice, ceremonial washing, or some other
  ritual action.

- sudo touch /etc/NIXOS_LUSTRATE
  ⇒ on next reboot, during stage 1, everything but /nix and /boot
  is moved to /old-root
- echo "etc/passwd" | sudo tee -a /etc/NIXOS_LUSTRATE
  ⇒ on next reboot, during stage 1, everything but /nix and /boot
  is moved to /old-root; except /etc/passwd is copied back.

Useful for installing NixOS in place on another distro. For instance:

$ nix-env -iE '_: with import <nixpkgs/nixos> { configuration = {}; }; with config.system.build; [ nixos-generate-config manual.manpages ]'
$ sudo mkdir /etc/nixos
$ sudo `which nixos-generate-config`

… edit the configuration files in /etc/nixos using man configuration.nix
  if needed

  maybe add: users.extraUsers.root.initialHashedPassword = "" ?

… Build the entire NixOS system and link it to the system profile:
$ nix-env -p /nix/var/nix/profiles/system -f '<nixpkgs/nixos>' -A system --set

… If you were using a single user install:
$ sudo chown -R 0.0 /nix

… NixOS is about to take over
$ sudo touch /etc/NIXOS
$ sudo touch /etc/NIXOS_LUSTRATE

… Let's keep the configuration files we just created
$ echo etc/nixos | sudo tee -a /etc/NIXOS_LUSTRATE

$ sudo mv -v /boot /boot.bak &&
  sudo /nix/var/nix/profiles/system/bin/switch-to-configuration boot
$ sudo reboot

… NixOS boots, Stage 1 moves all the old distro stuff in /old-root.

nixos/modules/system/boot/stage-1-init.sh

@@ -312,8 +312,50 @@ mountFS() {
         echo "retrying..."
         n=$((n + 1))
     done
+
+    [ "$mountPoint" == "/" ] &&
+        [ -f "/mnt-root/etc/NIXOS_LUSTRATE" ] &&
+        lustrateRoot "/mnt-root"
 }
 
+lustrateRoot () {
+    local root="$1"
+
+    echo
+    echo -e "\e[1;33m<<< NixOS is now lustrating the root filesystem (cruft goes to /old-root) >>>\e[0m"
+    echo
+
+    mkdir -m 0755 -p "$root/old-root.tmp"
+
+    echo
+    echo "Moving impurities out of the way:"
+    for d in "$root"/*
+    do
+        [ "$d" == "$root/nix"          ] && continue
+        [ "$d" == "$root/boot"         ] && continue # Don't render the system unbootable
+        [ "$d" == "$root/old-root.tmp" ] && continue
+
+        mv -v "$d" "$root/old-root.tmp"
+    done
+
+    # Use .tmp to make sure subsequent invokations don't clash
+    mv -v "$root/old-root.tmp" "$root/old-root"
+
+    mkdir -m 0755 -p "$root/etc"
+    touch "$root/etc/NIXOS"
+
+    exec 4< "$root/old-root/etc/NIXOS_LUSTRATE"
+
+    echo
+    echo "Restoring selected impurities:"
+    while read -u 4 keeper; do
+        dirname="$(dirname "$keeper")"
+        mkdir -m 0755 -p "$root/$dirname"
+        cp -av "$root/old-root/$keeper" "$root/$keeper"
+    done
+
+    exec 4>&-
+}
 
 # Function for waiting a device to appear.
 waitDevice() {

nixos/modules/system/boot/stage-1.nix

@@ -131,9 +131,16 @@ let
   # The initrd only has to mount / or any FS marked as necessary for
   # booting (such as the FS containing /nix/store, or an FS needed for
   # mounting /, like / on a loopback).
-  fileSystems = filter
-    (fs: fs.neededForBoot || elem fs.mountPoint [ "/" "/nix" "/nix/store" "/var" "/var/log" "/var/lib" "/etc" ])
-    (attrValues config.fileSystems);
+  #
+  # We need to guarantee that / is the first filesystem in the list so
+  # that if and when lustrateRoot is invoked, nothing else is mounted
+  fileSystems = let
+    filterNeeded = filter
+      (fs: fs.mountPoint != "/" && (fs.neededForBoot || elem fs.mountPoint [ "/nix" "/nix/store" "/var" "/var/log" "/var/lib" "/etc" ]));
+    filterRoot = filter
+      (fs: fs.mountPoint == "/");
+    allFileSystems = attrValues config.fileSystems;
+  in (filterRoot allFileSystems) ++ (filterNeeded allFileSystems);
 
 
   udevRules = pkgs.stdenv.mkDerivation {