Merge pull request #115590 from grahamc/iscsi

NixOS: services.{openiscsi, target}, boot.iscsi-initiator: init

nixos/modules/module-list.nix

@@ -697,6 +697,9 @@
   ./services/networking/iodine.nix
   ./services/networking/iperf3.nix
   ./services/networking/ircd-hybrid/default.nix
+  ./services/networking/iscsi/initiator.nix
+  ./services/networking/iscsi/root-initiator.nix
+  ./services/networking/iscsi/target.nix
   ./services/networking/iwd.nix
   ./services/networking/jicofo.nix
   ./services/networking/jitsi-videobridge.nix

nixos/modules/services/networking/iscsi/initiator.nix

@@ -0,0 +1,84 @@
+{ config, lib, pkgs, ... }: with lib;
+let
+  cfg = config.services.openiscsi;
+in
+{
+  options.services.openiscsi = with types; {
+    enable = mkEnableOption "the openiscsi iscsi daemon";
+    enableAutoLoginOut = mkEnableOption ''
+      automatic login and logout of all automatic targets.
+      You probably do not want this.
+    '';
+    discoverPortal = mkOption {
+      type = nullOr str;
+      default = null;
+      description = "Portal to discover targets on";
+    };
+    name = mkOption {
+      type = str;
+      description = "Name of this iscsi initiator";
+      example = "iqn.2020-08.org.linux-iscsi.initiatorhost:example";
+    };
+    package = mkOption {
+      type = package;
+      description = "openiscsi package to use";
+      default = pkgs.openiscsi;
+      defaultText = "pkgs.openiscsi";
+    };
+
+    extraConfig = mkOption {
+      type = str;
+      default = "";
+      description = "Lines to append to default iscsid.conf";
+    };
+
+    extraConfigFile = mkOption {
+      description = ''
+        Append an additional file's contents to /etc/iscsid.conf. Use a non-store path
+        and store passwords in this file.
+      '';
+      default = null;
+      type = nullOr str;
+    };
+  };
+
+  config = mkIf cfg.enable {
+    environment.etc."iscsi/iscsid.conf.fragment".source = pkgs.runCommand "iscsid.conf" {} ''
+      cat "${cfg.package}/etc/iscsi/iscsid.conf" > $out
+      cat << 'EOF' >> $out
+      ${cfg.extraConfig}
+      ${optionalString cfg.enableAutoLoginOut "node.startup = automatic"}
+      EOF
+    '';
+    environment.etc."iscsi/initiatorname.iscsi".text = "InitiatorName=${cfg.name}";
+
+    system.activationScripts.iscsid = let
+      extraCfgDumper = optionalString (cfg.extraConfigFile != null) ''
+        if [ -f "${cfg.extraConfigFile}" ]; then
+          printf "\n# The following is from ${cfg.extraConfigFile}:\n"
+          cat "${cfg.extraConfigFile}"
+        else
+          echo "Warning: services.openiscsi.extraConfigFile ${cfg.extraConfigFile} does not exist!" >&2
+        fi
+      '';
+    in ''
+      (
+        cat ${config.environment.etc."iscsi/iscsid.conf.fragment".source}
+        ${extraCfgDumper}
+      ) > /etc/iscsi/iscsid.conf
+    '';
+
+    systemd.packages = [ cfg.package ];
+
+    systemd.services."iscsid".wantedBy = [ "multi-user.target" ];
+    systemd.sockets."iscsid".wantedBy = [ "sockets.target" ];
+
+    systemd.services."iscsi" = mkIf cfg.enableAutoLoginOut {
+      wantedBy = [ "remote-fs.target" ];
+      serviceConfig.ExecStartPre = mkIf (cfg.discoverPortal != null) "${cfg.package}/bin/iscsiadm --mode discoverydb --type sendtargets --portal ${escapeShellArg cfg.discoverPortal} --discover";
+    };
+
+    environment.systemPackages = [ cfg.package ];
+    boot.kernelModules = [ "iscsi_tcp" ];
+  };
+}

nixos/modules/services/networking/iscsi/root-initiator.nix

@@ -0,0 +1,181 @@
+{ config, lib, pkgs, ... }: with lib;
+let
+  cfg = config.boot.iscsi-initiator;
+in
+{
+  # If you're booting entirely off another machine you may want to add
+  # this snippet to always boot the latest "system" version. It is not
+  # enabled by default in case you have an initrd on a local disk:
+  #
+  #     boot.initrd.postMountCommands = ''
+  #       ln -sfn /nix/var/nix/profiles/system/init /mnt-root/init
+  #       stage2Init=/init
+  #     '';
+  #
+  # Note: Theoretically you might want to connect to multiple portals and
+  # log in to multiple targets, however the authors of this module so far
+  # don't have the need or expertise to reasonably implement it. Also,
+  # consider carefully before making your boot chain depend on multiple
+  # machines to be up.
+  options.boot.iscsi-initiator = with types; {
+    name = mkOption {
+      description = ''
+        Name of the iSCSI initiator to boot from. Note, booting from iscsi
+        requires networkd based networking.
+      '';
+      default = null;
+      example = "iqn.2020-08.org.linux-iscsi.initiatorhost:example";
+      type = nullOr str;
+    };
+
+    discoverPortal = mkOption {
+      description = ''
+        iSCSI portal to boot from.
+      '';
+      default = null;
+      example = "192.168.1.1:3260";
+      type = nullOr str;
+    };
+
+    target = mkOption {
+      description = ''
+        Name of the iSCSI target to boot from.
+      '';
+      default = null;
+      example = "iqn.2020-08.org.linux-iscsi.targethost:example";
+      type = nullOr str;
+    };
+
+    logLevel = mkOption {
+      description = ''
+        Higher numbers elicits more logs.
+      '';
+      default = 1;
+      example = 8;
+      type = int;
+    };
+
+    loginAll = mkOption {
+      description = ''
+        Do not log into a specific target on the portal, but to all that we discover.
+        This overrides setting target.
+      '';
+      type = bool;
+      default = false;
+    };
+
+    extraConfig = mkOption {
+      description = "Extra lines to append to /etc/iscsid.conf";
+      default = null;
+      type = nullOr lines;
+    };
+
+    extraConfigFile = mkOption {
+      description = ''
+        Append an additional file's contents to `/etc/iscsid.conf`. Use a non-store path
+        and store passwords in this file. Note: the file specified here must be available
+        in the initrd, see: `boot.initrd.secrets`.
+      '';
+      default = null;
+      type = nullOr str;
+    };
+  };
+
+  config = mkIf (cfg.name != null) {
+    # The "scripted" networking configuration (ie: non-networkd)
+    # doesn't properly order the start and stop of the interfaces, and the
+    # network interfaces are torn down before unmounting disks. Since this
+    # module is specifically for very-early-boot network mounts, we need
+    # the network to stay on.
+    #
+    # We could probably fix the scripted options to properly order, but I'm
+    # not inclined to invest that time today. Hopefully this gets users far
+    # enough along and they can just use networkd.
+    networking.useNetworkd = true;
+    networking.useDHCP = false; # Required to set useNetworkd = true
+
+    boot.initrd = {
+      network.enable = true;
+
+      # By default, the stage-1 disables the network and resets the interfaces
+      # on startup. Since our startup disks are on the network, we can't let
+      # the network not work.
+      network.flushBeforeStage2 = false;
+
+      kernelModules = [ "iscsi_tcp" ];
+
+      extraUtilsCommands = ''
+        copy_bin_and_libs ${pkgs.openiscsi}/bin/iscsid
+        copy_bin_and_libs ${pkgs.openiscsi}/bin/iscsiadm
+        ${optionalString (!config.boot.initrd.network.ssh.enable) "cp -pv ${pkgs.glibc.out}/lib/libnss_files.so.* $out/lib"}
+
+        mkdir -p $out/etc/iscsi
+        cp ${config.environment.etc.hosts.source} $out/etc/hosts
+        cp ${pkgs.openiscsi}/etc/iscsi/iscsid.conf $out/etc/iscsi/iscsid.fragment.conf
+        chmod +w $out/etc/iscsi/iscsid.fragment.conf
+        cat << 'EOF' >> $out/etc/iscsi/iscsid.fragment.conf
+        ${optionalString (cfg.extraConfig != null) cfg.extraConfig}
+        EOF
+      '';
+
+      extraUtilsCommandsTest = ''
+        $out/bin/iscsiadm --version
+      '';
+
+      preLVMCommands = let
+        extraCfgDumper = optionalString (cfg.extraConfigFile != null) ''
+          if [ -f "${cfg.extraConfigFile}" ]; then
+            printf "\n# The following is from ${cfg.extraConfigFile}:\n"
+            cat "${cfg.extraConfigFile}"
+          else
+            echo "Warning: boot.iscsi-initiator.extraConfigFile ${cfg.extraConfigFile} does not exist!" >&2
+          fi
+        '';
+      in ''
+        ${optionalString (!config.boot.initrd.network.ssh.enable) ''
+        # stolen from initrd-ssh.nix
+        echo 'root:x:0:0:root:/root:/bin/ash' > /etc/passwd
+        echo 'passwd: files' > /etc/nsswitch.conf
+      ''}
+
+        cp -f $extraUtils/etc/hosts /etc/hosts
+
+        mkdir -p /etc/iscsi /run/lock/iscsi
+        echo "InitiatorName=${cfg.name}" > /etc/iscsi/initiatorname.iscsi
+
+        (
+          cat "$extraUtils/etc/iscsi/iscsid.fragment.conf"
+          printf "\n"
+          ${optionalString cfg.loginAll ''echo "node.startup = automatic"''}
+          ${extraCfgDumper}
+        ) > /etc/iscsi/iscsid.conf
+
+        iscsid --foreground --no-pid-file --debug ${toString cfg.logLevel} &
+        iscsiadm --mode discoverydb \
+          --type sendtargets \
+          --discover \
+          --portal ${escapeShellArg cfg.discoverPortal} \
+          --debug ${toString cfg.logLevel}
+
+        ${if cfg.loginAll then ''
+        iscsiadm --mode node --loginall all
+      '' else ''
+        iscsiadm --mode node --targetname ${escapeShellArg cfg.target} --login
+      ''}
+        pkill -9 iscsid
+      '';
+    };
+
+    services.openiscsi = {
+      enable = true;
+      inherit (cfg) name;
+    };
+
+    assertions = [
+      {
+        assertion = cfg.loginAll -> cfg.target == null;
+        message = "iSCSI target name is set while login on all portals is enabled.";
+      }
+    ];
+  };
+}

nixos/modules/services/networking/iscsi/target.nix

@@ -0,0 +1,53 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.services.target;
+in
+{
+  ###### interface
+  options = {
+    services.target = with types; {
+      enable = mkEnableOption "the kernel's LIO iscsi target";
+
+      config = mkOption {
+        type = attrs;
+        default = {};
+        description = ''
+          Content of /etc/target/saveconfig.json
+          This file is normally read and written by targetcli
+        '';
+      };
+    };
+  };
+
+  ###### implementation
+  config = mkIf cfg.enable {
+    environment.etc."target/saveconfig.json" = {
+      text = builtins.toJSON cfg.config;
+      mode = "0600";
+    };
+
+    environment.systemPackages = with pkgs; [ targetcli ];
+
+    boot.kernelModules = [ "configfs" "target_core_mod" "iscsi_target_mod" ];
+
+    systemd.services.iscsi-target = {
+      enable = true;
+      after = [ "network.target" "local-fs.target" ];
+      requires = [ "sys-kernel-config.mount" ];
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        Type = "oneshot";
+        ExecStart = "${pkgs.python3.pkgs.rtslib}/bin/targetctl restore";
+        ExecStop = "${pkgs.python3.pkgs.rtslib}/bin/targetctl clear";
+        RemainAfterExit = "yes";
+      };
+    };
+
+    systemd.tmpfiles.rules = [
+      "d /etc/target 0700 root root - -"
+    ];
+  };
+}

nixos/tests/all-tests.nix

@@ -185,6 +185,7 @@ in
   iodine = handleTest ./iodine.nix {};
   ipfs = handleTest ./ipfs.nix {};
   ipv6 = handleTest ./ipv6.nix {};
+  iscsi-root = handleTest ./iscsi-root.nix {};
   jackett = handleTest ./jackett.nix {};
   jellyfin = handleTest ./jellyfin.nix {};
   jenkins = handleTest ./jenkins.nix {};

nixos/tests/iscsi-root.nix

@@ -0,0 +1,161 @@
+import ./make-test-python.nix (
+  { pkgs, lib, ... }:
+    let
+      initiatorName = "iqn.2020-08.org.linux-iscsi.initiatorhost:example";
+      targetName = "iqn.2003-01.org.linux-iscsi.target.x8664:sn.acf8fd9c23af";
+    in
+      {
+        name = "iscsi";
+        meta = {
+          maintainers = pkgs.lib.teams.deshaw.members
+          ++ (with pkgs.lib.maintainers; [ ajs124 ]);
+        };
+
+        nodes = {
+          target = { config, pkgs, lib, ... }: {
+            services.target = {
+              enable = true;
+              config = {
+                fabric_modules = [];
+                storage_objects = [
+                  {
+                    dev = "/dev/vdb";
+                    name = "test";
+                    plugin = "block";
+                    write_back = true;
+                    wwn = "92b17c3f-6b40-4168-b082-ceeb7b495522";
+                  }
+                ];
+                targets = [
+                  {
+                    fabric = "iscsi";
+                    tpgs = [
+                      {
+                        enable = true;
+                        attributes = {
+                          authentication = 0;
+                          generate_node_acls = 1;
+                        };
+                        luns = [
+                          {
+                            alias = "94dfe06967";
+                            alua_tg_pt_gp_name = "default_tg_pt_gp";
+                            index = 0;
+                            storage_object = "/backstores/block/test";
+                          }
+                        ];
+                        node_acls = [
+                          {
+                            mapped_luns = [
+                              {
+                                alias = "d42f5bdf8a";
+                                index = 0;
+                                tpg_lun = 0;
+                                write_protect = false;
+                              }
+                            ];
+                            node_wwn = initiatorName;
+                          }
+                        ];
+                        portals = [
+                          {
+                            ip_address = "0.0.0.0";
+                            iser = false;
+                            offload = false;
+                            port = 3260;
+                          }
+                        ];
+                        tag = 1;
+                      }
+                    ];
+                    wwn = targetName;
+                  }
+                ];
+              };
+            };
+
+            networking.firewall.allowedTCPPorts = [ 3260 ];
+            networking.firewall.allowedUDPPorts = [ 3260 ];
+
+            virtualisation.memorySize = 2048;
+            virtualisation.emptyDiskImages = [ 2048 ];
+          };
+
+          initiatorAuto = { nodes, config, pkgs, ... }: {
+            services.openiscsi = {
+              enable = true;
+              enableAutoLoginOut = true;
+              discoverPortal = "target";
+              name = initiatorName;
+            };
+
+            environment.systemPackages = with pkgs; [
+              xfsprogs
+            ];
+
+            system.extraDependencies = [ nodes.initiatorRootDisk.config.system.build.toplevel ];
+
+            nix.binaryCaches = lib.mkForce [];
+            nix.extraOptions = ''
+              hashed-mirrors =
+              connect-timeout = 1
+            '';
+          };
+
+          initiatorRootDisk = { config, pkgs, modulesPath, lib, ... }: {
+            boot.loader.grub.enable = false;
+            boot.kernelParams = lib.mkOverride 5 (
+              [
+                "boot.shell_on_fail"
+                "console=tty1"
+                "ip=${config.networking.primaryIPAddress}:::255.255.255.0::ens9:none"
+              ]
+            );
+
+            # defaults to true, puts some code in the initrd that tries to mount an overlayfs on /nix/store
+            virtualisation.writableStore = false;
+
+            fileSystems = lib.mkOverride 5 {
+              "/" = {
+                fsType = "xfs";
+                device = "/dev/sda";
+                options = [ "_netdev" ];
+              };
+            };
+
+            boot.iscsi-initiator = {
+              discoverPortal = "target";
+              name = initiatorName;
+              target = targetName;
+            };
+          };
+        };
+
+        testScript = { nodes, ... }: ''
+          target.start()
+          target.wait_for_unit("iscsi-target.service")
+
+          initiatorAuto.start()
+
+          initiatorAuto.wait_for_unit("iscsid.service")
+          initiatorAuto.wait_for_unit("iscsi.service")
+          initiatorAuto.get_unit_info("iscsi")
+
+          initiatorAuto.succeed("set -x; while ! test -e /dev/sda; do sleep 1; done")
+
+          initiatorAuto.succeed("mkfs.xfs /dev/sda")
+          initiatorAuto.succeed("mkdir /mnt && mount /dev/sda /mnt")
+          initiatorAuto.succeed(
+              "nixos-install --no-bootloader --no-root-passwd --system ${nodes.initiatorRootDisk.config.system.build.toplevel}"
+          )
+          initiatorAuto.succeed("umount /mnt && rmdir /mnt")
+          initiatorAuto.shutdown()
+
+          initiatorRootDisk.start()
+          initiatorRootDisk.wait_for_unit("multi-user.target")
+          initiatorRootDisk.wait_for_unit("iscsid")
+          initiatorRootDisk.succeed("touch test")
+          initiatorRootDisk.shutdown()
+        '';
+      }
+)