nixos/mailman: refactor

- Add serve.enable option, which configures uwsgi and nginx to serve
  the mailman-web application;
- Configure services to log to the journal, where possible. Mailman
  Core does not provide any options for this, but will now log to
  /var/log/mailman;
- Use a unified python environment for all components, with an
  extraPackages option to allow use of postgres support and similar;
- Configure mailman's postfix module such that it can generate the
  domain and lmtp maps;
- Fix formatting for option examples;
- Provide a mailman-web user to run the uwsgi service by default
- Refactor Hyperkitty's periodic jobs to reduce repetition in the
  expressions;
- Remove service dependencies not related to functionality included in
  the module, such as httpd -- these should be configured in user config
  when used;
- Move static files root to /var/lib/mailman-web-static by default. This avoids
  permission issues when a static file web server attempts to access
  /var/lib/mailman which is private to mailman. The location can still
  be changed by setting services.mailman.webSettings.STATIC_ROOT;
- Remove the webRoot option, which seems to have been included by
  accident, being an unsuitable directory for serving via HTTP.
- Rename mailman-web.service to mailman-web-setup.service, since it
  doesn't actually serve mailman-web. There is now a
  mailman-uwsgi.service if serve.enable is set to true.

nixos/modules/services/mail/mailman.nix

@@ -6,6 +6,11 @@ let
 
   cfg = config.services.mailman;
 
+  pythonEnv = pkgs.python3.withPackages (ps:
+    [ps.mailman ps.mailman-web]
+    ++ lib.optional cfg.hyperkitty.enable ps.mailman-hyperkitty
+    ++ cfg.extraPythonPackages);
+
   # This deliberately doesn't use recursiveUpdate so users can
   # override the defaults.
   settings = {
@@ -13,12 +18,28 @@ let
     SERVER_EMAIL = cfg.siteOwner;
     ALLOWED_HOSTS = [ "localhost" "127.0.0.1" ] ++ cfg.webHosts;
     COMPRESS_OFFLINE = true;
-    STATIC_ROOT = "/var/lib/mailman-web/static";
+    STATIC_ROOT = "/var/lib/mailman-web-static";
     MEDIA_ROOT = "/var/lib/mailman-web/media";
+    LOGGING = {
+      version = 1;
+      disable_existing_loggers = true;
+      handlers.console.class = "logging.StreamHandler";
+      loggers.django = {
+        handlers = [ "console" ];
+        level = "INFO";
+      };
+    };
   } // cfg.webSettings;
 
   settingsJSON = pkgs.writeText "settings.json" (builtins.toJSON settings);
 
+  # TODO: Should this be RFC42-ised so that users can set additional options without modifying the module?
+  mtaConfig = pkgs.writeText "mailman-postfix.cfg" ''
+    [postfix]
+    postmap_command: ${pkgs.postfix}/bin/postmap
+    transport_file_type: hash
+  '';
+
   mailmanCfg = ''
     [mailman]
     site_owner: ${cfg.siteOwner}
@@ -29,11 +50,14 @@ let
     var_dir: /var/lib/mailman
     queue_dir: $var_dir/queue
     template_dir: $var_dir/templates
-    log_dir: $var_dir/log
+    log_dir: /var/log/mailman
     lock_dir: $var_dir/lock
     etc_dir: /etc
     ext_dir: $etc_dir/mailman.d
     pid_file: /run/mailman/master.pid
+
+    [mta]
+    configuration: ${mtaConfig}
   '' + optionalString cfg.hyperkitty.enable ''
 
     [archiver.hyperkitty]
@@ -84,7 +108,7 @@ in {
         type = types.package;
         default = pkgs.mailman;
         defaultText = "pkgs.mailman";
-        example = "pkgs.mailman.override { archivers = []; }";
+        example = literalExample "pkgs.mailman.override { archivers = []; }";
         description = "Mailman package to use";
       };
 
@@ -98,18 +122,6 @@ in {
         '';
       };
 
-      webRoot = mkOption {
-        type = types.path;
-        default = "${pkgs.mailman-web}/${pkgs.python3.sitePackages}";
-        defaultText = "\${pkgs.mailman-web}/\${pkgs.python3.sitePackages}";
-        description = ''
-          The web root for the Hyperkity + Postorius apps provided by Mailman.
-          This variable can be set, of course, but it mainly exists so that site
-          admins can refer to it in their own hand-written web server
-          configuration files.
-        '';
-      };
-
       webHosts = mkOption {
         type = types.listOf types.str;
         default = [];
@@ -124,7 +136,7 @@ in {
 
       webUser = mkOption {
         type = types.str;
-        default = config.services.httpd.user;
+        default = "mailman-web";
         description = ''
           User to run mailman-web as
         '';
@@ -138,6 +150,16 @@ in {
         '';
       };
 
+      serve = {
+        enable = mkEnableOption "Automatic nginx and uwsgi setup for mailman-web";
+      };
+
+      extraPythonPackages = mkOption {
+        description = "Packages to add to the python environment used by mailman and mailman-web";
+        type = types.listOf types.package;
+        default = [];
+      };
+
       hyperkitty = {
         enable = mkEnableOption "the Hyperkitty archiver for Mailman";
 
@@ -183,7 +205,17 @@ in {
       (requirePostfixHash [ "config" "local_recipient_maps" ] "postfix_lmtp")
     ];
 
-    users.users.mailman = { description = "GNU Mailman"; isSystemUser = true; };
+    users.users.mailman = {
+      description = "GNU Mailman";
+      isSystemUser = true;
+      group = "mailman";
+    };
+    users.users.mailman-web = lib.mkIf (cfg.webUser == "mailman-web") {
+      description = "GNU Mailman web interface";
+      isSystemUser = true;
+      group = "mailman";
+    };
+    users.groups.mailman = {};
 
     environment.etc."mailman.cfg".text = mailmanCfg;
 
@@ -205,7 +237,28 @@ in {
           globals().update(json.load(f))
     '';
 
-    environment.systemPackages = [ cfg.package ] ++ (with pkgs; [ mailman-web ]);
+    services.nginx = mkIf cfg.serve.enable {
+      enable = mkDefault true;
+      virtualHosts."${lib.head cfg.webHosts}" = {
+        serverAliases = cfg.webHosts;
+        locations = {
+          "/".extraConfig = "uwsgi_pass unix:/run/mailman-web.socket;";
+          "/static/".alias = settings.STATIC_ROOT + "/";
+        };
+      };
+    };
+
+    environment.systemPackages = [ (pkgs.buildEnv {
+      name = "mailman-tools";
+      # We don't want to pollute the system PATH with a python
+      # interpreter etc. so let's pick only the stuff we actually
+      # want from pythonEnv
+      pathsToLink = ["/bin"];
+      paths = [pythonEnv];
+      postBuild = ''
+        find $out/bin/ -mindepth 1 -not -name "mailman*" -delete
+      '';
+    }) ];
 
     services.postfix = {
       recipientDelimiter = "+";         # bake recipient addresses in mail envelopes via VERP
@@ -214,181 +267,151 @@ in {
       };
     };
 
-    systemd.services.mailman = {
+    systemd.sockets.mailman-uwsgi = lib.mkIf cfg.serve.enable {
-      description = "GNU Mailman Master Process";
+      wantedBy = ["sockets.target"];
-      after = [ "network.target" ];
+      before = ["nginx.service"];
-      restartTriggers = [ config.environment.etc."mailman.cfg".source ];
+      socketConfig.ListenStream = "/run/mailman-web.socket";
-      wantedBy = [ "multi-user.target" ];
-      serviceConfig = {
-        ExecStart = "${cfg.package}/bin/mailman start";
-        ExecStop = "${cfg.package}/bin/mailman stop";
-        User = "mailman";
-        Type = "forking";
-        RuntimeDirectory = "mailman";
-        PIDFile = "/run/mailman/master.pid";
-      };
     };
-
+    systemd.services = {
-    systemd.services.mailman-settings = {
+      mailman = {
-      description = "Generate settings files (including secrets) for Mailman";
+        description = "GNU Mailman Master Process";
-      before = [ "mailman.service" "mailman-web.service" "hyperkitty.service" "httpd.service" "uwsgi.service" ];
+        after = [ "network.target" ];
-      requiredBy = [ "mailman.service" "mailman-web.service" "hyperkitty.service" "httpd.service" "uwsgi.service" ];
+        restartTriggers = [ config.environment.etc."mailman.cfg".source ];
-      path = with pkgs; [ jq ];
+        wantedBy = [ "multi-user.target" ];
-      script = ''
+        serviceConfig = {
-        mailmanDir=/var/lib/mailman
+          ExecStart = "${pythonEnv}/bin/mailman start";
-        mailmanWebDir=/var/lib/mailman-web
+          ExecStop = "${pythonEnv}/bin/mailman stop";
-
+          User = "mailman";
-        mailmanCfg=$mailmanDir/mailman-hyperkitty.cfg
+          Group = "mailman";
-        mailmanWebCfg=$mailmanWebDir/settings_local.json
+          Type = "forking";
-
+          RuntimeDirectory = "mailman";
-        install -m 0700 -o mailman -g nogroup -d $mailmanDir
+          LogsDirectory = "mailman";
-        install -m 0700 -o ${cfg.webUser} -g nogroup -d $mailmanWebDir
+          PIDFile = "/run/mailman/master.pid";
-
+        };
-        if [ ! -e $mailmanWebCfg ]; then
-            hyperkittyApiKey=$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 64)
-            secretKey=$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 64)
-
-            mailmanWebCfgTmp=$(mktemp)
-            jq -n '.MAILMAN_ARCHIVER_KEY=$archiver_key | .SECRET_KEY=$secret_key' \
-                --arg archiver_key "$hyperkittyApiKey" \
-                --arg secret_key "$secretKey" \
-                >"$mailmanWebCfgTmp"
-            chown ${cfg.webUser} "$mailmanWebCfgTmp"
-            mv -n "$mailmanWebCfgTmp" $mailmanWebCfg
-        fi
-
-        hyperkittyApiKey="$(jq -r .MAILMAN_ARCHIVER_KEY $mailmanWebCfg)"
-        mailmanCfgTmp=$(mktemp)
-        sed "s/@API_KEY@/$hyperkittyApiKey/g" ${mailmanHyperkittyCfg} >"$mailmanCfgTmp"
-        chown mailman "$mailmanCfgTmp"
-        mv "$mailmanCfgTmp" $mailmanCfg
-      '';
-      serviceConfig = {
-        Type = "oneshot";
-        # RemainAfterExit makes restartIfChanged work for this service, so
-        # downstream services will get updated automatically when things like
-        # services.mailman.hyperkitty.baseUrl change.  Otherwise users have to
-        # restart things manually, which is confusing.
-        RemainAfterExit = "yes";
       };
-    };
 
-    systemd.services.mailman-web = {
+      mailman-settings = {
-      description = "Init Postorius DB";
+        description = "Generate settings files (including secrets) for Mailman";
-      before = [ "httpd.service" "uwsgi.service" ];
+        before = [ "mailman.service" "mailman-web-setup.service" "mailman-uwsgi.service" "hyperkitty.service" ];
-      requiredBy = [ "httpd.service" "uwsgi.service" ];
+        requiredBy = [ "mailman.service" "mailman-web-setup.service" "mailman-uwsgi.service" "hyperkitty.service" ];
-      restartTriggers = [ config.environment.etc."mailman3/settings.py".source ];
+        path = with pkgs; [ jq ];
-      script = ''
+        script = ''
-        ${pkgs.mailman-web}/bin/mailman-web migrate
+          mailmanDir=/var/lib/mailman
-        rm -rf static
+          mailmanWebDir=/var/lib/mailman-web
-        ${pkgs.mailman-web}/bin/mailman-web collectstatic
+
-        ${pkgs.mailman-web}/bin/mailman-web compress
+          mailmanCfg=$mailmanDir/mailman-hyperkitty.cfg
-      '';
+          mailmanWebCfg=$mailmanWebDir/settings_local.json
-      serviceConfig = {
+
-        User = cfg.webUser;
+          install -m 0775 -o mailman -g mailman -d /var/lib/mailman-web-static
-        Type = "oneshot";
+          install -m 0770 -o mailman -g mailman -d $mailmanDir
-        # Similar to mailman-settings.service, this makes restartTriggers work
+          install -m 0770 -o ${cfg.webUser} -g mailman -d $mailmanWebDir
-        # properly for this service.
+
-        RemainAfterExit = "yes";
+          if [ ! -e $mailmanWebCfg ]; then
-        WorkingDirectory = "/var/lib/mailman-web";
+              hyperkittyApiKey=$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 64)
+              secretKey=$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 64)
+
+              mailmanWebCfgTmp=$(mktemp)
+              jq -n '.MAILMAN_ARCHIVER_KEY=$archiver_key | .SECRET_KEY=$secret_key' \
+                  --arg archiver_key "$hyperkittyApiKey" \
+                  --arg secret_key "$secretKey" \
+                  >"$mailmanWebCfgTmp"
+              chown root:mailman "$mailmanWebCfgTmp"
+              chmod 440 "$mailmanWebCfgTmp"
+              mv -n "$mailmanWebCfgTmp" "$mailmanWebCfg"
+          fi
+
+          hyperkittyApiKey="$(jq -r .MAILMAN_ARCHIVER_KEY "$mailmanWebCfg")"
+          mailmanCfgTmp=$(mktemp)
+          sed "s/@API_KEY@/$hyperkittyApiKey/g" ${mailmanHyperkittyCfg} >"$mailmanCfgTmp"
+          chown mailman:mailman "$mailmanCfgTmp"
+          mv "$mailmanCfgTmp" "$mailmanCfg"
+        '';
       };
-    };
 
-    systemd.services.mailman-daily = {
+      mailman-web-setup = {
-      description = "Trigger daily Mailman events";
+        description = "Prepare mailman-web files and database";
-      startAt = "daily";
+        before = [ "uwsgi.service" "mailman-uwsgi.service" ];
-      restartTriggers = [ config.environment.etc."mailman.cfg".source ];
+        requiredBy = [ "mailman-uwsgi.service" ];
-      serviceConfig = {
+        restartTriggers = [ config.environment.etc."mailman3/settings.py".source ];
-        ExecStart = "${cfg.package}/bin/mailman digests --send";
+        script = ''
-        User = "mailman";
+          find "${settings.STATIC_ROOT}/" -mindepth 1 -delete
+          ${pythonEnv}/bin/mailman-web migrate
+          ${pythonEnv}/bin/mailman-web collectstatic
+          ${pythonEnv}/bin/mailman-web compress
+        '';
+        serviceConfig = {
+          User = cfg.webUser;
+          Group = "mailman";
+          Type = "oneshot";
+          WorkingDirectory = "/var/lib/mailman-web";
+        };
       };
-    };
 
-    systemd.services.hyperkitty = {
+      mailman-uwsgi = mkIf cfg.serve.enable (let
-      inherit (cfg.hyperkitty) enable;
+        uwsgiConfig.uwsgi = {
-      description = "GNU Hyperkitty QCluster Process";
+          type = "normal";
-      after = [ "network.target" ];
+          plugins = ["python3"];
-      restartTriggers = [ config.environment.etc."mailman3/settings.py".source ];
+          home = pythonEnv;
-      wantedBy = [ "mailman.service" "multi-user.target" ];
+          module = "mailman_web.wsgi";
-      serviceConfig = {
+        };
-        ExecStart = "${pkgs.mailman-web}/bin/mailman-web qcluster";
+        uwsgiConfigFile = pkgs.writeText "uwsgi-mailman.json" (builtins.toJSON uwsgiConfig);
-        User = cfg.webUser;
+      in {
-        WorkingDirectory = "/var/lib/mailman-web";
+        wantedBy = ["multi-user.target"];
+        requires = ["mailman-uwsgi.socket" "mailman-web-setup.service"];
+        restartTriggers = [ config.environment.etc."mailman3/settings.py".source ];
+        serviceConfig = {
+          # Since the mailman-web settings.py obstinately creates a logs
+          # dir in the cwd, change to the (writable) runtime directory before
+          # starting uwsgi.
+          ExecStart = "${pkgs.coreutils}/bin/env -C $RUNTIME_DIRECTORY ${pkgs.uwsgi.override { plugins = ["python3"]; }}/bin/uwsgi --json ${uwsgiConfigFile}";
+          User = cfg.webUser;
+          Group = "mailman";
+          RuntimeDirectory = "mailman-uwsgi";
+        };
+      });
+
+      mailman-daily = {
+        description = "Trigger daily Mailman events";
+        startAt = "daily";
+        restartTriggers = [ config.environment.etc."mailman.cfg".source ];
+        serviceConfig = {
+          ExecStart = "${pythonEnv}/bin/mailman digests --send";
+          User = "mailman";
+          Group = "mailman";
+        };
       };
-    };
 
-    systemd.services.hyperkitty-minutely = {
+      hyperkitty = lib.mkIf cfg.hyperkitty.enable {
-      inherit (cfg.hyperkitty) enable;
+        description = "GNU Hyperkitty QCluster Process";
-      description = "Trigger minutely Hyperkitty events";
+        after = [ "network.target" ];
-      startAt = "minutely";
+        restartTriggers = [ config.environment.etc."mailman3/settings.py".source ];
-      restartTriggers = [ config.environment.etc."mailman3/settings.py".source ];
+        wantedBy = [ "mailman.service" "multi-user.target" ];
-      serviceConfig = {
+        serviceConfig = {
-        ExecStart = "${pkgs.mailman-web}/bin/mailman-web runjobs minutely";
+          ExecStart = "${pythonEnv}/bin/mailman-web qcluster";
-        User = cfg.webUser;
+          User = cfg.webUser;
-        WorkingDirectory = "/var/lib/mailman-web";
+          Group = "mailman";
+          WorkingDirectory = "/var/lib/mailman-web";
+        };
       };
-    };
+    } // flip lib.mapAttrs' {
-
+      "minutely" = "minutely";
-    systemd.services.hyperkitty-quarter-hourly = {
+      "quarter_hourly" = "*:00/15";
-      inherit (cfg.hyperkitty) enable;
+      "hourly" = "hourly";
-      description = "Trigger quarter-hourly Hyperkitty events";
+      "daily" = "daily";
-      startAt = "*:00/15";
+      "weekly" = "weekly";
-      restartTriggers = [ config.environment.etc."mailman3/settings.py".source ];
+      "yearly" = "yearly";
-      serviceConfig = {
+    } (name: startAt:
-        ExecStart = "${pkgs.mailman-web}/bin/mailman-web runjobs quarter_hourly";
+      lib.nameValuePair "hyperkitty-${name}" (lib.mkIf cfg.hyperkitty.enable {
-        User = cfg.webUser;
+        description = "Trigger ${name} Hyperkitty events";
-        WorkingDirectory = "/var/lib/mailman-web";
+        inherit startAt;
-      };
+        restartTriggers = [ config.environment.etc."mailman3/settings.py".source ];
-    };
+        serviceConfig = {
-
+          ExecStart = "${pythonEnv}/bin/mailman-web runjobs minutely";
-    systemd.services.hyperkitty-hourly = {
+          User = cfg.webUser;
-      inherit (cfg.hyperkitty) enable;
+          Group = "mailman";
-      description = "Trigger hourly Hyperkitty events";
+          WorkingDirectory = "/var/lib/mailman-web";
-      startAt = "hourly";
+        };
-      restartTriggers = [ config.environment.etc."mailman3/settings.py".source ];
+      }));
-      serviceConfig = {
-        ExecStart = "${pkgs.mailman-web}/bin/mailman-web runjobs hourly";
-        User = cfg.webUser;
-        WorkingDirectory = "/var/lib/mailman-web";
-      };
-    };
-
-    systemd.services.hyperkitty-daily = {
-      inherit (cfg.hyperkitty) enable;
-      description = "Trigger daily Hyperkitty events";
-      startAt = "daily";
-      restartTriggers = [ config.environment.etc."mailman3/settings.py".source ];
-      serviceConfig = {
-        ExecStart = "${pkgs.mailman-web}/bin/mailman-web runjobs daily";
-        User = cfg.webUser;
-        WorkingDirectory = "/var/lib/mailman-web";
-      };
-    };
-
-    systemd.services.hyperkitty-weekly = {
-      inherit (cfg.hyperkitty) enable;
-      description = "Trigger weekly Hyperkitty events";
-      startAt = "weekly";
-      restartTriggers = [ config.environment.etc."mailman3/settings.py".source ];
-      serviceConfig = {
-        ExecStart = "${pkgs.mailman-web}/bin/mailman-web runjobs weekly";
-        User = cfg.webUser;
-        WorkingDirectory = "/var/lib/mailman-web";
-      };
-    };
-
-    systemd.services.hyperkitty-yearly = {
-      inherit (cfg.hyperkitty) enable;
-      description = "Trigger yearly Hyperkitty events";
-      startAt = "yearly";
-      restartTriggers = [ config.environment.etc."mailman3/settings.py".source ];
-      serviceConfig = {
-        ExecStart = "${pkgs.mailman-web}/bin/mailman-web runjobs yearly";
-        User = cfg.webUser;
-        WorkingDirectory = "/var/lib/mailman-web";
-      };
-    };
-
   };
 
 }