Merge pull request #115005 from rnhmjoj/privoxy

nixos/privoxy: add https and settings options

nixos/doc/manual/release-notes/rl-2105.xml

@@ -39,6 +39,24 @@
      (<link xlink:href="https://github.com/NixOS/nixpkgs/issues/75478">#7547</link>).
     </para>
    </listitem>
+   <listitem>
+    <para>
+     <link xlink:href="https://www.privoxy.org/">Privoxy</link> has been updated
+     to version 3.0.32 (See <link xlink:href="https://lists.privoxy.org/pipermail/privoxy-announce/2021-February/000007.html">announcement</link>).
+     Compared to the previous release, Privoxy has gained support for HTTPS
+     inspection (still experimental), Brotli decompression, several new filters
+     and lots of bug fixes, including security ones. In addition, the package
+     is now built with compression and external filters support, which were
+     previously disabled.
+    </para>
+    <para>
+     Regarding the NixOS module, new options for HTTPS inspection have been added
+     and <option>services.privoxy.extraConfig</option> has been replaced by the new
+     <xref linkend="opt-services.privoxy.settings"/>
+     (See <link xlink:href="https://github.com/NixOS/rfcs/blob/master/rfcs/0042-config-option.md">RFC 0042</link>
+     for the motivation).
+    </para>
+   </listitem>
   </itemizedlist>
  </section>
 

nixos/modules/services/networking/privoxy.nix

@@ -4,26 +4,46 @@ with lib;
 
 let
 
-  inherit (pkgs) privoxy;
-
   cfg = config.services.privoxy;
 
-  confFile = pkgs.writeText "privoxy.conf" (''
-    user-manual ${privoxy}/share/doc/privoxy/user-manual
-    confdir ${privoxy}/etc/
-    listen-address  ${cfg.listenAddress}
-    enable-edit-actions ${if (cfg.enableEditActions == true) then "1" else "0"}
-    ${concatMapStrings (f: "actionsfile ${f}\n") cfg.actionsFiles}
-    ${concatMapStrings (f: "filterfile ${f}\n") cfg.filterFiles}
-  '' + optionalString cfg.enableTor ''
-    forward-socks5t / 127.0.0.1:9063 .
-    toggle 1
-    enable-remote-toggle 0
-    enable-edit-actions 0
-    enable-remote-http-toggle 0
-  '' + ''
-    ${cfg.extraConfig}
-  '');
+  serialise = name: val:
+         if isList val then concatMapStrings (serialise name) val
+    else if isBool val then serialise name (if val then "1" else "0")
+    else "${name} ${toString val}\n";
+
+  configType = with types;
+    let atom = oneOf [ int bool string path ];
+    in attrsOf (either atom (listOf atom))
+    // { description = ''
+          privoxy configuration type. The format consists of an attribute
+          set of settings. Each setting can be either a value (integer, string,
+          boolean or path) or a list of such values.
+        '';
+       };
+
+  ageType = types.str // {
+    check = x:
+      isString x &&
+      (builtins.match "([0-9]+([smhdw]|min|ms|us)*)+" x != null);
+    description = "tmpfiles.d(5) age format";
+  };
+
+  configFile = pkgs.writeText "privoxy.conf"
+    (concatStrings (
+      # Relative paths in some options are relative to confdir. Privoxy seems
+      # to parse the options in order of appearance, so this must come first.
+      # Nix however doesn't preserve the order in attrsets, so we have to
+      # hardcode confdir here.
+      [ "confdir ${pkgs.privoxy}/etc\n" ]
+      ++ mapAttrsToList serialise cfg.settings
+    ));
+
+  inspectAction = pkgs.writeText "inspect-all-https.action"
+    ''
+      # Enable HTTPS inspection for all requests
+      {+https-inspection}
+      /
+    '';
 
 in
 
@@ -31,70 +51,130 @@ in
 
   ###### interface
 
-  options = {
+  options.services.privoxy = {
 
-    services.privoxy = {
+    enable = mkEnableOption "Privoxy, non-caching filtering proxy";
 
-      enable = mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Whether to enable the Privoxy non-caching filtering proxy.
-        '';
-      };
-
-      listenAddress = mkOption {
-        type = types.str;
-        default = "127.0.0.1:8118";
-        description = ''
-          Address the proxy server is listening to.
-        '';
-      };
-
-      actionsFiles = mkOption {
-        type = types.listOf types.str;
-        example = [ "match-all.action" "default.action" "/etc/privoxy/user.action" ];
-        default = [ "match-all.action" "default.action" ];
-        description = ''
-          List of paths to Privoxy action files.
-          These paths may either be absolute or relative to the privoxy configuration directory.
-        '';
-      };
-
-      filterFiles = mkOption {
-        type = types.listOf types.str;
-        example = [ "default.filter" "/etc/privoxy/user.filter" ];
-        default = [ "default.filter" ];
-        description = ''
-          List of paths to Privoxy filter files.
-          These paths may either be absolute or relative to the privoxy configuration directory.
-        '';
-      };
-
-      enableEditActions = mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Whether or not the web-based actions file editor may be used.
-        '';
-      };
-
-      enableTor = mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Whether to configure Privoxy to use Tor's faster SOCKS port,
-          suitable for HTTP.
-        '';
-      };
-
-      extraConfig = mkOption {
-        type = types.lines;
-        default = "" ;
-        description = ''
-          Extra configuration. Contents will be added verbatim to the configuration file.
-        '';
+    enableTor = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        Whether to configure Privoxy to use Tor's faster SOCKS port,
+        suitable for HTTP.
+      '';
+    };
+
+    inspectHttps = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        Whether to configure Privoxy to inspect HTTPS requests, meaning all
+        encrypted traffic will be filtered as well. This works by decrypting
+        and re-encrypting the requests using a per-domain generated certificate.
+
+        To issue per-domain certificates, Privoxy must be provided with a CA
+        certificate, using the <literal>ca-cert-file</literal>,
+        <literal>ca-key-file</literal> settings.
+
+        <warning><para>
+          The CA certificate must also be added to the system trust roots,
+          otherwise browsers will reject all Privoxy certificates as invalid.
+          You can do so by using the option
+          <option>security.pki.certificateFiles</option>.
+        </para></warning>
+      '';
+    };
+
+    certsLifetime = mkOption {
+      type = ageType;
+      default = "10d";
+      example = "12h";
+      description = ''
+        If <literal>inspectHttps</literal> is enabled, the time generated HTTPS
+        certificates will be stored in a temporary directory for reuse. Once
+        the lifetime has expired the directory will cleared and the certificate
+        will have to be generated again, on-demand.
+
+        Depending on the traffic, you may want to reduce the lifetime to limit
+        the disk usage, since Privoxy itself never deletes the certificates.
+
+        <note><para>The format is that of the <literal>tmpfiles.d(5)</literal>
+        Age parameter.</para></note>
+      '';
+    };
+
+    userActions = mkOption {
+      type = types.lines;
+      default = "";
+      description = ''
+        Actions to be included in a <literal>user.action</literal> file. This
+        will have a higher priority and can be used to override all other
+        actions.
+      '';
+    };
+
+    userFilters = mkOption {
+      type = types.lines;
+      default = "";
+      description = ''
+        Filters to be included in a <literal>user.filter</literal> file. This
+        will have a higher priority and can be used to override all other
+        filters definitions.
+      '';
+    };
+
+    settings = mkOption {
+      type = types.submodule {
+        freeformType = configType;
+
+        options.listen-address = mkOption {
+          type = types.str;
+          default = "127.0.0.1:8118";
+          description = "Pair of address:port the proxy server is listening to.";
+        };
+
+        options.enable-edit-actions = mkOption {
+          type = types.bool;
+          default = false;
+          description = "Whether the web-based actions file editor may be used.";
+        };
+
+        options.actionsfile = mkOption {
+          type = types.listOf types.str;
+          # This must come after all other entries, in order to override the
+          # other actions/filters installed by Privoxy or the user.
+          apply = x: x ++ optional (cfg.userActions != "")
+            (toString (pkgs.writeText "user.actions" cfg.userActions));
+          default = [ "match-all.action" "default.action" ];
+          description = ''
+            List of paths to Privoxy action files. These paths may either be
+            absolute or relative to the privoxy configuration directory.
+          '';
+        };
+
+        options.filterfile = mkOption {
+          type = types.listOf types.str;
+          default = [ "default.filter" ];
+          apply = x: x ++ optional (cfg.userFilters != "")
+            (toString (pkgs.writeText "user.filter" cfg.userFilters));
+          description = ''
+            List of paths to Privoxy filter files. These paths may either be
+            absolute or relative to the privoxy configuration directory.
+          '';
+        };
       };
+      default = {};
+      example = literalExample ''
+        { listen-address = "[::]:8118";               # listen on IPv6 only
+          forward-socks5 = ".onion localhost:9050 ."; # forward .onion requests to Tor
+        }
+      '';
+      description = ''
+        This option is mapped to the main Privoxy configuration file.
+        Check out the Privoxy user manual at
+        <link xlink:href="file://${pkgs.privoxy}/share/doc/privoxy/user-manual/config.html"/>
+        for available settings and documentation.
+      '';
     };
 
   };
@@ -104,23 +184,34 @@ in
   config = mkIf cfg.enable {
 
     users.users.privoxy = {
+      description = "Privoxy daemon user";
       isSystemUser = true;
-      home = "/var/empty";
       group = "privoxy";
     };
 
     users.groups.privoxy = {};
 
+    systemd.tmpfiles.rules = with cfg.settings; [
+      "d ${certificate-directory} 0770 privoxy privoxy ${cfg.certsLifetime}"
+    ];
+
     systemd.services.privoxy = {
       description = "Filtering web proxy";
       after = [ "network.target" "nss-lookup.target" ];
       wantedBy = [ "multi-user.target" ];
-      serviceConfig.ExecStart = "${privoxy}/bin/privoxy --no-daemon --user privoxy ${confFile}";
-
-      serviceConfig.PrivateDevices = true;
-      serviceConfig.PrivateTmp = true;
-      serviceConfig.ProtectHome = true;
-      serviceConfig.ProtectSystem = "full";
+      serviceConfig = {
+        User = "privoxy";
+        Group = "privoxy";
+        ExecStart = "${pkgs.privoxy}/bin/privoxy --no-daemon ${configFile}";
+        PrivateDevices = true;
+        PrivateTmp = true;
+        ProtectHome = true;
+        ProtectSystem = "full";
+      };
+      unitConfig =  mkIf cfg.inspectHttps {
+        ConditionPathExists = with cfg.settings;
+          [ ca-cert-file ca-key-file ];
+      };
     };
 
     services.tor.settings.SOCKSPort = mkIf cfg.enableTor [
@@ -128,8 +219,48 @@ in
       { addr = "127.0.0.1"; port = 9063; IsolateDestAddr = false; }
     ];
 
+    services.privoxy.settings = {
+      user-manual = "${pkgs.privoxy}/share/doc/privoxy/user-manual";
+      # This is needed for external filters
+      temporary-directory = "/tmp";
+      filterfile = [ "default.filter" ];
+      actionsfile =
+        [ "match-all.action"
+          "default.action"
+        ] ++ optional cfg.inspectHttps (toString inspectAction);
+    } // (optionalAttrs cfg.enableTor {
+      forward-socks5 = "127.0.0.1:9063 .";
+      toggle = true;
+      enable-remote-toggle = false;
+      enable-edit-actions = false;
+      enable-remote-http-toggle = false;
+    }) // (optionalAttrs cfg.inspectHttps {
+      # This allows setting absolute key/crt paths
+      ca-directory = "/var/empty";
+      certificate-directory = "/run/privoxy/certs";
+      trusted-cas-file = "/etc/ssl/certs/ca-certificates.crt";
+    });
+
   };
 
+  imports =
+    let
+      top = x: [ "services" "privoxy" x ];
+      setting = x: [ "services" "privoxy" "settings" x ];
+    in
+    [ (mkRenamedOptionModule (top "enableEditActions") (setting "enable-edit-actions"))
+      (mkRenamedOptionModule (top "listenAddress") (setting "listen-address"))
+      (mkRenamedOptionModule (top "actionsFiles") (setting "actionsfile"))
+      (mkRenamedOptionModule (top "filterFiles") (setting "filterfile"))
+      (mkRemovedOptionModule (top "extraConfig")
+      ''
+        Use services.privoxy.settings instead.
+        This is part of the general move to use structured settings instead of raw
+        text for config as introduced by RFC0042:
+        https://github.com/NixOS/rfcs/blob/master/rfcs/0042-config-option.md
+      '')
+    ];
+
   meta.maintainers = with lib.maintainers; [ rnhmjoj ];
 
 }

nixos/tests/all-tests.nix

@@ -326,6 +326,7 @@ in
   predictable-interface-names = handleTest ./predictable-interface-names.nix {};
   printing = handleTest ./printing.nix {};
   privacyidea = handleTest ./privacyidea.nix {};
+  privoxy = handleTest ./privoxy.nix {};
   prometheus = handleTest ./prometheus.nix {};
   prometheus-exporters = handleTest ./prometheus-exporters.nix {};
   prosody = handleTest ./xmpp/prosody.nix {};

nixos/tests/privoxy.nix

@@ -0,0 +1,113 @@
+import ./make-test-python.nix ({ lib, pkgs, ... }:
+
+let
+  # Note: For some reason Privoxy can't issue valid
+  # certificates if the CA is generated using gnutls :(
+  certs = pkgs.runCommand "example-certs"
+    { buildInputs = [ pkgs.openssl ]; }
+    ''
+      mkdir $out
+
+      # generate CA keypair
+      openssl req -new -nodes -x509 \
+        -extensions v3_ca -keyout $out/ca.key \
+        -out $out/ca.crt -days 365 \
+        -subj "/O=Privoxy CA/CN=Privoxy CA"
+
+      # generate server key/signing request
+      openssl genrsa -out $out/server.key 3072
+      openssl req -new -key $out/server.key \
+        -out server.csr -sha256 \
+        -subj "/O=An unhappy server./CN=example.com"
+
+      # sign the request/generate the certificate
+      openssl x509 -req -in server.csr -CA $out/ca.crt \
+      -CAkey $out/ca.key -CAcreateserial -out $out/server.crt \
+      -days 500 -sha256
+    '';
+in
+
+{
+  name = "privoxy";
+  meta = with lib.maintainers; {
+    maintainers = [ rnhmjoj ];
+  };
+
+  machine = { ... }: {
+    services.nginx.enable = true;
+    services.nginx.virtualHosts."example.com" = {
+      addSSL = true;
+      sslCertificate = "${certs}/server.crt";
+      sslCertificateKey = "${certs}/server.key";
+      locations."/".root = pkgs.writeTextFile
+        { name = "bad-day";
+          destination = "/how-are-you/index.html";
+          text = "I've had a bad day!\n";
+        };
+      locations."/ads".extraConfig = ''
+        return 200 "Hot Nixpkgs PRs in your area. Click here!\n";
+      '';
+    };
+
+    services.privoxy = {
+      enable = true;
+      inspectHttps = true;
+      settings = {
+        ca-cert-file = "${certs}/ca.crt";
+        ca-key-file  = "${certs}/ca.key";
+        debug = 65536;
+      };
+      userActions = ''
+        {+filter{positive}}
+        example.com
+
+        {+block{Fake ads}}
+        example.com/ads
+      '';
+      userFilters = ''
+        FILTER: positive This is a filter example.
+        s/bad/great/ig
+      '';
+    };
+
+    security.pki.certificateFiles = [ "${certs}/ca.crt" ];
+
+    networking.hosts."::1" = [ "example.com" ];
+    networking.proxy.httpProxy = "http://localhost:8118";
+    networking.proxy.httpsProxy = "http://localhost:8118";
+  };
+
+  testScript =
+    ''
+      with subtest("Privoxy is running"):
+          machine.wait_for_unit("privoxy")
+          machine.wait_for_open_port("8118")
+          machine.succeed("curl -f http://config.privoxy.org")
+
+      with subtest("Privoxy can filter http requests"):
+          machine.wait_for_open_port("80")
+          assert "great day" in machine.succeed(
+              "curl -sfL http://example.com/how-are-you? | tee /dev/stderr"
+          )
+
+      with subtest("Privoxy can filter https requests"):
+          machine.wait_for_open_port("443")
+          assert "great day" in machine.succeed(
+              "curl -sfL https://example.com/how-are-you? | tee /dev/stderr"
+          )
+
+      with subtest("Blocks are working"):
+          machine.wait_for_open_port("443")
+          machine.fail("curl -f https://example.com/ads 1>&2")
+          machine.succeed("curl -f https://example.com/PRIVOXY-FORCE/ads 1>&2")
+
+      with subtest("Temporary certificates are cleaned"):
+          # Count current certificates
+          machine.succeed("test $(ls /run/privoxy/certs | wc -l) -gt 0")
+          # Forward in time 12 days, trigger the timer..
+          machine.succeed("date -s \"$(date --date '12 days')\"")
+          machine.systemctl("start systemd-tmpfiles-clean")
+          # ...and count again
+          machine.succeed("test $(ls /run/privoxy/certs | wc -l) -eq 0")
+    '';
+})

pkgs/tools/networking/privoxy/default.nix

@@ -1,4 +1,9 @@
-{ lib, stdenv, fetchurl, autoreconfHook, zlib, pcre, w3m, man }:
+{ lib, stdenv
+, nixosTests
+, fetchurl, autoreconfHook
+, zlib, pcre, w3m, man
+, mbedtls, brotli
+}:
 
 stdenv.mkDerivation rec {
 
@@ -13,18 +18,28 @@ stdenv.mkDerivation rec {
   hardeningEnable = [ "pie" ];
 
   nativeBuildInputs = [ autoreconfHook w3m man ];
-  buildInputs = [ zlib pcre ];
+  buildInputs = [ zlib pcre mbedtls brotli ];
 
-  makeFlags = [ "STRIP="];
+  makeFlags = [ "STRIP=" ];
+  configureFlags = [
+    "--with-mbedtls"
+    "--with-brotli"
+    "--enable-external-filters"
+    "--enable-compression"
+  ];
 
   postInstall = ''
-    rm -rf $out/var
+    rm -r $out/var
   '';
 
+  passthru.tests.privoxy = nixosTests.privoxy;
+
   meta = with lib; {
     homepage = "https://www.privoxy.org/";
     description = "Non-caching web proxy with advanced filtering capabilities";
-    license = licenses.gpl2Plus;
+    # When linked with mbedtls, the license becomes GPLv3 (or later), otherwise
+    # GPLv2 (or later). See https://www.privoxy.org/user-manual/copyright.html
+    license = licenses.gpl3Plus;
     platforms = platforms.all;
     maintainers = [ maintainers.phreedom ];
   };