Merge pull request #109393 from attila-lendvai/attila-bee.master

maintainers/maintainer-list.nix

@@ -868,6 +868,12 @@
     githubId = 706854;
     name = "Etienne Laurin";
   };
+  attila-lendvai = {
+    name = "Attila Lendvai";
+    email = "attila@lendvai.name";
+    github = "attila-lendvai";
+    githubId = 840345;
+  };
   auntie = {
     email = "auntieNeo@gmail.com";
     github = "auntieNeo";

nixos/modules/module-list.nix

@@ -609,6 +609,8 @@
   ./services/networking/atftpd.nix
   ./services/networking/avahi-daemon.nix
   ./services/networking/babeld.nix
+  ./services/networking/bee.nix
+  ./services/networking/bee-clef.nix
   ./services/networking/biboumi.nix
   ./services/networking/bind.nix
   ./services/networking/bitcoind.nix

nixos/modules/services/networking/bee-clef.nix

@@ -0,0 +1,107 @@
+{ config, lib, pkgs, ... }:
+
+# NOTE for now nothing is installed into /etc/bee-clef/. the config files are used as read-only from the nix store.
+
+with lib;
+let
+  cfg = config.services.bee-clef;
+in {
+  meta = {
+    maintainers = with maintainers; [ attila-lendvai ];
+  };
+
+  ### interface
+
+  options = {
+    services.bee-clef = {
+      enable = mkEnableOption "clef external signer instance for Ethereum Swarm Bee";
+
+      dataDir = mkOption {
+        type = types.nullOr types.str;
+        default = "/var/lib/bee-clef";
+        description = ''
+          Data dir for bee-clef. Beware that some helper scripts may not work when changed!
+          The service itself should work fine, though.
+        '';
+      };
+
+      passwordFile = mkOption {
+        type = types.nullOr types.str;
+        default = "/var/lib/bee-clef/password";
+        description = "Password file for bee-clef.";
+      };
+
+      user = mkOption {
+        type = types.str;
+        default = "bee-clef";
+        description = ''
+          User the bee-clef daemon should execute under.
+        '';
+      };
+
+      group = mkOption {
+        type = types.str;
+        default = "bee-clef";
+        description = ''
+          Group the bee-clef daemon should execute under.
+        '';
+      };
+    };
+  };
+
+  ### implementation
+
+  config = mkIf cfg.enable {
+    # if we ever want to have rules.js under /etc/bee-clef/
+    # environment.etc."bee-clef/rules.js".source = ${pkgs.bee-clef}/rules.js
+
+    systemd.packages = [ pkgs.bee-clef ]; # include the upstream bee-clef.service file
+
+    systemd.tmpfiles.rules = [
+        "d '${cfg.dataDir}/'         0750 ${cfg.user} ${cfg.group}"
+        "d '${cfg.dataDir}/keystore' 0700 ${cfg.user} ${cfg.group}"
+      ];
+
+    systemd.services.bee-clef = {
+      path = [
+        # these are needed for the ensure-clef-account script
+        pkgs.coreutils
+        pkgs.gnused
+        pkgs.gawk
+      ];
+
+      wantedBy = [ "bee.service" "multi-user.target" ];
+
+      serviceConfig = {
+        User = cfg.user;
+        Group = cfg.group;
+        ExecStartPre = ''${pkgs.bee-clef}/share/bee-clef/ensure-clef-account "${cfg.dataDir}" "${pkgs.bee-clef}/share/bee-clef/"'';
+        ExecStart = [
+          "" # this hides/overrides what's in the original entry
+          "${pkgs.bee-clef}/share/bee-clef/bee-clef-service start"
+        ];
+        ExecStop = [
+          "" # this hides/overrides what's in the original entry
+          "${pkgs.bee-clef}/share/bee-clef/bee-clef-service stop"
+        ];
+        Environment = [
+          "CONFIGDIR=${cfg.dataDir}"
+          "PASSWORD_FILE=${cfg.passwordFile}"
+        ];
+      };
+    };
+
+    users.users = optionalAttrs (cfg.user == "bee-clef") {
+      bee-clef = {
+        group = cfg.group;
+        home = cfg.dataDir;
+        isSystemUser = true;
+        description = "Daemon user for the bee-clef service";
+      };
+    };
+
+    users.groups = optionalAttrs (cfg.group == "bee-clef") {
+      bee-clef = {};
+    };
+  };
+}

nixos/modules/services/networking/bee.nix

@@ -0,0 +1,149 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+let
+  cfg = config.services.bee;
+  format = pkgs.formats.yaml {};
+  configFile = format.generate "bee.yaml" cfg.settings;
+in {
+  meta = {
+    # doc = ./bee.xml;
+    maintainers = with maintainers; [ attila-lendvai ];
+  };
+
+  ### interface
+
+  options = {
+    services.bee = {
+      enable = mkEnableOption "Ethereum Swarm Bee";
+
+      package = mkOption {
+        type = types.package;
+        default = pkgs.bee;
+        defaultText = "pkgs.bee";
+        example = "pkgs.bee-unstable";
+        description = "The package providing the bee binary for the service.";
+      };
+
+      settings = mkOption {
+        type = format.type;
+        description = ''
+          Ethereum Swarm Bee configuration. Refer to
+          <link xlink:href="https://gateway.ethswarm.org/bzz/docs.swarm.eth/docs/installation/configuration/"/>
+          for details on supported values.
+        '';
+      };
+
+      daemonNiceLevel = mkOption {
+        type = types.int;
+        default = 0;
+        description = ''
+          Daemon process priority for bee.
+          0 is the default Unix process priority, 19 is the lowest.
+        '';
+      };
+
+      user = mkOption {
+        type = types.str;
+        default = "bee";
+        description = ''
+          User the bee binary should execute under.
+        '';
+      };
+
+      group = mkOption {
+        type = types.str;
+        default = "bee";
+        description = ''
+          Group the bee binary should execute under.
+        '';
+      };
+    };
+  };
+
+  ### implementation
+
+  config = mkIf cfg.enable {
+    assertions = [
+      { assertion = (hasAttr "password" cfg.settings) != true;
+        message = ''
+          `services.bee.settings.password` is insecure. Use `services.bee.settings.password-file` or `systemd.services.bee.serviceConfig.EnvironmentFile` instead.
+        '';
+      }
+      { assertion = (hasAttr "swap-endpoint" cfg.settings) || (cfg.settings.swap-enable or true == false);
+        message = ''
+          In a swap-enabled network a working Ethereum blockchain node is required. You must specify one using `services.bee.settings.swap-endpoint`, or disable `services.bee.settings.swap-enable` = false.
+        '';
+      }
+    ];
+
+    warnings = optional (! config.services.bee-clef.enable) "The bee service requires an external signer. Consider setting `config.services.bee-clef.enable` = true";
+
+    services.bee.settings = {
+      data-dir             = lib.mkDefault "/var/lib/bee";
+      password-file        = lib.mkDefault "/var/lib/bee/password";
+      clef-signer-enable   = lib.mkDefault true;
+      clef-signer-endpoint = lib.mkDefault "/var/lib/bee-clef/clef.ipc";
+      swap-endpoint        = lib.mkDefault "https://rpc.slock.it/goerli";
+    };
+
+    systemd.packages = [ cfg.package ]; # include the upstream bee.service file
+
+    systemd.tmpfiles.rules = [
+      "d '${cfg.settings.data-dir}' 0750 ${cfg.user} ${cfg.group}"
+    ];
+
+    systemd.services.bee = {
+      requires = optional config.services.bee-clef.enable
+        "bee-clef.service";
+
+      wantedBy = [ "multi-user.target" ];
+
+      serviceConfig = {
+        Nice = cfg.daemonNiceLevel;
+        User = cfg.user;
+        Group = cfg.group;
+        ExecStart = [
+          "" # this hides/overrides what's in the original entry
+          "${cfg.package}/bin/bee --config=${configFile} start"
+        ];
+      };
+
+      preStart = with cfg.settings; ''
+        if ! test -f ${password-file}; then
+          < /dev/urandom tr -dc _A-Z-a-z-0-9 2> /dev/null | head -c32 > ${password-file}
+          chmod 0600 ${password-file}
+          echo "Initialized ${password-file} from /dev/urandom"
+        fi
+        if [ ! -f ${data-dir}/keys/libp2p.key ]; then
+          ${cfg.package}/bin/bee init --config=${configFile} >/dev/null
+          echo "
+Logs:   journalctl -f -u bee.service
+
+Bee has SWAP enabled by default and it needs ethereum endpoint to operate.
+It is recommended to use external signer with bee.
+Check documentation for more info:
+- SWAP https://docs.ethswarm.org/docs/installation/manual#swap-bandwidth-incentives
+- External signer https://docs.ethswarm.org/docs/installation/bee-clef
+
+After you finish configuration run 'sudo bee-get-addr'."
+        fi
+      '';
+    };
+
+    users.users = optionalAttrs (cfg.user == "bee") {
+      bee = {
+        group = cfg.group;
+        home = cfg.settings.data-dir;
+        isSystemUser = true;
+        description = "Daemon user for Ethereum Swarm Bee";
+        extraGroups = optional config.services.bee-clef.enable
+          config.services.bee-clef.group;
+      };
+    };
+
+    users.groups = optionalAttrs (cfg.group == "bee") {
+      bee = {};
+    };
+  };
+}

pkgs/applications/networking/bee/0001-clef-service-accept-default-CONFIGDIR-from-the-envir.patch

@@ -0,0 +1,44 @@
+From 04933c578f51aa1f536991318dc5aede57f81c0d Mon Sep 17 00:00:00 2001
+From: Attila Lendvai <attila@lendvai.name>
+Date: Sat, 30 Jan 2021 14:02:02 +0100
+Subject: [PATCH 1/2] clef-service: accept default CONFIGDIR from the
+ environment
+
+---
+ packaging/bee-clef-service | 15 ++++++++++-----
+ 1 file changed, 10 insertions(+), 5 deletions(-)
+
+diff --git a/packaging/bee-clef-service b/packaging/bee-clef-service
+index 10bcd92..34c7edd 100755
+--- a/packaging/bee-clef-service
++++ b/packaging/bee-clef-service
+@@ -1,16 +1,21 @@
+ #!/usr/bin/env sh
+ 
+ start() {
+-    KEYSTORE=/var/lib/bee-clef/keystore
+-    CONFIGDIR=/var/lib/bee-clef
++    if [ -z "$CONFIGDIR" ]; then
++        CONFIGDIR=/var/lib/bee-clef
++    fi
++    if [ -z "$PASSWORD_FILE" ]; then
++        PASSWORD_FILE=${CONFIGDIR}/password
++    fi
++    KEYSTORE=${CONFIGDIR}/keystore
++    SECRET=$(cat ${PASSWORD_FILE})
+     CHAINID=5
+-    SECRET=$(cat /var/lib/bee-clef/password)
+     # clef with every start sets permissions back to 600
+-    (sleep 4; chmod 660 /var/lib/bee-clef/clef.ipc) &
++    (sleep 4; chmod 660 ${CONFIGDIR}/clef.ipc) &
+     ( sleep 2; cat << EOF
+ { "jsonrpc": "2.0", "id":1, "result": { "text":"$SECRET" } }
+ EOF
+-) | clef --stdio-ui --keystore $KEYSTORE --configdir $CONFIGDIR --chainid $CHAINID --rules /etc/bee-clef/rules.js --nousb --4bytedb-custom /etc/bee-clef/4byte.json --pcscdpath "" --auditlog "" --loglevel 3 --ipcpath /var/lib/bee-clef
++) | clef --stdio-ui --keystore $KEYSTORE --configdir $CONFIGDIR --chainid $CHAINID --rules /etc/bee-clef/rules.js --nousb --4bytedb-custom /etc/bee-clef/4byte.json --pcscdpath "" --auditlog "" --loglevel 3 --ipcpath ${CONFIGDIR}
+ }
+ 
+ stop() {
+-- 
+2.29.2
+

pkgs/applications/networking/bee/0002-nix-diff-for-substituteAll.patch

@@ -0,0 +1,25 @@
+From 1a1ab986245e8b74648a1a0adb5d1c7019561d18 Mon Sep 17 00:00:00 2001
+From: Attila Lendvai <attila@lendvai.name>
+Date: Sat, 30 Jan 2021 15:24:57 +0100
+Subject: [PATCH 2/2] nix diff for substituteAll
+
+---
+ packaging/bee-clef-service | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/packaging/bee-clef-service b/packaging/bee-clef-service
+index 34c7edd..31e9d95 100755
+--- a/packaging/bee-clef-service
++++ b/packaging/bee-clef-service
+@@ -15,7 +15,7 @@ start() {
+     ( sleep 2; cat << EOF
+ { "jsonrpc": "2.0", "id":1, "result": { "text":"$SECRET" } }
+ EOF
+-) | clef --stdio-ui --keystore $KEYSTORE --configdir $CONFIGDIR --chainid $CHAINID --rules /etc/bee-clef/rules.js --nousb --4bytedb-custom /etc/bee-clef/4byte.json --pcscdpath "" --auditlog "" --loglevel 3 --ipcpath ${CONFIGDIR}
++) | @clefBinary@ --stdio-ui --keystore $KEYSTORE --configdir $CONFIGDIR --chainid $CHAINID --rules @out@/share/bee-clef/rules.js --nousb --4bytedb-custom @out@/share/bee-clef/4byte.json --pcscdpath "" --auditlog "" --loglevel 3 --ipcpath ${CONFIGDIR}
+ }
+ 
+ stop() {
+-- 
+2.29.2
+

pkgs/applications/networking/bee/bee-clef.nix

@@ -0,0 +1,57 @@
+{ version ? "release", stdenv, lib, substituteAll, fetchFromGitHub, go-ethereum }:
+
+stdenv.mkDerivation rec {
+  pname = "bee-clef";
+  version = "0.4.7";
+
+  src = fetchFromGitHub {
+    owner = "ethersphere";
+    repo = "bee-clef";
+    rev = "refs/tags/v${version}";
+    sha256 = "1sfwql0kvnir8b9ggpqcyc0ar995gxgfbhqb1xpfzp6wl0g3g4zz";
+  };
+
+  buildInputs = [ go-ethereum ];
+
+  clefBinary = "${go-ethereum}/bin/clef";
+
+  patches = [
+    ./0001-clef-service-accept-default-CONFIGDIR-from-the-envir.patch
+    ./0002-nix-diff-for-substituteAll.patch
+  ];
+
+  dontBuild = true;
+
+  installPhase = ''
+    mkdir -p $out/bin/
+    mkdir -p $out/share/bee-clef/
+    mkdir -p $out/lib/systemd/system/
+    cp packaging/bee-clef.service $out/lib/systemd/system/
+    substituteAll packaging/bee-clef-service $out/share/bee-clef/bee-clef-service
+    substituteAll ${./ensure-clef-account} $out/share/bee-clef/ensure-clef-account
+    substituteAll packaging/bee-clef-keys $out/bin/bee-clef-keys
+    cp packaging/rules.js packaging/4byte.json $out/share/bee-clef/
+    chmod +x $out/bin/bee-clef-keys
+    chmod +x $out/share/bee-clef/bee-clef-service
+    chmod +x $out/share/bee-clef/ensure-clef-account
+    patchShebangs $out/
+  '';
+
+  meta = with lib; {
+    # homepage = "https://gateway.ethswarm.org/bzz/docs.swarm.eth/docs/installation/bee-clef/";
+    homepage = "https://docs.ethswarm.org/docs/installation/bee-clef";
+    description = "External signer for Ethereum Swarm Bee";
+    longDescription = ''
+      clef is go-ethereum's external signer.
+
+      bee-clef is a package that starts up a vanilla clef instance as a systemd service,
+      but configured in such a way that is suitable for bee (relaxed security for
+      automated operation).
+
+      This package contains the files necessary to run the bee-clef service.
+    '';
+    license = with licenses; [ bsd3 ];
+    maintainers = with maintainers; [ attila-lendvai ];
+    platforms = go-ethereum.meta.platforms;
+  };
+}

pkgs/applications/networking/bee/bee.nix

@@ -0,0 +1,77 @@
+{ version ? "release", stdenv, lib, fetchFromGitHub, buildGoModule, coreutils }:
+
+let
+
+  versionSpec = rec {
+    unstable = rec {
+      pname = "bee-unstable";
+      version = "2021-01-30";
+      rev = "824636a2c2629c329ab10275cef6a0b7395343ad";
+      goVersionString = "g" + builtins.substring 0 7 rev;     # this seems to be some kind of standard of git describe...
+      sha256 = "0ly1yqjq29arbak8lchdradf39l5bmxpbfir6ljjc7nyqdxz0sxg";
+      vendorSha256 = "0w1db7xpissdpf8i5bb96z92zbasj5x9kk3kcisxn0dwla6n55n3";
+    };
+    release = rec {
+      pname = "bee";
+      version = "0.4.2";
+      rev = "refs/tags/v${version}";
+      sha256 = "1jg7aivsgdb9bm87dlmwpf1g6gla8j6v55xmzs8h5xmwqcybbmag";
+      vendorSha256 = "0w1db7xpissdpf8i5bb96z92zbasj5x9kk3kcisxn0dwla6n55n3";
+    };
+    "0.4.2" = release;
+    "0.4.1" = rec {
+      pname = "bee";
+      version = "0.4.1";
+      rev = "refs/tags/v${version}";
+      sha256 = "1bmgbav52pcb5p7cgq9756512fzfqhjybyr0dv538plkqx47mpv7";
+      vendorSha256 = "0j393va4jrg9q3wlc9mgkbpgnn2w2s3k2hcn8phzj8d5fl4n4v2h";
+    };
+  }.${version};
+
+in
+
+buildGoModule {
+  inherit (versionSpec) pname version vendorSha256;
+
+  src = fetchFromGitHub {
+    owner = "ethersphere";
+    repo = "bee";
+    inherit (versionSpec) rev sha256;
+  };
+
+  nativeBuildInputs = [ coreutils ];
+
+  subPackages = [ "cmd/bee" ];
+
+  # no symbol table, no debug info, and pass the commit for the version string
+  buildFlags = lib.optionalString ( lib.hasAttr "goVersionString" versionSpec)
+    "-ldflags -s -ldflags -w -ldflags -X=github.com/ethersphere/bee.commit=${versionSpec.goVersionString}";
+
+  # Mimic the bee Makefile: without disabling CGO, two (transitive and
+  # unused) dependencies would fail to compile.
+  preBuild = ''
+    export CGO_ENABLED=0
+  '';
+
+  postInstall = ''
+    mkdir -p $out/lib/systemd/system
+    cp packaging/bee.service $out/lib/systemd/system/
+    cp packaging/bee-get-addr $out/bin/
+    chmod +x $out/bin/bee-get-addr
+    patchShebangs $out/bin/
+  '';
+
+  meta = with lib; {
+    homepage = "https://swarm.ethereum.org/";
+    description = "Ethereum Swarm Bee";
+    longDescription = ''
+      A decentralised storage and communication system for a sovereign digital society.
+
+      Swarm is a system of peer-to-peer networked nodes that create a decentralised storage and communication service. The system is economically self-sustaining due to a built-in incentive system enforced through smart contracts on the Ethereum blockchain.
+
+      Bee is a Swarm node implementation, written in Go.
+    '';
+    license = with licenses; [ bsd3 ];
+    maintainers = with maintainers; [ attila-lendvai ];
+  };
+}

pkgs/applications/networking/bee/ensure-clef-account

@@ -0,0 +1,47 @@
+#!/usr/bin/env sh
+
+set -e
+
+# NOTE This file is called by the systemd service in its preStart
+# hook, but it's not Nix specific in any way. Ideally, the same file
+# should be called from the postinst scripts of the other packages,
+# but... the world is not ideal.
+
+# What follows was extracted from, and should be in sync with
+# https://github.com/ethersphere/bee-clef/tree/master/packaging
+
+DATA_DIR="$1"
+CONFIG_DIR="$2"
+PASSWORD_FILE=${DATA_DIR}/password
+MASTERSEED=${DATA_DIR}/masterseed.json
+KEYSTORE=${DATA_DIR}/keystore
+
+echo "ensure-clef-account $DATA_DIR $CONFIG_DIR"
+
+if ! test -f ${PASSWORD_FILE}; then
+    < /dev/urandom tr -dc _A-Z-a-z-0-9 2> /dev/null | head -c32 > ${PASSWORD_FILE}
+    chmod 0400 ${PASSWORD_FILE}
+    echo "Initialized ${PASSWORD_FILE} from /dev/urandom"
+fi
+
+if ! test -f ${MASTERSEED}; then
+    parse_json() { echo $1|sed -e 's/[{}]/''/g'|sed -e 's/", "/'\",\"'/g'|sed -e 's/" ,"/'\",\"'/g'|sed -e 's/" , "/'\",\"'/g'|sed -e 's/","/'\"---SEPERATOR---\"'/g'|awk -F=':' -v RS='---SEPERATOR---' "\$1~/\"$2\"/ {print}"|sed -e "s/\"$2\"://"|tr -d "\n\t"|sed -e 's/\\"/"/g'|sed -e 's/\\\\/\\/g'|sed -e 's/^[ \t]*//g'|sed -e 's/^"//' -e 's/"$//' ; }
+    SECRET=$(cat ${PASSWORD_FILE})
+    CLEF="@clefBinary@ --configdir ${DATA_DIR} --keystore ${KEYSTORE} --stdio-ui"
+    $CLEF init >/dev/null << EOF
+$SECRET
+$SECRET
+EOF
+    $CLEF newaccount >/dev/null << EOF
+$SECRET
+EOF
+    $CLEF setpw 0x$(parse_json $(cat ${KEYSTORE}/*) address) >/dev/null << EOF
+$SECRET
+$SECRET
+$SECRET
+EOF
+    $CLEF attest $(sha256sum ${CONFIG_DIR}/rules.js | cut -d' ' -f1 | tr -d '\n') >/dev/null << EOF
+$SECRET
+EOF
+    echo "Clef data dir initialized"
+fi

pkgs/top-level/all-packages.nix

@@ -2908,6 +2908,16 @@ in
 
   beanstalkd = callPackage ../servers/beanstalkd { };
 
+  bee = callPackage ../applications/networking/bee/bee.nix {
+    version = "release";
+  };
+
+  bee-unstable = bee.override {
+    version = "unstable";
+  };
+
+  bee-clef = callPackage ../applications/networking/bee/bee-clef.nix { };
+
   beets = callPackage ../tools/audio/beets {
     pythonPackages = python3Packages;
   };