From b244a4718598aa28a5bf6807410354fb2bdb9f08 Mon Sep 17 00:00:00 2001 From: Jaka Hudoklin Date: Sun, 11 Aug 2013 12:16:19 +0200 Subject: [PATCH 1/2] Add graphite, scalable realtime graphing service --- modules/misc/ids.nix | 2 + modules/module-list.nix | 1 + modules/services/monitoring/graphite.nix | 235 +++++++++++++++++++++++ 3 files changed, 238 insertions(+) create mode 100644 modules/services/monitoring/graphite.nix diff --git a/modules/misc/ids.nix b/modules/misc/ids.nix index 922bbb531719..66df0b22db47 100644 --- a/modules/misc/ids.nix +++ b/modules/misc/ids.nix @@ -80,6 +80,7 @@ in smtpq = 64; supybot = 65; iodined = 66; + graphite = 67; # When adding a uid, make sure it doesn't match an existing gid. @@ -143,6 +144,7 @@ in smtpq = 64; supybot = 65; iodined = 66; + graphite = 67; # When adding a gid, make sure it doesn't match an existing uid. diff --git a/modules/module-list.nix b/modules/module-list.nix index 2d14a143e226..37e68f06d3a2 100644 --- a/modules/module-list.nix +++ b/modules/module-list.nix @@ -121,6 +121,7 @@ ./services/misc/synergy.nix ./services/monitoring/apcupsd.nix ./services/monitoring/dd-agent.nix + ./services/monitoring/graphite.nix ./services/monitoring/monit.nix ./services/monitoring/nagios/default.nix ./services/monitoring/smartd.nix diff --git a/modules/services/monitoring/graphite.nix b/modules/services/monitoring/graphite.nix new file mode 100644 index 000000000000..43fc7be00634 --- /dev/null +++ b/modules/services/monitoring/graphite.nix @@ -0,0 +1,235 @@ +{ config, pkgs, ... }: + +with pkgs.lib; + +let + cfg = config.services.graphite; + writeTextOrNull = f: t: if t == null then null else pkgs.writeText f t; + + dataDir = "/var/db/graphite"; + +in { + + ###### interface + + options.services.graphite = { + carbon = mkOption { + description = "Content of carbon configuration file"; + default = ""; + type = types.uniq types.string; + }; + + enableCarbonCache = mkOption { + description = "Whether to enable carbon cache, the graphite storage daemon"; + default = false; + type = types.uniq types.bool; + }; + + storageAggregation = mkOption { + description = "Defines how to aggregate data to lower-precision retentions"; + default = null; + type = types.uniq (types.nullOr types.string); + example = '' + [all_min] + pattern = \.min$ + xFilesFactor = 0.1 + aggregationMethod = min + ''; + }; + + storageSchemas = mkOption { + description = "Defines retention rates for storing metrics"; + default = ""; + type = types.uniq (types.nullOr types.string); + example = '' + [apache_busyWorkers] + pattern = ^servers\.www.*\.workers\.busyWorkers$ + retentions = 15s:7d,1m:21d,15m:5y + ''; + }; + + blacklist = mkOption { + description = "Any metrics received which match one of the experssions will be dropped"; + default = null; + type = types.uniq (types.nullOr types.string); + example = "^some\.noisy\.metric\.prefix\..*"; + }; + + whitelist = mkOption { + description = "Only metrics received which match one of the experssions will be persisted"; + default = null; + type = types.uniq (types.nullOr types.string); + example = ".*"; + }; + + rewriteRules = mkOption { + description = "Regular expression patterns that can be used to rewrite metric names in a search and replace fashion"; + default = null; + type = types.uniq (types.nullOr types.string); + example = '' + [post] + _sum$ = + _avg$ = + ''; + }; + + enableCarbonRelay = mkOption { + description = "Whether to enable carbon relay, the carbon replication and sharding service"; + default = false; + type = types.uniq types.bool; + }; + + relayRules = mkOption { + description = "Relay rules are used to send certain metrics to a certain backend."; + default = null; + type = types.uniq (types.nullOr types.string); + example = '' + [example] + pattern = ^mydata\.foo\..+ + servers = 10.1.2.3, 10.1.2.4:2004, myserver.mydomain.com + ''; + }; + + enableCarbonAggregator = mkOption { + description = "Whether to enable carbon agregator, the carbon buffering service"; + default = false; + type = types.uniq types.bool; + }; + + aggregationRules = mkOption { + description = "Defines if and how received metrics will be agregated"; + default = null; + type = types.uniq (types.nullOr types.string); + example = '' + .applications..all.requests (60) = sum .applications..*.requests + .applications..all.latency (60) = avg .applications..*.latency + ''; + }; + + enableGraphiteWeb = mkOption { + description = "Whether to enable graphite web frontend"; + default = false; + type = types.uniq types.bool; + }; + + }; + + ###### implementation + + config = mkIf (cfg.enableCarbonAggregator || cfg.enableCarbonCache || cfg.enableCarbonRelay || cfg.enableGraphiteWeb) { + environment.etc = lists.filter (el: el.source != null) [ + { source = writeTextOrNull "carbon.conf" cfg.carbon; + target = "graphite/carbon.conf"; } + { source = writeTextOrNull "storage-agregation.conf" cfg.storageAggregation; + target = "graphite/storage-agregation.conf"; } + { source = writeTextOrNull "storage-schemas.conf" cfg.storageSchemas; + target = "graphite/storage-schemas.conf"; } + { source = writeTextOrNull "blacklist.conf" cfg.blacklist; + target = "graphite/blacklist.conf"; } + { source = writeTextOrNull "whitelist.conf" cfg.whitelist; + target = "graphite/whitelist.conf"; } + { source = writeTextOrNull "rewrite-rules.conf" cfg.rewriteRules; + target = "graphite/rewrite-rules.conf"; } + { source = writeTextOrNull "relay-rules.conf" cfg.relayRules; + target = "graphite/relay-rules.conf"; } + { source = writeTextOrNull "aggregation-rules.conf" cfg.aggregationRules; + target = "graphite/aggregation-rules.conf"; } + ]; + + systemd.services.carbonCache = mkIf cfg.enableCarbonCache { + description = "Graphite data storage backend"; + wantedBy = [ "multi-user.target" ]; + after = [ "network-interfaces.target" ]; + environment = { + GRAPHITE_CONF_DIR = "/etc/graphite/"; + GRAPHITE_STORAGE_DIR = "/var/db/graphite/"; + }; + serviceConfig = { + ExecStart = "${pkgs.pythonPackages.carbon}/bin/carbon-cache.py --debug --pidfile /tmp/carbonCache.pid start"; + User = "graphite"; + Group = "graphite"; + }; + restartTriggers = [ + pkgs.pythonPackages.carbon cfg.carbon + cfg.storageAggregation cfg.storageSchemas cfg.rewriteRules + ]; + preStart = '' + mkdir -p ${dataDir}/whisper + ''; + }; + + systemd.services.carbonAggregator = mkIf cfg.enableCarbonAggregator { + description = "Carbon data aggregator"; + wantedBy = [ "multi-user.target" ]; + after = [ "network-interfaces.target" ]; + environment = { + GRAPHITE_CONF_DIR = "/etc/graphite/"; + GRAPHITE_STORAGE_DIR = "${dataDir}"; + }; + serviceConfig = { + ExecStart = "${pkgs.pythonPackages.carbon}/bin/carbon-aggregator.py --debug --pidfile /tmp/carbonAggregator.pid start"; + User = "graphite"; + Group = "graphite"; + }; + restartTriggers = [ pkgs.pythonPackages.carbon cfg.carbon cfg.aggregationRules ]; + }; + + systemd.services.carbonRelay = mkIf cfg.enableCarbonRelay { + description = "Carbon data relay"; + wantedBy = [ "multi-user.target" ]; + after = [ "network-interfaces.target" ]; + environment = { + GRAPHITE_CONF_DIR = "/etc/graphite/"; + GRAPHITE_STORAGE_DIR = "${dataDir}"; + }; + serviceConfig = { + ExecStart = "${pkgs.pythonPackages.carbon}/bin/carbon-relay.py --debug --pidfile /tmp/carbonRelay.pid start"; + User = "graphite"; + Group = "graphite"; + }; + restartTriggers = [ pkgs.pythonPackages.carbon cfg.carbon cfg.relayRules ]; + }; + + systemd.services.graphiteWeb = mkIf cfg.enableGraphiteWeb { + description = "Graphite web interface"; + wantedBy = [ "multi-user.target" ]; + after = [ "network-interfaces.target" ]; + environment = { + GRAPHITE_CONF_DIR = "/etc/graphite/"; + GRAPHITE_STORAGE_DIR = "${dataDir}"; + }; + serviceConfig = { + ExecStart = "${pkgs.pythonPackages.graphite_web}/bin/run-graphite-devel-server.py ${pkgs.pythonPackages.graphite_web}"; + User = "graphite"; + Group = "graphite"; + }; + preStart = '' + if ! test -e ${dataDir}/db-created; then + mkdir -p ${dataDir}/{whisper/,log/webapp/} + + # populate database + ${pkgs.pythonPackages.graphite_web}/bin/manage-graphite.py syncdb --noinput + + # create index + ${pkgs.pythonPackages.graphite_web}/bin/build-index.sh + + touch ${dataDir}/db-created + fi + ''; + restartTriggers = [ pkgs.pythonPackages.graphite_web ]; + }; + + environment.systemPackages = [ + pkgs.pythonPackages.carbon pkgs.pythonPackages.graphite_web pkgs.pythonPackages.django_1_3 + ]; + + users.extraUsers = singleton { + name = "graphite"; + uid = config.ids.uids.graphite; + description = "Graphite daemon user"; + home = "${dataDir}"; + createHome = true; + }; + users.extraGroups.graphite.gid = config.ids.gids.graphite; + }; +} From 4628fd8434582592c3b5929bb8244b07c4007f3f Mon Sep 17 00:00:00 2001 From: Jaka Hudoklin Date: Mon, 19 Aug 2013 04:14:53 +0200 Subject: [PATCH 2/2] graphite: Refactor options, serve with waitress --- modules/services/monitoring/graphite.nix | 243 +++++++++++++---------- 1 file changed, 135 insertions(+), 108 deletions(-) diff --git a/modules/services/monitoring/graphite.nix b/modules/services/monitoring/graphite.nix index 43fc7be00634..8ed045e5eb5e 100644 --- a/modules/services/monitoring/graphite.nix +++ b/modules/services/monitoring/graphite.nix @@ -13,130 +13,145 @@ in { ###### interface options.services.graphite = { - carbon = mkOption { - description = "Content of carbon configuration file"; - default = ""; - type = types.uniq types.string; + web = { + enable = mkOption { + description = "Whether to enable graphite web frontend"; + default = false; + type = types.uniq types.bool; + }; + + host = mkOption { + description = "Graphite web frontend listen address"; + default = "127.0.0.1"; + types = type.uniq types.string; + }; + + port = mkOption { + description = "Graphite web frontend port"; + default = "8080"; + types = type.uniq types.string; + }; }; - enableCarbonCache = mkOption { - description = "Whether to enable carbon cache, the graphite storage daemon"; - default = false; - type = types.uniq types.bool; - }; + carbon = { + config = mkOption { + description = "Content of carbon configuration file"; + default = ""; + type = types.uniq types.string; + }; - storageAggregation = mkOption { - description = "Defines how to aggregate data to lower-precision retentions"; - default = null; - type = types.uniq (types.nullOr types.string); - example = '' - [all_min] - pattern = \.min$ - xFilesFactor = 0.1 - aggregationMethod = min - ''; - }; + enableCache = mkOption { + description = "Whether to enable carbon cache, the graphite storage daemon"; + default = false; + type = types.uniq types.bool; + }; - storageSchemas = mkOption { - description = "Defines retention rates for storing metrics"; - default = ""; - type = types.uniq (types.nullOr types.string); - example = '' - [apache_busyWorkers] - pattern = ^servers\.www.*\.workers\.busyWorkers$ - retentions = 15s:7d,1m:21d,15m:5y - ''; - }; + storageAggregation = mkOption { + description = "Defines how to aggregate data to lower-precision retentions"; + default = null; + type = types.uniq (types.nullOr types.string); + example = '' + [all_min] + pattern = \.min$ + xFilesFactor = 0.1 + aggregationMethod = min + ''; + }; - blacklist = mkOption { - description = "Any metrics received which match one of the experssions will be dropped"; - default = null; - type = types.uniq (types.nullOr types.string); - example = "^some\.noisy\.metric\.prefix\..*"; - }; + storageSchemas = mkOption { + description = "Defines retention rates for storing metrics"; + default = ""; + type = types.uniq (types.nullOr types.string); + example = '' + [apache_busyWorkers] + pattern = ^servers\.www.*\.workers\.busyWorkers$ + retentions = 15s:7d,1m:21d,15m:5y + ''; + }; - whitelist = mkOption { - description = "Only metrics received which match one of the experssions will be persisted"; - default = null; - type = types.uniq (types.nullOr types.string); - example = ".*"; - }; + blacklist = mkOption { + description = "Any metrics received which match one of the experssions will be dropped"; + default = null; + type = types.uniq (types.nullOr types.string); + example = "^some\.noisy\.metric\.prefix\..*"; + }; - rewriteRules = mkOption { - description = "Regular expression patterns that can be used to rewrite metric names in a search and replace fashion"; - default = null; - type = types.uniq (types.nullOr types.string); - example = '' - [post] - _sum$ = - _avg$ = - ''; - }; + whitelist = mkOption { + description = "Only metrics received which match one of the experssions will be persisted"; + default = null; + type = types.uniq (types.nullOr types.string); + example = ".*"; + }; - enableCarbonRelay = mkOption { - description = "Whether to enable carbon relay, the carbon replication and sharding service"; - default = false; - type = types.uniq types.bool; - }; + rewriteRules = mkOption { + description = "Regular expression patterns that can be used to rewrite metric names in a search and replace fashion"; + default = null; + type = types.uniq (types.nullOr types.string); + example = '' + [post] + _sum$ = + _avg$ = + ''; + }; - relayRules = mkOption { - description = "Relay rules are used to send certain metrics to a certain backend."; - default = null; - type = types.uniq (types.nullOr types.string); - example = '' - [example] - pattern = ^mydata\.foo\..+ - servers = 10.1.2.3, 10.1.2.4:2004, myserver.mydomain.com - ''; - }; + enableRelay = mkOption { + description = "Whether to enable carbon relay, the carbon replication and sharding service"; + default = false; + type = types.uniq types.bool; + }; - enableCarbonAggregator = mkOption { - description = "Whether to enable carbon agregator, the carbon buffering service"; - default = false; - type = types.uniq types.bool; - }; + relayRules = mkOption { + description = "Relay rules are used to send certain metrics to a certain backend."; + default = null; + type = types.uniq (types.nullOr types.string); + example = '' + [example] + pattern = ^mydata\.foo\..+ + servers = 10.1.2.3, 10.1.2.4:2004, myserver.mydomain.com + ''; + }; - aggregationRules = mkOption { - description = "Defines if and how received metrics will be agregated"; - default = null; - type = types.uniq (types.nullOr types.string); - example = '' - .applications..all.requests (60) = sum .applications..*.requests - .applications..all.latency (60) = avg .applications..*.latency - ''; - }; + enableAggregator = mkOption { + description = "Whether to enable carbon agregator, the carbon buffering service"; + default = false; + type = types.uniq types.bool; + }; - enableGraphiteWeb = mkOption { - description = "Whether to enable graphite web frontend"; - default = false; - type = types.uniq types.bool; + aggregationRules = mkOption { + description = "Defines if and how received metrics will be agregated"; + default = null; + type = types.uniq (types.nullOr types.string); + example = '' + .applications..all.requests (60) = sum .applications..*.requests + .applications..all.latency (60) = avg .applications..*.latency + ''; + }; }; - }; ###### implementation - config = mkIf (cfg.enableCarbonAggregator || cfg.enableCarbonCache || cfg.enableCarbonRelay || cfg.enableGraphiteWeb) { + config = mkIf (cfg.carbon.enableAggregator || cfg.carbon.enableCache || cfg.cabon.enableRelay || cfg.web.enable) { environment.etc = lists.filter (el: el.source != null) [ - { source = writeTextOrNull "carbon.conf" cfg.carbon; + { source = writeTextOrNull "carbon.conf" cfg.carbon.config; target = "graphite/carbon.conf"; } - { source = writeTextOrNull "storage-agregation.conf" cfg.storageAggregation; + { source = writeTextOrNull "storage-agregation.conf" cfg.carbon.storageAggregation; target = "graphite/storage-agregation.conf"; } - { source = writeTextOrNull "storage-schemas.conf" cfg.storageSchemas; + { source = writeTextOrNull "storage-schemas.conf" cfg.carbon.storageSchemas; target = "graphite/storage-schemas.conf"; } - { source = writeTextOrNull "blacklist.conf" cfg.blacklist; + { source = writeTextOrNull "blacklist.conf" cfg.carbon.blacklist; target = "graphite/blacklist.conf"; } - { source = writeTextOrNull "whitelist.conf" cfg.whitelist; + { source = writeTextOrNull "whitelist.conf" cfg.carbon.whitelist; target = "graphite/whitelist.conf"; } - { source = writeTextOrNull "rewrite-rules.conf" cfg.rewriteRules; + { source = writeTextOrNull "rewrite-rules.conf" cfg.carbon.rewriteRules; target = "graphite/rewrite-rules.conf"; } - { source = writeTextOrNull "relay-rules.conf" cfg.relayRules; + { source = writeTextOrNull "relay-rules.conf" cfg.carbon.relayRules; target = "graphite/relay-rules.conf"; } - { source = writeTextOrNull "aggregation-rules.conf" cfg.aggregationRules; + { source = writeTextOrNull "aggregation-rules.conf" cfg.carbon.aggregationRules; target = "graphite/aggregation-rules.conf"; } ]; - systemd.services.carbonCache = mkIf cfg.enableCarbonCache { + systemd.services.carbonCache = mkIf cfg.carbon.enableCache { description = "Graphite data storage backend"; wantedBy = [ "multi-user.target" ]; after = [ "network-interfaces.target" ]; @@ -150,15 +165,15 @@ in { Group = "graphite"; }; restartTriggers = [ - pkgs.pythonPackages.carbon cfg.carbon - cfg.storageAggregation cfg.storageSchemas cfg.rewriteRules + pkgs.pythonPackages.carbon cfg.carbon.config cfg.carbon.storageAggregation + cfg.carbon.storageSchemas cfg.carbon.rewriteRules ]; preStart = '' mkdir -p ${dataDir}/whisper ''; }; - systemd.services.carbonAggregator = mkIf cfg.enableCarbonAggregator { + systemd.services.carbonAggregator = mkIf cfg.carbon.enableAggregator { description = "Carbon data aggregator"; wantedBy = [ "multi-user.target" ]; after = [ "network-interfaces.target" ]; @@ -171,10 +186,12 @@ in { User = "graphite"; Group = "graphite"; }; - restartTriggers = [ pkgs.pythonPackages.carbon cfg.carbon cfg.aggregationRules ]; + restartTriggers = [ + pkgs.pythonPackages.carbon cfg.carbon.config cfg.carbon.aggregationRules + ]; }; - systemd.services.carbonRelay = mkIf cfg.enableCarbonRelay { + systemd.services.carbonRelay = mkIf cfg.carbon.enableRelay { description = "Carbon data relay"; wantedBy = [ "multi-user.target" ]; after = [ "network-interfaces.target" ]; @@ -187,19 +204,26 @@ in { User = "graphite"; Group = "graphite"; }; - restartTriggers = [ pkgs.pythonPackages.carbon cfg.carbon cfg.relayRules ]; + restartTriggers = [ + pkgs.pythonPackages.carbon cfg.carbon.config cfg.carbon.relayRules + ]; }; - systemd.services.graphiteWeb = mkIf cfg.enableGraphiteWeb { + systemd.services.graphiteWeb = mkIf cfg.web.enable { description = "Graphite web interface"; wantedBy = [ "multi-user.target" ]; after = [ "network-interfaces.target" ]; environment = { + PYTHONPATH = "${pkgs.python27Packages.graphite_web}/lib/python2.7/site-packages"; + DJANGO_SETTINGS_MODULE = "graphite.settings"; GRAPHITE_CONF_DIR = "/etc/graphite/"; GRAPHITE_STORAGE_DIR = "${dataDir}"; }; serviceConfig = { - ExecStart = "${pkgs.pythonPackages.graphite_web}/bin/run-graphite-devel-server.py ${pkgs.pythonPackages.graphite_web}"; + ExecStart = '' + ${pkgs.python27Packages.waitress}/bin/waitress-serve \ + --host=${cfg.web.host} --port=${cfg.web.port} \ + --call django.core.handlers.wsgi:WSGIHandler''; User = "graphite"; Group = "graphite"; }; @@ -208,19 +232,22 @@ in { mkdir -p ${dataDir}/{whisper/,log/webapp/} # populate database - ${pkgs.pythonPackages.graphite_web}/bin/manage-graphite.py syncdb --noinput + ${pkgs.python27Packages.graphite_web}/bin/manage-graphite.py syncdb --noinput # create index - ${pkgs.pythonPackages.graphite_web}/bin/build-index.sh + ${pkgs.python27Packages.graphite_web}/bin/build-index.sh touch ${dataDir}/db-created fi ''; - restartTriggers = [ pkgs.pythonPackages.graphite_web ]; + restartTriggers = [ + pkgs.python27Packages.graphite_web pkgs.python27Packages.waitress + ]; }; environment.systemPackages = [ - pkgs.pythonPackages.carbon pkgs.pythonPackages.graphite_web pkgs.pythonPackages.django_1_3 + pkgs.pythonPackages.carbon pkgs.python27Packages.graphite_web + pkgs.python27Packages.waitress ]; users.extraUsers = singleton {