From 400a046084ae2d6f61020b3b494d2b3632e28b2b Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Mon, 28 Oct 2013 13:51:43 +0100 Subject: [PATCH] Added index and indices aggregate stats exported. Added current version of dashboards --- dashboards/Indices.json | 740 +++++++++++ dashboards/nodes.json | 1173 +++++++++++++++++ dashboards/shards.json | 799 +++++++++++ .../monitor/StatsExportersService.java | 36 +- .../monitor/exporter/ESExporter.java | 187 ++- .../monitor/exporter/StatsExporter.java | 8 + 6 files changed, 2899 insertions(+), 44 deletions(-) create mode 100644 dashboards/Indices.json create mode 100644 dashboards/nodes.json create mode 100644 dashboards/shards.json diff --git a/dashboards/Indices.json b/dashboards/Indices.json new file mode 100644 index 000000000000..0dbcec0a3a96 --- /dev/null +++ b/dashboards/Indices.json @@ -0,0 +1,740 @@ +{ + "title": "Indices", + "services": { + "query": { + "idQueue": [ + ], + "list": { + }, + "ids": [ + ] + }, + "filter": { + "idQueue": [], + "list": { + "0": { + "type": "time", + "field": "@timestamp", + "from": "now-1h", + "to": "now", + "mandate": "must", + "active": true, + "alias": "", + "id": 0 + }, + "1": { + "type": "querystring", + "mandate": "must", + "active": true, + "alias": "shard starts", + "query": "_type:shard_stats", + "id": 1 + } + }, + "ids": [ + 0, + 1 + ] + } + }, + "rows": [ + { + "title": "Node filtering", + "height": "50px", + "editable": true, + "collapse": false, + "collapsable": true, + "panels": [ + { + "error": false, + "span": 4, + "editable": true, + "type": "derivequeries", + "loadingEditor": false, + "loading": false, + "label": "Search", + "query": "*", + "field": "index", + "fields": [], + "spyable": true, + "rest": false, + "size": 5, + "mode": "terms only", + "exclude": [], + "history": [ + null + ], + "queries": [ + ], + "selected_query": null, + "use_select": false, + "remember": 10, + "title": "Indices" + }, + { + "error": false, + "span": 4, + "editable": true, + "type": "derivefilters", + "loadingEditor": false, + "loading": false, + "label": "Search", + "query": "*", + "field": "node.name", + "fields": [], + "spyable": true, + "rest": false, + "size": 5, + "mode": "terms only", + "exclude": [], + "history": [ + null + ], + "filters": [ + ], + "selected_filters": [], + "remember": 10, + "title": "Nodes" + } + ], + "notice": false + }, + { + "title": "Data", + "height": "150px", + "editable": true, + "collapse": false, + "collapsable": true, + "panels": [ + { + "span": 4, + "editable": true, + "type": "histogram", + "loadingEditor": false, + "mode": "total", + "time_field": "@timestamp", + "queries": { + "mode": "all", + "ids": [ + 1, + 4, + 5, + 0, + 6 + ] + }, + "value_field": "docs.count", + "auto_int": true, + "resolution": 100, + "interval": "30s", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "fill": 0, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": false, + "lines": true, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "options": true, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": false + }, + "title": "Documents", + "extra_filter": "primary:true" + }, + { + "span": 4, + "editable": true, + "type": "histogram", + "loadingEditor": false, + "mode": "total", + "time_field": "@timestamp", + "queries": { + "mode": "all", + "ids": [ + 1, + 4, + 5, + 0, + 6 + ] + }, + "value_field": "docs.deleted", + "auto_int": true, + "resolution": 100, + "interval": "30s", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "fill": 0, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": false, + "lines": true, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "options": true, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": false + }, + "title": "Deleted docs" + }, + { + "span": 4, + "editable": true, + "type": "histogram", + "loadingEditor": false, + "mode": "total", + "time_field": "@timestamp", + "queries": { + "mode": "all", + "ids": [ + 1, + 4, + 5, + 0, + 6 + ] + }, + "value_field": "store.size_in_bytes", + "auto_int": true, + "resolution": 100, + "interval": "30s", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "fill": 0, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": false, + "lines": true, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "options": true, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": false + }, + "title": "Index size" + } + ], + "notice": false + }, + { + "title": "Indexing", + "height": "150px", + "editable": true, + "collapse": false, + "collapsable": true, + "panels": [ + { + "span": 4, + "editable": true, + "type": "histogram", + "loadingEditor": false, + "mode": "total", + "time_field": "@timestamp", + "queries": { + "mode": "all", + "ids": [ + 1, + 4, + 5, + 0, + 6 + ] + }, + "value_field": "indexing.index_total", + "auto_int": true, + "resolution": 100, + "interval": "30s", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "fill": 0, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": false, + "lines": true, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "options": true, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": false + }, + "title": "Indexing requests" + }, + { + "span": 4, + "editable": true, + "type": "histogram", + "loadingEditor": false, + "mode": "total", + "time_field": "@timestamp", + "queries": { + "mode": "all", + "ids": [ + 1, + 4, + 5, + 0, + 6 + ] + }, + "value_field": "indexing.index_time_in_millis", + "auto_int": true, + "resolution": 100, + "interval": "30s", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "fill": 0, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": false, + "lines": true, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "options": true, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": false + }, + "title": "Indexing time" + } + ], + "notice": false + }, + { + "title": "Search & Get", + "height": "150px", + "editable": true, + "collapse": false, + "collapsable": true, + "panels": [ + { + "span": 4, + "editable": true, + "type": "histogram", + "loadingEditor": false, + "mode": "total", + "time_field": "@timestamp", + "queries": { + "mode": "all", + "ids": [ + 1, + 4, + 5, + 0, + 6 + ] + }, + "value_field": "search.query_total", + "auto_int": true, + "resolution": 100, + "interval": "30s", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "fill": 0, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": false, + "lines": true, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "options": true, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": false + }, + "title": "Search requests" + }, + { + "span": 4, + "editable": true, + "type": "histogram", + "loadingEditor": false, + "mode": "total", + "time_field": "@timestamp", + "queries": { + "mode": "all", + "ids": [ + 1, + 4, + 5, + 0, + 6 + ] + }, + "value_field": "get.total", + "auto_int": true, + "resolution": 100, + "interval": "30s", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "fill": 0, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": false, + "lines": true, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "options": true, + "tooltip": { + "value_type": "individual", + "query_as_alias": false + }, + "title": "GET REQUEST" + } + ], + "notice": false + }, + { + "title": "System health", + "height": "150px", + "editable": true, + "collapse": false, + "collapsable": true, + "panels": [ + { + "span": 4, + "editable": true, + "type": "histogram", + "loadingEditor": false, + "mode": "total", + "time_field": "@timestamp", + "queries": { + "mode": "all", + "ids": [ + 1, + 4, + 5, + 0, + 6 + ] + }, + "value_field": "search.open_contexts", + "auto_int": true, + "resolution": 100, + "interval": "30s", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "fill": 0, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": false, + "lines": true, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "options": true, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": false + }, + "title": "Open search context" + }, + { + "span": 4, + "editable": true, + "type": "histogram", + "loadingEditor": false, + "mode": "max", + "time_field": "@timestamp", + "queries": { + "mode": "all", + "ids": [ + 1, + 4, + 5, + 0, + 6 + ] + }, + "value_field": "refresh.total_time_in_millis", + "auto_int": true, + "resolution": 100, + "interval": "30s", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "fill": 0, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": false, + "lines": true, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "options": true, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": false + }, + "title": "Avg Refresh time" + } + ], + "notice": false + } + ], + "editable": true, + "failover": false, + "index": { + "interval": "day", + "pattern": "[es_monitor-]YYYY.MM.DD", + "default": "_all" + }, + "style": "dark", + "panel_hints": true, + "pulldowns": [ + { + "type": "query", + "collapse": false, + "notice": false, + "enable": true, + "query": "*", + "pinned": true, + "history": [ + "index:\"index2\" AND (_type:shard_stats)", + "index:\"es_monitor-2013.10.16\" AND (_type:shard_stats)", + "index:\"kibana-int\" AND (_type:shard_stats)", + "index:\"index\" AND (_type:shard_stats)", + "index:\"es_monitor-2013.10.15\" AND (_type:shard_stats)" + ], + "remember": 10 + }, + { + "type": "filtering", + "collapse": false, + "notice": true, + "enable": true + } + ], + "nav": [ + { + "type": "timepicker", + "collapse": false, + "notice": false, + "enable": true, + "status": "Stable", + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "timefield": "@timestamp", + "now": true, + "filter_id": 0 + } + ], + "loader": { + "save_gist": true, + "save_elasticsearch": true, + "save_local": true, + "save_default": true, + "save_temp": true, + "save_temp_ttl_enable": true, + "save_temp_ttl": "30d", + "load_gist": true, + "load_elasticsearch": true, + "load_elasticsearch_size": 20, + "load_local": true, + "hide": false + }, + "refresh": false +} \ No newline at end of file diff --git a/dashboards/nodes.json b/dashboards/nodes.json new file mode 100644 index 000000000000..fe6a3589b811 --- /dev/null +++ b/dashboards/nodes.json @@ -0,0 +1,1173 @@ +{ + "title": "Nodes", + "services": { + "query": { + "idQueue": [ + ], + "list": { + }, + "ids": [ + ] + }, + "filter": { + "idQueue": [], + "list": { + "0": { + "type": "time", + "field": "@timestamp", + "from": "now-1h", + "to": "now", + "mandate": "must", + "active": true, + "alias": "", + "id": 0 + }, + "1": { + "type": "querystring", + "mandate": "must", + "active": true, + "alias": "node starts", + "query": "_type:node_stats", + "id": 1 + } + }, + "ids": [ + 0, + 1 + ] + } + }, + "rows": [ + { + "title": "Server", + "height": "150px", + "editable": true, + "collapse": true, + "collapsable": true, + "panels": [ + { + "span": 4, + "editable": true, + "type": "histogram", + "loadingEditor": false, + "mode": "max", + "time_field": "@timestamp", + "queries": { + "mode": "all", + "ids": [ + 1, + 2 + ] + }, + "value_field": "os.cpu.user", + "auto_int": true, + "resolution": 100, + "interval": "30s", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "fill": 0, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": false, + "lines": true, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "options": true, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": false + }, + "extra_filter": "", + "title": "CPU (user)" + }, + { + "span": 4, + "editable": true, + "type": "histogram", + "loadingEditor": false, + "mode": "max", + "time_field": "@timestamp", + "queries": { + "mode": "all", + "ids": [ + 1, + 2 + ] + }, + "value_field": "os.mem.used_percent", + "auto_int": true, + "resolution": 100, + "interval": "30s", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "fill": 0, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": false, + "lines": true, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "options": true, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": false + }, + "extra_filter": "", + "title": "Memory usage (%)" + }, + { + "span": 4, + "editable": true, + "type": "histogram", + "loadingEditor": false, + "mode": "max", + "time_field": "@timestamp", + "queries": { + "mode": "all", + "ids": [ + 1, + 2 + ] + }, + "value_field": "os.load_average.1m", + "auto_int": true, + "resolution": 100, + "interval": "30s", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "fill": 0, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": false, + "lines": true, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "options": true, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": false + }, + "extra_filter": "", + "title": "Load (1m)" + } + ], + "notice": false + }, + { + "title": "JVM Memory", + "height": "150px", + "editable": true, + "collapse": true, + "collapsable": true, + "panels": [ + { + "span": 4, + "editable": true, + "type": "histogram", + "loadingEditor": false, + "mode": "max", + "time_field": "@timestamp", + "queries": { + "mode": "all", + "ids": [ + 1, + 2 + ] + }, + "value_field": "jvm.mem.heap_used_in_bytes", + "auto_int": true, + "resolution": 100, + "interval": "30s", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "fill": 0, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": false, + "lines": true, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "options": true, + "tooltip": { + "value_type": "individual", + "query_as_alias": false + }, + "title": "Heap", + "extra_filter": "" + }, + { + "span": 4, + "editable": true, + "type": "histogram", + "loadingEditor": false, + "mode": "max", + "time_field": "@timestamp", + "queries": { + "mode": "all", + "ids": [ + 1, + 2 + ] + }, + "value_field": "jvm.gc.collection_time_in_millis", + "auto_int": true, + "resolution": 100, + "interval": "30s", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "fill": 0, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": false, + "lines": true, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "options": true, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": false + }, + "title": "GC duration", + "extra_filter": "" + }, + { + "span": 4, + "editable": true, + "type": "histogram", + "loadingEditor": false, + "mode": "max", + "time_field": "@timestamp", + "queries": { + "mode": "all", + "ids": [ + 1, + 2 + ] + }, + "value_field": "jvm.gc.collection_count", + "auto_int": true, + "resolution": 100, + "interval": "30s", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "fill": 0, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": false, + "lines": true, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "options": true, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": false + }, + "title": "GC counts", + "extra_filter": "" + } + ], + "notice": false + }, + { + "title": "Caches", + "height": "150px", + "editable": true, + "collapse": true, + "collapsable": true, + "panels": [ + { + "span": 4, + "editable": true, + "type": "histogram", + "loadingEditor": false, + "mode": "max", + "time_field": "@timestamp", + "queries": { + "mode": "all", + "ids": [ + 1, + 2 + ] + }, + "value_field": "indices.fielddata.memory_size_in_bytes", + "auto_int": true, + "resolution": 100, + "interval": "30s", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "fill": 0, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": false, + "lines": true, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "options": true, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": false + }, + "title": "Field Data", + "extra_filter": "" + }, + { + "span": 4, + "editable": true, + "type": "histogram", + "loadingEditor": false, + "mode": "max", + "time_field": "@timestamp", + "queries": { + "mode": "all", + "ids": [ + 1, + 2 + ] + }, + "value_field": "indices.filter_cache.memory_size_in_bytes", + "auto_int": true, + "resolution": 100, + "interval": "30s", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "fill": 0, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": false, + "lines": true, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "options": true, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": false + }, + "title": "Filter cache", + "extra_filter": "" + }, + { + "span": 4, + "editable": true, + "type": "histogram", + "loadingEditor": false, + "mode": "max", + "time_field": "@timestamp", + "queries": { + "mode": "all", + "ids": [ + 1, + 2 + ] + }, + "value_field": "indices.id_cache.memory_size_in_bytes", + "auto_int": true, + "resolution": 100, + "interval": "30s", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "fill": 0, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": false, + "lines": true, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "options": true, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": false + }, + "title": "Id cache", + "extra_filter": "" + } + ], + "notice": false + }, + { + "title": "Disk IO", + "height": "150px", + "editable": true, + "collapse": true, + "collapsable": true, + "panels": [ + { + "span": 4, + "editable": true, + "type": "histogram", + "loadingEditor": false, + "mode": "max", + "time_field": "@timestamp", + "queries": { + "mode": "all", + "ids": [ + 1, + 2 + ] + }, + "value_field": "fs.data.disk_read_size_in_bytes", + "auto_int": true, + "resolution": 100, + "interval": "30s", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "fill": 0, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": false, + "lines": true, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "options": true, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": false + }, + "extra_filter": "", + "title": "Disk reads (bytes)" + }, + { + "span": 4, + "editable": true, + "type": "histogram", + "loadingEditor": false, + "mode": "max", + "time_field": "@timestamp", + "queries": { + "mode": "all", + "ids": [ + 1, + 2 + ] + }, + "value_field": "fs.data.disk_write_size_in_bytes", + "auto_int": true, + "resolution": 100, + "interval": "30s", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "fill": 0, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": false, + "lines": true, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "options": true, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": false + }, + "extra_filter": "", + "title": "Disk writes (bytes)" + } + ], + "notice": false + }, + { + "title": "Network", + "height": "150px", + "editable": true, + "collapse": true, + "collapsable": true, + "panels": [ + { + "span": 4, + "editable": true, + "type": "histogram", + "loadingEditor": false, + "mode": "max", + "time_field": "@timestamp", + "queries": { + "mode": "all", + "ids": [ + 1, + 2 + ] + }, + "value_field": "http.current_open", + "auto_int": true, + "resolution": 100, + "interval": "30s", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "fill": 0, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": false, + "lines": true, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "options": true, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": false + }, + "extra_filter": "", + "title": "HTTP currently open" + }, + { + "span": 4, + "editable": true, + "type": "histogram", + "loadingEditor": false, + "mode": "max", + "time_field": "@timestamp", + "queries": { + "mode": "all", + "ids": [ + 1, + 2 + ] + }, + "value_field": "http.total_opened", + "auto_int": true, + "resolution": 100, + "interval": "30s", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "fill": 0, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": false, + "lines": true, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "options": true, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": false + }, + "extra_filter": "", + "title": "HTTP opened" + }, + { + "span": 4, + "editable": true, + "type": "histogram", + "loadingEditor": false, + "mode": "max", + "time_field": "@timestamp", + "queries": { + "mode": "all", + "ids": [ + 1, + 2 + ] + }, + "value_field": "transport.server_open", + "auto_int": true, + "resolution": 100, + "interval": "30s", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "fill": 0, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": false, + "lines": true, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "options": true, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": false + }, + "extra_filter": "", + "title": "Transport open" + } + ], + "notice": false + }, + { + "title": "Data", + "height": "150px", + "editable": true, + "collapse": true, + "collapsable": true, + "panels": [ + { + "span": 4, + "editable": true, + "type": "histogram", + "loadingEditor": false, + "mode": "max", + "time_field": "@timestamp", + "queries": { + "mode": "all", + "ids": [ + 1, + 2 + ] + }, + "value_field": "indices.docs.count", + "auto_int": true, + "resolution": 100, + "interval": "30s", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "fill": 0, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": false, + "lines": true, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "options": true, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": false + }, + "title": "Documents", + "extra_filter": "" + }, + { + "span": 4, + "editable": true, + "type": "histogram", + "loadingEditor": false, + "mode": "max", + "time_field": "@timestamp", + "queries": { + "mode": "all", + "ids": [ + 1, + 2 + ] + }, + "value_field": "indices.merges.current_size_in_bytes", + "auto_int": true, + "resolution": 100, + "interval": "30s", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "fill": 0, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": false, + "lines": true, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "options": true, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": false + }, + "title": "Merges (bytes)", + "extra_filter": "" + }, + { + "span": 4, + "editable": true, + "type": "histogram", + "loadingEditor": false, + "mode": "max", + "time_field": "@timestamp", + "queries": { + "mode": "all", + "ids": [ + 1, + 2 + ] + }, + "value_field": "indices.store.size_in_bytes", + "auto_int": true, + "resolution": 100, + "interval": "30s", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "fill": 0, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": false, + "lines": true, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "options": true, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": false + }, + "title": "Size", + "extra_filter": "" + } + ], + "notice": false + }, + { + "title": "internal", + "height": "150px", + "editable": true, + "collapse": false, + "collapsable": true, + "panels": [ + { + "error": false, + "span": 4, + "editable": true, + "type": "derivequeries", + "loadingEditor": false, + "loading": false, + "label": "Search", + "query": "*", + "field": "node.name", + "fields": [], + "spyable": true, + "rest": false, + "size": 5, + "mode": "terms only", + "exclude": [], + "history": [ + null, + null + ], + "use_select": false, + "remember": 10, + "title": "Node names", + "queries": [ + { + "alias": "Screech", + "query": "node.name:\"Screech\"", + "id": 1, + "color": "#EAB839", + "pin": false, + "type": "lucene" + }, + { + "alias": "Paradigm", + "query": "node.name:\"Paradigm\"", + "id": 2, + "color": "#6ED0E0", + "pin": false, + "type": "lucene" + } + ], + "selected_query": {} + } + ], + "notice": false + } + ], + "editable": true, + "failover": false, + "index": { + "interval": "day", + "pattern": "[es_monitor-]YYYY.MM.DD", + "default": "_all" + }, + "style": "dark", + "panel_hints": true, + "pulldowns": [ + { + "type": "query", + "collapse": false, + "notice": false, + "enable": true, + "query": "*", + "pinned": true, + "history": [ + "node.name:\"Paradigm\"", + "node.name:\"Screech\"", + "node.name:\"Gideon Mace\" AND (_type:node_stats)", + "node.name:\"Proteus\" AND (_type:node_stats)", + "*" + ], + "remember": 10 + }, + { + "type": "filtering", + "collapse": true, + "notice": false, + "enable": true + } + ], + "nav": [ + { + "type": "timepicker", + "collapse": false, + "notice": false, + "enable": true, + "status": "Stable", + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "timefield": "@timestamp", + "now": true, + "filter_id": 0 + } + ], + "loader": { + "save_gist": true, + "save_elasticsearch": true, + "save_local": true, + "save_default": true, + "save_temp": true, + "save_temp_ttl_enable": true, + "save_temp_ttl": "30d", + "load_gist": true, + "load_elasticsearch": true, + "load_elasticsearch_size": 20, + "load_local": true, + "hide": false + }, + "refresh": false +} \ No newline at end of file diff --git a/dashboards/shards.json b/dashboards/shards.json new file mode 100644 index 000000000000..d01a48bd8192 --- /dev/null +++ b/dashboards/shards.json @@ -0,0 +1,799 @@ +{ + "title": "Shards", + "services": { + "query": { + "idQueue": [ + ], + "list": { + }, + "ids": [ + ] + }, + "filter": { + "idQueue": [], + "list": { + "0": { + "type": "querystring", + "mandate": "must", + "active": true, + "alias": "shard starts", + "query": "_type:shard_stats", + "id": 0 + } + }, + "ids": [ + 0 + ] + } + }, + "rows": [ + { + "title": "Node filtering", + "height": "50px", + "editable": true, + "collapse": false, + "collapsable": true, + "panels": [ + { + "error": false, + "span": 4, + "editable": true, + "type": "derivequeries", + "loadingEditor": false, + "loading": false, + "label": "Search", + "query": "*", + "field": "index", + "fields": [], + "spyable": true, + "rest": false, + "size": 5, + "mode": "terms only", + "exclude": [], + "history": [ + null, + null + ], + "queries":[], + "selected_query": null, + "use_select": false, + "remember": 10, + "title": "Indices" + }, + { + "error": false, + "span": 4, + "editable": true, + "type": "derivefilters", + "loadingEditor": false, + "loading": false, + "label": "Search", + "query": "*", + "field": "node.name", + "fields": [], + "spyable": true, + "rest": false, + "size": 5, + "mode": "terms only", + "exclude": [], + "history": [ + null + ], + "filters": [], + "selected_filters": [], + "remember": 10, + "title": "Nodes" + } + ], + "notice": false + }, + { + "title": "Data", + "height": "150px", + "editable": true, + "collapse": false, + "collapsable": true, + "panels": [ + { + "span": 4, + "editable": true, + "type": "histogram", + "loadingEditor": false, + "mode": "max", + "time_field": "@timestamp", + "queries": { + "mode": "all", + "ids": [ + 1, + 2, + 3, + 4, + 5 + ] + }, + "value_field": "docs.count", + "auto_int": true, + "resolution": 100, + "interval": "30s", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "fill": 0, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": false, + "lines": true, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "options": true, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": false + }, + "title": "Docs per shard", + "extra_filter": "" + }, + { + "span": 4, + "editable": true, + "type": "histogram", + "loadingEditor": false, + "mode": "max", + "time_field": "@timestamp", + "queries": { + "mode": "all", + "ids": [ + 1, + 2, + 3, + 4, + 5 + ] + }, + "value_field": "docs.deleted", + "auto_int": true, + "resolution": 100, + "interval": "30s", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "fill": 0, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": false, + "lines": true, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "options": true, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": false + }, + "title": "Deleted docs", + "extra_filter": "" + }, + { + "span": 4, + "editable": true, + "type": "histogram", + "loadingEditor": false, + "mode": "max", + "time_field": "@timestamp", + "queries": { + "mode": "all", + "ids": [ + 1, + 2, + 3, + 4, + 5 + ] + }, + "value_field": "store.size_in_bytes", + "auto_int": true, + "resolution": 100, + "interval": "30s", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "fill": 0, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": false, + "lines": true, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "options": true, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": false + }, + "title": "Shard size", + "extra_filter": "" + } + ], + "notice": false + }, + { + "title": "Indexing", + "height": "150px", + "editable": true, + "collapse": false, + "collapsable": true, + "panels": [ + { + "span": 4, + "editable": true, + "type": "histogram", + "loadingEditor": false, + "mode": "max", + "time_field": "@timestamp", + "queries": { + "mode": "all", + "ids": [ + 1, + 2, + 3, + 4, + 5 + ] + }, + "value_field": "indexing.index_total", + "auto_int": true, + "resolution": 100, + "interval": "30s", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "fill": 0, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": false, + "lines": true, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "options": true, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": false + }, + "title": "Indexing requests", + "extra_filter": "" + }, + { + "span": 4, + "editable": true, + "type": "histogram", + "loadingEditor": false, + "mode": "max", + "time_field": "@timestamp", + "queries": { + "mode": "all", + "ids": [ + 1, + 2, + 3, + 4, + 5 + ] + }, + "value_field": "indexing.index_time_in_millis", + "auto_int": true, + "resolution": 100, + "interval": "30s", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "fill": 0, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": false, + "lines": true, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "options": true, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": false + }, + "title": "Indexing time", + "extra_filter": "" + } + ], + "notice": false + }, + { + "title": "Search & Get", + "height": "150px", + "editable": true, + "collapse": false, + "collapsable": true, + "panels": [ + { + "span": 4, + "editable": true, + "type": "histogram", + "loadingEditor": false, + "mode": "max", + "time_field": "@timestamp", + "queries": { + "mode": "all", + "ids": [ + 1, + 2, + 3, + 4, + 5 + ] + }, + "value_field": "search.query_total", + "auto_int": true, + "resolution": 100, + "interval": "30s", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "fill": 0, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": false, + "lines": true, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "options": true, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": false + }, + "title": "Search requests", + "extra_filter": "" + }, + { + "span": 4, + "editable": true, + "type": "histogram", + "loadingEditor": false, + "mode": "max", + "time_field": "@timestamp", + "queries": { + "mode": "all", + "ids": [ + 1, + 2, + 3, + 4, + 5 + ] + }, + "value_field": "get.total", + "auto_int": true, + "resolution": 100, + "interval": "30s", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "fill": 0, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": false, + "lines": true, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "options": true, + "tooltip": { + "value_type": "individual", + "query_as_alias": false + }, + "title": "GET REQUEST", + "extra_filter": "" + } + ], + "notice": false + }, + { + "title": "System health", + "height": "150px", + "editable": true, + "collapse": false, + "collapsable": true, + "panels": [ + { + "span": 4, + "editable": true, + "type": "histogram", + "loadingEditor": false, + "mode": "max", + "time_field": "@timestamp", + "queries": { + "mode": "all", + "ids": [ + 1, + 2, + 3, + 4, + 5 + ] + }, + "value_field": "fielddata.memory_size_in_bytes", + "auto_int": true, + "resolution": 100, + "interval": "30s", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "fill": 0, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": false, + "lines": true, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "options": true, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": false + }, + "extra_filter": "", + "title": "Field data" + }, + { + "span": 4, + "editable": true, + "type": "histogram", + "loadingEditor": false, + "mode": "max", + "time_field": "@timestamp", + "queries": { + "mode": "all", + "ids": [ + 1, + 2, + 3, + 4, + 5 + ] + }, + "value_field": "refresh.total_time_in_millis", + "auto_int": true, + "resolution": 100, + "interval": "30s", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "fill": 0, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": false, + "lines": true, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "options": true, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": false + }, + "title": "Avg Refresh time", + "extra_filter": "" + }, + { + "span": 4, + "editable": true, + "type": "histogram", + "loadingEditor": false, + "mode": "total", + "time_field": "@timestamp", + "queries": { + "mode": "all", + "ids": [ + 1, + 2, + 3, + 4, + 5 + ] + }, + "value_field": "search.open_contexts", + "auto_int": true, + "resolution": 100, + "interval": "30s", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "fill": 0, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": false, + "lines": true, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "options": true, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": false + }, + "title": "Open search context", + "extra_filter": "" + } + ], + "notice": false + } + ], + "editable": true, + "failover": false, + "index": { + "interval": "day", + "pattern": "[es_monitor-]YYYY.MM.DD", + "default": "_all" + }, + "style": "dark", + "panel_hints": true, + "pulldowns": [ + { + "type": "query", + "collapse": false, + "notice": false, + "enable": true, + "query": "*", + "pinned": true, + "history": [ + "index:\"es_monitor-2013.10.21\"", + "index:\"index\"", + "index:\"es_monitor-2013.10.01\"", + "index:\"kibana-int\"", + "index:\"es_monitor-2013.10.17\"", + "index:\"index2\" AND (_type:shard_stats)", + "index:\"es_monitor-2013.10.16\" AND (_type:shard_stats)", + "index:\"kibana-int\" AND (_type:shard_stats)", + "index:\"index\" AND (_type:shard_stats)", + "index:\"es_monitor-2013.10.15\" AND (_type:shard_stats)" + ], + "remember": 10 + }, + { + "type": "filtering", + "collapse": false, + "notice": true, + "enable": true + } + ], + "nav": [ + { + "type": "timepicker", + "collapse": false, + "notice": false, + "enable": true, + "status": "Stable", + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "timefield": "@timestamp", + "now": true, + "filter_id": 0 + } + ], + "loader": { + "save_gist": true, + "save_elasticsearch": true, + "save_local": true, + "save_default": true, + "save_temp": true, + "save_temp_ttl_enable": true, + "save_temp_ttl": "30d", + "load_gist": true, + "load_elasticsearch": true, + "load_elasticsearch_size": 20, + "load_local": true, + "hide": false + }, + "refresh": false +} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/enterprise/monitor/StatsExportersService.java b/src/main/java/org/elasticsearch/enterprise/monitor/StatsExportersService.java index 57775ff13d87..02d497721985 100644 --- a/src/main/java/org/elasticsearch/enterprise/monitor/StatsExportersService.java +++ b/src/main/java/org/elasticsearch/enterprise/monitor/StatsExportersService.java @@ -22,8 +22,10 @@ package org.elasticsearch.enterprise.monitor; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.admin.indices.stats.ShardStats; -import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.common.collect.ImmutableSet; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; @@ -43,6 +45,8 @@ public class StatsExportersService extends AbstractLifecycleComponent implement final NodeStatsRenderer nodeStatsRenderer; final ShardStatsRenderer shardStatsRenderer; + final IndexStatsRenderer indexStatsRenderer; + final IndicesStatsRenderer indicesStatsRenderer; public ESExporter(Settings settings, Discovery discovery) { super(settings); @@ -90,6 +92,8 @@ public class ESExporter extends AbstractLifecycleComponent implement nodeStatsRenderer = new NodeStatsRenderer(); shardStatsRenderer = new ShardStatsRenderer(); + indexStatsRenderer = new IndexStatsRenderer(); + indicesStatsRenderer = new IndicesStatsRenderer(); logger.info("ESExporter initialized. Targets: {}, index prefix [{}], index time format [{}]", hosts, indexPrefix, indexTimeFormat); } @@ -112,52 +116,86 @@ public class ESExporter extends AbstractLifecycleComponent implement exportXContent("shard_stats", shardStatsRenderer); } + @Override + public void exportIndicesStats(IndicesStatsResponse indicesStats) { + Map perIndexStats = indicesStats.getIndices(); + indexStatsRenderer.reset(perIndexStats.values().toArray(new IndexStats[perIndexStats.size()])); + indicesStatsRenderer.reset(indicesStats.getTotal(), indicesStats.getPrimaries()); + logger.debug("exporting index_stats + indices_stats"); + HttpURLConnection conn = openExportingConnection(); + if (conn == null) { + return; + } + try { + addXContentRendererToConnection(conn, "index_stats", indexStatsRenderer); + addXContentRendererToConnection(conn, "indices_stats", indicesStatsRenderer); + sendCloseExportingConnection(conn); + } catch (IOException e) { + logger.error("error sending data", e); + return; + } + } + + private HttpURLConnection openExportingConnection() { + if (!checkedForIndexTemplate) { + if (!checkForIndexTemplate()) { + logger.debug("no template defined yet. skipping"); + return null; + } + } + + logger.trace("setting up an export connection"); + HttpURLConnection conn = openConnection("POST", "/_bulk", XContentType.SMILE.restContentType()); + if (conn == null) { + logger.error("could not connect to any configured elasticsearch instances: [{}]", hosts); + } + return conn; + } + + private void addXContentRendererToConnection(HttpURLConnection conn, String type, + MultiXContentRenderer renderer) throws IOException { + OutputStream os = conn.getOutputStream(); + // TODO: find a way to disable builder's substream flushing or something neat solution + for (int i = 0; i < renderer.length(); i++) { + XContentBuilder builder = XContentFactory.smileBuilder(os); + builder.startObject().startObject("index") + .field("_index", getIndexName()).field("_type", type).endObject().endObject(); + builder.flush(); + os.write(SmileXContent.smileXContent.streamSeparator()); + + builder = XContentFactory.smileBuilder(os); + builder.humanReadable(false); + renderer.render(i, builder); + builder.flush(); + os.write(SmileXContent.smileXContent.streamSeparator()); + } + } + + private void sendCloseExportingConnection(HttpURLConnection conn) throws IOException { + logger.trace("sending exporting content"); + OutputStream os = conn.getOutputStream(); + os.close(); + + if (conn.getResponseCode() != 200) { + logConnectionError("remote target didn't respond with 200 OK", conn); + } else { + conn.getInputStream().close(); // close and release to connection pool. + } + } private void exportXContent(String type, MultiXContentRenderer xContentRenderer) { if (xContentRenderer.length() == 0) { return; } - if (!checkedForIndexTemplate) { - if (!checkForIndexTemplate()) { - logger.debug("no template defined yet. skipping"); - return; - } - ; - } - logger.debug("exporting {}", type); - HttpURLConnection conn = openConnection("POST", "/_bulk", XContentType.SMILE.restContentType()); + HttpURLConnection conn = openExportingConnection(); if (conn == null) { - logger.error("could not connect to any configured elasticsearch instances: [{}]", hosts); return; } try { - OutputStream os = conn.getOutputStream(); - // TODO: find a way to disable builder's substream flushing or something neat solution - for (int i = 0; i < xContentRenderer.length(); i++) { - XContentBuilder builder = XContentFactory.smileBuilder(os); - builder.startObject().startObject("index") - .field("_index", getIndexName()).field("_type", type).endObject().endObject(); - builder.flush(); - os.write(SmileXContent.smileXContent.streamSeparator()); - - builder = XContentFactory.smileBuilder(os); - builder.humanReadable(false); - xContentRenderer.render(i, builder); - builder.flush(); - os.write(SmileXContent.smileXContent.streamSeparator()); - - } - os.close(); - - if (conn.getResponseCode() != 200) { - logConnectionError("remote target didn't respond with 200 OK", conn); - } else { - conn.getInputStream().close(); // close and release to connection pool. - } - - + addXContentRendererToConnection(conn, type, xContentRenderer); + sendCloseExportingConnection(conn); } catch (IOException e) { logger.error("error sending data", e); return; @@ -282,7 +320,11 @@ public class ESExporter extends AbstractLifecycleComponent implement private void addNodeInfo(XContentBuilder builder) throws IOException { - builder.startObject("node"); + addNodeInfo(builder, "node"); + } + + private void addNodeInfo(XContentBuilder builder, String fieldname) throws IOException { + builder.startObject(fieldname); DiscoveryNode node = discovery.localNode(); builder.field("id", node.id()); builder.field("name", node.name()); @@ -367,5 +409,72 @@ public class ESExporter extends AbstractLifecycleComponent implement } } + class IndexStatsRenderer implements MultiXContentRenderer { + + IndexStats[] stats; + long collectionTime; + ToXContent.Params xContentParams = ToXContent.EMPTY_PARAMS; + + public void reset(IndexStats[] stats) { + this.stats = stats; + collectionTime = System.currentTimeMillis(); + } + + @Override + public int length() { + return stats == null ? 0 : stats.length; + } + + @Override + public void render(int index, XContentBuilder builder) throws IOException { + builder.startObject(); + builder.field("@timestamp", defaultDatePrinter.print(collectionTime)); + IndexStats indexStats = stats[index]; + builder.field("index", indexStats.getIndex()); + addNodeInfo(builder, "_source_node"); + builder.startObject("primaries"); + indexStats.getPrimaries().toXContent(builder, xContentParams); + builder.endObject(); + builder.startObject("total"); + indexStats.getTotal().toXContent(builder, xContentParams); + builder.endObject(); + builder.endObject(); + } + } + + class IndicesStatsRenderer implements MultiXContentRenderer { + + CommonStats totalStats; + CommonStats primariesStats; + long collectionTime; + ToXContent.Params xContentParams = ToXContent.EMPTY_PARAMS; + + public void reset(CommonStats totalStats, CommonStats primariesStats) { + this.totalStats = totalStats; + this.primariesStats = primariesStats; + collectionTime = System.currentTimeMillis(); + } + + @Override + public int length() { + return totalStats == null ? 0 : 1; + } + + @Override + public void render(int index, XContentBuilder builder) throws IOException { + assert index == 0; + builder.startObject(); + builder.field("@timestamp", defaultDatePrinter.print(collectionTime)); + addNodeInfo(builder, "_source_node"); + builder.startObject("primaries"); + primariesStats.toXContent(builder, xContentParams); + builder.endObject(); + builder.startObject("total"); + totalStats.toXContent(builder, xContentParams); + builder.endObject(); + builder.endObject(); + } + } + } diff --git a/src/main/java/org/elasticsearch/enterprise/monitor/exporter/StatsExporter.java b/src/main/java/org/elasticsearch/enterprise/monitor/exporter/StatsExporter.java index ffb3c8f4c5fe..7a2263781e04 100644 --- a/src/main/java/org/elasticsearch/enterprise/monitor/exporter/StatsExporter.java +++ b/src/main/java/org/elasticsearch/enterprise/monitor/exporter/StatsExporter.java @@ -20,9 +20,14 @@ package org.elasticsearch.enterprise.monitor.exporter; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; +import org.elasticsearch.action.admin.indices.stats.CommonStats; +import org.elasticsearch.action.admin.indices.stats.IndexStats; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.common.component.LifecycleComponent; +import java.util.Map; + public interface StatsExporter extends LifecycleComponent { String name(); @@ -30,4 +35,7 @@ public interface StatsExporter extends LifecycleComponent { void exportNodeStats(NodeStats nodeStats); void exportShardStats(ShardStats[] shardStatsArray); + + void exportIndicesStats(IndicesStatsResponse indicesStats); + }