maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Added index and indices aggregate stats exported.

Browse source 
Added current version of dashboards
This commit is contained in:
Boaz Leskes 2013-10-28 13:51:43 +01:00
parent 7b74b4713a
commit 400a046084
6 changed files with 2899 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

740
dashboards/Indices.json Normal file
View file

@ -0,0 +1,740 @@
{
"title": "Indices",
"services": {
"query": {
"idQueue": [
],
"list": {
},
"ids": [
]
},
"filter": {
"idQueue": [],
"list": {
"0": {
"type": "time",
"field": "@timestamp",
"from": "now-1h",
"to": "now",
"mandate": "must",
"active": true,
"alias": "",
"id": 0
},
"1": {
"type": "querystring",
"mandate": "must",
"active": true,
"alias": "shard starts",
"query": "_type:shard_stats",
"id": 1
}
},
"ids": [
0,
1
]
}
},
"rows": [
{
"title": "Node filtering",
"height": "50px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"error": false,
"span": 4,
"editable": true,
"type": "derivequeries",
"loadingEditor": false,
"loading": false,
"label": "Search",
"query": "*",
"field": "index",
"fields": [],
"spyable": true,
"rest": false,
"size": 5,
"mode": "terms only",
"exclude": [],
"history": [
null
],
"queries": [
],
"selected_query": null,
"use_select": false,
"remember": 10,
"title": "Indices"
},
{
"error": false,
"span": 4,
"editable": true,
"type": "derivefilters",
"loadingEditor": false,
"loading": false,
"label": "Search",
"query": "*",
"field": "node.name",
"fields": [],
"spyable": true,
"rest": false,
"size": 5,
"mode": "terms only",
"exclude": [],
"history": [
null
],
"filters": [
],
"selected_filters": [],
"remember": 10,
"title": "Nodes"
}
],
"notice": false
},
{
"title": "Data",
"height": "150px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"span": 4,
"editable": true,
"type": "histogram",
"loadingEditor": false,
"mode": "total",
"time_field": "@timestamp",
"queries": {
"mode": "all",
"ids": [
1,
4,
5,
0,
6
]
},
"value_field": "docs.count",
"auto_int": true,
"resolution": 100,
"interval": "30s",
"intervals": [
"auto",
"1s",
"1m",
"5m",
"10m",
"30m",
"1h",
"3h",
"12h",
"1d",
"1w",
"1M",
"1y"
],
"fill": 0,
"linewidth": 3,
"timezone": "browser",
"spyable": true,
"zoomlinks": true,
"bars": false,
"stack": false,
"points": false,
"lines": true,
"legend": true,
"x-axis": true,
"y-axis": true,
"percentage": false,
"interactive": true,
"options": true,
"tooltip": {
"value_type": "cumulative",
"query_as_alias": false
},
"title": "Documents",
"extra_filter": "primary:true"
},
{
"span": 4,
"editable": true,
"type": "histogram",
"loadingEditor": false,
"mode": "total",
"time_field": "@timestamp",
"queries": {
"mode": "all",
"ids": [
1,
4,
5,
0,
6
]
},
"value_field": "docs.deleted",
"auto_int": true,
"resolution": 100,
"interval": "30s",
"intervals": [
"auto",
"1s",
"1m",
"5m",
"10m",
"30m",
"1h",
"3h",
"12h",
"1d",
"1w",
"1M",
"1y"
],
"fill": 0,
"linewidth": 3,
"timezone": "browser",
"spyable": true,
"zoomlinks": true,
"bars": false,
"stack": false,
"points": false,
"lines": true,
"legend": true,
"x-axis": true,
"y-axis": true,
"percentage": false,
"interactive": true,
"options": true,
"tooltip": {
"value_type": "cumulative",
"query_as_alias": false
},
"title": "Deleted docs"
},
{
"span": 4,
"editable": true,
"type": "histogram",
"loadingEditor": false,
"mode": "total",
"time_field": "@timestamp",
"queries": {
"mode": "all",
"ids": [
1,
4,
5,
0,
6
]
},
"value_field": "store.size_in_bytes",
"auto_int": true,
"resolution": 100,
"interval": "30s",
"intervals": [
"auto",
"1s",
"1m",
"5m",
"10m",
"30m",
"1h",
"3h",
"12h",
"1d",
"1w",
"1M",
"1y"
],
"fill": 0,
"linewidth": 3,
"timezone": "browser",
"spyable": true,
"zoomlinks": true,
"bars": false,
"stack": false,
"points": false,
"lines": true,
"legend": true,
"x-axis": true,
"y-axis": true,
"percentage": false,
"interactive": true,
"options": true,
"tooltip": {
"value_type": "cumulative",
"query_as_alias": false
},
"title": "Index size"
}
],
"notice": false
},
{
"title": "Indexing",
"height": "150px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"span": 4,
"editable": true,
"type": "histogram",
"loadingEditor": false,
"mode": "total",
"time_field": "@timestamp",
"queries": {
"mode": "all",
"ids": [
1,
4,
5,
0,
6
]
},
"value_field": "indexing.index_total",
"auto_int": true,
"resolution": 100,
"interval": "30s",
"intervals": [
"auto",
"1s",
"1m",
"5m",
"10m",
"30m",
"1h",
"3h",
"12h",
"1d",
"1w",
"1M",
"1y"
],
"fill": 0,
"linewidth": 3,
"timezone": "browser",
"spyable": true,
"zoomlinks": true,
"bars": false,
"stack": false,
"points": false,
"lines": true,
"legend": true,
"x-axis": true,
"y-axis": true,
"percentage": false,
"interactive": true,
"options": true,
"tooltip": {
"value_type": "cumulative",
"query_as_alias": false
},
"title": "Indexing requests"
},
{
"span": 4,
"editable": true,
"type": "histogram",
"loadingEditor": false,
"mode": "total",
"time_field": "@timestamp",
"queries": {
"mode": "all",
"ids": [
1,
4,
5,
0,
6
]
},
"value_field": "indexing.index_time_in_millis",
"auto_int": true,
"resolution": 100,
"interval": "30s",
"intervals": [
"auto",
"1s",
"1m",
"5m",
"10m",
"30m",
"1h",
"3h",
"12h",
"1d",
"1w",
"1M",
"1y"
],
"fill": 0,
"linewidth": 3,
"timezone": "browser",
"spyable": true,
"zoomlinks": true,
"bars": false,
"stack": false,
"points": false,
"lines": true,
"legend": true,
"x-axis": true,
"y-axis": true,
"percentage": false,
"interactive": true,
"options": true,
"tooltip": {
"value_type": "cumulative",
"query_as_alias": false
},
"title": "Indexing time"
}
],
"notice": false
},
{
"title": "Search & Get",
"height": "150px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"span": 4,
"editable": true,
"type": "histogram",
"loadingEditor": false,
"mode": "total",
"time_field": "@timestamp",
"queries": {
"mode": "all",
"ids": [
1,
4,
5,
0,
6
]
},
"value_field": "search.query_total",
"auto_int": true,
"resolution": 100,
"interval": "30s",
"intervals": [
"auto",
"1s",
"1m",
"5m",
"10m",
"30m",
"1h",
"3h",
"12h",
"1d",
"1w",
"1M",
"1y"
],
"fill": 0,
"linewidth": 3,
"timezone": "browser",
"spyable": true,
"zoomlinks": true,
"bars": false,
"stack": false,
"points": false,
"lines": true,
"legend": true,
"x-axis": true,
"y-axis": true,
"percentage": false,
"interactive": true,
"options": true,
"tooltip": {
"value_type": "cumulative",
"query_as_alias": false
},
"title": "Search requests"
},
{
"span": 4,
"editable": true,
"type": "histogram",
"loadingEditor": false,
"mode": "total",
"time_field": "@timestamp",
"queries": {
"mode": "all",
"ids": [
1,
4,
5,
0,
6
]
},
"value_field": "get.total",
"auto_int": true,
"resolution": 100,
"interval": "30s",
"intervals": [
"auto",
"1s",
"1m",
"5m",
"10m",
"30m",
"1h",
"3h",
"12h",
"1d",
"1w",
"1M",
"1y"
],
"fill": 0,
"linewidth": 3,
"timezone": "browser",
"spyable": true,
"zoomlinks": true,
"bars": false,
"stack": false,
"points": false,
"lines": true,
"legend": true,
"x-axis": true,
"y-axis": true,
"percentage": false,
"interactive": true,
"options": true,
"tooltip": {
"value_type": "individual",
"query_as_alias": false
},
"title": "GET REQUEST"
}
],
"notice": false
},
{
"title": "System health",
"height": "150px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"span": 4,
"editable": true,
"type": "histogram",
"loadingEditor": false,
"mode": "total",
"time_field": "@timestamp",
"queries": {
"mode": "all",
"ids": [
1,
4,
5,
0,
6
]
},
"value_field": "search.open_contexts",
"auto_int": true,
"resolution": 100,
"interval": "30s",
"intervals": [
"auto",
"1s",
"1m",
"5m",
"10m",
"30m",
"1h",
"3h",
"12h",
"1d",
"1w",
"1M",
"1y"
],
"fill": 0,
"linewidth": 3,
"timezone": "browser",
"spyable": true,
"zoomlinks": true,
"bars": false,
"stack": false,
"points": false,
"lines": true,
"legend": true,
"x-axis": true,
"y-axis": true,
"percentage": false,
"interactive": true,
"options": true,
"tooltip": {
"value_type": "cumulative",
"query_as_alias": false
},
"title": "Open search context"
},
{
"span": 4,
"editable": true,
"type": "histogram",
"loadingEditor": false,
"mode": "max",
"time_field": "@timestamp",
"queries": {
"mode": "all",
"ids": [
1,
4,
5,
0,
6
]
},
"value_field": "refresh.total_time_in_millis",
"auto_int": true,
"resolution": 100,
"interval": "30s",
"intervals": [
"auto",
"1s",
"1m",
"5m",
"10m",
"30m",
"1h",
"3h",
"12h",
"1d",
"1w",
"1M",
"1y"
],
"fill": 0,
"linewidth": 3,
"timezone": "browser",
"spyable": true,
"zoomlinks": true,
"bars": false,
"stack": false,
"points": false,
"lines": true,
"legend": true,
"x-axis": true,
"y-axis": true,
"percentage": false,
"interactive": true,
"options": true,
"tooltip": {
"value_type": "cumulative",
"query_as_alias": false
},
"title": "Avg Refresh time"
}
],
"notice": false
}
],
"editable": true,
"failover": false,
"index": {
"interval": "day",
"pattern": "[es_monitor-]YYYY.MM.DD",
"default": "_all"
},
"style": "dark",
"panel_hints": true,
"pulldowns": [
{
"type": "query",
"collapse": false,
"notice": false,
"enable": true,
"query": "*",
"pinned": true,
"history": [
"index:\"index2\" AND (_type:shard_stats)",
"index:\"es_monitor-2013.10.16\" AND (_type:shard_stats)",
"index:\"kibana-int\" AND (_type:shard_stats)",
"index:\"index\" AND (_type:shard_stats)",
"index:\"es_monitor-2013.10.15\" AND (_type:shard_stats)"
],
"remember": 10
},
{
"type": "filtering",
"collapse": false,
"notice": true,
"enable": true
}
],
"nav": [
{
"type": "timepicker",
"collapse": false,
"notice": false,
"enable": true,
"status": "Stable",
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
],
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"timefield": "@timestamp",
"now": true,
"filter_id": 0
}
],
"loader": {
"save_gist": true,
"save_elasticsearch": true,
"save_local": true,
"save_default": true,
"save_temp": true,
"save_temp_ttl_enable": true,
"save_temp_ttl": "30d",
"load_gist": true,
"load_elasticsearch": true,
"load_elasticsearch_size": 20,
"load_local": true,
"hide": false
},
"refresh": false
}

1173
dashboards/nodes.json Normal file
View file

File diff suppressed because it is too large Load diff

799
dashboards/shards.json Normal file
View file

@ -0,0 +1,799 @@
{
"title": "Shards",
"services": {
"query": {
"idQueue": [
],
"list": {
},
"ids": [
]
},
"filter": {
"idQueue": [],
"list": {
"0": {
"type": "querystring",
"mandate": "must",
"active": true,
"alias": "shard starts",
"query": "_type:shard_stats",
"id": 0
}
},
"ids": [
0
]
}
},
"rows": [
{
"title": "Node filtering",
"height": "50px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"error": false,
"span": 4,
"editable": true,
"type": "derivequeries",
"loadingEditor": false,
"loading": false,
"label": "Search",
"query": "*",
"field": "index",
"fields": [],
"spyable": true,
"rest": false,
"size": 5,
"mode": "terms only",
"exclude": [],
"history": [
null,
null
],
"queries":[],
"selected_query": null,
"use_select": false,
"remember": 10,
"title": "Indices"
},
{
"error": false,
"span": 4,
"editable": true,
"type": "derivefilters",
"loadingEditor": false,
"loading": false,
"label": "Search",
"query": "*",
"field": "node.name",
"fields": [],
"spyable": true,
"rest": false,
"size": 5,
"mode": "terms only",
"exclude": [],
"history": [
null
],
"filters": [],
"selected_filters": [],
"remember": 10,
"title": "Nodes"
}
],
"notice": false
},
{
"title": "Data",
"height": "150px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"span": 4,
"editable": true,
"type": "histogram",
"loadingEditor": false,
"mode": "max",
"time_field": "@timestamp",
"queries": {
"mode": "all",
"ids": [
1,
2,
3,
4,
5
]
},
"value_field": "docs.count",
"auto_int": true,
"resolution": 100,
"interval": "30s",
"intervals": [
"auto",
"1s",
"1m",
"5m",
"10m",
"30m",
"1h",
"3h",
"12h",
"1d",
"1w",
"1M",
"1y"
],
"fill": 0,
"linewidth": 3,
"timezone": "browser",
"spyable": true,
"zoomlinks": true,
"bars": false,
"stack": false,
"points": false,
"lines": true,
"legend": true,
"x-axis": true,
"y-axis": true,
"percentage": false,
"interactive": true,
"options": true,
"tooltip": {
"value_type": "cumulative",
"query_as_alias": false
},
"title": "Docs per shard",
"extra_filter": ""
},
{
"span": 4,
"editable": true,
"type": "histogram",
"loadingEditor": false,
"mode": "max",
"time_field": "@timestamp",
"queries": {
"mode": "all",
"ids": [
1,
2,
3,
4,
5
]
},
"value_field": "docs.deleted",
"auto_int": true,
"resolution": 100,
"interval": "30s",
"intervals": [
"auto",
"1s",
"1m",
"5m",
"10m",
"30m",
"1h",
"3h",
"12h",
"1d",
"1w",
"1M",
"1y"
],
"fill": 0,
"linewidth": 3,
"timezone": "browser",
"spyable": true,
"zoomlinks": true,
"bars": false,
"stack": false,
"points": false,
"lines": true,
"legend": true,
"x-axis": true,
"y-axis": true,
"percentage": false,
"interactive": true,
"options": true,
"tooltip": {
"value_type": "cumulative",
"query_as_alias": false
},
"title": "Deleted docs",
"extra_filter": ""
},
{
"span": 4,
"editable": true,
"type": "histogram",
"loadingEditor": false,
"mode": "max",
"time_field": "@timestamp",
"queries": {
"mode": "all",
"ids": [
1,
2,
3,
4,
5
]
},
"value_field": "store.size_in_bytes",
"auto_int": true,
"resolution": 100,
"interval": "30s",
"intervals": [
"auto",
"1s",
"1m",
"5m",
"10m",
"30m",
"1h",
"3h",
"12h",
"1d",
"1w",
"1M",
"1y"
],
"fill": 0,
"linewidth": 3,
"timezone": "browser",
"spyable": true,
"zoomlinks": true,
"bars": false,
"stack": false,
"points": false,
"lines": true,
"legend": true,
"x-axis": true,
"y-axis": true,
"percentage": false,
"interactive": true,
"options": true,
"tooltip": {
"value_type": "cumulative",
"query_as_alias": false
},
"title": "Shard size",
"extra_filter": ""
}
],
"notice": false
},
{
"title": "Indexing",
"height": "150px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"span": 4,
"editable": true,
"type": "histogram",
"loadingEditor": false,
"mode": "max",
"time_field": "@timestamp",
"queries": {
"mode": "all",
"ids": [
1,
2,
3,
4,
5
]
},
"value_field": "indexing.index_total",
"auto_int": true,
"resolution": 100,
"interval": "30s",
"intervals": [
"auto",
"1s",
"1m",
"5m",
"10m",
"30m",
"1h",
"3h",
"12h",
"1d",
"1w",
"1M",
"1y"
],
"fill": 0,
"linewidth": 3,
"timezone": "browser",
"spyable": true,
"zoomlinks": true,
"bars": false,
"stack": false,
"points": false,
"lines": true,
"legend": true,
"x-axis": true,
"y-axis": true,
"percentage": false,
"interactive": true,
"options": true,
"tooltip": {
"value_type": "cumulative",
"query_as_alias": false
},
"title": "Indexing requests",
"extra_filter": ""
},
{
"span": 4,
"editable": true,
"type": "histogram",
"loadingEditor": false,
"mode": "max",
"time_field": "@timestamp",
"queries": {
"mode": "all",
"ids": [
1,
2,
3,
4,
5
]
},
"value_field": "indexing.index_time_in_millis",
"auto_int": true,
"resolution": 100,
"interval": "30s",
"intervals": [
"auto",
"1s",
"1m",
"5m",
"10m",
"30m",
"1h",
"3h",
"12h",
"1d",
"1w",
"1M",
"1y"
],
"fill": 0,
"linewidth": 3,
"timezone": "browser",
"spyable": true,
"zoomlinks": true,
"bars": false,
"stack": false,
"points": false,
"lines": true,
"legend": true,
"x-axis": true,
"y-axis": true,
"percentage": false,
"interactive": true,
"options": true,
"tooltip": {
"value_type": "cumulative",
"query_as_alias": false
},
"title": "Indexing time",
"extra_filter": ""
}
],
"notice": false
},
{
"title": "Search & Get",
"height": "150px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"span": 4,
"editable": true,
"type": "histogram",
"loadingEditor": false,
"mode": "max",
"time_field": "@timestamp",
"queries": {
"mode": "all",
"ids": [
1,
2,
3,
4,
5
]
},
"value_field": "search.query_total",
"auto_int": true,
"resolution": 100,
"interval": "30s",
"intervals": [
"auto",
"1s",
"1m",
"5m",
"10m",
"30m",
"1h",
"3h",
"12h",
"1d",
"1w",
"1M",
"1y"
],
"fill": 0,
"linewidth": 3,
"timezone": "browser",
"spyable": true,
"zoomlinks": true,
"bars": false,
"stack": false,
"points": false,
"lines": true,
"legend": true,
"x-axis": true,
"y-axis": true,
"percentage": false,
"interactive": true,
"options": true,
"tooltip": {
"value_type": "cumulative",
"query_as_alias": false
},
"title": "Search requests",
"extra_filter": ""
},
{
"span": 4,
"editable": true,
"type": "histogram",
"loadingEditor": false,
"mode": "max",
"time_field": "@timestamp",
"queries": {
"mode": "all",
"ids": [
1,
2,
3,
4,
5
]
},
"value_field": "get.total",
"auto_int": true,
"resolution": 100,
"interval": "30s",
"intervals": [
"auto",
"1s",
"1m",
"5m",
"10m",
"30m",
"1h",
"3h",
"12h",
"1d",
"1w",
"1M",
"1y"
],
"fill": 0,
"linewidth": 3,
"timezone": "browser",
"spyable": true,
"zoomlinks": true,
"bars": false,
"stack": false,
"points": false,
"lines": true,
"legend": true,
"x-axis": true,
"y-axis": true,
"percentage": false,
"interactive": true,
"options": true,
"tooltip": {
"value_type": "individual",
"query_as_alias": false
},
"title": "GET REQUEST",
"extra_filter": ""
}
],
"notice": false
},
{
"title": "System health",
"height": "150px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"span": 4,
"editable": true,
"type": "histogram",
"loadingEditor": false,
"mode": "max",
"time_field": "@timestamp",
"queries": {
"mode": "all",
"ids": [
1,
2,
3,
4,
5
]
},
"value_field": "fielddata.memory_size_in_bytes",
"auto_int": true,
"resolution": 100,
"interval": "30s",
"intervals": [
"auto",
"1s",
"1m",
"5m",
"10m",
"30m",
"1h",
"3h",
"12h",
"1d",
"1w",
"1M",
"1y"
],
"fill": 0,
"linewidth": 3,
"timezone": "browser",
"spyable": true,
"zoomlinks": true,
"bars": false,
"stack": false,
"points": false,
"lines": true,
"legend": true,
"x-axis": true,
"y-axis": true,
"percentage": false,
"interactive": true,
"options": true,
"tooltip": {
"value_type": "cumulative",
"query_as_alias": false
},
"extra_filter": "",
"title": "Field data"
},
{
"span": 4,
"editable": true,
"type": "histogram",
"loadingEditor": false,
"mode": "max",
"time_field": "@timestamp",
"queries": {
"mode": "all",
"ids": [
1,
2,
3,
4,
5
]
},
"value_field": "refresh.total_time_in_millis",
"auto_int": true,
"resolution": 100,
"interval": "30s",
"intervals": [
"auto",
"1s",
"1m",
"5m",
"10m",
"30m",
"1h",
"3h",
"12h",
"1d",
"1w",
"1M",
"1y"
],
"fill": 0,
"linewidth": 3,
"timezone": "browser",
"spyable": true,
"zoomlinks": true,
"bars": false,
"stack": false,
"points": false,
"lines": true,
"legend": true,
"x-axis": true,
"y-axis": true,
"percentage": false,
"interactive": true,
"options": true,
"tooltip": {
"value_type": "cumulative",
"query_as_alias": false
},
"title": "Avg Refresh time",
"extra_filter": ""
},
{
"span": 4,
"editable": true,
"type": "histogram",
"loadingEditor": false,
"mode": "total",
"time_field": "@timestamp",
"queries": {
"mode": "all",
"ids": [
1,
2,
3,
4,
5
]
},
"value_field": "search.open_contexts",
"auto_int": true,
"resolution": 100,
"interval": "30s",
"intervals": [
"auto",
"1s",
"1m",
"5m",
"10m",
"30m",
"1h",
"3h",
"12h",
"1d",
"1w",
"1M",
"1y"
],
"fill": 0,
"linewidth": 3,
"timezone": "browser",
"spyable": true,
"zoomlinks": true,
"bars": false,
"stack": false,
"points": false,
"lines": true,
"legend": true,
"x-axis": true,
"y-axis": true,
"percentage": false,
"interactive": true,
"options": true,
"tooltip": {
"value_type": "cumulative",
"query_as_alias": false
},
"title": "Open search context",
"extra_filter": ""
}
],
"notice": false
}
],
"editable": true,
"failover": false,
"index": {
"interval": "day",
"pattern": "[es_monitor-]YYYY.MM.DD",
"default": "_all"
},
"style": "dark",
"panel_hints": true,
"pulldowns": [
{
"type": "query",
"collapse": false,
"notice": false,
"enable": true,
"query": "*",
"pinned": true,
"history": [
"index:\"es_monitor-2013.10.21\"",
"index:\"index\"",
"index:\"es_monitor-2013.10.01\"",
"index:\"kibana-int\"",
"index:\"es_monitor-2013.10.17\"",
"index:\"index2\" AND (_type:shard_stats)",
"index:\"es_monitor-2013.10.16\" AND (_type:shard_stats)",
"index:\"kibana-int\" AND (_type:shard_stats)",
"index:\"index\" AND (_type:shard_stats)",
"index:\"es_monitor-2013.10.15\" AND (_type:shard_stats)"
],
"remember": 10
},
{
"type": "filtering",
"collapse": false,
"notice": true,
"enable": true
}
],
"nav": [
{
"type": "timepicker",
"collapse": false,
"notice": false,
"enable": true,
"status": "Stable",
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
],
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"timefield": "@timestamp",
"now": true,
"filter_id": 0
}
],
"loader": {
"save_gist": true,
"save_elasticsearch": true,
"save_local": true,
"save_default": true,
"save_temp": true,
"save_temp_ttl_enable": true,
"save_temp_ttl": "30d",
"load_gist": true,
"load_elasticsearch": true,
"load_elasticsearch_size": 20,
"load_local": true,
"hide": false
},
"refresh": false
}

36
src/main/java/org/elasticsearch/enterprise/monitor/StatsExportersService.java
View file

@ -22,8 +22,10 @@ package org.elasticsearch.enterprise.monitor;
import org.elasticsearch.ElasticSearchException;
import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
import org.elasticsearch.action.admin.indices.stats.ShardStats;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.common.collect.ImmutableSet;
import org.elasticsearch.common.component.AbstractLifecycleComponent;
import org.elasticsearch.common.inject.Inject;
@ -43,6 +45,8 @@ public class StatsExportersService extends AbstractLifecycleComponent<StatsExpor
private final InternalIndicesService indicesService;
private final NodeService nodeService;
private final ClusterService clusterService;
private final Client client;
private volatile ExportingWorker exp;
private volatile Thread thread;
@ -52,11 +56,15 @@ public class StatsExportersService extends AbstractLifecycleComponent<StatsExpor
@Inject
public StatsExportersService(Settings settings, IndicesService indicesService,
NodeService nodeService, Discovery discovery) {
NodeService nodeService, ClusterService clusterService,
Client client,
Discovery discovery) {
super(settings);
this.indicesService = (InternalIndicesService) indicesService;
this.clusterService = clusterService;
this.nodeService = nodeService;
this.interval = componentSettings.getAsTime("interval", TimeValue.timeValueSeconds(5));
this.client = client;
StatsExporter esExporter = new ESExporter(settings.getComponentSettings(ESExporter.class), discovery);
this.exporters = ImmutableSet.of(esExporter);
@ -94,7 +102,7 @@ public class StatsExportersService extends AbstractLifecycleComponent<StatsExpor
@Override
public void run() {
while (!closed) {
// sleep first to allow node to complete initialization before collectiont the first start
// sleep first to allow node to complete initialization before collecting the first start
try {
Thread.sleep(interval.millis());
} catch (InterruptedException e) {
@ -111,7 +119,7 @@ public class StatsExportersService extends AbstractLifecycleComponent<StatsExpor
try {
e.exportNodeStats(nodeStats);
} catch (Throwable t) {
logger.error("StatsExporter {} has thrown an exception:", t, e.name());
logger.error("StatsExporter [{}] has thrown an exception:", t, e.name());
}
}
@ -123,9 +131,27 @@ public class StatsExportersService extends AbstractLifecycleComponent<StatsExpor
try {
e.exportShardStats(shardStatsArray);
} catch (Throwable t) {
logger.error("StatsExporter {} has thrown an exception:", t, e.name());
logger.error("StatsExporter [{}] has thrown an exception:", t, e.name());
}
}
if (clusterService.state().nodes().localNodeMaster()) {
logger.debug("local node is master, exporting aggregated stats");
IndicesStatsResponse indicesStatsResponse = client.admin().indices().prepareStats().all().get();
for (StatsExporter e : exporters) {
try {
e.exportIndicesStats(indicesStatsResponse);
} catch (Throwable t) {
logger.error("StatsExporter [{}] has thrown an exception:", t, e.name());
}
}
}
} catch (Throwable t) {
logger.error("Background thread had an uncaught exception:", t);
}

187
src/main/java/org/elasticsearch/enterprise/monitor/exporter/ESExporter.java
View file

@ -21,8 +21,10 @@ package org.elasticsearch.enterprise.monitor.exporter;
import org.elasticsearch.ElasticSearchException;
import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
import org.elasticsearch.action.admin.indices.stats.CommonStats;
import org.elasticsearch.action.admin.indices.stats.IndexStats;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
import org.elasticsearch.action.admin.indices.stats.ShardStats;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.common.collect.ImmutableMap;
@ -43,8 +45,6 @@ import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.smile.SmileXContent;
import org.elasticsearch.discovery.Discovery;
import org.elasticsearch.discovery.DiscoveryService;
import org.elasticsearch.node.service.NodeService;
import java.io.IOException;
import java.io.InputStream;
@ -72,6 +72,8 @@ public class ESExporter extends AbstractLifecycleComponent<ESExporter> implement
final NodeStatsRenderer nodeStatsRenderer;
final ShardStatsRenderer shardStatsRenderer;
final IndexStatsRenderer indexStatsRenderer;
final IndicesStatsRenderer indicesStatsRenderer;
public ESExporter(Settings settings, Discovery discovery) {
super(settings);
@ -90,6 +92,8 @@ public class ESExporter extends AbstractLifecycleComponent<ESExporter> implement
nodeStatsRenderer = new NodeStatsRenderer();
shardStatsRenderer = new ShardStatsRenderer();
indexStatsRenderer = new IndexStatsRenderer();
indicesStatsRenderer = new IndicesStatsRenderer();
logger.info("ESExporter initialized. Targets: {}, index prefix [{}], index time format [{}]", hosts, indexPrefix, indexTimeFormat);
}
@ -112,52 +116,86 @@ public class ESExporter extends AbstractLifecycleComponent<ESExporter> implement
exportXContent("shard_stats", shardStatsRenderer);
}
@Override
public void exportIndicesStats(IndicesStatsResponse indicesStats) {
Map<String, IndexStats> perIndexStats = indicesStats.getIndices();
indexStatsRenderer.reset(perIndexStats.values().toArray(new IndexStats[perIndexStats.size()]));
indicesStatsRenderer.reset(indicesStats.getTotal(), indicesStats.getPrimaries());
logger.debug("exporting index_stats + indices_stats");
HttpURLConnection conn = openExportingConnection();
if (conn == null) {
return;
}
try {
addXContentRendererToConnection(conn, "index_stats", indexStatsRenderer);
addXContentRendererToConnection(conn, "indices_stats", indicesStatsRenderer);
sendCloseExportingConnection(conn);
} catch (IOException e) {
logger.error("error sending data", e);
return;
}
}
private HttpURLConnection openExportingConnection() {
if (!checkedForIndexTemplate) {
if (!checkForIndexTemplate()) {
logger.debug("no template defined yet. skipping");
return null;
}
}
logger.trace("setting up an export connection");
HttpURLConnection conn = openConnection("POST", "/_bulk", XContentType.SMILE.restContentType());
if (conn == null) {
logger.error("could not connect to any configured elasticsearch instances: [{}]", hosts);
}
return conn;
}
private void addXContentRendererToConnection(HttpURLConnection conn, String type,
MultiXContentRenderer renderer) throws IOException {
OutputStream os = conn.getOutputStream();
// TODO: find a way to disable builder's substream flushing or something neat solution
for (int i = 0; i < renderer.length(); i++) {
XContentBuilder builder = XContentFactory.smileBuilder(os);
builder.startObject().startObject("index")
.field("_index", getIndexName()).field("_type", type).endObject().endObject();
builder.flush();
os.write(SmileXContent.smileXContent.streamSeparator());
builder = XContentFactory.smileBuilder(os);
builder.humanReadable(false);
renderer.render(i, builder);
builder.flush();
os.write(SmileXContent.smileXContent.streamSeparator());
}
}
private void sendCloseExportingConnection(HttpURLConnection conn) throws IOException {
logger.trace("sending exporting content");
OutputStream os = conn.getOutputStream();
os.close();
if (conn.getResponseCode() != 200) {
logConnectionError("remote target didn't respond with 200 OK", conn);
} else {
conn.getInputStream().close(); // close and release to connection pool.
}
}
private void exportXContent(String type, MultiXContentRenderer xContentRenderer) {
if (xContentRenderer.length() == 0) {
return;
}
if (!checkedForIndexTemplate) {
if (!checkForIndexTemplate()) {
logger.debug("no template defined yet. skipping");
return;
}
;
}
logger.debug("exporting {}", type);
HttpURLConnection conn = openConnection("POST", "/_bulk", XContentType.SMILE.restContentType());
HttpURLConnection conn = openExportingConnection();
if (conn == null) {
logger.error("could not connect to any configured elasticsearch instances: [{}]", hosts);
return;
}
try {
OutputStream os = conn.getOutputStream();
// TODO: find a way to disable builder's substream flushing or something neat solution
for (int i = 0; i < xContentRenderer.length(); i++) {
XContentBuilder builder = XContentFactory.smileBuilder(os);
builder.startObject().startObject("index")
.field("_index", getIndexName()).field("_type", type).endObject().endObject();
builder.flush();
os.write(SmileXContent.smileXContent.streamSeparator());
builder = XContentFactory.smileBuilder(os);
builder.humanReadable(false);
xContentRenderer.render(i, builder);
builder.flush();
os.write(SmileXContent.smileXContent.streamSeparator());
}
os.close();
if (conn.getResponseCode() != 200) {
logConnectionError("remote target didn't respond with 200 OK", conn);
} else {
conn.getInputStream().close(); // close and release to connection pool.
}
addXContentRendererToConnection(conn, type, xContentRenderer);
sendCloseExportingConnection(conn);
} catch (IOException e) {
logger.error("error sending data", e);
return;
@ -282,7 +320,11 @@ public class ESExporter extends AbstractLifecycleComponent<ESExporter> implement
private void addNodeInfo(XContentBuilder builder) throws IOException {
builder.startObject("node");
addNodeInfo(builder, "node");
}
private void addNodeInfo(XContentBuilder builder, String fieldname) throws IOException {
builder.startObject(fieldname);
DiscoveryNode node = discovery.localNode();
builder.field("id", node.id());
builder.field("name", node.name());
@ -367,5 +409,72 @@ public class ESExporter extends AbstractLifecycleComponent<ESExporter> implement
}
}
class IndexStatsRenderer implements MultiXContentRenderer {
IndexStats[] stats;
long collectionTime;
ToXContent.Params xContentParams = ToXContent.EMPTY_PARAMS;
public void reset(IndexStats[] stats) {
this.stats = stats;
collectionTime = System.currentTimeMillis();
}
@Override
public int length() {
return stats == null ? 0 : stats.length;
}
@Override
public void render(int index, XContentBuilder builder) throws IOException {
builder.startObject();
builder.field("@timestamp", defaultDatePrinter.print(collectionTime));
IndexStats indexStats = stats[index];
builder.field("index", indexStats.getIndex());
addNodeInfo(builder, "_source_node");
builder.startObject("primaries");
indexStats.getPrimaries().toXContent(builder, xContentParams);
builder.endObject();
builder.startObject("total");
indexStats.getTotal().toXContent(builder, xContentParams);
builder.endObject();
builder.endObject();
}
}
class IndicesStatsRenderer implements MultiXContentRenderer {
CommonStats totalStats;
CommonStats primariesStats;
long collectionTime;
ToXContent.Params xContentParams = ToXContent.EMPTY_PARAMS;
public void reset(CommonStats totalStats, CommonStats primariesStats) {
this.totalStats = totalStats;
this.primariesStats = primariesStats;
collectionTime = System.currentTimeMillis();
}
@Override
public int length() {
return totalStats == null ? 0 : 1;
}
@Override
public void render(int index, XContentBuilder builder) throws IOException {
assert index == 0;
builder.startObject();
builder.field("@timestamp", defaultDatePrinter.print(collectionTime));
addNodeInfo(builder, "_source_node");
builder.startObject("primaries");
primariesStats.toXContent(builder, xContentParams);
builder.endObject();
builder.startObject("total");
totalStats.toXContent(builder, xContentParams);
builder.endObject();
builder.endObject();
}
}
}

8
src/main/java/org/elasticsearch/enterprise/monitor/exporter/StatsExporter.java
View file

@ -20,9 +20,14 @@ package org.elasticsearch.enterprise.monitor.exporter;
import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
import org.elasticsearch.action.admin.indices.stats.CommonStats;
import org.elasticsearch.action.admin.indices.stats.IndexStats;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
import org.elasticsearch.action.admin.indices.stats.ShardStats;
import org.elasticsearch.common.component.LifecycleComponent;
import java.util.Map;
public interface StatsExporter<T> extends LifecycleComponent<T> {
String name();
@ -30,4 +35,7 @@ public interface StatsExporter<T> extends LifecycleComponent<T> {
void exportNodeStats(NodeStats nodeStats);
void exportShardStats(ShardStats[] shardStatsArray);
void exportIndicesStats(IndicesStatsResponse indicesStats);
}