maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[Context view] Incrementally increase context time window (#16878)

Browse source 
This PR tries to reduce the Elasticsearch cluster load for index patterns with many indices.

**Theory of operation**

Before this PR, Elasticsearch had to perform the query and sorting for every shard matching the index pattern. In order to avoid that in a time-base indexing scheme, the queries should include a `range` filter. This enables Elasticsearch to rewrite most of the shard accesses to `match_none`. But since the context view operates on document counts, the time intervals need to be determined heuristically:

* start of the interval is the second end of the previous query iteration or `anchor_time` if it is the first iteration
* end of the interval is `interval_start +/- n days` with `n in {1, 7, 30, 365, 10000}` or unlimited if insufficient hits were returned for all `n`

This date arithmetic introduces the assumption that the primary sorting field is a date or at least numeric. Therefore, the `sortingField` has been renamed to `timeField` to make those new assumptions explicit.

**Other notes**

As an additional optimization, the queries are now executed in a `constant_score` filter context to enable caching by Elasticsearch.

Tests for `fetchSuccessors` and `fetchPredecessors` were added.

Changes in ElasticSearch required a concurrent fix of #17696, which is also included. It now splits up the anchor `uid` into `anchorType` and `anchorId` and uses them in an `ids` query.

**Testing**

The fact that only a small subset of the shards are involved in a query should be observable using the `skipped` shard count in the response.

fixes #15143
fixes #17696
This commit is contained in:
Felix Stürmer 2018-06-18 12:31:54 +02:00 committed by GitHub
parent ef29b891a4
commit cf16b801fd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 739 additions and 200 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

78
src/core_plugins/kibana/public/context/api/__tests__/_stubs.js Normal file
View file

@ -0,0 +1,78 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import sinon from 'sinon';
export function createCourierStub() {
return {
indexPatterns: {
get: sinon.spy(indexPatternId =>
Promise.resolve({
id: indexPatternId,
})
),
},
};
}
export function createSearchSourceStubProvider(hits, timeField) {
const searchSourceStub = {
_stubHits: hits,
_stubTimeField: timeField,
_createStubHit: (timestamp, tiebreaker = 0) => ({
[searchSourceStub._stubTimeField]: timestamp,
sort: [timestamp, tiebreaker],
}),
};
searchSourceStub.filter = sinon.stub().returns(searchSourceStub);
searchSourceStub.inherits = sinon.stub().returns(searchSourceStub);
searchSourceStub.set = sinon.stub().returns(searchSourceStub);
searchSourceStub.get = sinon.spy(key => {
const previousSetCall = searchSourceStub.set.withArgs(key).lastCall;
return previousSetCall ? previousSetCall.args[1] : null;
});
searchSourceStub.fetchAsRejectablePromise = sinon.spy(() => {
const timeField = searchSourceStub._stubTimeField;
const lastQuery = searchSourceStub.set.withArgs('query').lastCall.args[1];
const timeRange = lastQuery.query.constant_score.filter.range[timeField];
const lastSort = searchSourceStub.set.withArgs('sort').lastCall.args[1];
const sortDirection = lastSort[0][timeField];
const sortFunction =
sortDirection === 'asc'
? (first, second) => first[timeField] - second[timeField]
: (first, second) => second[timeField] - first[timeField];
const filteredHits = searchSourceStub._stubHits
.filter(
hit =>
hit[timeField] >= timeRange.gte && hit[timeField] <= timeRange.lte
)
.sort(sortFunction);
return Promise.resolve({
hits: {
hits: filteredHits,
total: filteredHits.length,
},
});
});
return function SearchSourceStubProvider() {
return searchSourceStub;
};
}

41
src/core_plugins/kibana/public/context/api/__tests__/anchor.js
View file

@ -21,6 +21,7 @@ import expect from 'expect.js';
import ngMock from 'ng_mock';
import sinon from 'sinon';
import { createCourierStub } from './_stubs';
import { SearchSourceProvider } from 'ui/courier/data_source/search_source';
import { fetchAnchorProvider } from '../anchor';
@ -49,7 +50,7 @@ describe('context app', function () {
it('should use the `fetchAsRejectablePromise` method of the SearchSource', function () {
const searchSourceStub = new SearchSourceStub();
return fetchAnchor('INDEX_PATTERN_ID', 'UID', [{ '@timestamp': 'desc' }, { '_doc': 'asc' }])
return fetchAnchor('INDEX_PATTERN_ID', 'doc', 'id', [{ '@timestamp': 'desc' }, { '_doc': 'asc' }])
.then(() => {
expect(searchSourceStub.fetchAsRejectablePromise.calledOnce).to.be(true);
});
@ -58,7 +59,7 @@ describe('context app', function () {
it('should configure the SearchSource to not inherit from the implicit root', function () {
const searchSourceStub = new SearchSourceStub();
return fetchAnchor('INDEX_PATTERN_ID', 'UID', [{ '@timestamp': 'desc' }, { '_doc': 'asc' }])
return fetchAnchor('INDEX_PATTERN_ID', 'doc', 'id', [{ '@timestamp': 'desc' }, { '_doc': 'asc' }])
.then(() => {
const inheritsSpy = searchSourceStub.inherits;
expect(inheritsSpy.calledOnce).to.be(true);
@ -69,7 +70,7 @@ describe('context app', function () {
it('should set the SearchSource index pattern', function () {
const searchSourceStub = new SearchSourceStub();
return fetchAnchor('INDEX_PATTERN_ID', 'UID', [{ '@timestamp': 'desc' }, { '_doc': 'asc' }])
return fetchAnchor('INDEX_PATTERN_ID', 'doc', 'id', [{ '@timestamp': 'desc' }, { '_doc': 'asc' }])
.then(() => {
const setIndexSpy = searchSourceStub.set.withArgs('index');
expect(setIndexSpy.calledOnce).to.be(true);
@ -80,7 +81,7 @@ describe('context app', function () {
it('should set the SearchSource version flag to true', function () {
const searchSourceStub = new SearchSourceStub();
return fetchAnchor('INDEX_PATTERN_ID', 'UID', [{ '@timestamp': 'desc' }, { '_doc': 'asc' }])
return fetchAnchor('INDEX_PATTERN_ID', 'doc', 'id', [{ '@timestamp': 'desc' }, { '_doc': 'asc' }])
.then(() => {
const setVersionSpy = searchSourceStub.set.withArgs('version');
expect(setVersionSpy.calledOnce).to.be(true);
@ -91,7 +92,7 @@ describe('context app', function () {
it('should set the SearchSource size to 1', function () {
const searchSourceStub = new SearchSourceStub();
return fetchAnchor('INDEX_PATTERN_ID', 'UID', [{ '@timestamp': 'desc' }, { '_doc': 'asc' }])
return fetchAnchor('INDEX_PATTERN_ID', 'doc', 'id', [{ '@timestamp': 'desc' }, { '_doc': 'asc' }])
.then(() => {
const setSizeSpy = searchSourceStub.set.withArgs('size');
expect(setSizeSpy.calledOnce).to.be(true);
@ -99,17 +100,22 @@ describe('context app', function () {
});
});
it('should set the SearchSource query to a _uid terms query', function () {
it('should set the SearchSource query to an ids query', function () {
const searchSourceStub = new SearchSourceStub();
return fetchAnchor('INDEX_PATTERN_ID', 'UID', [{ '@timestamp': 'desc' }, { '_doc': 'asc' }])
return fetchAnchor('INDEX_PATTERN_ID', 'doc', 'id', [{ '@timestamp': 'desc' }, { '_doc': 'asc' }])
.then(() => {
const setQuerySpy = searchSourceStub.set.withArgs('query');
expect(setQuerySpy.calledOnce).to.be(true);
expect(setQuerySpy.firstCall.args[1]).to.eql({
query: {
terms: {
_uid: ['UID'],
constant_score: {
filter: {
ids: {
type: 'doc',
values: ['id'],
},
}
}
},
language: 'lucene'
@ -120,7 +126,7 @@ describe('context app', function () {
it('should set the SearchSource sort order', function () {
const searchSourceStub = new SearchSourceStub();
return fetchAnchor('INDEX_PATTERN_ID', 'UID', [{ '@timestamp': 'desc' }, { '_doc': 'asc' }])
return fetchAnchor('INDEX_PATTERN_ID', 'doc', 'id', [{ '@timestamp': 'desc' }, { '_doc': 'asc' }])
.then(() => {
const setSortSpy = searchSourceStub.set.withArgs('sort');
expect(setSortSpy.calledOnce).to.be(true);
@ -135,7 +141,7 @@ describe('context app', function () {
const searchSourceStub = new SearchSourceStub();
searchSourceStub._stubHits = [];
return fetchAnchor('INDEX_PATTERN_ID', 'UID', [{ '@timestamp': 'desc' }, { '_doc': 'asc' }])
return fetchAnchor('INDEX_PATTERN_ID', 'doc', 'id', [{ '@timestamp': 'desc' }, { '_doc': 'asc' }])
.then(
() => {
expect().fail('expected the promise to be rejected');
@ -153,7 +159,7 @@ describe('context app', function () {
{ property2: 'value2' },
];
return fetchAnchor('INDEX_PATTERN_ID', 'UID', [{ '@timestamp': 'desc' }, { '_doc': 'asc' }])
return fetchAnchor('INDEX_PATTERN_ID', 'doc', 'id', [{ '@timestamp': 'desc' }, { '_doc': 'asc' }])
.then((anchorDocument) => {
expect(anchorDocument).to.have.property('property1', 'value1');
expect(anchorDocument).to.have.property('$$_isAnchor', true);
@ -162,17 +168,6 @@ describe('context app', function () {
});
});
function createCourierStub() {
return {
indexPatterns: {
get: sinon.spy((indexPatternId) => Promise.resolve({
id: indexPatternId,
})),
},
};
}
function createSearchSourceStubProvider(hits) {
const searchSourceStub = {
_stubHits: hits,

186
src/core_plugins/kibana/public/context/api/__tests__/predecessors.js Normal file
View file

@ -0,0 +1,186 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import expect from 'expect.js';
import ngMock from 'ng_mock';
import * as _ from 'lodash';
import { createCourierStub, createSearchSourceStubProvider } from './_stubs';
import { SearchSourceProvider } from 'ui/courier/data_source/search_source';
import { fetchContextProvider } from '../context';
const MS_PER_DAY = 24 * 60 * 60 * 1000;
describe('context app', function () {
beforeEach(ngMock.module('kibana'));
describe('function fetchPredecessors', function () {
let fetchPredecessors;
let getSearchSourceStub;
beforeEach(ngMock.module(function createServiceStubs($provide) {
$provide.value('courier', createCourierStub());
}));
beforeEach(ngMock.inject(function createPrivateStubs(Private) {
getSearchSourceStub = createSearchSourceStubProvider([], '@timestamp', MS_PER_DAY * 8);
Private.stub(SearchSourceProvider, getSearchSourceStub);
fetchPredecessors = Private(fetchContextProvider).fetchPredecessors;
}));
it('should perform exactly one query when enough hits are returned', function () {
const searchSourceStub = getSearchSourceStub();
searchSourceStub._stubHits = [
searchSourceStub._createStubHit(MS_PER_DAY * 3000 + 2),
searchSourceStub._createStubHit(MS_PER_DAY * 3000 + 1),
searchSourceStub._createStubHit(MS_PER_DAY * 3000),
searchSourceStub._createStubHit(MS_PER_DAY * 2000),
searchSourceStub._createStubHit(MS_PER_DAY * 1000),
];
return fetchPredecessors(
'INDEX_PATTERN_ID',
'@timestamp',
'desc',
MS_PER_DAY * 3000,
'_doc',
'asc',
0,
3,
[]
)
.then((hits) => {
expect(searchSourceStub.fetchAsRejectablePromise.calledOnce).to.be(true);
expect(hits).to.eql(searchSourceStub._stubHits.slice(0, 3));
});
});
it('should perform multiple queries with the last being unrestricted when too few hits are returned', function () {
const searchSourceStub = getSearchSourceStub();
searchSourceStub._stubHits = [
searchSourceStub._createStubHit(MS_PER_DAY * 3010),
searchSourceStub._createStubHit(MS_PER_DAY * 3002),
searchSourceStub._createStubHit(MS_PER_DAY * 3000),
searchSourceStub._createStubHit(MS_PER_DAY * 2998),
searchSourceStub._createStubHit(MS_PER_DAY * 2990),
];
return fetchPredecessors(
'INDEX_PATTERN_ID',
'@timestamp',
'desc',
MS_PER_DAY * 3000,
'_doc',
'asc',
0,
6,
[]
)
.then((hits) => {
const intervals = searchSourceStub.set.args
.filter(([property]) => property === 'query')
.map(([, { query }]) => _.get(query, ['constant_score', 'filter', 'range', '@timestamp']));
expect(intervals.every(({ gte, lte }) => (gte && lte) ? gte < lte : true)).to.be(true);
// should have started at the given time
expect(intervals[0].gte).to.eql(MS_PER_DAY * 3000);
// should have ended with a half-open interval
expect(_.last(intervals)).to.only.have.key('gte');
expect(intervals.length).to.be.greaterThan(1);
expect(hits).to.eql(searchSourceStub._stubHits.slice(0, 3));
});
});
it('should perform multiple queries until the expected hit count is returned', function () {
const searchSourceStub = getSearchSourceStub();
searchSourceStub._stubHits = [
searchSourceStub._createStubHit(MS_PER_DAY * 1700),
searchSourceStub._createStubHit(MS_PER_DAY * 1200),
searchSourceStub._createStubHit(MS_PER_DAY * 1100),
searchSourceStub._createStubHit(MS_PER_DAY * 1000),
];
return fetchPredecessors(
'INDEX_PATTERN_ID',
'@timestamp',
'desc',
MS_PER_DAY * 1000,
'_doc',
'asc',
0,
3,
[]
)
.then((hits) => {
const intervals = searchSourceStub.set.args
.filter(([property]) => property === 'query')
.map(([, { query }]) => _.get(query, ['constant_score', 'filter', 'range', '@timestamp']));
// should have started at the given time
expect(intervals[0].gte).to.eql(MS_PER_DAY * 1000);
// should have stopped before reaching MS_PER_DAY * 1700
expect(_.last(intervals).lte).to.be.lessThan(MS_PER_DAY * 1700);
expect(intervals.length).to.be.greaterThan(1);
expect(hits).to.eql(searchSourceStub._stubHits.slice(-3));
});
});
it('should return an empty array when no hits were found', function () {
return fetchPredecessors(
'INDEX_PATTERN_ID',
'@timestamp',
'desc',
MS_PER_DAY * 3,
'_doc',
'asc',
0,
3,
[]
)
.then((hits) => {
expect(hits).to.eql([]);
});
});
it('should configure the SearchSource to not inherit from the implicit root', function () {
const searchSourceStub = getSearchSourceStub();
return fetchPredecessors(
'INDEX_PATTERN_ID',
'@timestamp',
'desc',
MS_PER_DAY * 3,
'_doc',
'asc',
0,
3,
[]
)
.then(() => {
const inheritsSpy = searchSourceStub.inherits;
expect(inheritsSpy.alwaysCalledWith(false)).to.be(true);
expect(inheritsSpy.called).to.be(true);
});
});
});
});

188
src/core_plugins/kibana/public/context/api/__tests__/successors.js Normal file
View file

@ -0,0 +1,188 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import expect from 'expect.js';
import ngMock from 'ng_mock';
import * as _ from 'lodash';
import { createCourierStub, createSearchSourceStubProvider } from './_stubs';
import { SearchSourceProvider } from 'ui/courier/data_source/search_source';
import { fetchContextProvider } from '../context';
const MS_PER_DAY = 24 * 60 * 60 * 1000;
describe('context app', function () {
beforeEach(ngMock.module('kibana'));
describe('function fetchSuccessors', function () {
let fetchSuccessors;
let getSearchSourceStub;
beforeEach(ngMock.module(function createServiceStubs($provide) {
$provide.value('courier', createCourierStub());
}));
beforeEach(ngMock.inject(function createPrivateStubs(Private) {
getSearchSourceStub = createSearchSourceStubProvider([], '@timestamp');
Private.stub(SearchSourceProvider, getSearchSourceStub);
fetchSuccessors = Private(fetchContextProvider).fetchSuccessors;
}));
it('should perform exactly one query when enough hits are returned', function () {
const searchSourceStub = getSearchSourceStub();
searchSourceStub._stubHits = [
searchSourceStub._createStubHit(MS_PER_DAY * 5000),
searchSourceStub._createStubHit(MS_PER_DAY * 4000),
searchSourceStub._createStubHit(MS_PER_DAY * 3000),
searchSourceStub._createStubHit(MS_PER_DAY * 3000 - 1),
searchSourceStub._createStubHit(MS_PER_DAY * 3000 - 2),
];
return fetchSuccessors(
'INDEX_PATTERN_ID',
'@timestamp',
'desc',
MS_PER_DAY * 3000,
'_doc',
'asc',
0,
3,
[]
)
.then((hits) => {
expect(searchSourceStub.fetchAsRejectablePromise.calledOnce).to.be(true);
expect(hits).to.eql(searchSourceStub._stubHits.slice(-3));
});
});
it('should perform multiple queries with the last being unrestricted when too few hits are returned', function () {
const searchSourceStub = getSearchSourceStub();
searchSourceStub._stubHits = [
searchSourceStub._createStubHit(MS_PER_DAY * 3010),
searchSourceStub._createStubHit(MS_PER_DAY * 3002),
searchSourceStub._createStubHit(MS_PER_DAY * 3000),
searchSourceStub._createStubHit(MS_PER_DAY * 2998),
searchSourceStub._createStubHit(MS_PER_DAY * 2990),
];
return fetchSuccessors(
'INDEX_PATTERN_ID',
'@timestamp',
'desc',
MS_PER_DAY * 3000,
'_doc',
'asc',
0,
6,
[]
)
.then((hits) => {
const intervals = searchSourceStub.set.args
.filter(([property]) => property === 'query')
.map(([, { query }]) => _.get(query, ['constant_score', 'filter', 'range', '@timestamp']));
expect(intervals.every(({ gte, lte }) => (gte && lte) ? gte < lte : true)).to.be(true);
// should have started at the given time
expect(intervals[0].lte).to.eql(MS_PER_DAY * 3000);
// should have ended with a half-open interval
expect(_.last(intervals)).to.only.have.key('lte');
expect(intervals.length).to.be.greaterThan(1);
expect(hits).to.eql(searchSourceStub._stubHits.slice(-3));
});
});
it('should perform multiple queries until the expected hit count is returned', function () {
const searchSourceStub = getSearchSourceStub();
searchSourceStub._stubHits = [
searchSourceStub._createStubHit(MS_PER_DAY * 3000),
searchSourceStub._createStubHit(MS_PER_DAY * 3000 - 1),
searchSourceStub._createStubHit(MS_PER_DAY * 3000 - 2),
searchSourceStub._createStubHit(MS_PER_DAY * 2800),
searchSourceStub._createStubHit(MS_PER_DAY * 2200),
searchSourceStub._createStubHit(MS_PER_DAY * 1000),
];
return fetchSuccessors(
'INDEX_PATTERN_ID',
'@timestamp',
'desc',
MS_PER_DAY * 3000,
'_doc',
'asc',
0,
4,
[]
)
.then((hits) => {
const intervals = searchSourceStub.set.args
.filter(([property]) => property === 'query')
.map(([, { query }]) => _.get(query, ['constant_score', 'filter', 'range', '@timestamp']));
// should have started at the given time
expect(intervals[0].lte).to.eql(MS_PER_DAY * 3000);
// should have stopped before reaching MS_PER_DAY * 2200
expect(_.last(intervals).gte).to.be.greaterThan(MS_PER_DAY * 2200);
expect(intervals.length).to.be.greaterThan(1);
expect(hits).to.eql(searchSourceStub._stubHits.slice(0, 4));
});
});
it('should return an empty array when no hits were found', function () {
return fetchSuccessors(
'INDEX_PATTERN_ID',
'@timestamp',
'desc',
MS_PER_DAY * 3,
'_doc',
'asc',
0,
3,
[]
)
.then((hits) => {
expect(hits).to.eql([]);
});
});
it('should configure the SearchSource to not inherit from the implicit root', function () {
const searchSourceStub = getSearchSourceStub();
return fetchSuccessors(
'INDEX_PATTERN_ID',
'@timestamp',
'desc',
MS_PER_DAY * 3,
'_doc',
'asc',
0,
3,
[]
)
.then(() => {
const inheritsSpy = searchSourceStub.inherits;
expect(inheritsSpy.alwaysCalledWith(false)).to.be(true);
expect(inheritsSpy.called).to.be(true);
});
});
});
});

28
src/core_plugins/kibana/public/context/api/anchor.js
View file

@ -21,11 +21,15 @@ import _ from 'lodash';
import { SearchSourceProvider } from 'ui/courier/data_source/search_source';
function fetchAnchorProvider(courier, Private) {
export function fetchAnchorProvider(courier, Private) {
const SearchSource = Private(SearchSourceProvider);
return async function fetchAnchor(indexPatternId, uid, sort) {
return async function fetchAnchor(
indexPatternId,
anchorType,
anchorId,
sort
) {
const indexPattern = await courier.indexPatterns.get(indexPatternId);
const searchSource = new SearchSource()
@ -35,11 +39,16 @@ function fetchAnchorProvider(courier, Private) {
.set('size', 1)
.set('query', {
query: {
terms: {
_uid: [uid],
}
constant_score: {
filter: {
ids: {
type: anchorType,
values: [anchorId],
},
},
},
},
language: 'lucene'
language: 'lucene',
})
.set('sort', sort);
@ -55,8 +64,3 @@ function fetchAnchorProvider(courier, Private) {
};
};
}
export {
fetchAnchorProvider,
};

245
src/core_plugins/kibana/public/context/api/context.js
View file

@ -16,15 +16,40 @@
* specific language governing permissions and limitations
* under the License.
*/
// @ts-check
import _ from 'lodash';
// @ts-ignore
import { SearchSourceProvider } from 'ui/courier/data_source/search_source';
import { reverseSortDirective } from './utils/sorting';
import { reverseSortDirection } from './utils/sorting';
/**
* @typedef {Object} SearchResult
* @prop {{ total: number, hits: any[] }} hits
* @prop {Object} aggregations
*/
/**
* @typedef {Object} SearchSourceT
* @prop {function(): Promise<SearchResult>} fetchAsRejectablePromise
* @prop {function(string, any): SearchSourceT} set
* @prop {function(any): SearchSourceT} inherits
*/
/**
* @typedef {'asc' | 'desc'} SortDirection
*/
const DAY_MILLIS = 24 * 60 * 60 * 1000;
// look from 1 day up to 10000 days into the past and future
const LOOKUP_OFFSETS = [0, 1, 7, 30, 365, 10000].map((days) => days * DAY_MILLIS);
function fetchContextProvider(courier, Private) {
/**
* @type {{new(): SearchSourceT}}
*/
const SearchSource = Private(SearchSourceProvider);
return {
@ -32,59 +57,205 @@ function fetchContextProvider(courier, Private) {
fetchSuccessors,
};
async function fetchSuccessors(indexPatternId, anchorDocument, contextSort, size, filters) {
const successorsSearchSource = await createSearchSource(
indexPatternId,
anchorDocument,
contextSort,
size,
filters,
);
const results = await performQuery(successorsSearchSource);
return results;
async function fetchSuccessors(
indexPatternId,
timeField,
timeSortDirection,
timeValue,
tieBreakerField,
tieBreakerSortDirection,
tieBreakerValue,
size,
filters
) {
const searchSource = await createSearchSource(indexPatternId, filters);
const offsetSign = timeSortDirection === 'asc' ? 1 : -1;
// ending with `null` opens the last interval
const intervals = asPairs([...LOOKUP_OFFSETS.map(offset => timeValue + offset * offsetSign), null]);
let successors = [];
for (const [startTimeValue, endTimeValue] of intervals) {
const remainingSize = size - successors.length;
if (remainingSize <= 0) {
break;
}
const [afterTimeValue, afterTieBreakerValue] = successors.length > 0
? successors[successors.length - 1].sort
: [timeValue, tieBreakerValue];
const hits = await fetchHitsInInterval(
searchSource,
timeField,
timeSortDirection,
startTimeValue,
endTimeValue,
afterTimeValue,
tieBreakerField,
tieBreakerSortDirection,
afterTieBreakerValue,
remainingSize
);
successors = [...successors, ...hits];
}
return successors;
}
async function fetchPredecessors(indexPatternId, anchorDocument, contextSort, size, filters) {
const predecessorsSort = contextSort.map(reverseSortDirective);
const predecessorsSearchSource = await createSearchSource(
indexPatternId,
anchorDocument,
predecessorsSort,
size,
filters,
);
const reversedResults = await performQuery(predecessorsSearchSource);
const results = reversedResults.slice().reverse();
return results;
async function fetchPredecessors(
indexPatternId,
timeField,
timeSortDirection,
timeValue,
tieBreakerField,
tieBreakerSortDirection,
tieBreakerValue,
size,
filters
) {
const searchSource = await createSearchSource(indexPatternId, filters);
const offsetSign = timeSortDirection === 'desc' ? 1 : -1;
// ending with `null` opens the last interval
const intervals = asPairs([...LOOKUP_OFFSETS.map(offset => timeValue + offset * offsetSign), null]);
let predecessors = [];
for (const [startTimeValue, endTimeValue] of intervals) {
const remainingSize = size - predecessors.length;
if (remainingSize <= 0) {
break;
}
const [afterTimeValue, afterTieBreakerValue] = predecessors.length > 0
? predecessors[0].sort
: [timeValue, tieBreakerValue];
const hits = await fetchHitsInInterval(
searchSource,
timeField,
reverseSortDirection(timeSortDirection),
startTimeValue,
endTimeValue,
afterTimeValue,
tieBreakerField,
reverseSortDirection(tieBreakerSortDirection),
afterTieBreakerValue,
remainingSize
);
predecessors = [...hits.slice().reverse(), ...predecessors];
}
return predecessors;
}
async function createSearchSource(indexPatternId, anchorDocument, sort, size, filters) {
/**
* @param {string} indexPatternId
* @param {any[]} filters
* @returns {Promise<Object>}
*/
async function createSearchSource(indexPatternId, filters) {
const indexPattern = await courier.indexPatterns.get(indexPatternId);
return new SearchSource()
.inherits(false)
.set('index', indexPattern)
.set('version', true)
.set('size', size)
.set('filter', filters)
.set('filter', filters);
}
/**
* Fetch the hits between `(afterTimeValue, tieBreakerValue)` and
* `endTimeValue` from the `searchSource` using the given `timeField` and
* `tieBreakerField` fields up to a maximum of `maxCount` documents. The
* documents are sorted by `(timeField, tieBreakerField)` using the
* respective `timeSortDirection` and `tieBreakerSortDirection`.
*
* The `searchSource` is assumed to have the appropriate index pattern
* and filters set.
*
* @param {SearchSourceT} searchSource
* @param {string} timeField
* @param {SortDirection} timeSortDirection
* @param {number} startTimeValue
* @param {number | null} endTimeValue
* @param {number} [afterTimeValue=startTimeValue]
* @param {string} tieBreakerField
* @param {SortDirection} tieBreakerSortDirection
* @param {number} tieBreakerValue
* @param {number} maxCount
* @returns {Promise<object[]>}
*/
async function fetchHitsInInterval(
searchSource,
timeField,
timeSortDirection,
startTimeValue,
endTimeValue,
afterTimeValue,
tieBreakerField,
tieBreakerSortDirection,
tieBreakerValue,
maxCount
) {
const startRange = {
[timeSortDirection === 'asc' ? 'gte' : 'lte']: startTimeValue,
};
const endRange = endTimeValue === null ? {} : {
[timeSortDirection === 'asc' ? 'lte' : 'gte']: endTimeValue,
};
const response = await searchSource
.set('size', maxCount)
.set('query', {
query: {
match_all: {},
constant_score: {
filter: {
range: {
[timeField]: {
...startRange,
...endRange,
}
},
},
},
},
language: 'lucene'
})
.set('searchAfter', anchorDocument.sort)
.set('sort', sort);
}
.set('searchAfter', [
afterTimeValue !== null ? afterTimeValue : startTimeValue,
tieBreakerValue,
])
.set('sort', [
{ [timeField]: timeSortDirection },
{ [tieBreakerField]: tieBreakerSortDirection },
])
.set('version', true)
.fetchAsRejectablePromise();
async function performQuery(searchSource) {
const response = await searchSource.fetchAsRejectablePromise();
return _.get(response, ['hits', 'hits'], []);
return response.hits ? response.hits.hits : [];
}
}
/**
* Generate a sequence of pairs from the iterable that looks like
* `[[x_0, x_1], [x_1, x_2], [x_2, x_3], ..., [x_(n-1), x_n]]`.
*
* @param {Iterable<any>} iterable
* @returns {IterableIterator<(any[])>}
*/
function* asPairs(iterable) {
let currentPair = [];
for (const value of iterable) {
currentPair = [...currentPair, value].slice(-2);
if (currentPair.length === 2) {
yield currentPair;
}
}
}
export {
fetchContextProvider,

27
src/core_plugins/kibana/public/context/api/utils/__tests__/sorting.js
View file

@ -21,7 +21,6 @@ import expect from 'expect.js';
import {
reverseSortDirection,
reverseSortDirective
} from '../sorting';
@ -31,31 +30,5 @@ describe('context app', function () {
expect(reverseSortDirection('asc')).to.eql('desc');
expect(reverseSortDirection('desc')).to.eql('asc');
});
it('should reverse a direction given in an option object', function () {
expect(reverseSortDirection({ order: 'asc' })).to.eql({ order: 'desc' });
expect(reverseSortDirection({ order: 'desc' })).to.eql({ order: 'asc' });
});
it('should preserve other properties than `order` in an option object', function () {
expect(reverseSortDirection({
order: 'asc',
other: 'field',
})).to.have.property('other', 'field');
});
});
describe('function reverseSortDirective', function () {
it('should return direction `asc` when given just `_score`', function () {
expect(reverseSortDirective('_score')).to.eql({ _score: 'asc' });
});
it('should return direction `desc` when given just a field name', function () {
expect(reverseSortDirective('field1')).to.eql({ field1: 'desc' });
});
it('should reverse direction when given an object', function () {
expect(reverseSortDirective({ field1: 'asc' })).to.eql({ field1: 'desc' });
});
});
});

27
src/core_plugins/kibana/public/context/api/utils/ids.js
View file

@ -1,27 +0,0 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
function getDocumentUid(type, id) {
return `${type}#${id}`;
}
export {
getDocumentUid,
};

63
src/core_plugins/kibana/public/context/api/utils/sorting.js
View file

@ -17,9 +17,6 @@
* under the License.
*/
import _ from 'lodash';
/**
* The list of field names that are allowed for sorting, but not included in
* index pattern fields.
@ -47,79 +44,25 @@ function getFirstSortableField(indexPattern, fieldNames) {
return sortableFields[0];
}
/**
* A sort directive in object or string form.
*
* @typedef {(SortDirectiveString|SortDirectiveObject)} SortDirective
*/
/**
* A sort directive in object form.
*
* @typedef {Object.<FieldName, (SortDirection|SortOptions)>} SortDirectiveObject
*/
/**
* A sort order string.
*
* @typedef {('asc'|'desc')} SortDirection
*/
/**
* A field name.
*
* @typedef {string} FieldName
*/
/**
* A sort options object
*
* @typedef {Object} SortOptions
* @property {SortDirection} order
*/
/**
* Return a copy of the directive with the sort direction reversed. If the
* field name is '_score', it inverts the default sort direction in the same
* way as Elasticsearch itself.
*
* @param {SortDirective} sortDirective - The directive to reverse the
* sort direction of
*
* @returns {SortDirective}
*/
function reverseSortDirective(sortDirective) {
if (_.isString(sortDirective)) {
return {
[sortDirective]: (sortDirective === '_score' ? 'asc' : 'desc'),
};
} else if (_.isPlainObject(sortDirective)) {
return _.mapValues(sortDirective, reverseSortDirection);
} else {
return sortDirective;
}
}
/**
* Return the reversed sort direction.
*
* @param {(SortDirection|SortOptions)} sortDirection
* @param {(SortDirection)} sortDirection
*
* @returns {(SortDirection|SortOptions)}
* @returns {(SortDirection)}
*/
function reverseSortDirection(sortDirection) {
if (_.isPlainObject(sortDirection)) {
return _.assign({}, sortDirection, {
order: reverseSortDirection(sortDirection.order),
});
} else {
return (sortDirection === 'asc' ? 'desc' : 'asc');
}
return (sortDirection === 'asc' ? 'desc' : 'asc');
}
export {
getFirstSortableField,
reverseSortDirection,
reverseSortDirective,
};

4
src/core_plugins/kibana/public/context/app.html
View file

@ -8,7 +8,9 @@
</div>
<div class="kuiLocalNavRow kuiLocalNavRow--secondary">
<div class="kuiLocalTabs">
<div class="kuiLocalTab kuiLocalTab-isSelected" ng-bind="contextApp.state.queryParameters.anchorUid"></div>
<div class="kuiLocalTab kuiLocalTab-isSelected" >
{{ contextApp.state.queryParameters.anchorType }}#{{ contextApp.state.queryParameters.anchorId }}
</div>
</div>
</div>
</div>

6
src/core_plugins/kibana/public/context/app.js
View file

@ -52,7 +52,8 @@ module.directive('contextApp', function ContextApp() {
controllerAs: 'contextApp',
restrict: 'E',
scope: {
anchorUid: '=',
anchorType: '=',
anchorId: '=',
columns: '=',
indexPattern: '=',
filters: '=',
@ -106,7 +107,8 @@ function ContextAppController($scope, config, Private, timefilter) {
const { queryParameters } = this.state;
if (
(newQueryParameters.indexPatternId !== queryParameters.indexPatternId)
|| (newQueryParameters.anchorUid !== queryParameters.anchorUid)
|| (newQueryParameters.anchorType !== queryParameters.anchorType)
|| (newQueryParameters.anchorId !== queryParameters.anchorId)
|| (!_.isEqual(newQueryParameters.sort, queryParameters.sort))
) {
this.actions.fetchAllRowsWithNewQueryParameters(_.cloneDeep(newQueryParameters));

3
src/core_plugins/kibana/public/context/index.html
View file

@ -1,5 +1,6 @@
<context-app
anchor-uid="contextAppRoute.anchorUid"
anchor-type="contextAppRoute.anchorType"
anchor-id="contextAppRoute.anchorId"
columns="contextAppRoute.state.columns"
discover-url="contextAppRoute.discoverUrl"
index-pattern="contextAppRoute.indexPattern"

4
src/core_plugins/kibana/public/context/index.js
View file

@ -23,7 +23,6 @@ import { FilterBarQueryFilterProvider } from 'ui/filter_bar/query_filter';
import uiRoutes from 'ui/routes';
import './app';
import { getDocumentUid } from './api/utils/ids';
import contextAppRouteTemplate from './index.html';
@ -64,7 +63,8 @@ function ContextAppRouteController(
this.filters = _.cloneDeep(queryFilter.getFilters());
});
this.anchorUid = getDocumentUid($routeParams.type, $routeParams.id);
this.anchorType = $routeParams.type;
this.anchorId = $routeParams.id;
this.indexPattern = indexPattern;
this.discoverUrl = chrome.getNavLinkById('kibana:discover').lastSubUrl;
this.filters = _.cloneDeep(queryFilter.getFilters());

28
src/core_plugins/kibana/public/context/query/actions.js
View file

@ -61,7 +61,7 @@ export function QueryActionsProvider(courier, Notifier, Private, Promise) {
);
const fetchAnchorRow = (state) => () => {
const { queryParameters: { indexPatternId, anchorUid, sort, tieBreakerField } } = state;
const { queryParameters: { indexPatternId, anchorType, anchorId, sort, tieBreakerField } } = state;
if (!tieBreakerField) {
return Promise.reject(setFailedStatus(state)('anchor', {
@ -72,7 +72,7 @@ export function QueryActionsProvider(courier, Notifier, Private, Promise) {
setLoadingStatus(state)('anchor');
return Promise.try(() => (
fetchAnchor(indexPatternId, anchorUid, [_.zipObject([sort]), { [tieBreakerField]: 'asc' }])
fetchAnchor(indexPatternId, anchorType, anchorId, [_.zipObject([sort]), { [tieBreakerField]: 'asc' }])
))
.then(
(anchorDocument) => {
@ -103,7 +103,17 @@ export function QueryActionsProvider(courier, Notifier, Private, Promise) {
setLoadingStatus(state)('predecessors');
return Promise.try(() => (
fetchPredecessors(indexPatternId, anchor, [_.zipObject([sort]), { [tieBreakerField]: 'asc' }], predecessorCount, filters)
fetchPredecessors(
indexPatternId,
sort[0],
sort[1],
anchor.sort[0],
tieBreakerField,
'asc',
anchor.sort[1],
predecessorCount,
filters
)
))
.then(
(predecessorDocuments) => {
@ -134,7 +144,17 @@ export function QueryActionsProvider(courier, Notifier, Private, Promise) {
setLoadingStatus(state)('successors');
return Promise.try(() => (
fetchSuccessors(indexPatternId, anchor, [_.zipObject([sort]), { [tieBreakerField]: 'asc' }], successorCount, filters)
fetchSuccessors(
indexPatternId,
sort[0],
sort[1],
anchor.sort[0],
tieBreakerField,
'asc',
anchor.sort[1],
successorCount,
filters
)
))
.then(
(successorDocuments) => {

6
src/core_plugins/kibana/public/context/query_parameters/__tests__/action_set_query_parameters.js
View file

@ -46,7 +46,8 @@ describe('context app', function () {
});
setQueryParameters(state)({
anchorUid: 'ANCHOR_UID',
anchorType: 'ANCHOR_TYPE',
anchorId: 'ANCHOR_ID',
columns: ['column'],
defaultStepSize: 3,
filters: ['filter'],
@ -58,7 +59,8 @@ describe('context app', function () {
expect(state.queryParameters).to.eql({
additionalParameter: 'ADDITIONAL_PARAMETER',
anchorUid: 'ANCHOR_UID',
anchorType: 'ANCHOR_TYPE',
anchorId: 'ANCHOR_ID',
columns: ['column'],
defaultStepSize: 3,
filters: ['filter'],

3
src/core_plugins/kibana/public/context/query_parameters/state.js
View file

@ -19,7 +19,8 @@
export function createInitialQueryParametersState(defaultStepSize, tieBreakerField) {
return {
anchorUid: null,
anchorType: null,
anchorId: null,
columns: [],
defaultStepSize,
filters: [],

2
test/functional/config.js
View file

@ -57,7 +57,7 @@ export default async function ({ readConfigFile }) {
testFiles: [
require.resolve('./apps/console'),
require.resolve('./apps/getting_started'),
// require.resolve('./apps/context'),
require.resolve('./apps/context'),
require.resolve('./apps/dashboard'),
require.resolve('./apps/discover'),
require.resolve('./apps/home'),