maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[Reporting] Add output size stats to telemetry metrics (#112037)

Browse source 
* [Reporting] Add output size stats to telemetry metrics

* fix types

* add output_size for each jobtype

* add size metrics for each job type

* use more mock data in unit tests

* clean up test

* update test snapshots

* update telemetry mapping

* SizeMetrics => SizePercentiles

* DocCount interface

* fix tests

* Update get_export_stats.ts

* update snapshots
This commit is contained in:
Tim Sullivan 2021-09-23 16:14:35 -07:00 committed by GitHub
parent 09e7093b7d
commit 03007d0150
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 1281 additions and 477 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

942
x-pack/plugins/reporting/server/usage/__snapshots__/reporting_usage_collector.test.ts.snap generated
View file

File diff suppressed because it is too large Load diff

81
x-pack/plugins/reporting/server/usage/get_export_stats.test.ts
View file

@ -11,6 +11,15 @@ import { getExportTypesHandler } from './get_export_type_handler';
import { FeatureAvailabilityMap } from './types';
let featureMap: FeatureAvailabilityMap;
const sizesAggResponse = {
'1.0': 5093470.0,
'5.0': 5093470.0,
'25.0': 5093470.0,
'50.0': 8514532.0,
'75.0': 1.1935594e7,
'95.0': 1.1935594e7,
'99.0': 1.1935594e7,
};
beforeEach(() => {
featureMap = { PNG: true, csv: true, csv_searchsource: true, printable_pdf: true };
@ -67,14 +76,19 @@ test('Model of job status and status-by-pdf-app', () => {
test('Model of jobTypes', () => {
const result = getExportStats(
{
PNG: { available: true, total: 3 },
PNG: { available: true, total: 3, sizes: sizesAggResponse },
printable_pdf: {
available: true,
total: 3,
sizes: sizesAggResponse,
app: { dashboard: 0, visualization: 0, 'canvas workpad': 3 },
layout: { preserve_layout: 3, print: 0 },
},
csv_searchsource: { available: true, total: 3 },
csv_searchsource: {
available: true,
total: 3,
sizes: sizesAggResponse,
},
},
featureMap,
exportTypesHandler
@ -95,6 +109,15 @@ test('Model of jobTypes', () => {
"preserve_layout": 0,
"print": 0,
},
"output_size": Object {
"1.0": 5093470,
"25.0": 5093470,
"5.0": 5093470,
"50.0": 8514532,
"75.0": 11935594,
"95.0": 11935594,
"99.0": 11935594,
},
"total": 3,
}
`);
@ -131,6 +154,15 @@ test('Model of jobTypes', () => {
"preserve_layout": 0,
"print": 0,
},
"output_size": Object {
"1.0": 5093470,
"25.0": 5093470,
"5.0": 5093470,
"50.0": 8514532,
"75.0": 11935594,
"95.0": 11935594,
"99.0": 11935594,
},
"total": 3,
}
`);
@ -149,6 +181,15 @@ test('Model of jobTypes', () => {
"preserve_layout": 3,
"print": 0,
},
"output_size": Object {
"1.0": 5093470,
"25.0": 5093470,
"5.0": 5093470,
"50.0": 8514532,
"75.0": 11935594,
"95.0": 11935594,
"99.0": 11935594,
},
"total": 3,
}
`);
@ -156,7 +197,14 @@ test('Model of jobTypes', () => {
test('PNG counts, provided count of deprecated jobs explicitly', () => {
const result = getExportStats(
{ PNG: { available: true, total: 15, deprecated: 5 } },
{
PNG: {
available: true,
total: 15,
deprecated: 5,
sizes: sizesAggResponse,
},
},
featureMap,
exportTypesHandler
);
@ -175,6 +223,15 @@ test('PNG counts, provided count of deprecated jobs explicitly', () => {
"preserve_layout": 0,
"print": 0,
},
"output_size": Object {
"1.0": 5093470,
"25.0": 5093470,
"5.0": 5093470,
"50.0": 8514532,
"75.0": 11935594,
"95.0": 11935594,
"99.0": 11935594,
},
"total": 15,
}
`);
@ -182,7 +239,14 @@ test('PNG counts, provided count of deprecated jobs explicitly', () => {
test('CSV counts, provides all jobs implicitly deprecated due to jobtype', () => {
const result = getExportStats(
{ csv: { available: true, total: 15, deprecated: 0 } },
{
csv: {
available: true,
total: 15,
deprecated: 0,
sizes: sizesAggResponse,
},
},
featureMap,
exportTypesHandler
);
@ -201,6 +265,15 @@ test('CSV counts, provides all jobs implicitly deprecated due to jobtype', () =>
"preserve_layout": 0,
"print": 0,
},
"output_size": Object {
"1.0": 5093470,
"25.0": 5093470,
"5.0": 5093470,
"50.0": 8514532,
"75.0": 11935594,
"95.0": 11935594,
"99.0": 11935594,
},
"total": 15,
}
`);

3
x-pack/plugins/reporting/server/usage/get_export_stats.ts
View file

@ -33,6 +33,7 @@ function getAvailableTotalForFeature(
available: isAvailable(featureAvailability, typeKey),
total: jobType.total,
deprecated,
output_size: jobType.sizes,
app: { ...defaultTotalsForFeature.app, ...jobType.app },
layout: { ...defaultTotalsForFeature.layout, ...jobType.layout },
};
@ -56,6 +57,7 @@ export const getExportStats = (
_all: rangeAll,
status: rangeStatus,
statuses: rangeStatusByApp,
output_size: outputSize,
...rangeStats
} = rangeStatsInput;
@ -84,6 +86,7 @@ export const getExportStats = (
_all: rangeAll || 0,
status: { completed: 0, failed: 0, ...rangeStatus },
statuses: rangeStatusByApp,
output_size: outputSize,
} as RangeStats;
return resultStats;

34
x-pack/plugins/reporting/server/usage/get_reporting_usage.ts
View file

@ -13,6 +13,7 @@ import type { GetLicense } from './';
import { getExportStats } from './get_export_stats';
import { getExportTypesHandler } from './get_export_type_handler';
import type {
AggregationBuckets,
AggregationResultBuckets,
AvailableTotal,
FeatureAvailabilityMap,
@ -33,6 +34,8 @@ const OBJECT_TYPES_FIELD = 'meta.objectType.keyword';
const STATUS_TYPES_KEY = 'statusTypes';
const STATUS_BY_APP_KEY = 'statusByApp';
const STATUS_TYPES_FIELD = 'status';
const OUTPUT_SIZES_KEY = 'sizes';
const OUTPUT_SIZES_FIELD = 'output.size';
const DEFAULT_TERMS_SIZE = 10;
const PRINTABLE_PDF_JOBTYPE = 'printable_pdf';
@ -64,13 +67,14 @@ const getAppStatuses = (buckets: StatusByAppBucket[]) =>
}, {});
function getAggStats(aggs: AggregationResultBuckets): Partial<RangeStats> {
const { buckets: jobBuckets } = aggs[JOB_TYPES_KEY];
const { buckets: jobBuckets } = aggs[JOB_TYPES_KEY] as AggregationBuckets;
const jobTypes = jobBuckets.reduce((accum: JobTypes, bucket) => {
const { key, doc_count: count, isDeprecated } = bucket;
const { key, doc_count: count, isDeprecated, sizes } = bucket;
const deprecatedCount = isDeprecated?.doc_count;
const total: Omit<AvailableTotal, 'available'> = {
total: count,
deprecated: deprecatedCount,
sizes: sizes?.values,
};
return { ...accum, [key]: total };
}, {} as JobTypes);
@ -97,7 +101,13 @@ function getAggStats(aggs: AggregationResultBuckets): Partial<RangeStats> {
statusByApp = getAppStatuses(statusAppBuckets);
}
return { _all: all, status: statusTypes, statuses: statusByApp, ...jobTypes };
return {
_all: all,
status: statusTypes,
statuses: statusByApp,
output_size: get(aggs[OUTPUT_SIZES_KEY], 'values') ?? undefined,
...jobTypes,
};
}
type RangeStatSets = Partial<RangeStats> & {
@ -135,7 +145,6 @@ export async function getReportingUsage(
exportTypesRegistry: ExportTypesRegistry
): Promise<ReportingUsageType> {
const reportingIndex = config.get('index');
const params = {
index: `${reportingIndex}-*`,
filterPath: 'aggregations.*.buckets',
@ -152,8 +161,14 @@ export async function getReportingUsage(
aggs: {
[JOB_TYPES_KEY]: {
terms: { field: JOB_TYPES_FIELD, size: DEFAULT_TERMS_SIZE },
aggs: { isDeprecated: { filter: { term: { [OBJECT_TYPE_DEPRECATED_KEY]: true } } } },
aggs: {
isDeprecated: { filter: { term: { [OBJECT_TYPE_DEPRECATED_KEY]: true } } },
[OUTPUT_SIZES_KEY]: {
percentiles: { field: OUTPUT_SIZES_FIELD },
},
},
},
[STATUS_TYPES_KEY]: { terms: { field: STATUS_TYPES_FIELD, size: DEFAULT_TERMS_SIZE } },
[STATUS_BY_APP_KEY]: {
terms: { field: 'status', size: DEFAULT_TERMS_SIZE },
@ -161,19 +176,24 @@ export async function getReportingUsage(
jobTypes: {
terms: { field: JOB_TYPES_FIELD, size: DEFAULT_TERMS_SIZE },
aggs: {
appNames: { terms: { field: OBJECT_TYPES_FIELD, size: DEFAULT_TERMS_SIZE } }, // NOTE Discover/CSV export is missing the 'meta.objectType' field, so Discover/CSV results are missing for this agg
appNames: { terms: { field: OBJECT_TYPES_FIELD, size: DEFAULT_TERMS_SIZE } },
},
},
},
},
[OBJECT_TYPES_KEY]: {
filter: { term: { jobtype: PRINTABLE_PDF_JOBTYPE } },
aggs: { pdf: { terms: { field: OBJECT_TYPES_FIELD, size: DEFAULT_TERMS_SIZE } } },
aggs: {
pdf: { terms: { field: OBJECT_TYPES_FIELD, size: DEFAULT_TERMS_SIZE } },
},
},
[LAYOUT_TYPES_KEY]: {
filter: { term: { jobtype: PRINTABLE_PDF_JOBTYPE } },
aggs: { pdf: { terms: { field: LAYOUT_TYPES_FIELD, size: DEFAULT_TERMS_SIZE } } },
},
[OUTPUT_SIZES_KEY]: {
percentiles: { field: OUTPUT_SIZES_FIELD },
},
},
},
},

227
x-pack/plugins/reporting/server/usage/reporting_usage_collector.test.ts
View file

@ -18,7 +18,6 @@ import {
getReportingUsageCollector,
registerReportingUsageCollector,
} from './reporting_usage_collector';
import { SearchResponse } from './types';
const exportTypesRegistry = getExportTypesRegistry();
@ -190,7 +189,7 @@ describe('data modeling', () => {
beforeAll(async () => {
mockCore = await createMockReportingCore(createMockConfigSchema());
});
test('with normal looking usage data', async () => {
test('with usage data from the reporting/archived_reports es archive', async () => {
const plugins = getPluginsMock();
const collector = getReportingUsageCollector(
mockCore,
@ -202,39 +201,37 @@ describe('data modeling', () => {
}
);
collectorFetchContext = getMockFetchClients(
getResponseMock(
{
getResponseMock({
aggregations: {
ranges: {
meta: {},
buckets: {
all: {
doc_count: 12,
jobTypes: { buckets: [ { doc_count: 9, key: 'printable_pdf' }, { doc_count: 3, key: 'PNG' }, ], },
layoutTypes: { doc_count: 9, pdf: { buckets: [{ doc_count: 9, key: 'preserve_layout' }] }, },
objectTypes: { doc_count: 9, pdf: { buckets: [ { doc_count: 6, key: 'canvas workpad' }, { doc_count: 3, key: 'visualization' }, ], }, },
statusByApp: { buckets: [ { doc_count: 10, jobTypes: { buckets: [ { appNames: { buckets: [ { doc_count: 6, key: 'canvas workpad' }, { doc_count: 3, key: 'visualization' }, ], }, doc_count: 9, key: 'printable_pdf', }, { appNames: { buckets: [{ doc_count: 1, key: 'visualization' }] }, doc_count: 1, key: 'PNG', }, ], }, key: 'completed', }, { doc_count: 1, jobTypes: { buckets: [ { appNames: { buckets: [{ doc_count: 1, key: 'dashboard' }] }, doc_count: 1, key: 'PNG', }, ], }, key: 'completed_with_warnings', }, { doc_count: 1, jobTypes: { buckets: [ { appNames: { buckets: [{ doc_count: 1, key: 'dashboard' }] }, doc_count: 1, key: 'PNG', }, ], }, key: 'failed', }, ], },
statusTypes: { buckets: [ { doc_count: 10, key: 'completed' }, { doc_count: 1, key: 'completed_with_warnings' }, { doc_count: 1, key: 'failed' }, ], },
doc_count: 11,
layoutTypes: { doc_count: 6, pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'preserve_layout', doc_count: 5 }, { key: 'print', doc_count: 1 }, ] } },
statusByApp: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'completed', doc_count: 6, jobTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'csv_searchsource', doc_count: 3, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'search', doc_count: 3 }, ] } }, { key: 'printable_pdf', doc_count: 2, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'dashboard', doc_count: 2 }, ] } }, { key: 'csv', doc_count: 1, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'search', doc_count: 1 }, ] } }, ] } }, { key: 'completed_with_warnings', doc_count: 2, jobTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'PNG', doc_count: 1, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'dashboard', doc_count: 1 }, ] } }, { key: 'printable_pdf', doc_count: 1, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'dashboard', doc_count: 1 }, ] } }, ] } }, { key: 'failed', doc_count: 2, jobTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'printable_pdf', doc_count: 2, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'dashboard', doc_count: 2 }, ] } }, ] } }, { key: 'pending', doc_count: 1, jobTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'printable_pdf', doc_count: 1, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'dashboard', doc_count: 1 }, ] } }, ] } }, ] },
objectTypes: { doc_count: 6, pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'dashboard', doc_count: 6 }, ] } },
statusTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'completed', doc_count: 6 }, { key: 'completed_with_warnings', doc_count: 2 }, { key: 'failed', doc_count: 2 }, { key: 'pending', doc_count: 1 }, ] },
jobTypes: { meta: {}, doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'printable_pdf', doc_count: 6, isDeprecated: { meta: {}, doc_count: 0 }, sizeMax: { value: 1713303.0 }, sizeAvg: { value: 957215.0 }, sizeMin: { value: 43226.0 } }, { key: 'csv_searchsource', doc_count: 3, isDeprecated: { meta: {}, doc_count: 0 }, sizeMax: { value: 7557.0 }, sizeAvg: { value: 3684.6666666666665 }, sizeMin: { value: 204.0 } }, { key: 'PNG', doc_count: 1, isDeprecated: { meta: {}, doc_count: 0 }, sizeMax: { value: 37748.0 }, sizeAvg: { value: 37748.0 }, sizeMin: { value: 37748.0 } }, { key: 'csv', doc_count: 1, isDeprecated: { meta: {}, doc_count: 0 }, sizeMax: { value: 231.0 }, sizeAvg: { value: 231.0 }, sizeMin: { value: 231.0 } }, ] },
sizeMax: { value: 1713303.0 },
sizeMin: { value: 204.0 },
sizeAvg: { value: 365084.75 },
},
last7Days: {
doc_count: 1,
jobTypes: { buckets: [{ doc_count: 1, key: 'PNG' }] },
layoutTypes: { doc_count: 0, pdf: { buckets: [] } },
objectTypes: { doc_count: 0, pdf: { buckets: [] } },
statusByApp: { buckets: [ { doc_count: 1, jobTypes: { buckets: [ { appNames: { buckets: [{ doc_count: 1, key: 'dashboard' }] }, doc_count: 1, key: 'PNG', }, ], }, key: 'completed_with_warnings', }, ], },
statusTypes: { buckets: [{ doc_count: 1, key: 'completed_with_warnings' }] },
},
lastDay: {
doc_count: 1,
jobTypes: { buckets: [{ doc_count: 1, key: 'PNG' }] },
layoutTypes: { doc_count: 0, pdf: { buckets: [] } },
objectTypes: { doc_count: 0, pdf: { buckets: [] } },
statusByApp: { buckets: [ { doc_count: 1, jobTypes: { buckets: [ { appNames: { buckets: [{ doc_count: 1, key: 'dashboard' }] }, doc_count: 1, key: 'PNG', }, ], }, key: 'completed_with_warnings', }, ], },
statusTypes: { buckets: [{ doc_count: 1, key: 'completed_with_warnings' }] },
doc_count: 0,
layoutTypes: { doc_count: 0, pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] } },
statusByApp: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] },
objectTypes: { doc_count: 0, pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] } },
statusTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] },
jobTypes: { meta: {}, doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] },
sizeMax: { value: null },
sizeMin: { value: null },
sizeAvg: { value: null },
},
},
},
}, // prettier-ignore
},
} as SearchResponse) // prettier-ignore
})
);
const usageStats = await collector.fetch(collectorFetchContext);
expect(usageStats).toMatchSnapshot();
@ -258,121 +255,21 @@ describe('data modeling', () => {
buckets: {
all: {
doc_count: 9,
layoutTypes: {
doc_count: 0,
pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] },
},
statusByApp: {
doc_count_error_upper_bound: 0,
sum_other_doc_count: 0,
buckets: [
{
key: 'completed',
doc_count: 9,
jobTypes: {
doc_count_error_upper_bound: 0,
sum_other_doc_count: 0,
buckets: [
{
key: 'csv_searchsource',
doc_count: 5,
appNames: {
doc_count_error_upper_bound: 0,
sum_other_doc_count: 0,
buckets: [{ key: 'search', doc_count: 5 }],
},
},
{
key: 'csv',
doc_count: 4,
appNames: {
doc_count_error_upper_bound: 0,
sum_other_doc_count: 0,
buckets: [{ key: 'search', doc_count: 4 }],
},
},
],
},
},
],
},
objectTypes: {
doc_count: 0,
pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] },
},
statusTypes: {
doc_count_error_upper_bound: 0,
sum_other_doc_count: 0,
buckets: [{ key: 'completed', doc_count: 9 }],
},
jobTypes: {
doc_count_error_upper_bound: 0,
sum_other_doc_count: 0,
buckets: [
{ key: 'csv_searchsource', doc_count: 5, isDeprecated: { doc_count: 0 } },
{ key: 'csv', doc_count: 4, isDeprecated: { doc_count: 4 } },
],
},
layoutTypes: { doc_count: 0, pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] } },
statusByApp: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'completed', doc_count: 9, jobTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'csv_searchsource', doc_count: 5, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [{ key: 'search', doc_count: 5 }] } }, { key: 'csv', doc_count: 4, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [{ key: 'search', doc_count: 4 }] } }, ] } }, ] },
objectTypes: { doc_count: 0, pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] } },
statusTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [{ key: 'completed', doc_count: 9 }] },
jobTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'csv_searchsource', doc_count: 5, isDeprecated: { doc_count: 0 } }, { key: 'csv', doc_count: 4, isDeprecated: { doc_count: 4 } }, ] },
},
last7Days: {
doc_count: 9,
layoutTypes: {
doc_count: 0,
pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] },
},
statusByApp: {
doc_count_error_upper_bound: 0,
sum_other_doc_count: 0,
buckets: [
{
key: 'completed',
doc_count: 9,
jobTypes: {
doc_count_error_upper_bound: 0,
sum_other_doc_count: 0,
buckets: [
{
key: 'csv_searchsource',
doc_count: 5,
appNames: {
doc_count_error_upper_bound: 0,
sum_other_doc_count: 0,
buckets: [{ key: 'search', doc_count: 5 }],
},
},
{
key: 'csv',
doc_count: 4,
appNames: {
doc_count_error_upper_bound: 0,
sum_other_doc_count: 0,
buckets: [{ key: 'search', doc_count: 4 }],
},
},
],
},
},
],
},
objectTypes: {
doc_count: 0,
pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] },
},
statusTypes: {
doc_count_error_upper_bound: 0,
sum_other_doc_count: 0,
buckets: [{ key: 'completed', doc_count: 9 }],
},
jobTypes: {
doc_count_error_upper_bound: 0,
sum_other_doc_count: 0,
buckets: [
{ key: 'csv_searchsource', doc_count: 5, isDeprecated: { doc_count: 0 } },
{ key: 'csv', doc_count: 4, isDeprecated: { doc_count: 4 } },
],
},
layoutTypes: { doc_count: 0, pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] } },
statusByApp: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'completed', doc_count: 9, jobTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'csv_searchsource', doc_count: 5, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [{ key: 'search', doc_count: 5 }] } }, { key: 'csv', doc_count: 4, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [{ key: 'search', doc_count: 4 }] } }, ] } }, ] },
objectTypes: { doc_count: 0, pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] } },
statusTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [{ key: 'completed', doc_count: 9 }] },
jobTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'csv_searchsource', doc_count: 5, isDeprecated: { doc_count: 0 } }, { key: 'csv', doc_count: 4, isDeprecated: { doc_count: 4 } }, ] },
},
},
}, // prettier-ignore
},
},
})
@ -393,39 +290,30 @@ describe('data modeling', () => {
}
);
collectorFetchContext = getMockFetchClients(
getResponseMock(
{
getResponseMock({
aggregations: {
ranges: {
buckets: {
all: {
doc_count: 4,
layoutTypes: { doc_count: 2, pdf: { buckets: [{ key: 'preserve_layout', doc_count: 2 }] }, },
statusByApp: { buckets: [ { key: 'completed', doc_count: 4, jobTypes: { buckets: [ { key: 'printable_pdf', doc_count: 2, appNames: { buckets: [ { key: 'canvas workpad', doc_count: 1 }, { key: 'dashboard', doc_count: 1 }, ], }, }, { key: 'PNG', doc_count: 1, appNames: { buckets: [{ key: 'dashboard', doc_count: 1 }] }, }, { key: 'csv', doc_count: 1, appNames: { buckets: [] } }, ], }, }, ], },
objectTypes: { doc_count: 2, pdf: { buckets: [ { key: 'canvas workpad', doc_count: 1 }, { key: 'dashboard', doc_count: 1 }, ], }, },
layoutTypes: { doc_count: 2, pdf: { buckets: [{ key: 'preserve_layout', doc_count: 2 }] } },
statusByApp: { buckets: [ { key: 'completed', doc_count: 4, jobTypes: { buckets: [ { key: 'printable_pdf', doc_count: 2, appNames: { buckets: [ { key: 'canvas workpad', doc_count: 1 }, { key: 'dashboard', doc_count: 1 }, ] } }, { key: 'PNG', doc_count: 1, appNames: { buckets: [{ key: 'dashboard', doc_count: 1 }] } }, { key: 'csv', doc_count: 1, appNames: { buckets: [] } }, ] } }, ] },
objectTypes: { doc_count: 2, pdf: { buckets: [ { key: 'canvas workpad', doc_count: 1 }, { key: 'dashboard', doc_count: 1 }, ] } },
statusTypes: { buckets: [{ key: 'completed', doc_count: 4 }] },
jobTypes: { buckets: [ { key: 'printable_pdf', doc_count: 2 }, { key: 'PNG', doc_count: 1 }, { key: 'csv', doc_count: 1 }, ], },
jobTypes: { buckets: [ { key: 'printable_pdf', doc_count: 2 }, { key: 'PNG', doc_count: 1 }, { key: 'csv', doc_count: 1 }, ] },
},
last7Days: {
doc_count: 4,
layoutTypes: { doc_count: 2, pdf: { buckets: [{ key: 'preserve_layout', doc_count: 2 }] }, },
statusByApp: { buckets: [ { key: 'completed', doc_count: 4, jobTypes: { buckets: [ { key: 'printable_pdf', doc_count: 2, appNames: { buckets: [ { key: 'canvas workpad', doc_count: 1 }, { key: 'dashboard', doc_count: 1 }, ], }, }, { key: 'PNG', doc_count: 1, appNames: { buckets: [{ key: 'dashboard', doc_count: 1 }] }, }, { key: 'csv', doc_count: 1, appNames: { buckets: [] } }, ], }, }, ], },
objectTypes: { doc_count: 2, pdf: { buckets: [ { key: 'canvas workpad', doc_count: 1 }, { key: 'dashboard', doc_count: 1 }, ], }, },
layoutTypes: { doc_count: 2, pdf: { buckets: [{ key: 'preserve_layout', doc_count: 2 }] } },
statusByApp: { buckets: [ { key: 'completed', doc_count: 4, jobTypes: { buckets: [ { key: 'printable_pdf', doc_count: 2, appNames: { buckets: [ { key: 'canvas workpad', doc_count: 1 }, { key: 'dashboard', doc_count: 1 }, ] } }, { key: 'PNG', doc_count: 1, appNames: { buckets: [{ key: 'dashboard', doc_count: 1 }] } }, { key: 'csv', doc_count: 1, appNames: { buckets: [] } }, ] } }, ] },
objectTypes: { doc_count: 2, pdf: { buckets: [ { key: 'canvas workpad', doc_count: 1 }, { key: 'dashboard', doc_count: 1 }, ] } },
statusTypes: { buckets: [{ key: 'completed', doc_count: 4 }] },
jobTypes: { buckets: [ { key: 'printable_pdf', doc_count: 2 }, { key: 'PNG', doc_count: 1 }, { key: 'csv', doc_count: 1 }, ], },
jobTypes: { buckets: [ { key: 'printable_pdf', doc_count: 2 }, { key: 'PNG', doc_count: 1 }, { key: 'csv', doc_count: 1 }, ] },
},
lastDay: {
doc_count: 4,
layoutTypes: { doc_count: 2, pdf: { buckets: [{ key: 'preserve_layout', doc_count: 2 }] }, },
statusByApp: { buckets: [ { key: 'completed', doc_count: 4, jobTypes: { buckets: [ { key: 'printable_pdf', doc_count: 2, appNames: { buckets: [ { key: 'canvas workpad', doc_count: 1 }, { key: 'dashboard', doc_count: 1 }, ], }, }, { key: 'PNG', doc_count: 1, appNames: { buckets: [{ key: 'dashboard', doc_count: 1 }] }, }, { key: 'csv', doc_count: 1, appNames: { buckets: [] } }, ], }, }, ], },
objectTypes: { doc_count: 2, pdf: { buckets: [ { key: 'canvas workpad', doc_count: 1 }, { key: 'dashboard', doc_count: 1 }, ], }, },
statusTypes: { buckets: [{ key: 'completed', doc_count: 4 }] },
jobTypes: { buckets: [ { key: 'printable_pdf', doc_count: 2 }, { key: 'PNG', doc_count: 1 }, { key: 'csv', doc_count: 1 }, ], },
},
},
}, // prettier-ignore
},
},
} as SearchResponse) // prettier-ignore
})
);
const usageStats = await collector.fetch(collectorFetchContext);
expect(usageStats).toMatchSnapshot();
@ -445,9 +333,9 @@ describe('data modeling', () => {
collectorFetchContext = getMockFetchClients(
getResponseMock({
aggregations: {
ranges: {
buckets: {
aggregations: {
ranges: {
buckets: {
all: {
doc_count: 0,
jobTypes: { buckets: [] },
@ -455,6 +343,9 @@ describe('data modeling', () => {
objectTypes: { doc_count: 0, pdf: { buckets: [] } },
statusByApp: { buckets: [] },
statusTypes: { buckets: [] },
sizeMax: { value: null},
sizeMin: { value: null },
sizeAvg: { value: null},
},
last7Days: {
doc_count: 0,
@ -463,19 +354,15 @@ describe('data modeling', () => {
objectTypes: { doc_count: 0, pdf: { buckets: [] } },
statusByApp: { buckets: [] },
statusTypes: { buckets: [] },
sizeMax: { value: null},
sizeMin: { value: null },
sizeAvg: { value: null},
},
lastDay: {
doc_count: 0,
jobTypes: { buckets: [] },
layoutTypes: { doc_count: 0, pdf: { buckets: [] } },
objectTypes: { doc_count: 0, pdf: { buckets: [] } },
statusByApp: { buckets: [] },
statusTypes: { buckets: [] },
},
}, // prettier-ignore
},
},
},
} as SearchResponse) // prettier-ignore
})
);
const usageStats = await collector.fetch(collectorFetchContext);
expect(usageStats).toMatchSnapshot();

13
x-pack/plugins/reporting/server/usage/schema.ts
View file

@ -14,6 +14,7 @@ import {
LayoutCounts,
RangeStats,
ReportingUsageType,
SizePercentiles,
} from './types';
const appCountsSchema: MakeSchemaFrom<AppCounts> = {
@ -39,10 +40,21 @@ const byAppCountsSchema: MakeSchemaFrom<ByAppCounts> = {
printable_pdf_v2: appCountsSchema,
};
const sizesSchema: MakeSchemaFrom<SizePercentiles> = {
'1.0': { type: 'long' },
'5.0': { type: 'long' },
'25.0': { type: 'long' },
'50.0': { type: 'long' },
'75.0': { type: 'long' },
'95.0': { type: 'long' },
'99.0': { type: 'long' },
};
const availableTotalSchema: MakeSchemaFrom<AvailableTotal> = {
available: { type: 'boolean' },
total: { type: 'long' },
deprecated: { type: 'long' },
sizes: sizesSchema,
app: appCountsSchema,
layout: layoutCountsSchema,
};
@ -74,6 +86,7 @@ const rangeStatsSchema: MakeSchemaFrom<RangeStats> = {
pending: byAppCountsSchema,
processing: byAppCountsSchema,
},
output_size: sizesSchema,
};
export const reportingSchema: MakeSchemaFrom<ReportingUsageType> = {

58
x-pack/plugins/reporting/server/usage/types.ts
View file

@ -5,45 +5,57 @@
* 2.0.
*/
export interface KeyCountBucket {
key: string;
export interface SizePercentiles {
'1.0': number | null;
'5.0': number | null;
'25.0': number | null;
'50.0': number | null;
'75.0': number | null;
'95.0': number | null;
'99.0': number | null;
}
interface DocCount {
doc_count: number;
isDeprecated?: {
doc_count: number;
};
}
interface SizeStats {
sizes?: { values: SizePercentiles };
}
export interface KeyCountBucket extends DocCount, SizeStats {
key: string;
isDeprecated?: DocCount;
}
export interface AggregationBuckets {
buckets: KeyCountBucket[];
}
export interface StatusByAppBucket {
export interface StatusByAppBucket extends DocCount {
key: string;
doc_count: number;
jobTypes: {
buckets: Array<{
doc_count: number;
key: string;
appNames: AggregationBuckets;
}>;
buckets: Array<
{
key: string;
appNames: AggregationBuckets;
} & DocCount
>;
};
}
export interface AggregationResultBuckets {
jobTypes: AggregationBuckets;
export interface AggregationResultBuckets extends DocCount, SizeStats {
jobTypes?: AggregationBuckets;
layoutTypes: {
doc_count: number;
pdf: AggregationBuckets;
};
pdf?: AggregationBuckets;
} & DocCount;
objectTypes: {
doc_count: number;
pdf: AggregationBuckets;
};
pdf?: AggregationBuckets;
} & DocCount;
statusTypes: AggregationBuckets;
statusByApp: {
buckets: StatusByAppBucket[];
};
doc_count: number;
}
export interface SearchResponse {
@ -61,6 +73,7 @@ export interface AvailableTotal {
available: boolean;
total: number;
deprecated?: number;
sizes?: SizePercentiles;
app?: {
search?: number;
dashboard?: number;
@ -110,7 +123,8 @@ type StatusByAppCounts = {
export type RangeStats = JobTypes & {
_all: number;
status: StatusCounts;
statuses: StatusByAppCounts;
statuses?: StatusByAppCounts;
output_size?: SizePercentiles;
};
export type ReportingUsageType = RangeStats & {

400
x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json
View file

@ -4051,6 +4051,31 @@
"deprecated": {
"type": "long"
},
"sizes": {
"properties": {
"1.0": {
"type": "long"
},
"5.0": {
"type": "long"
},
"25.0": {
"type": "long"
},
"50.0": {
"type": "long"
},
"75.0": {
"type": "long"
},
"95.0": {
"type": "long"
},
"99.0": {
"type": "long"
}
}
},
"app": {
"properties": {
"search": {
@ -4093,6 +4118,31 @@
"deprecated": {
"type": "long"
},
"sizes": {
"properties": {
"1.0": {
"type": "long"
},
"5.0": {
"type": "long"
},
"25.0": {
"type": "long"
},
"50.0": {
"type": "long"
},
"75.0": {
"type": "long"
},
"95.0": {
"type": "long"
},
"99.0": {
"type": "long"
}
}
},
"app": {
"properties": {
"search": {
@ -4135,6 +4185,31 @@
"deprecated": {
"type": "long"
},
"sizes": {
"properties": {
"1.0": {
"type": "long"
},
"5.0": {
"type": "long"
},
"25.0": {
"type": "long"
},
"50.0": {
"type": "long"
},
"75.0": {
"type": "long"
},
"95.0": {
"type": "long"
},
"99.0": {
"type": "long"
}
}
},
"app": {
"properties": {
"search": {
@ -4177,6 +4252,31 @@
"deprecated": {
"type": "long"
},
"sizes": {
"properties": {
"1.0": {
"type": "long"
},
"5.0": {
"type": "long"
},
"25.0": {
"type": "long"
},
"50.0": {
"type": "long"
},
"75.0": {
"type": "long"
},
"95.0": {
"type": "long"
},
"99.0": {
"type": "long"
}
}
},
"app": {
"properties": {
"search": {
@ -4219,6 +4319,31 @@
"deprecated": {
"type": "long"
},
"sizes": {
"properties": {
"1.0": {
"type": "long"
},
"5.0": {
"type": "long"
},
"25.0": {
"type": "long"
},
"50.0": {
"type": "long"
},
"75.0": {
"type": "long"
},
"95.0": {
"type": "long"
},
"99.0": {
"type": "long"
}
}
},
"app": {
"properties": {
"search": {
@ -4261,6 +4386,31 @@
"deprecated": {
"type": "long"
},
"sizes": {
"properties": {
"1.0": {
"type": "long"
},
"5.0": {
"type": "long"
},
"25.0": {
"type": "long"
},
"50.0": {
"type": "long"
},
"75.0": {
"type": "long"
},
"95.0": {
"type": "long"
},
"99.0": {
"type": "long"
}
}
},
"app": {
"properties": {
"search": {
@ -4303,6 +4453,31 @@
"deprecated": {
"type": "long"
},
"sizes": {
"properties": {
"1.0": {
"type": "long"
},
"5.0": {
"type": "long"
},
"25.0": {
"type": "long"
},
"50.0": {
"type": "long"
},
"75.0": {
"type": "long"
},
"95.0": {
"type": "long"
},
"99.0": {
"type": "long"
}
}
},
"app": {
"properties": {
"search": {
@ -4940,6 +5115,31 @@
}
}
},
"output_size": {
"properties": {
"1.0": {
"type": "long"
},
"5.0": {
"type": "long"
},
"25.0": {
"type": "long"
},
"50.0": {
"type": "long"
},
"75.0": {
"type": "long"
},
"95.0": {
"type": "long"
},
"99.0": {
"type": "long"
}
}
},
"available": {
"type": "boolean"
},
@ -4962,6 +5162,31 @@
"deprecated": {
"type": "long"
},
"sizes": {
"properties": {
"1.0": {
"type": "long"
},
"5.0": {
"type": "long"
},
"25.0": {
"type": "long"
},
"50.0": {
"type": "long"
},
"75.0": {
"type": "long"
},
"95.0": {
"type": "long"
},
"99.0": {
"type": "long"
}
}
},
"app": {
"properties": {
"search": {
@ -5004,6 +5229,31 @@
"deprecated": {
"type": "long"
},
"sizes": {
"properties": {
"1.0": {
"type": "long"
},
"5.0": {
"type": "long"
},
"25.0": {
"type": "long"
},
"50.0": {
"type": "long"
},
"75.0": {
"type": "long"
},
"95.0": {
"type": "long"
},
"99.0": {
"type": "long"
}
}
},
"app": {
"properties": {
"search": {
@ -5046,6 +5296,31 @@
"deprecated": {
"type": "long"
},
"sizes": {
"properties": {
"1.0": {
"type": "long"
},
"5.0": {
"type": "long"
},
"25.0": {
"type": "long"
},
"50.0": {
"type": "long"
},
"75.0": {
"type": "long"
},
"95.0": {
"type": "long"
},
"99.0": {
"type": "long"
}
}
},
"app": {
"properties": {
"search": {
@ -5088,6 +5363,31 @@
"deprecated": {
"type": "long"
},
"sizes": {
"properties": {
"1.0": {
"type": "long"
},
"5.0": {
"type": "long"
},
"25.0": {
"type": "long"
},
"50.0": {
"type": "long"
},
"75.0": {
"type": "long"
},
"95.0": {
"type": "long"
},
"99.0": {
"type": "long"
}
}
},
"app": {
"properties": {
"search": {
@ -5130,6 +5430,31 @@
"deprecated": {
"type": "long"
},
"sizes": {
"properties": {
"1.0": {
"type": "long"
},
"5.0": {
"type": "long"
},
"25.0": {
"type": "long"
},
"50.0": {
"type": "long"
},
"75.0": {
"type": "long"
},
"95.0": {
"type": "long"
},
"99.0": {
"type": "long"
}
}
},
"app": {
"properties": {
"search": {
@ -5172,6 +5497,31 @@
"deprecated": {
"type": "long"
},
"sizes": {
"properties": {
"1.0": {
"type": "long"
},
"5.0": {
"type": "long"
},
"25.0": {
"type": "long"
},
"50.0": {
"type": "long"
},
"75.0": {
"type": "long"
},
"95.0": {
"type": "long"
},
"99.0": {
"type": "long"
}
}
},
"app": {
"properties": {
"search": {
@ -5214,6 +5564,31 @@
"deprecated": {
"type": "long"
},
"sizes": {
"properties": {
"1.0": {
"type": "long"
},
"5.0": {
"type": "long"
},
"25.0": {
"type": "long"
},
"50.0": {
"type": "long"
},
"75.0": {
"type": "long"
},
"95.0": {
"type": "long"
},
"99.0": {
"type": "long"
}
}
},
"app": {
"properties": {
"search": {
@ -5850,6 +6225,31 @@
}
}
}
},
"output_size": {
"properties": {
"1.0": {
"type": "long"
},
"5.0": {
"type": "long"
},
"25.0": {
"type": "long"
},
"50.0": {
"type": "long"
},
"75.0": {
"type": "long"
},
"95.0": {
"type": "long"
},
"99.0": {
"type": "long"
}
}
}
}
}