[Reporting] Remove any, improve telemetry schema for completeness (#111212)

* [Reporting] Remove `any`, improve telemetry schema for completeness

* remove another any

* rename file per exported function

* test variable name

* use variable for DRY

* update reporting telemetry contract

* added csv_searchsource_immediate to telemetry

* fix types

* update jest snapshots

* remove tests on large literal objects

Co-authored-by: Jean-Louis Leysens <jloleysens@gmail.com>
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Tim Sullivan 2021-09-08 16:39:42 -07:00 committed by GitHub
parent af06aec5b0
commit c4c653606a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 2276 additions and 394 deletions

View file

@ -184,7 +184,7 @@ export class CsvGenerator {
data: dataTableCell,
}: {
column: string;
data: any;
data: unknown;
}): string => {
let cell: string[] | string | object;
// check truthiness to guard against _score, _type, etc

View file

@ -1,97 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { uniq } from 'lodash';
import {
CSV_JOB_TYPE,
CSV_JOB_TYPE_DEPRECATED,
DEPRECATED_JOB_TYPES,
PDF_JOB_TYPE,
PNG_JOB_TYPE,
} from '../../common/constants';
import { AvailableTotal, ExportType, FeatureAvailabilityMap, RangeStats } from './types';
const jobTypeIsDeprecated = (jobType: string) => DEPRECATED_JOB_TYPES.includes(jobType);
function getForFeature(
range: Partial<RangeStats>,
typeKey: ExportType,
featureAvailability: FeatureAvailabilityMap,
additional?: any
): AvailableTotal & typeof additional {
const isAvailable = (feature: ExportType) => !!featureAvailability[feature];
const jobType = range[typeKey] || { total: 0, ...additional, deprecated: 0 };
// merge the additional stats for the jobType
type AdditionalType = { [K in keyof typeof additional]: K };
const filledAdditional: AdditionalType = {};
if (additional) {
Object.keys(additional).forEach((k) => {
filledAdditional[k] = { ...additional[k], ...jobType[k] };
});
}
// if the type itself is deprecated, all jobs are deprecated, otherwise only some of them might be
const deprecated = jobTypeIsDeprecated(typeKey) ? jobType.total : jobType.deprecated || 0;
return {
available: isAvailable(typeKey),
total: jobType.total,
deprecated,
...filledAdditional,
};
}
/*
* Decorates range stats (stats for last day, last 7 days, etc) with feature
* availability booleans, and zero-filling for unused features
*
* This function builds the result object for all export types found in the
* Reporting data, even if the type is unknown to this Kibana instance.
*/
export const decorateRangeStats = (
rangeStats: Partial<RangeStats> = {},
featureAvailability: FeatureAvailabilityMap
): RangeStats => {
const {
_all: rangeAll,
status: rangeStatus,
statuses: rangeStatusByApp,
[PDF_JOB_TYPE]: rangeStatsPdf,
...rangeStatsBasic
} = rangeStats;
// combine the known types with any unknown type found in reporting data
const keysBasic = uniq([
CSV_JOB_TYPE,
CSV_JOB_TYPE_DEPRECATED,
PNG_JOB_TYPE,
...Object.keys(rangeStatsBasic),
]) as ExportType[];
const rangeBasic = keysBasic.reduce((accum, currentKey) => {
return {
...accum,
[currentKey]: getForFeature(rangeStatsBasic, currentKey, featureAvailability),
};
}, {}) as Partial<RangeStats>;
const rangePdf = {
[PDF_JOB_TYPE]: getForFeature(rangeStats, PDF_JOB_TYPE, featureAvailability, {
app: { dashboard: 0, visualization: 0 },
layout: { preserve_layout: 0, print: 0 },
}),
};
const resultStats = {
_all: rangeAll || 0,
status: { completed: 0, failed: 0, ...rangeStatus },
statuses: rangeStatusByApp,
...rangePdf,
...rangeBasic,
} as RangeStats;
return resultStats;
};

View file

@ -5,7 +5,9 @@
* 2.0.
*/
import { decorateRangeStats } from './decorate_range_stats';
import { getExportTypesRegistry } from '../lib';
import { getExportStats } from './get_export_stats';
import { getExportTypesHandler } from './get_export_type_handler';
import { FeatureAvailabilityMap } from './types';
let featureMap: FeatureAvailabilityMap;
@ -14,8 +16,10 @@ beforeEach(() => {
featureMap = { PNG: true, csv: true, csv_searchsource: true, printable_pdf: true };
});
const exportTypesHandler = getExportTypesHandler(getExportTypesRegistry());
test('Model of job status and status-by-pdf-app', () => {
const result = decorateRangeStats(
const result = getExportStats(
{
status: { completed: 0, processing: 1, pending: 2, failed: 3 },
statuses: {
@ -24,7 +28,8 @@ test('Model of job status and status-by-pdf-app', () => {
failed: { printable_pdf: { visualization: 2, dashboard: 2, 'canvas workpad': 1 } },
},
},
featureMap
featureMap,
exportTypesHandler
);
expect(result.status).toMatchInlineSnapshot(`
@ -60,7 +65,7 @@ test('Model of job status and status-by-pdf-app', () => {
});
test('Model of jobTypes', () => {
const result = decorateRangeStats(
const result = getExportStats(
{
PNG: { available: true, total: 3 },
printable_pdf: {
@ -71,27 +76,61 @@ test('Model of jobTypes', () => {
},
csv_searchsource: { available: true, total: 3 },
},
featureMap
featureMap,
exportTypesHandler
);
expect(result.PNG).toMatchInlineSnapshot(`
Object {
"app": Object {
"canvas workpad": 0,
"dashboard": 0,
"search": 0,
"visualization": 0,
},
"available": true,
"deprecated": 0,
"layout": Object {
"canvas": 0,
"preserve_layout": 0,
"print": 0,
},
"total": 3,
}
`);
expect(result.csv).toMatchInlineSnapshot(`
Object {
"app": Object {
"canvas workpad": 0,
"dashboard": 0,
"search": 0,
"visualization": 0,
},
"available": true,
"deprecated": 0,
"layout": Object {
"canvas": 0,
"preserve_layout": 0,
"print": 0,
},
"total": 0,
}
`);
expect(result.csv_searchsource).toMatchInlineSnapshot(`
Object {
"app": Object {
"canvas workpad": 0,
"dashboard": 0,
"search": 0,
"visualization": 0,
},
"available": true,
"deprecated": 0,
"layout": Object {
"canvas": 0,
"preserve_layout": 0,
"print": 0,
},
"total": 3,
}
`);
@ -100,11 +139,13 @@ test('Model of jobTypes', () => {
"app": Object {
"canvas workpad": 3,
"dashboard": 0,
"search": 0,
"visualization": 0,
},
"available": true,
"deprecated": 0,
"layout": Object {
"canvas": 0,
"preserve_layout": 3,
"print": 0,
},
@ -114,28 +155,52 @@ test('Model of jobTypes', () => {
});
test('PNG counts, provided count of deprecated jobs explicitly', () => {
const result = decorateRangeStats(
const result = getExportStats(
{ PNG: { available: true, total: 15, deprecated: 5 } },
featureMap
featureMap,
exportTypesHandler
);
expect(result.PNG).toMatchInlineSnapshot(`
Object {
"app": Object {
"canvas workpad": 0,
"dashboard": 0,
"search": 0,
"visualization": 0,
},
"available": true,
"deprecated": 5,
"layout": Object {
"canvas": 0,
"preserve_layout": 0,
"print": 0,
},
"total": 15,
}
`);
});
test('CSV counts, provides all jobs implicitly deprecated due to jobtype', () => {
const result = decorateRangeStats(
const result = getExportStats(
{ csv: { available: true, total: 15, deprecated: 0 } },
featureMap
featureMap,
exportTypesHandler
);
expect(result.csv).toMatchInlineSnapshot(`
Object {
"app": Object {
"canvas workpad": 0,
"dashboard": 0,
"search": 0,
"visualization": 0,
},
"available": true,
"deprecated": 15,
"layout": Object {
"canvas": 0,
"preserve_layout": 0,
"print": 0,
},
"total": 15,
}
`);

View file

@ -0,0 +1,90 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { DEPRECATED_JOB_TYPES } from '../../common/constants';
import { ExportTypesHandler } from './get_export_type_handler';
import { AvailableTotal, FeatureAvailabilityMap, RangeStats } from './types';
const jobTypeIsDeprecated = (jobType: string) => DEPRECATED_JOB_TYPES.includes(jobType);
const defaultTotalsForFeature: Omit<AvailableTotal, 'available'> = {
total: 0,
deprecated: 0,
app: { 'canvas workpad': 0, search: 0, visualization: 0, dashboard: 0 },
layout: { canvas: 0, print: 0, preserve_layout: 0 },
};
const isAvailable = (featureAvailability: FeatureAvailabilityMap, feature: string) =>
!!featureAvailability[feature];
function getAvailableTotalForFeature(
jobType: AvailableTotal,
typeKey: string,
featureAvailability: FeatureAvailabilityMap
): AvailableTotal {
// if the type itself is deprecated, all jobs are deprecated, otherwise only some of them might be
const deprecated = jobTypeIsDeprecated(typeKey) ? jobType.total : jobType.deprecated || 0;
// merge the additional stats for the jobType
const availableTotal = {
available: isAvailable(featureAvailability, typeKey),
total: jobType.total,
deprecated,
app: { ...defaultTotalsForFeature.app, ...jobType.app },
layout: { ...defaultTotalsForFeature.layout, ...jobType.layout },
};
return availableTotal as AvailableTotal;
}
/*
* Decorates range stats (stats for last day, last 7 days, etc) with feature
* availability booleans, and zero-filling for unused features
*
* This function builds the result object for all export types found in the
* Reporting data, even if the type is unknown to this Kibana instance.
*/
export const getExportStats = (
rangeStatsInput: Partial<RangeStats> = {},
featureAvailability: FeatureAvailabilityMap,
exportTypesHandler: ExportTypesHandler
) => {
const {
_all: rangeAll,
status: rangeStatus,
statuses: rangeStatusByApp,
...rangeStats
} = rangeStatsInput;
// combine the known types with any unknown type found in reporting data
const statsForExportType = exportTypesHandler.getJobTypes().reduce((accum, exportType) => {
const availableTotal = rangeStats[exportType as keyof typeof rangeStats];
if (!availableTotal) {
return {
...accum,
[exportType]: {
available: isAvailable(featureAvailability, exportType),
...defaultTotalsForFeature,
},
};
}
return {
...accum,
[exportType]: getAvailableTotalForFeature(availableTotal, exportType, featureAvailability),
};
}, {});
const resultStats = {
...statsForExportType,
_all: rangeAll || 0,
status: { completed: 0, failed: 0, ...rangeStatus },
statuses: rangeStatusByApp,
} as RangeStats;
return resultStats;
};

View file

@ -15,6 +15,13 @@ import { FeaturesAvailability } from './';
*/
export function getExportTypesHandler(exportTypesRegistry: ExportTypesRegistry) {
return {
/*
* Allow usage collection to loop through each registered job type
*/
getJobTypes() {
return exportTypesRegistry.getAll().map(({ jobType }) => jobType);
},
/*
* Based on the X-Pack license and which export types are available,
* returns an object where the keys are the export types and the values are
@ -46,3 +53,5 @@ export function getExportTypesHandler(exportTypesRegistry: ExportTypesRegistry)
},
};
}
export type ExportTypesHandler = ReturnType<typeof getExportTypesHandler>;

View file

@ -10,7 +10,7 @@ import { get } from 'lodash';
import type { ReportingConfig } from '../';
import type { ExportTypesRegistry } from '../lib/export_types_registry';
import type { GetLicense } from './';
import { decorateRangeStats } from './decorate_range_stats';
import { getExportStats } from './get_export_stats';
import { getExportTypesHandler } from './get_export_type_handler';
import type {
AggregationResultBuckets,
@ -108,11 +108,16 @@ type RangeStatSets = Partial<RangeStats> & {
type ESResponse = Partial<estypes.SearchResponse>;
async function handleResponse(response: ESResponse): Promise<Partial<RangeStatSets>> {
const buckets = get(response, 'aggregations.ranges.buckets');
const buckets = get(response, 'aggregations.ranges.buckets') as Record<
'all' | 'last7Days',
AggregationResultBuckets
>;
if (!buckets) {
return {};
}
const { last7Days, all } = buckets as any;
const { all, last7Days } = buckets;
const last7DaysUsage = last7Days ? getAggStats(last7Days) : {};
const allUsage = all ? getAggStats(all) : {};
@ -196,8 +201,8 @@ export async function getReportingUsage(
available: true,
browser_type: browserType,
enabled: true,
last7Days: decorateRangeStats(last7Days, availability),
...decorateRangeStats(all, availability),
last7Days: getExportStats(last7Days, availability, exportTypesHandler),
...getExportStats(all, availability, exportTypesHandler),
};
}
);

View file

@ -18,7 +18,7 @@ import {
getReportingUsageCollector,
registerReportingUsageCollector,
} from './reporting_usage_collector';
import { ReportingUsageType, SearchResponse } from './types';
import { SearchResponse } from './types';
const exportTypesRegistry = getExportTypesRegistry();
@ -478,161 +478,6 @@ describe('data modeling', () => {
const usageStats = await collector.fetch(collectorFetchContext);
expect(usageStats).toMatchSnapshot();
});
test('Cast various example data to the TypeScript definition', () => {
const check = (obj: ReportingUsageType) => {
return typeof obj;
};
// just check that the example objects can be cast to ReportingUsageType
check({
PNG: { available: true, total: 7 },
PNGV2: { available: true, total: 7 },
_all: 21,
available: true,
browser_type: 'chromium',
csv: { available: true, total: 4 },
csv_searchsource: { available: true, total: 4 },
enabled: true,
last7Days: {
PNG: { available: true, total: 0 },
PNGV2: { available: true, total: 0 },
_all: 0,
csv: { available: true, total: 0 },
csv_searchsource: { available: true, total: 0 },
printable_pdf: {
app: { dashboard: 0, visualization: 0 },
available: true,
layout: { preserve_layout: 0, print: 0 },
total: 0,
},
printable_pdf_v2: {
app: { dashboard: 0, visualization: 0 },
available: true,
layout: { preserve_layout: 0, print: 0 },
total: 0,
},
status: { completed: 0, failed: 0 },
statuses: {},
},
printable_pdf: {
app: { 'canvas workpad': 3, dashboard: 3, visualization: 4 },
available: true,
layout: { preserve_layout: 7, print: 3 },
total: 10,
},
printable_pdf_v2: {
app: { 'canvas workpad': 3, dashboard: 3, visualization: 4 },
available: true,
layout: { preserve_layout: 7, print: 3 },
total: 10,
},
status: { completed: 21, failed: 0 },
statuses: {
completed: {
PNG: { dashboard: 3, visualization: 4 },
PNGV2: { dashboard: 3, visualization: 4 },
csv: {},
printable_pdf: { 'canvas workpad': 3, dashboard: 3, visualization: 4 },
printable_pdf_v2: { 'canvas workpad': 3, dashboard: 3, visualization: 4 },
},
},
});
check({
PNG: { available: true, total: 3 },
PNGV2: { available: true, total: 3 },
_all: 4,
available: true,
browser_type: 'chromium',
csv: { available: true, total: 0 },
csv_searchsource: { available: true, total: 0 },
enabled: true,
last7Days: {
PNG: { available: true, total: 3 },
PNGV2: { available: true, total: 3 },
_all: 4,
csv: { available: true, total: 0 },
csv_searchsource: { available: true, total: 0 },
printable_pdf: {
app: { 'canvas workpad': 1, dashboard: 0, visualization: 0 },
available: true,
layout: { preserve_layout: 1, print: 0 },
total: 1,
},
printable_pdf_v2: {
app: { 'canvas workpad': 1, dashboard: 0, visualization: 0 },
available: true,
layout: { preserve_layout: 1, print: 0 },
total: 1,
},
status: { completed: 4, failed: 0 },
statuses: {
completed: { PNG: { visualization: 3 }, printable_pdf: { 'canvas workpad': 1 } },
},
},
printable_pdf: {
app: { 'canvas workpad': 1, dashboard: 0, visualization: 0 },
available: true,
layout: { preserve_layout: 1, print: 0 },
total: 1,
},
printable_pdf_v2: {
app: { 'canvas workpad': 1, dashboard: 0, visualization: 0 },
available: true,
layout: { preserve_layout: 1, print: 0 },
total: 1,
},
status: { completed: 4, failed: 0 },
statuses: {
completed: { PNG: { visualization: 3 }, printable_pdf: { 'canvas workpad': 1 } },
},
});
check({
available: true,
browser_type: 'chromium',
enabled: true,
last7Days: {
_all: 0,
status: { completed: 0, failed: 0 },
statuses: {},
printable_pdf: {
available: true,
total: 0,
app: { dashboard: 0, visualization: 0 },
layout: { preserve_layout: 0, print: 0 },
},
printable_pdf_v2: {
available: true,
total: 0,
app: { dashboard: 0, visualization: 0 },
layout: { preserve_layout: 0, print: 0 },
},
csv: { available: true, total: 0 },
csv_searchsource: { available: true, total: 0 },
PNG: { available: true, total: 0 },
PNGV2: { available: true, total: 0 },
},
_all: 0,
status: { completed: 0, failed: 0 },
statuses: {},
printable_pdf: {
available: true,
total: 0,
app: { dashboard: 0, visualization: 0 },
layout: { preserve_layout: 0, print: 0 },
},
printable_pdf_v2: {
available: true,
total: 0,
app: { dashboard: 0, visualization: 0 },
layout: { preserve_layout: 0, print: 0 },
},
csv: { available: true, total: 0 },
csv_searchsource: { available: true, total: 0 },
PNG: { available: true, total: 0 },
PNGV2: { available: true, total: 0 },
});
});
});
describe('Ready for collection observable', () => {

View file

@ -11,19 +11,28 @@ import {
AvailableTotal,
ByAppCounts,
JobTypes,
LayoutCounts,
RangeStats,
ReportingUsageType,
} from './types';
const appCountsSchema: MakeSchemaFrom<AppCounts> = {
search: { type: 'long' },
'canvas workpad': { type: 'long' },
dashboard: { type: 'long' },
visualization: { type: 'long' },
};
const layoutCountsSchema: MakeSchemaFrom<LayoutCounts> = {
canvas: { type: 'long' },
print: { type: 'long' },
preserve_layout: { type: 'long' },
};
const byAppCountsSchema: MakeSchemaFrom<ByAppCounts> = {
csv: appCountsSchema,
csv_searchsource: appCountsSchema,
csv_searchsource_immediate: appCountsSchema,
PNG: appCountsSchema,
PNGV2: appCountsSchema,
printable_pdf: appCountsSchema,
@ -34,29 +43,18 @@ const availableTotalSchema: MakeSchemaFrom<AvailableTotal> = {
available: { type: 'boolean' },
total: { type: 'long' },
deprecated: { type: 'long' },
app: appCountsSchema,
layout: layoutCountsSchema,
};
const jobTypesSchema: MakeSchemaFrom<JobTypes> = {
csv: availableTotalSchema,
csv_searchsource: availableTotalSchema,
csv_searchsource_immediate: availableTotalSchema,
PNG: availableTotalSchema,
PNGV2: availableTotalSchema,
printable_pdf: {
...availableTotalSchema,
app: appCountsSchema,
layout: {
print: { type: 'long' },
preserve_layout: { type: 'long' },
},
},
printable_pdf_v2: {
...availableTotalSchema,
app: appCountsSchema,
layout: {
print: { type: 'long' },
preserve_layout: { type: 'long' },
},
},
printable_pdf: availableTotalSchema,
printable_pdf_v2: availableTotalSchema,
};
const rangeStatsSchema: MakeSchemaFrom<RangeStats> = {

View file

@ -61,37 +61,40 @@ export interface AvailableTotal {
available: boolean;
total: number;
deprecated?: number;
app?: {
search?: number;
dashboard?: number;
visualization?: number;
'canvas workpad'?: number;
};
layout?: {
print?: number;
preserve_layout?: number;
canvas?: number;
};
}
// FIXME: find a way to get this from exportTypesHandler or common/constants
type BaseJobTypes =
| 'csv'
| 'csv_searchsource'
| 'csv_searchsource_immediate'
| 'PNG'
| 'PNGV2'
| 'printable_pdf'
| 'printable_pdf_v2';
export interface LayoutCounts {
canvas: number;
print: number;
preserve_layout: number;
}
type AppNames = 'canvas workpad' | 'dashboard' | 'visualization';
export type AppCounts = {
[A in AppNames]?: number;
[A in 'canvas workpad' | 'dashboard' | 'visualization' | 'search']?: number;
};
export type JobTypes = { [K in BaseJobTypes]: AvailableTotal } & {
printable_pdf: AvailableTotal & {
app: AppCounts;
layout: LayoutCounts;
};
} & {
printable_pdf_v2: AvailableTotal & {
app: AppCounts;
layout: LayoutCounts;
};
};
export type JobTypes = { [K in BaseJobTypes]: AvailableTotal };
export type ByAppCounts = { [J in BaseJobTypes]?: AppCounts };
@ -117,8 +120,7 @@ export type ReportingUsageType = RangeStats & {
last7Days: RangeStats;
};
export type ExportType = 'csv' | 'csv_searchsource' | 'printable_pdf' | 'PNG';
export type FeatureAvailabilityMap = { [F in ExportType]: boolean };
export type FeatureAvailabilityMap = Record<string, boolean>;
export interface ReportingUsageSearchResponse {
aggregations: {