maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[Usage Collection] [schema] Explicit "array" definition (#78141)

Browse source 
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
This commit is contained in:
Alejandro Fernández Haro 2020-09-28 16:13:21 +01:00 committed by GitHub
parent e111c2ab3e
commit db78d70df3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 362 additions and 240 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
packages/kbn-telemetry-tools/src/tools/__fixture__/mock_schema.json
View file

@ -35,16 +35,19 @@
}
},
"my_array": {
"properties": {
"total": {
"type": "number"
},
"type": {
"type": "boolean"
"type": "array",
"items": {
"properties": {
"total": {
"type": "number"
},
"type": {
"type": "boolean"
}
}
}
},
"my_str_array": { "type": "keyword" }
"my_str_array": { "type": "array", "items": { "type": "keyword" } }
}
}
}

33
packages/kbn-telemetry-tools/src/tools/__fixture__/parsed_working_collector.ts
View file

@ -55,12 +55,15 @@ export const parsedWorkingCollector: ParsedUsageCollection = [
},
},
my_array: {
total: {
type: 'number',
type: 'array',
items: {
total: {
type: 'number',
},
type: { type: 'boolean' },
},
type: { type: 'boolean' },
},
my_str_array: { type: 'keyword' },
my_str_array: { type: 'array', items: { type: 'keyword' } },
},
},
fetch: {
@ -91,18 +94,22 @@ export const parsedWorkingCollector: ParsedUsageCollection = [
},
},
my_array: {
total: {
kind: SyntaxKind.NumberKeyword,
type: 'NumberKeyword',
},
type: {
kind: SyntaxKind.BooleanKeyword,
type: 'BooleanKeyword',
items: {
total: {
kind: SyntaxKind.NumberKeyword,
type: 'NumberKeyword',
},
type: {
kind: SyntaxKind.BooleanKeyword,
type: 'BooleanKeyword',
},
},
},
my_str_array: {
kind: SyntaxKind.StringKeyword,
type: 'StringKeyword',
items: {
kind: SyntaxKind.StringKeyword,
type: 'StringKeyword',
},
},
},
},

40
packages/kbn-telemetry-tools/src/tools/__snapshots__/extract_collectors.test.ts.snap generated
View file

@ -153,13 +153,15 @@ Array [
"type": "StringKeyword",
},
"my_array": Object {
"total": Object {
"kind": 143,
"type": "NumberKeyword",
},
"type": Object {
"kind": 131,
"type": "BooleanKeyword",
"items": Object {
"total": Object {
"kind": 143,
"type": "NumberKeyword",
},
"type": Object {
"kind": 131,
"type": "BooleanKeyword",
},
},
},
"my_index_signature_prop": Object {
@ -183,8 +185,10 @@ Array [
"type": "StringKeyword",
},
"my_str_array": Object {
"kind": 146,
"type": "StringKeyword",
"items": Object {
"kind": 146,
"type": "StringKeyword",
},
},
},
"typeName": "Usage",
@ -195,12 +199,15 @@ Array [
"type": "keyword",
},
"my_array": Object {
"total": Object {
"type": "number",
},
"type": Object {
"type": "boolean",
"items": Object {
"total": Object {
"type": "number",
},
"type": Object {
"type": "boolean",
},
},
"type": "array",
},
"my_index_signature_prop": Object {
"avg": Object {
@ -228,7 +235,10 @@ Array [
"type": "text",
},
"my_str_array": Object {
"type": "keyword",
"items": Object {
"type": "keyword",
},
"type": "array",
},
},
},

18
packages/kbn-telemetry-tools/src/tools/manage_schema.ts
View file

@ -28,7 +28,7 @@ export type AllowedSchemaTypes =
| 'date'
| 'float';
export function compatibleSchemaTypes(type: AllowedSchemaTypes) {
export function compatibleSchemaTypes(type: AllowedSchemaTypes | 'array') {
switch (type) {
case 'keyword':
case 'text':
@ -40,6 +40,8 @@ export function compatibleSchemaTypes(type: AllowedSchemaTypes) {
case 'float':
case 'long':
return 'number';
case 'array':
return 'array';
default:
throw new Error(`Unknown schema type ${type}`);
}
@ -66,10 +68,22 @@ export function isObjectMapping(entity: any) {
return false;
}
function isArrayMapping(entity: any): entity is { type: 'array'; items: object } {
return typeof entity === 'object' && entity.type === 'array' && typeof entity.items === 'object';
}
function getValueMapping(value: any) {
return isObjectMapping(value) ? transformToEsMapping(value) : value;
}
function transformToEsMapping(usageMappingValue: any) {
const fieldMapping: any = { properties: {} };
for (const [key, value] of Object.entries(usageMappingValue)) {
fieldMapping.properties[key] = isObjectMapping(value) ? transformToEsMapping(value) : value;
if (isArrayMapping(value)) {
fieldMapping.properties[key] = { ...value, items: getValueMapping(value.items) };
} else {
fieldMapping.properties[key] = getValueMapping(value);
}
}
return fieldMapping;
}

4
packages/kbn-telemetry-tools/src/tools/serializer.test.ts
View file

@ -84,8 +84,8 @@ describe('getDescriptor', () => {
expect(descriptor).toEqual({
prop1: { kind: TelemetryKinds.MomentDate, type: 'MomentDate' },
prop2: { kind: TelemetryKinds.MomentDate, type: 'MomentDate' },
prop3: { kind: TelemetryKinds.MomentDate, type: 'MomentDate' },
prop4: { kind: TelemetryKinds.Date, type: 'Date' },
prop3: { items: { kind: TelemetryKinds.MomentDate, type: 'MomentDate' } },
prop4: { items: { kind: TelemetryKinds.Date, type: 'Date' } },
});
});

2
packages/kbn-telemetry-tools/src/tools/serializer.ts
View file

@ -139,7 +139,7 @@ export function getDescriptor(node: ts.Node, program: ts.Program): Descriptor |
}
if (ts.isArrayTypeNode(node)) {
return getDescriptor(node.elementType, program);
return { items: getDescriptor(node.elementType, program) };
}
if (ts.isLiteralTypeNode(node)) {

11
src/fixtures/telemetry_collectors/working_collector.ts
View file

@ -90,12 +90,15 @@ export const myCollector = makeUsageCollector<Usage>({
type: { type: 'boolean' },
},
my_array: {
total: {
type: 'number',
type: 'array',
items: {
total: {
type: 'number',
},
type: { type: 'boolean' },
},
type: { type: 'boolean' },
},
my_str_array: { type: 'keyword' },
my_str_array: { type: 'array', items: { type: 'keyword' } },
my_index_signature_prop: {
count: { type: 'number' },
avg: { type: 'number' },

4
src/plugins/home/server/services/sample_data/usage/collector.ts
View file

@ -38,12 +38,12 @@ export async function makeSampleDataUsageCollector(
fetch: fetchProvider(index),
isReady: () => true,
schema: {
installed: { type: 'keyword' },
installed: { type: 'array', items: { type: 'keyword' } },
last_install_date: { type: 'date' },
last_install_set: { type: 'keyword' },
last_uninstall_date: { type: 'date' },
last_uninstall_set: { type: 'keyword' },
uninstalled: { type: 'keyword' },
uninstalled: { type: 'array', items: { type: 'keyword' } },
},
});

10
src/plugins/telemetry/schema/oss_plugins.json
View file

@ -29,7 +29,10 @@
"sample-data": {
"properties": {
"installed": {
"type": "keyword"
"type": "array",
"items": {
"type": "keyword"
}
},
"last_install_date": {
"type": "date"
@ -44,7 +47,10 @@
"type": "keyword"
},
"uninstalled": {
"type": "keyword"
"type": "array",
"items": {
"type": "keyword"
}
}
}
},

22
src/plugins/usage_collection/README.md
View file

@ -140,6 +140,14 @@ The `AllowedSchemaTypes` is the list of allowed schema types for the usage field
'keyword', 'text', 'number', 'boolean', 'long', 'date', 'float'
```
### Arrays
If any of your properties is an array, the schema definition must follow the convention below:
```
{ type: 'array', items: {...mySchemaDefinitionOfTheEntriesInTheArray} }
```
### Example
```ts
@ -152,6 +160,8 @@ export const myCollector = makeUsageCollector<Usage>({
some_obj: {
total: 123,
},
some_array: ['value1', 'value2'],
some_array_of_obj: [{total: 123}],
};
},
schema: {
@ -163,6 +173,18 @@ export const myCollector = makeUsageCollector<Usage>({
type: 'number',
},
},
some_array: {
type: 'array',
items: { type: 'keyword' }
},
some_array_of_obj: {
type: 'array',
items: {
total: {
type: 'number',
},
},
},
},
});
```

20
src/plugins/usage_collection/server/collector/collector.test.ts
View file

@ -153,7 +153,10 @@ describe('collector', () => {
isReady: () => false,
fetch: () => ({ testPass: [{ name: 'a', value: 100 }] }),
schema: {
testPass: { name: { type: 'keyword' }, value: { type: 'long' } },
testPass: {
type: 'array',
items: { name: { type: 'keyword' }, value: { type: 'long' } },
},
},
});
expect(collector).toBeDefined();
@ -166,7 +169,10 @@ describe('collector', () => {
fetch: () => ({ testPass: [{ name: 'a', value: 100 }], otherProp: 1 }),
// @ts-expect-error
schema: {
testPass: { name: { type: 'keyword' }, value: { type: 'long' } },
testPass: {
type: 'array',
items: { name: { type: 'keyword' }, value: { type: 'long' } },
},
},
});
expect(collector).toBeDefined();
@ -185,7 +191,10 @@ describe('collector', () => {
},
// @ts-expect-error
schema: {
testPass: { name: { type: 'keyword' }, value: { type: 'long' } },
testPass: {
type: 'array',
items: { name: { type: 'keyword' }, value: { type: 'long' } },
},
},
});
expect(collector).toBeDefined();
@ -203,7 +212,10 @@ describe('collector', () => {
return { otherProp: 1 };
},
schema: {
testPass: { name: { type: 'keyword' }, value: { type: 'long' } },
testPass: {
type: 'array',
items: { name: { type: 'keyword' }, value: { type: 'long' } },
},
otherProp: { type: 'long' },
},
});

2
src/plugins/usage_collection/server/collector/collector.ts
View file

@ -40,7 +40,7 @@ export type RecursiveMakeSchemaFrom<U> = U extends object
export type MakeSchemaFrom<Base> = {
[Key in keyof Base]: Base[Key] extends Array<infer U>
? RecursiveMakeSchemaFrom<U>
? { type: 'array'; items: RecursiveMakeSchemaFrom<U> }
: RecursiveMakeSchemaFrom<Base[Key]>;
};

9
x-pack/plugins/ingest_manager/server/collectors/register.ts
View file

@ -50,9 +50,12 @@ export function registerIngestManagerUsageCollector(
offline: { type: 'long' },
},
packages: {
name: { type: 'keyword' },
version: { type: 'keyword' },
enabled: { type: 'boolean' },
type: 'array',
items: {
name: { type: 'keyword' },
version: { type: 'keyword' },
enabled: { type: 'boolean' },
},
},
},
});

55
x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_usage_collector.test.ts
View file

@ -80,33 +80,36 @@ describe('getMonitoringUsageCollector', () => {
expect(args[0].schema).toStrictEqual({
hasMonitoringData: { type: 'boolean' },
clusters: {
license: { type: 'keyword' },
clusterUuid: { type: 'keyword' },
metricbeatUsed: { type: 'boolean' },
elasticsearch: {
enabled: { type: 'boolean' },
count: { type: 'long' },
metricbeatUsed: { type: 'boolean' },
},
kibana: {
enabled: { type: 'boolean' },
count: { type: 'long' },
metricbeatUsed: { type: 'boolean' },
},
logstash: {
enabled: { type: 'boolean' },
count: { type: 'long' },
metricbeatUsed: { type: 'boolean' },
},
beats: {
enabled: { type: 'boolean' },
count: { type: 'long' },
metricbeatUsed: { type: 'boolean' },
},
apm: {
enabled: { type: 'boolean' },
count: { type: 'long' },
type: 'array',
items: {
license: { type: 'keyword' },
clusterUuid: { type: 'keyword' },
metricbeatUsed: { type: 'boolean' },
elasticsearch: {
enabled: { type: 'boolean' },
count: { type: 'long' },
metricbeatUsed: { type: 'boolean' },
},
kibana: {
enabled: { type: 'boolean' },
count: { type: 'long' },
metricbeatUsed: { type: 'boolean' },
},
logstash: {
enabled: { type: 'boolean' },
count: { type: 'long' },
metricbeatUsed: { type: 'boolean' },
},
beats: {
enabled: { type: 'boolean' },
count: { type: 'long' },
metricbeatUsed: { type: 'boolean' },
},
apm: {
enabled: { type: 'boolean' },
count: { type: 'long' },
metricbeatUsed: { type: 'boolean' },
},
},
},
});

109
x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_usage_collector.ts
View file

@ -28,68 +28,71 @@ export function getMonitoringUsageCollector(
type: 'boolean',
},
clusters: {
license: {
type: 'keyword',
},
clusterUuid: {
type: 'keyword',
},
metricbeatUsed: {
type: 'boolean',
},
elasticsearch: {
enabled: {
type: 'boolean',
type: 'array',
items: {
license: {
type: 'keyword',
},
count: {
type: 'long',
clusterUuid: {
type: 'keyword',
},
metricbeatUsed: {
type: 'boolean',
},
},
kibana: {
enabled: {
type: 'boolean',
elasticsearch: {
enabled: {
type: 'boolean',
},
count: {
type: 'long',
},
metricbeatUsed: {
type: 'boolean',
},
},
count: {
type: 'long',
kibana: {
enabled: {
type: 'boolean',
},
count: {
type: 'long',
},
metricbeatUsed: {
type: 'boolean',
},
},
metricbeatUsed: {
type: 'boolean',
logstash: {
enabled: {
type: 'boolean',
},
count: {
type: 'long',
},
metricbeatUsed: {
type: 'boolean',
},
},
},
logstash: {
enabled: {
type: 'boolean',
beats: {
enabled: {
type: 'boolean',
},
count: {
type: 'long',
},
metricbeatUsed: {
type: 'boolean',
},
},
count: {
type: 'long',
},
metricbeatUsed: {
type: 'boolean',
},
},
beats: {
enabled: {
type: 'boolean',
},
count: {
type: 'long',
},
metricbeatUsed: {
type: 'boolean',
},
},
apm: {
enabled: {
type: 'boolean',
},
count: {
type: 'long',
},
metricbeatUsed: {
type: 'boolean',
apm: {
enabled: {
type: 'boolean',
},
count: {
type: 'long',
},
metricbeatUsed: {
type: 'boolean',
},
},
},
},

10
x-pack/plugins/security/server/usage_collector/security_usage_collector.ts
View file

@ -62,10 +62,16 @@ export function registerSecurityUsageCollector({ usageCollection, config, licens
type: 'number',
},
enabledAuthProviders: {
type: 'keyword',
type: 'array',
items: {
type: 'keyword',
},
},
httpAuthSchemes: {
type: 'keyword',
type: 'array',
items: {
type: 'keyword',
},
},
},
fetch: () => {

11
x-pack/plugins/security_solution/server/usage/collector.ts
View file

@ -59,10 +59,13 @@ export const registerCollector: RegisterCollector = ({
total_installed: { type: 'long' },
active_within_last_24_hours: { type: 'long' },
os: {
full_name: { type: 'keyword' },
platform: { type: 'keyword' },
version: { type: 'keyword' },
count: { type: 'long' },
type: 'array',
items: {
full_name: { type: 'keyword' },
platform: { type: 'keyword' },
version: { type: 'keyword' },
count: { type: 'long' },
},
},
policies: {
malware: {

217
x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json
View file

@ -141,15 +141,18 @@
}
},
"packages": {
"properties": {
"name": {
"type": "keyword"
},
"version": {
"type": "keyword"
},
"enabled": {
"type": "boolean"
"type": "array",
"items": {
"properties": {
"name": {
"type": "keyword"
},
"version": {
"type": "keyword"
},
"enabled": {
"type": "boolean"
}
}
}
}
@ -546,78 +549,81 @@
"type": "boolean"
},
"clusters": {
"properties": {
"license": {
"type": "keyword"
},
"clusterUuid": {
"type": "keyword"
},
"metricbeatUsed": {
"type": "boolean"
},
"elasticsearch": {
"properties": {
"enabled": {
"type": "boolean"
},
"count": {
"type": "long"
},
"metricbeatUsed": {
"type": "boolean"
"type": "array",
"items": {
"properties": {
"license": {
"type": "keyword"
},
"clusterUuid": {
"type": "keyword"
},
"metricbeatUsed": {
"type": "boolean"
},
"elasticsearch": {
"properties": {
"enabled": {
"type": "boolean"
},
"count": {
"type": "long"
},
"metricbeatUsed": {
"type": "boolean"
}
}
}
},
"kibana": {
"properties": {
"enabled": {
"type": "boolean"
},
"count": {
"type": "long"
},
"metricbeatUsed": {
"type": "boolean"
},
"kibana": {
"properties": {
"enabled": {
"type": "boolean"
},
"count": {
"type": "long"
},
"metricbeatUsed": {
"type": "boolean"
}
}
}
},
"logstash": {
"properties": {
"enabled": {
"type": "boolean"
},
"count": {
"type": "long"
},
"metricbeatUsed": {
"type": "boolean"
},
"logstash": {
"properties": {
"enabled": {
"type": "boolean"
},
"count": {
"type": "long"
},
"metricbeatUsed": {
"type": "boolean"
}
}
}
},
"beats": {
"properties": {
"enabled": {
"type": "boolean"
},
"count": {
"type": "long"
},
"metricbeatUsed": {
"type": "boolean"
},
"beats": {
"properties": {
"enabled": {
"type": "boolean"
},
"count": {
"type": "long"
},
"metricbeatUsed": {
"type": "boolean"
}
}
}
},
"apm": {
"properties": {
"enabled": {
"type": "boolean"
},
"count": {
"type": "long"
},
"metricbeatUsed": {
"type": "boolean"
},
"apm": {
"properties": {
"enabled": {
"type": "boolean"
},
"count": {
"type": "long"
},
"metricbeatUsed": {
"type": "boolean"
}
}
}
}
@ -720,18 +726,21 @@
"type": "long"
},
"os": {
"properties": {
"full_name": {
"type": "keyword"
},
"platform": {
"type": "keyword"
},
"version": {
"type": "keyword"
},
"count": {
"type": "long"
"type": "array",
"items": {
"properties": {
"full_name": {
"type": "keyword"
},
"platform": {
"type": "keyword"
},
"version": {
"type": "keyword"
},
"count": {
"type": "long"
}
}
}
},
@ -771,10 +780,16 @@
"type": "number"
},
"enabledAuthProviders": {
"type": "keyword"
"type": "array",
"items": {
"type": "keyword"
}
},
"httpAuthSchemes": {
"type": "keyword"
"type": "array",
"items": {
"type": "keyword"
}
}
}
},
@ -906,16 +921,28 @@
"type": "boolean"
},
"autorefreshInterval": {
"type": "long"
"type": "array",
"items": {
"type": "long"
}
},
"dateRangeEnd": {
"type": "date"
"type": "array",
"items": {
"type": "date"
}
},
"dateRangeStart": {
"type": "date"
"type": "array",
"items": {
"type": "date"
}
},
"monitor_frequency": {
"type": "long"
"type": "array",
"items": {
"type": "long"
}
},
"monitor_name_stats": {
"properties": {

8
x-pack/plugins/uptime/server/lib/adapters/telemetry/kibana_telemetry_adapter.ts
View file

@ -47,10 +47,10 @@ export class KibanaTelemetryAdapter {
autoRefreshEnabled: {
type: 'boolean',
},
autorefreshInterval: { type: 'long' },
dateRangeEnd: { type: 'date' },
dateRangeStart: { type: 'date' },
monitor_frequency: { type: 'long' },
autorefreshInterval: { type: 'array', items: { type: 'long' } },
dateRangeEnd: { type: 'array', items: { type: 'date' } },
dateRangeStart: { type: 'array', items: { type: 'date' } },
monitor_frequency: { type: 'array', items: { type: 'long' } },
monitor_name_stats: {
avg_length: { type: 'float' },
max_length: { type: 'long' },