[Usage collection] Collect non-default kibana configs (#97368)

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Ahmad Bamieh 2021-04-20 18:02:27 +03:00 committed by GitHub
parent db7f279a03
commit 0f4538195f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
31 changed files with 1026 additions and 27 deletions

View file

@ -2,6 +2,8 @@
{
"output": "src/plugins/telemetry/schema/oss_plugins.json",
"root": "src/plugins/",
"exclude": []
"exclude": [
"src/plugins/kibana_usage_collection/server/collectors/config_usage/register_config_usage_collector.ts"
]
}
]

View file

@ -0,0 +1,15 @@
<!-- Do not edit this file. It is automatically generated by API Documenter. -->
[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [MakeUsageFromSchema](./kibana-plugin-core-server.makeusagefromschema.md)
## MakeUsageFromSchema type
List of configuration values that will be exposed to usage collection. If parent node or actual config path is set to `true` then the actual value of these configs will be reoprted. If parent node or actual config path is set to `false` then the config will be reported as \[redacted\].
<b>Signature:</b>
```typescript
export declare type MakeUsageFromSchema<T> = {
[Key in keyof T]?: T[Key] extends Maybe<object[]> ? false : T[Key] extends Maybe<any[]> ? boolean : T[Key] extends Maybe<object> ? MakeUsageFromSchema<T[Key]> | boolean : boolean;
};
```

View file

@ -272,6 +272,7 @@ The plugin integrates with the core system via lifecycle events: `setup`<!-- -->
| [LegacyElasticsearchClientConfig](./kibana-plugin-core-server.legacyelasticsearchclientconfig.md) | |
| [LifecycleResponseFactory](./kibana-plugin-core-server.lifecycleresponsefactory.md) | Creates an object containing redirection or error response with error details, HTTP headers, and other data transmitted to the client. |
| [LoggerConfigType](./kibana-plugin-core-server.loggerconfigtype.md) | |
| [MakeUsageFromSchema](./kibana-plugin-core-server.makeusagefromschema.md) | List of configuration values that will be exposed to usage collection. If parent node or actual config path is set to <code>true</code> then the actual value of these configs will be reoprted. If parent node or actual config path is set to <code>false</code> then the config will be reported as \[redacted\]. |
| [MetricsServiceStart](./kibana-plugin-core-server.metricsservicestart.md) | APIs to retrieves metrics gathered and exposed by the core platform. |
| [MIGRATION\_ASSISTANCE\_INDEX\_ACTION](./kibana-plugin-core-server.migration_assistance_index_action.md) | |
| [MIGRATION\_DEPRECATION\_LEVEL](./kibana-plugin-core-server.migration_deprecation_level.md) | |

View file

@ -0,0 +1,17 @@
<!-- Do not edit this file. It is automatically generated by API Documenter. -->
[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [PluginConfigDescriptor](./kibana-plugin-core-server.pluginconfigdescriptor.md) &gt; [exposeToUsage](./kibana-plugin-core-server.pluginconfigdescriptor.exposetousage.md)
## PluginConfigDescriptor.exposeToUsage property
Expose non-default configs to usage collection to be sent via telemetry. set a config to `true` to report the actual changed config value. set a config to `false` to report the changed config value as \[redacted\].
All changed configs except booleans and numbers will be reported as \[redacted\] unless otherwise specified.
[MakeUsageFromSchema](./kibana-plugin-core-server.makeusagefromschema.md)
<b>Signature:</b>
```typescript
exposeToUsage?: MakeUsageFromSchema<T>;
```

View file

@ -46,5 +46,6 @@ export const config: PluginConfigDescriptor<ConfigType> = {
| --- | --- | --- |
| [deprecations](./kibana-plugin-core-server.pluginconfigdescriptor.deprecations.md) | <code>ConfigDeprecationProvider</code> | Provider for the to apply to the plugin configuration. |
| [exposeToBrowser](./kibana-plugin-core-server.pluginconfigdescriptor.exposetobrowser.md) | <code>{</code><br/><code> [P in keyof T]?: boolean;</code><br/><code> }</code> | List of configuration properties that will be available on the client-side plugin. |
| [exposeToUsage](./kibana-plugin-core-server.pluginconfigdescriptor.exposetousage.md) | <code>MakeUsageFromSchema&lt;T&gt;</code> | Expose non-default configs to usage collection to be sent via telemetry. set a config to <code>true</code> to report the actual changed config value. set a config to <code>false</code> to report the changed config value as \[redacted\].<!-- -->All changed configs except booleans and numbers will be reported as \[redacted\] unless otherwise specified.[MakeUsageFromSchema](./kibana-plugin-core-server.makeusagefromschema.md) |
| [schema](./kibana-plugin-core-server.pluginconfigdescriptor.schema.md) | <code>PluginConfigSchema&lt;T&gt;</code> | Schema to use to validate the plugin configuration.[PluginConfigSchema](./kibana-plugin-core-server.pluginconfigschema.md) |

View file

@ -139,6 +139,7 @@ const createStartContractMock = () => {
},
})
),
getConfigsUsageData: jest.fn(),
};
return startContract;

View file

@ -35,7 +35,35 @@ describe('CoreUsageDataService', () => {
});
let service: CoreUsageDataService;
const configService = configServiceMock.create();
const mockConfig = {
unused_config: {},
elasticsearch: { username: 'kibana_system', password: 'changeme' },
plugins: { paths: ['pluginA', 'pluginAB', 'pluginB'] },
server: { port: 5603, basePath: '/zvt', rewriteBasePath: true },
logging: { json: false },
pluginA: {
enabled: true,
objectConfig: {
debug: true,
username: 'some_user',
},
arrayOfNumbers: [1, 2, 3],
},
pluginAB: {
enabled: false,
},
pluginB: {
arrayOfObjects: [
{ propA: 'a', propB: 'b' },
{ propA: 'a2', propB: 'b2' },
],
},
};
const configService = configServiceMock.create({
getConfig$: mockConfig,
});
configService.atPath.mockImplementation((path) => {
if (path === 'elasticsearch') {
return new BehaviorSubject(RawElasticsearchConfig.schema.validate({}));
@ -146,6 +174,7 @@ describe('CoreUsageDataService', () => {
const { getCoreUsageData } = service.start({
savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry),
exposedConfigsToUsage: new Map(),
elasticsearch,
});
expect(getCoreUsageData()).resolves.toMatchInlineSnapshot(`
@ -281,6 +310,453 @@ describe('CoreUsageDataService', () => {
`);
});
});
describe('getConfigsUsageData', () => {
const elasticsearch = elasticsearchServiceMock.createStart();
const typeRegistry = savedObjectsServiceMock.createTypeRegistryMock();
let exposedConfigsToUsage: Map<string, Record<string, boolean>>;
beforeEach(() => {
exposedConfigsToUsage = new Map();
});
it('loops over all used configs once each', async () => {
configService.getUsedPaths.mockResolvedValue([
'pluginA.objectConfig.debug',
'logging.json',
]);
exposedConfigsToUsage.set('pluginA', {
objectConfig: true,
});
const { getConfigsUsageData } = service.start({
savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry),
exposedConfigsToUsage,
elasticsearch,
});
const mockGetMarkedAsSafe = jest.fn().mockReturnValue({});
// @ts-expect-error
service.getMarkedAsSafe = mockGetMarkedAsSafe;
await getConfigsUsageData();
expect(mockGetMarkedAsSafe).toBeCalledTimes(2);
expect(mockGetMarkedAsSafe.mock.calls).toMatchInlineSnapshot(`
Array [
Array [
Map {
"pluginA" => Object {
"objectConfig": true,
},
},
"pluginA.objectConfig.debug",
"pluginA",
],
Array [
Map {
"pluginA" => Object {
"objectConfig": true,
},
},
"logging.json",
undefined,
],
]
`);
});
it('plucks pluginId from config path correctly', async () => {
exposedConfigsToUsage.set('pluginA', {
enabled: false,
});
exposedConfigsToUsage.set('pluginAB', {
enabled: false,
});
configService.getUsedPaths.mockResolvedValue(['pluginA.enabled', 'pluginAB.enabled']);
const { getConfigsUsageData } = service.start({
savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry),
exposedConfigsToUsage,
elasticsearch,
});
await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(`
Object {
"pluginA.enabled": "[redacted]",
"pluginAB.enabled": "[redacted]",
}
`);
});
it('returns an object of plugin config usage', async () => {
exposedConfigsToUsage.set('unused_config', { never_reported: true });
exposedConfigsToUsage.set('server', { basePath: true });
exposedConfigsToUsage.set('pluginA', { elasticsearch: false });
exposedConfigsToUsage.set('plugins', { paths: false });
exposedConfigsToUsage.set('pluginA', { arrayOfNumbers: false });
configService.getUsedPaths.mockResolvedValue([
'elasticsearch.username',
'elasticsearch.password',
'plugins.paths',
'server.port',
'server.basePath',
'server.rewriteBasePath',
'logging.json',
'pluginA.enabled',
'pluginA.objectConfig.debug',
'pluginA.objectConfig.username',
'pluginA.arrayOfNumbers',
'pluginAB.enabled',
'pluginB.arrayOfObjects',
]);
const { getConfigsUsageData } = service.start({
savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry),
exposedConfigsToUsage,
elasticsearch,
});
await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(`
Object {
"elasticsearch.password": "[redacted]",
"elasticsearch.username": "[redacted]",
"logging.json": false,
"pluginA.arrayOfNumbers": "[redacted]",
"pluginA.enabled": true,
"pluginA.objectConfig.debug": true,
"pluginA.objectConfig.username": "[redacted]",
"pluginAB.enabled": false,
"pluginB.arrayOfObjects": "[redacted]",
"plugins.paths": "[redacted]",
"server.basePath": "/zvt",
"server.port": 5603,
"server.rewriteBasePath": true,
}
`);
});
describe('config explicitly exposed to usage', () => {
it('returns [redacted] on unsafe complete match', async () => {
exposedConfigsToUsage.set('pluginA', {
'objectConfig.debug': false,
});
exposedConfigsToUsage.set('server', {
basePath: false,
});
configService.getUsedPaths.mockResolvedValue([
'pluginA.objectConfig.debug',
'server.basePath',
]);
const { getConfigsUsageData } = service.start({
savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry),
exposedConfigsToUsage,
elasticsearch,
});
await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(`
Object {
"pluginA.objectConfig.debug": "[redacted]",
"server.basePath": "[redacted]",
}
`);
});
it('returns config value on safe complete match', async () => {
exposedConfigsToUsage.set('server', {
basePath: true,
});
configService.getUsedPaths.mockResolvedValue(['server.basePath']);
const { getConfigsUsageData } = service.start({
savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry),
exposedConfigsToUsage,
elasticsearch,
});
await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(`
Object {
"server.basePath": "/zvt",
}
`);
});
it('returns [redacted] on unsafe parent match', async () => {
exposedConfigsToUsage.set('pluginA', {
objectConfig: false,
});
configService.getUsedPaths.mockResolvedValue([
'pluginA.objectConfig.debug',
'pluginA.objectConfig.username',
]);
const { getConfigsUsageData } = service.start({
savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry),
exposedConfigsToUsage,
elasticsearch,
});
await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(`
Object {
"pluginA.objectConfig.debug": "[redacted]",
"pluginA.objectConfig.username": "[redacted]",
}
`);
});
it('returns config value on safe parent match', async () => {
exposedConfigsToUsage.set('pluginA', {
objectConfig: true,
});
configService.getUsedPaths.mockResolvedValue([
'pluginA.objectConfig.debug',
'pluginA.objectConfig.username',
]);
const { getConfigsUsageData } = service.start({
savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry),
exposedConfigsToUsage,
elasticsearch,
});
await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(`
Object {
"pluginA.objectConfig.debug": true,
"pluginA.objectConfig.username": "some_user",
}
`);
});
it('returns [redacted] on explicitly marked as safe array of objects', async () => {
exposedConfigsToUsage.set('pluginB', {
arrayOfObjects: true,
});
configService.getUsedPaths.mockResolvedValue(['pluginB.arrayOfObjects']);
const { getConfigsUsageData } = service.start({
savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry),
exposedConfigsToUsage,
elasticsearch,
});
await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(`
Object {
"pluginB.arrayOfObjects": "[redacted]",
}
`);
});
it('returns values on explicitly marked as safe array of numbers', async () => {
exposedConfigsToUsage.set('pluginA', {
arrayOfNumbers: true,
});
configService.getUsedPaths.mockResolvedValue(['pluginA.arrayOfNumbers']);
const { getConfigsUsageData } = service.start({
savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry),
exposedConfigsToUsage,
elasticsearch,
});
await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(`
Object {
"pluginA.arrayOfNumbers": Array [
1,
2,
3,
],
}
`);
});
it('returns values on explicitly marked as safe array of strings', async () => {
exposedConfigsToUsage.set('plugins', {
paths: true,
});
configService.getUsedPaths.mockResolvedValue(['plugins.paths']);
const { getConfigsUsageData } = service.start({
savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry),
exposedConfigsToUsage,
elasticsearch,
});
await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(`
Object {
"plugins.paths": Array [
"pluginA",
"pluginAB",
"pluginB",
],
}
`);
});
});
describe('config not explicitly exposed to usage', () => {
it('returns [redacted] for string configs', async () => {
exposedConfigsToUsage.set('pluginA', {
objectConfig: false,
});
configService.getUsedPaths.mockResolvedValue([
'pluginA.objectConfig.debug',
'pluginA.objectConfig.username',
]);
const { getConfigsUsageData } = service.start({
savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry),
exposedConfigsToUsage,
elasticsearch,
});
await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(`
Object {
"pluginA.objectConfig.debug": "[redacted]",
"pluginA.objectConfig.username": "[redacted]",
}
`);
});
it('returns config value on safe parent match', async () => {
configService.getUsedPaths.mockResolvedValue([
'elasticsearch.password',
'elasticsearch.username',
'pluginA.objectConfig.username',
]);
const { getConfigsUsageData } = service.start({
savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry),
exposedConfigsToUsage,
elasticsearch,
});
await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(`
Object {
"elasticsearch.password": "[redacted]",
"elasticsearch.username": "[redacted]",
"pluginA.objectConfig.username": "[redacted]",
}
`);
});
it('returns [redacted] on implicit array of objects', async () => {
configService.getUsedPaths.mockResolvedValue(['pluginB.arrayOfObjects']);
const { getConfigsUsageData } = service.start({
savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry),
exposedConfigsToUsage,
elasticsearch,
});
await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(`
Object {
"pluginB.arrayOfObjects": "[redacted]",
}
`);
});
it('returns values on implicit array of numbers', async () => {
configService.getUsedPaths.mockResolvedValue(['pluginA.arrayOfNumbers']);
const { getConfigsUsageData } = service.start({
savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry),
exposedConfigsToUsage,
elasticsearch,
});
await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(`
Object {
"pluginA.arrayOfNumbers": Array [
1,
2,
3,
],
}
`);
});
it('returns [redacted] on implicit array of strings', async () => {
configService.getUsedPaths.mockResolvedValue(['plugins.paths']);
const { getConfigsUsageData } = service.start({
savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry),
exposedConfigsToUsage,
elasticsearch,
});
await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(`
Object {
"plugins.paths": "[redacted]",
}
`);
});
it('returns config value for numbers', async () => {
configService.getUsedPaths.mockResolvedValue(['server.port']);
const { getConfigsUsageData } = service.start({
savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry),
exposedConfigsToUsage,
elasticsearch,
});
await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(`
Object {
"server.port": 5603,
}
`);
});
it('returns config value for booleans', async () => {
configService.getUsedPaths.mockResolvedValue([
'pluginA.objectConfig.debug',
'logging.json',
]);
const { getConfigsUsageData } = service.start({
savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry),
exposedConfigsToUsage,
elasticsearch,
});
await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(`
Object {
"logging.json": false,
"pluginA.objectConfig.debug": true,
}
`);
});
it('ignores exposed to usage configs but not used', async () => {
exposedConfigsToUsage.set('pluginA', {
objectConfig: true,
});
configService.getUsedPaths.mockResolvedValue(['logging.json']);
const { getConfigsUsageData } = service.start({
savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry),
exposedConfigsToUsage,
elasticsearch,
});
await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(`
Object {
"logging.json": false,
}
`);
});
});
});
});
describe('setup and stop', () => {

View file

@ -7,7 +7,9 @@
*/
import { Subject } from 'rxjs';
import { takeUntil } from 'rxjs/operators';
import { takeUntil, first } from 'rxjs/operators';
import { get } from 'lodash';
import { hasConfigPathIntersection } from '@kbn/config';
import { CoreService } from 'src/core/types';
import { Logger, SavedObjectsServiceStart, SavedObjectTypeRegistry } from 'src/core/server';
@ -16,11 +18,12 @@ import { ElasticsearchConfigType } from '../elasticsearch/elasticsearch_config';
import { HttpConfigType, InternalHttpServiceSetup } from '../http';
import { LoggingConfigType } from '../logging';
import { SavedObjectsConfigType } from '../saved_objects/saved_objects_config';
import {
import type {
CoreServicesUsageData,
CoreUsageData,
CoreUsageDataStart,
CoreUsageDataSetup,
ConfigUsageData,
} from './types';
import { isConfigured } from './is_configured';
import { ElasticsearchServiceStart } from '../elasticsearch';
@ -30,6 +33,8 @@ import { CORE_USAGE_STATS_TYPE } from './constants';
import { CoreUsageStatsClient } from './core_usage_stats_client';
import { MetricsServiceSetup, OpsMetrics } from '..';
export type ExposedConfigsToUsage = Map<string, Record<string, boolean>>;
export interface SetupDeps {
http: InternalHttpServiceSetup;
metrics: MetricsServiceSetup;
@ -39,6 +44,7 @@ export interface SetupDeps {
export interface StartDeps {
savedObjects: SavedObjectsServiceStart;
elasticsearch: ElasticsearchServiceStart;
exposedConfigsToUsage: ExposedConfigsToUsage;
}
/**
@ -266,6 +272,110 @@ export class CoreUsageDataService implements CoreService<CoreUsageDataSetup, Cor
};
}
private getMarkedAsSafe(
exposedConfigsToUsage: ExposedConfigsToUsage,
usedPath: string,
pluginId?: string
): { explicitlyMarked: boolean; isSafe: boolean } {
if (pluginId) {
const exposeDetails = exposedConfigsToUsage.get(pluginId) || {};
const exposeKeyDetails = Object.keys(exposeDetails).find((exposeKey) => {
const fullPath = `${pluginId}.${exposeKey}`;
return hasConfigPathIntersection(usedPath, fullPath);
});
if (exposeKeyDetails) {
const explicitlyMarkedAsSafe = exposeDetails[exposeKeyDetails];
if (typeof explicitlyMarkedAsSafe === 'boolean') {
return {
explicitlyMarked: true,
isSafe: explicitlyMarkedAsSafe,
};
}
}
}
return { explicitlyMarked: false, isSafe: false };
}
private async getNonDefaultKibanaConfigs(
exposedConfigsToUsage: ExposedConfigsToUsage
): Promise<ConfigUsageData> {
const config = await this.configService.getConfig$().pipe(first()).toPromise();
const nonDefaultConfigs = config.toRaw();
const usedPaths = await this.configService.getUsedPaths();
const exposedConfigsKeys = [...exposedConfigsToUsage.keys()];
return usedPaths.reduce((acc, usedPath) => {
const rawConfigValue = get(nonDefaultConfigs, usedPath);
const pluginId = exposedConfigsKeys.find(
(exposedConfigsKey) =>
usedPath === exposedConfigsKey || usedPath.startsWith(`${exposedConfigsKey}.`)
);
const { explicitlyMarked, isSafe } = this.getMarkedAsSafe(
exposedConfigsToUsage,
usedPath,
pluginId
);
// explicitly marked as safe
if (explicitlyMarked && isSafe) {
// report array of objects as redacted even if explicitly marked as safe.
// TS typings prevent explicitly marking arrays of objects as safe
// this makes sure to report redacted even if TS was bypassed.
if (
Array.isArray(rawConfigValue) &&
rawConfigValue.some((item) => typeof item === 'object')
) {
acc[usedPath] = '[redacted]';
} else {
acc[usedPath] = rawConfigValue;
}
}
// explicitly marked as unsafe
if (explicitlyMarked && !isSafe) {
acc[usedPath] = '[redacted]';
}
/**
* not all types of values may contain sensitive values.
* Report boolean and number configs if not explicitly marked as unsafe.
*/
if (!explicitlyMarked) {
switch (typeof rawConfigValue) {
case 'number':
case 'boolean':
acc[usedPath] = rawConfigValue;
break;
case 'undefined':
acc[usedPath] = 'undefined';
break;
case 'object': {
// non-array object types are already handled
if (Array.isArray(rawConfigValue)) {
if (
rawConfigValue.every(
(item) => typeof item === 'number' || typeof item === 'boolean'
)
) {
acc[usedPath] = rawConfigValue;
break;
}
}
}
default: {
acc[usedPath] = '[redacted]';
}
}
}
return acc;
}, {} as Record<string, any | any[]>);
}
setup({ http, metrics, savedObjectsStartPromise }: SetupDeps) {
metrics
.getOpsMetrics$()
@ -326,10 +436,13 @@ export class CoreUsageDataService implements CoreService<CoreUsageDataSetup, Cor
return { registerType, getClient } as CoreUsageDataSetup;
}
start({ savedObjects, elasticsearch }: StartDeps) {
start({ savedObjects, elasticsearch, exposedConfigsToUsage }: StartDeps) {
return {
getCoreUsageData: () => {
return this.getCoreUsageData(savedObjects, elasticsearch);
getCoreUsageData: async () => {
return await this.getCoreUsageData(savedObjects, elasticsearch);
},
getConfigsUsageData: async () => {
return await this.getNonDefaultKibanaConfigs(exposedConfigsToUsage);
},
};
}

View file

@ -6,7 +6,7 @@
* Side Public License, v 1.
*/
export type { CoreUsageDataSetup, CoreUsageDataStart } from './types';
export type { CoreUsageDataSetup, ConfigUsageData, CoreUsageDataStart } from './types';
export { CoreUsageDataService } from './core_usage_data_service';
export { CoreUsageStatsClient } from './core_usage_stats_client';

View file

@ -122,6 +122,18 @@ export interface CoreUsageData extends CoreUsageStats {
environment: CoreEnvironmentUsageData;
}
/**
* Type describing Core's usage data payload
* @internal
*/
export type ConfigUsageData = Record<string, any | any[]>;
/**
* Type describing Core's usage data payload
* @internal
*/
export type ExposedConfigsToUsage = Map<string, Record<string, boolean>>;
/**
* Usage data from Core services
* @internal
@ -270,4 +282,5 @@ export interface CoreUsageDataStart {
* @internal
* */
getCoreUsageData(): Promise<CoreUsageData>;
getConfigsUsageData(): Promise<ConfigUsageData>;
}

View file

@ -64,6 +64,7 @@ import {
CoreUsageStats,
CoreUsageData,
CoreConfigUsageData,
ConfigUsageData,
CoreEnvironmentUsageData,
CoreServicesUsageData,
} from './core_usage_data';
@ -74,6 +75,7 @@ export type {
CoreConfigUsageData,
CoreEnvironmentUsageData,
CoreServicesUsageData,
ConfigUsageData,
};
export { bootstrap } from './bootstrap';
@ -256,6 +258,7 @@ export type {
PluginManifest,
PluginName,
SharedGlobalConfig,
MakeUsageFromSchema,
} from './plugins';
export {

View file

@ -19,6 +19,7 @@ const createStartContractMock = () => ({ contracts: new Map() });
const createServiceMock = (): PluginsServiceMock => ({
discover: jest.fn(),
getExposedPluginConfigsToUsage: jest.fn(),
setup: jest.fn().mockResolvedValue(createSetupContractMock()),
start: jest.fn().mockResolvedValue(createStartContractMock()),
stop: jest.fn(),

View file

@ -78,7 +78,7 @@ const createPlugin = (
manifest: {
id,
version,
configPath: `${configPath}${disabled ? '-disabled' : ''}`,
configPath: disabled ? configPath.concat('-disabled') : configPath,
kibanaVersion,
requiredPlugins,
requiredBundles,
@ -374,7 +374,6 @@ describe('PluginsService', () => {
expect(mockPluginSystem.addPlugin).toHaveBeenCalledTimes(2);
expect(mockPluginSystem.addPlugin).toHaveBeenCalledWith(firstPlugin);
expect(mockPluginSystem.addPlugin).toHaveBeenCalledWith(secondPlugin);
expect(mockDiscover).toHaveBeenCalledTimes(1);
expect(mockDiscover).toHaveBeenCalledWith(
{
@ -472,6 +471,88 @@ describe('PluginsService', () => {
expect(pluginPaths).toEqual(['/plugin-A-path', '/plugin-B-path']);
});
it('ppopulates pluginConfigUsageDescriptors with plugins exposeToUsage property', async () => {
const pluginA = createPlugin('plugin-with-expose-usage', {
path: 'plugin-with-expose-usage',
configPath: 'pathA',
});
jest.doMock(
join('plugin-with-expose-usage', 'server'),
() => ({
config: {
exposeToUsage: {
test: true,
nested: {
prop: true,
},
},
schema: schema.maybe(schema.any()),
},
}),
{
virtual: true,
}
);
const pluginB = createPlugin('plugin-with-array-configPath', {
path: 'plugin-with-array-configPath',
configPath: ['plugin', 'pathB'],
});
jest.doMock(
join('plugin-with-array-configPath', 'server'),
() => ({
config: {
exposeToUsage: {
test: true,
},
schema: schema.maybe(schema.any()),
},
}),
{
virtual: true,
}
);
jest.doMock(
join('plugin-without-expose', 'server'),
() => ({
config: {
schema: schema.maybe(schema.any()),
},
}),
{
virtual: true,
}
);
const pluginC = createPlugin('plugin-without-expose', {
path: 'plugin-without-expose',
configPath: 'pathC',
});
mockDiscover.mockReturnValue({
error$: from([]),
plugin$: from([pluginA, pluginB, pluginC]),
});
await pluginsService.discover({ environment: environmentSetup });
// eslint-disable-next-line dot-notation
expect(pluginsService['pluginConfigUsageDescriptors']).toMatchInlineSnapshot(`
Map {
"pathA" => Object {
"nested.prop": true,
"test": true,
},
"plugin.pathB" => Object {
"test": true,
},
}
`);
});
});
describe('#generateUiPluginsConfigs()', () => {
@ -624,6 +705,20 @@ describe('PluginsService', () => {
});
});
describe('#getExposedPluginConfigsToUsage', () => {
it('returns pluginConfigUsageDescriptors', () => {
// eslint-disable-next-line dot-notation
pluginsService['pluginConfigUsageDescriptors'].set('test', { enabled: true });
expect(pluginsService.getExposedPluginConfigsToUsage()).toMatchInlineSnapshot(`
Map {
"test" => Object {
"enabled": true,
},
}
`);
});
});
describe('#stop()', () => {
it('`stop` stops plugins system', async () => {
await pluginsService.stop();

View file

@ -9,7 +9,7 @@
import Path from 'path';
import { Observable } from 'rxjs';
import { filter, first, map, mergeMap, tap, toArray } from 'rxjs/operators';
import { pick } from '@kbn/std';
import { pick, getFlattenedObject } from '@kbn/std';
import { CoreService } from '../../types';
import { CoreContext } from '../core_context';
@ -75,6 +75,7 @@ export class PluginsService implements CoreService<PluginsServiceSetup, PluginsS
private readonly config$: Observable<PluginsConfig>;
private readonly pluginConfigDescriptors = new Map<PluginName, PluginConfigDescriptor>();
private readonly uiPluginInternalInfo = new Map<PluginName, InternalPluginInfo>();
private readonly pluginConfigUsageDescriptors = new Map<string, Record<string, any | any[]>>();
constructor(private readonly coreContext: CoreContext) {
this.log = coreContext.logger.get('plugins-service');
@ -109,6 +110,10 @@ export class PluginsService implements CoreService<PluginsServiceSetup, PluginsS
};
}
public getExposedPluginConfigsToUsage() {
return this.pluginConfigUsageDescriptors;
}
public async setup(deps: PluginsServiceSetupDeps) {
this.log.debug('Setting up plugins service');
@ -211,6 +216,12 @@ export class PluginsService implements CoreService<PluginsServiceSetup, PluginsS
configDescriptor.deprecations
);
}
if (configDescriptor.exposeToUsage) {
this.pluginConfigUsageDescriptors.set(
Array.isArray(plugin.configPath) ? plugin.configPath.join('.') : plugin.configPath,
getFlattenedObject(configDescriptor.exposeToUsage)
);
}
this.coreContext.configService.setSchema(plugin.configPath, configDescriptor.schema);
}
const isEnabled = await this.coreContext.configService.isEnabledAtPath(plugin.configPath);

View file

@ -18,6 +18,8 @@ import { ElasticsearchConfigType } from '../elasticsearch/elasticsearch_config';
import { SavedObjectsConfigType } from '../saved_objects/saved_objects_config';
import { CoreSetup, CoreStart } from '..';
type Maybe<T> = T | undefined;
/**
* Dedicated type for plugin configuration schema.
*
@ -70,8 +72,39 @@ export interface PluginConfigDescriptor<T = any> {
* {@link PluginConfigSchema}
*/
schema: PluginConfigSchema<T>;
/**
* Expose non-default configs to usage collection to be sent via telemetry.
* set a config to `true` to report the actual changed config value.
* set a config to `false` to report the changed config value as [redacted].
*
* All changed configs except booleans and numbers will be reported
* as [redacted] unless otherwise specified.
*
* {@link MakeUsageFromSchema}
*/
exposeToUsage?: MakeUsageFromSchema<T>;
}
/**
* List of configuration values that will be exposed to usage collection.
* If parent node or actual config path is set to `true` then the actual value
* of these configs will be reoprted.
* If parent node or actual config path is set to `false` then the config
* will be reported as [redacted].
*
* @public
*/
export type MakeUsageFromSchema<T> = {
[Key in keyof T]?: T[Key] extends Maybe<object[]>
? // arrays of objects are always redacted
false
: T[Key] extends Maybe<any[]>
? boolean
: T[Key] extends Maybe<object>
? MakeUsageFromSchema<T[Key]> | boolean
: boolean;
};
/**
* Dedicated type for plugin name/id that is supposed to make Map/Set/Arrays
* that use it as a key or value more obvious.

View file

@ -381,6 +381,9 @@ export { ConfigPath }
export { ConfigService }
// @internal
export type ConfigUsageData = Record<string, any | any[]>;
// @public
export interface ContextSetup {
createContextContainer(): IContextContainer;
@ -558,6 +561,8 @@ export interface CoreUsageData extends CoreUsageStats {
// @internal
export interface CoreUsageDataStart {
// (undocumented)
getConfigsUsageData(): Promise<ConfigUsageData>;
getCoreUsageData(): Promise<CoreUsageData>;
}
@ -1662,6 +1667,13 @@ export { LogMeta }
export { LogRecord }
// Warning: (ae-forgotten-export) The symbol "Maybe" needs to be exported by the entry point index.d.ts
//
// @public
export type MakeUsageFromSchema<T> = {
[Key in keyof T]?: T[Key] extends Maybe<object[]> ? false : T[Key] extends Maybe<any[]> ? boolean : T[Key] extends Maybe<object> ? MakeUsageFromSchema<T[Key]> | boolean : boolean;
};
// @public
export interface MetricsServiceSetup {
readonly collectionInterval: number;
@ -1848,6 +1860,7 @@ export interface PluginConfigDescriptor<T = any> {
exposeToBrowser?: {
[P in keyof T]?: boolean;
};
exposeToUsage?: MakeUsageFromSchema<T>;
schema: PluginConfigSchema<T>;
}
@ -3234,9 +3247,9 @@ export const validBodyOutput: readonly ["data", "stream"];
//
// src/core/server/elasticsearch/client/types.ts:94:7 - (ae-forgotten-export) The symbol "Explanation" needs to be exported by the entry point index.d.ts
// src/core/server/http/router/response.ts:297:3 - (ae-forgotten-export) The symbol "KibanaResponse" needs to be exported by the entry point index.d.ts
// src/core/server/plugins/types.ts:293:3 - (ae-forgotten-export) The symbol "KibanaConfigType" needs to be exported by the entry point index.d.ts
// src/core/server/plugins/types.ts:293:3 - (ae-forgotten-export) The symbol "SharedGlobalConfigKeys" needs to be exported by the entry point index.d.ts
// src/core/server/plugins/types.ts:296:3 - (ae-forgotten-export) The symbol "SavedObjectsConfigType" needs to be exported by the entry point index.d.ts
// src/core/server/plugins/types.ts:401:5 - (ae-unresolved-link) The @link reference could not be resolved: The package "kibana" does not have an export "create"
// src/core/server/plugins/types.ts:326:3 - (ae-forgotten-export) The symbol "KibanaConfigType" needs to be exported by the entry point index.d.ts
// src/core/server/plugins/types.ts:326:3 - (ae-forgotten-export) The symbol "SharedGlobalConfigKeys" needs to be exported by the entry point index.d.ts
// src/core/server/plugins/types.ts:329:3 - (ae-forgotten-export) The symbol "SavedObjectsConfigType" needs to be exported by the entry point index.d.ts
// src/core/server/plugins/types.ts:434:5 - (ae-unresolved-link) The @link reference could not be resolved: The package "kibana" does not have an export "create"
```

View file

@ -247,6 +247,7 @@ export class Server {
const coreUsageDataStart = this.coreUsageData.start({
elasticsearch: elasticsearchStart,
savedObjects: savedObjectsStart,
exposedConfigsToUsage: this.plugins.getExposedPluginConfigsToUsage(),
});
this.coreStart = {

View file

@ -4,6 +4,7 @@ This plugin registers the basic usage collectors from Kibana:
- [Application Usage](./server/collectors/application_usage/README.md)
- Core Metrics
- [Config Usage](./server/collectors/config_usage/README.md)
- CSP configuration
- Kibana: Number of Saved Objects per type
- Localization data
@ -11,8 +12,3 @@ This plugin registers the basic usage collectors from Kibana:
- Ops stats
- UI Counts
- UI Metrics

View file

@ -0,0 +1,64 @@
# Config Usage Collector
The config usage collector reports non-default kibana configs.
All non-default configs except booleans and numbers will be reported as `[redacted]` unless otherwise specified via `config.exposeToUsage` in the plugin config descriptor.
```ts
import { schema, TypeOf } from '@kbn/config-schema';
import { PluginConfigDescriptor } from 'src/core/server';
export const configSchema = schema.object({
usageCounters: schema.object({
enabled: schema.boolean({ defaultValue: true }),
retryCount: schema.number({ defaultValue: 1 }),
bufferDuration: schema.duration({ defaultValue: '5s' }),
}),
uiCounters: schema.object({
enabled: schema.boolean({ defaultValue: true }),
debug: schema.boolean({ defaultValue: schema.contextRef('dev') }),
}),
maximumWaitTimeForAllCollectorsInS: schema.number({
defaultValue: DEFAULT_MAXIMUM_WAIT_TIME_FOR_ALL_COLLECTORS_IN_S,
}),
});
export const config: PluginConfigDescriptor<ConfigType> = {
schema: configSchema,
exposeToUsage: {
uiCounters: true,
usageCounters: {
bufferDuration: true,
},
maximumWaitTimeForAllCollectorsInS: false,
},
};
```
In the above example setting `uiCounters: true` in the `exposeToUsage` property marks all configs
under the path `uiCounters` as safe. The collector will send the actual non-default config value
when setting an exact config or its parent path to `true`.
Settings the config path or its parent path to `false` will explicitly mark this config as unsafe.
The collector will send `[redacted]` for non-default configs
when setting an exact config or its parent path to `false`.
### Output of the collector
```json
{
"kibana_config_usage": {
"xpack.apm.serviceMapTraceIdBucketSize": 30,
"elasticsearch.username": "[redacted]",
"elasticsearch.password": "[redacted]",
"plugins.paths": "[redacted]",
"server.port": 5603,
"server.basePath": "[redacted]",
"server.rewriteBasePath": true,
"logging.json": false,
"usageCollection.uiCounters.debug": true
}
}
```
Note that arrays of objects will be reported as `[redacted]` and cannot be explicitly marked as safe.

View file

@ -0,0 +1,9 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
export { registerConfigUsageCollector } from './register_config_usage_collector';

View file

@ -0,0 +1,44 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import {
Collector,
createUsageCollectionSetupMock,
createCollectorFetchContextMock,
} from '../../../../usage_collection/server/mocks';
import { registerConfigUsageCollector } from './register_config_usage_collector';
import { coreUsageDataServiceMock, loggingSystemMock } from '../../../../../core/server/mocks';
import type { ConfigUsageData } from '../../../../../core/server';
const logger = loggingSystemMock.createLogger();
describe('kibana_config_usage', () => {
let collector: Collector<unknown>;
const usageCollectionMock = createUsageCollectionSetupMock();
usageCollectionMock.makeUsageCollector.mockImplementation((config) => {
collector = new Collector(logger, config);
return createUsageCollectionSetupMock().makeUsageCollector(config);
});
const collectorFetchContext = createCollectorFetchContextMock();
const coreUsageDataStart = coreUsageDataServiceMock.createStartContract();
const mockConfigUsage = (Symbol('config usage telemetry') as any) as ConfigUsageData;
coreUsageDataStart.getConfigsUsageData.mockResolvedValue(mockConfigUsage);
beforeAll(() => registerConfigUsageCollector(usageCollectionMock, () => coreUsageDataStart));
test('registered collector is set', () => {
expect(collector).not.toBeUndefined();
expect(collector.type).toBe('kibana_config_usage');
});
test('fetch', async () => {
expect(await collector.fetch(collectorFetchContext)).toEqual(mockConfigUsage);
});
});

View file

@ -0,0 +1,39 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { UsageCollectionSetup } from '../../../../usage_collection/server';
import { ConfigUsageData, CoreUsageDataStart } from '../../../../../core/server';
export function registerConfigUsageCollector(
usageCollection: UsageCollectionSetup,
getCoreUsageDataService: () => CoreUsageDataStart
) {
const collector = usageCollection.makeUsageCollector<ConfigUsageData | undefined>({
type: 'kibana_config_usage',
isReady: () => typeof getCoreUsageDataService() !== 'undefined',
/**
* No schema for this collector.
* This collector will collect non-default configs from all plugins.
* Mapping each config to the schema is inconvenient for developers
* and would result in 100's of extra field mappings.
*
* We'll experiment with flattened type and runtime fields before comitting to a schema.
*/
schema: {},
fetch: async () => {
const coreUsageDataService = getCoreUsageDataService();
if (!coreUsageDataService) {
return;
}
return await coreUsageDataService.getConfigsUsageData();
},
});
usageCollection.registerCollector(collector);
}

View file

@ -9,11 +9,11 @@
import {
Collector,
createUsageCollectionSetupMock,
createCollectorFetchContextMock,
} from '../../../../usage_collection/server/mocks';
import { createCollectorFetchContextMock } from 'src/plugins/usage_collection/server/mocks';
import { registerCoreUsageCollector } from '.';
import { registerCoreUsageCollector } from './core_usage_collector';
import { coreUsageDataServiceMock, loggingSystemMock } from '../../../../../core/server/mocks';
import { CoreUsageData } from 'src/core/server/';
import type { CoreUsageData } from '../../../../../core/server';
const logger = loggingSystemMock.createLogger();

View file

@ -15,6 +15,7 @@ export { registerCloudProviderUsageCollector } from './cloud';
export { registerCspCollector } from './csp';
export { registerCoreUsageCollector } from './core';
export { registerLocalizationUsageCollector } from './localization';
export { registerConfigUsageCollector } from './config_usage';
export {
registerUiCountersUsageCollector,
registerUiCounterSavedObjectType,

View file

@ -93,6 +93,10 @@ describe('kibana_usage_collection', () => {
"isReady": false,
"type": "core",
},
Object {
"isReady": false,
"type": "kibana_config_usage",
},
Object {
"isReady": true,
"type": "localization",

View file

@ -35,6 +35,7 @@ import {
registerUiCountersUsageCollector,
registerUiCounterSavedObjectType,
registerUiCountersRollups,
registerConfigUsageCollector,
registerUsageCountersRollups,
registerUsageCountersUsageCollector,
} from './collectors';
@ -122,6 +123,7 @@ export class KibanaUsageCollectionPlugin implements Plugin {
registerCloudProviderUsageCollector(usageCollection);
registerCspCollector(usageCollection, coreSetup.http);
registerCoreUsageCollector(usageCollection, getCoreUsageDataService);
registerConfigUsageCollector(usageCollection, getCoreUsageDataService);
registerLocalizationUsageCollector(usageCollection, coreSetup.i18n);
}
}

View file

@ -183,8 +183,8 @@
},
"plugins": {
"properties": {
"THIS_WILL_BE_REPLACED_BY_THE_PLUGINS_JSON": {
"type": "text"
"kibana_config_usage": {
"type": "pass_through"
}
}
}

View file

@ -38,4 +38,9 @@ export const config: PluginConfigDescriptor<ConfigType> = {
exposeToBrowser: {
uiCounters: true,
},
exposeToUsage: {
usageCounters: {
bufferDuration: true,
},
},
};

View file

@ -8,6 +8,7 @@
import expect from '@kbn/expect';
import supertestAsPromised from 'supertest-as-promised';
import { omit } from 'lodash';
import { basicUiCounters } from './__fixtures__/ui_counters';
import { basicUsageCounters } from './__fixtures__/usage_counters';
import type { FtrProviderContext } from '../../ftr_provider_context';
@ -86,6 +87,35 @@ export default function ({ getService }: FtrProviderContext) {
expect(stats.stack_stats.kibana.plugins.csp.strict).to.be(true);
expect(stats.stack_stats.kibana.plugins.csp.warnLegacyBrowsers).to.be(true);
expect(stats.stack_stats.kibana.plugins.csp.rulesChangedFromDefault).to.be(false);
expect(stats.stack_stats.kibana.plugins.kibana_config_usage).to.be.an('object');
// non-default kibana configs. Configs set at 'test/api_integration/config.js'.
expect(omit(stats.stack_stats.kibana.plugins.kibana_config_usage, 'server.port')).to.eql({
'elasticsearch.username': '[redacted]',
'elasticsearch.password': '[redacted]',
'elasticsearch.hosts': '[redacted]',
'elasticsearch.healthCheck.delay': 3600000,
'plugins.paths': '[redacted]',
'logging.json': false,
'server.xsrf.disableProtection': true,
'server.compression.referrerWhitelist': '[redacted]',
'server.maxPayload': 1679958,
'status.allowAnonymous': true,
'home.disableWelcomeScreen': true,
'data.search.aggs.shardDelay.enabled': true,
'security.showInsecureClusterWarning': false,
'telemetry.banner': false,
'telemetry.url': '[redacted]',
'telemetry.optInStatusUrl': '[redacted]',
'telemetry.optIn': false,
'newsfeed.service.urlRoot': '[redacted]',
'newsfeed.service.pathTemplate': '[redacted]',
'savedObjects.maxImportPayloadBytes': 10485760,
'savedObjects.maxImportExportSize': 10001,
'usageCollection.usageCounters.bufferDuration': 0,
});
expect(stats.stack_stats.kibana.plugins.kibana_config_usage['server.port']).to.be.a(
'number'
);
// Testing stack_stats.data
expect(stats.stack_stats.data).to.be.an('object');

View file

@ -8,8 +8,8 @@
import type { ObjectType, Type } from '@kbn/config-schema';
import { schema } from '@kbn/config-schema';
import { get } from 'lodash';
import { set } from '@elastic/safer-lodash-set';
import { get, merge } from 'lodash';
import type { AllowedSchemaTypes } from 'src/plugins/usage_collection/server';
/**
@ -125,11 +125,19 @@ export function assertTelemetryPayload(
stats: unknown
): void {
const fullSchema = telemetrySchema.root;
const mergedPluginsSchema = merge(
{},
get(fullSchema, 'properties.stack_stats.properties.kibana.properties.plugins'),
telemetrySchema.plugins
);
set(
fullSchema,
'properties.stack_stats.properties.kibana.properties.plugins',
telemetrySchema.plugins
mergedPluginsSchema
);
const ossTelemetryValidationSchema = convertSchemaToConfigSchema(fullSchema);
// Run @kbn/config-schema validation to the entire payload

View file

@ -53,6 +53,7 @@ export default function ({ getService }: FtrProviderContext) {
it('should pass the schema validation', () => {
const root = deepmerge(ossRootTelemetrySchema, xpackRootTelemetrySchema);
const plugins = deepmerge(ossPluginsTelemetrySchema, xpackPluginsTelemetrySchema);
try {
assertTelemetryPayload({ root, plugins }, stats);
} catch (err) {