2020-02-28 03:15:08 +01:00
|
|
|
/*
|
|
|
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
|
|
|
* or more contributor license agreements. Licensed under the Elastic License;
|
|
|
|
* you may not use this file except in compliance with the Elastic License.
|
|
|
|
*/
|
|
|
|
|
|
|
|
import { i18n } from '@kbn/i18n';
|
2020-03-12 11:04:40 +01:00
|
|
|
import {
|
|
|
|
CoreSetup,
|
2020-04-24 13:08:19 +02:00
|
|
|
CoreStart,
|
2020-03-12 11:04:40 +01:00
|
|
|
Plugin,
|
2020-06-29 17:43:31 +02:00
|
|
|
ILegacyScopedClusterClient,
|
2020-04-24 13:08:19 +02:00
|
|
|
KibanaRequest,
|
2020-03-12 11:04:40 +01:00
|
|
|
Logger,
|
|
|
|
PluginInitializerContext,
|
2020-06-29 17:43:31 +02:00
|
|
|
ILegacyCustomClusterClient,
|
2020-04-24 13:08:19 +02:00
|
|
|
CapabilitiesStart,
|
2020-03-12 11:04:40 +01:00
|
|
|
} from 'kibana/server';
|
2020-03-04 11:14:11 +01:00
|
|
|
import { PluginsSetup, RouteInitialization } from './types';
|
2020-03-13 20:16:41 +01:00
|
|
|
import { PLUGIN_ID, PLUGIN_ICON } from '../common/constants/app';
|
2020-04-24 13:08:19 +02:00
|
|
|
import { MlCapabilities } from '../common/types/capabilities';
|
2020-02-28 03:15:08 +01:00
|
|
|
|
|
|
|
import { elasticsearchJsPlugin } from './client/elasticsearch_ml';
|
2020-03-18 09:33:53 +01:00
|
|
|
import { initMlTelemetry } from './lib/telemetry';
|
2020-02-28 03:15:08 +01:00
|
|
|
import { initMlServerLog } from './client/log';
|
2020-03-04 11:14:11 +01:00
|
|
|
import { initSampleDataSets } from './lib/sample_data_sets';
|
2020-02-28 03:15:08 +01:00
|
|
|
|
|
|
|
import { annotationRoutes } from './routes/annotations';
|
|
|
|
import { calendars } from './routes/calendars';
|
|
|
|
import { dataFeedRoutes } from './routes/datafeeds';
|
|
|
|
import { dataFrameAnalyticsRoutes } from './routes/data_frame_analytics';
|
|
|
|
import { dataRecognizer } from './routes/modules';
|
|
|
|
import { dataVisualizerRoutes } from './routes/data_visualizer';
|
|
|
|
import { fieldsService } from './routes/fields_service';
|
|
|
|
import { fileDataVisualizerRoutes } from './routes/file_data_visualizer';
|
|
|
|
import { filtersRoutes } from './routes/filters';
|
|
|
|
import { indicesRoutes } from './routes/indices';
|
|
|
|
import { jobAuditMessagesRoutes } from './routes/job_audit_messages';
|
|
|
|
import { jobRoutes } from './routes/anomaly_detectors';
|
|
|
|
import { jobServiceRoutes } from './routes/job_service';
|
|
|
|
import { jobValidationRoutes } from './routes/job_validation';
|
|
|
|
import { notificationRoutes } from './routes/notification_settings';
|
|
|
|
import { resultsServiceRoutes } from './routes/results_service';
|
|
|
|
import { systemRoutes } from './routes/system';
|
2020-03-13 20:16:41 +01:00
|
|
|
import { MlLicense } from '../common/license';
|
2020-03-04 11:14:11 +01:00
|
|
|
import { MlServerLicense } from './lib/license';
|
2020-03-12 11:04:40 +01:00
|
|
|
import { createSharedServices, SharedServices } from './shared_services';
|
2020-04-29 19:25:48 +02:00
|
|
|
import { getPluginPrivileges } from '../common/types/capabilities';
|
2020-04-24 13:08:19 +02:00
|
|
|
import { setupCapabilitiesSwitcher } from './lib/capabilities';
|
2020-04-28 10:22:46 +02:00
|
|
|
import { registerKibanaSettings } from './lib/register_settings';
|
2020-02-28 03:15:08 +01:00
|
|
|
|
|
|
|
declare module 'kibana/server' {
|
|
|
|
interface RequestHandlerContext {
|
2020-07-01 18:45:36 +02:00
|
|
|
[PLUGIN_ID]?: {
|
2020-06-29 17:43:31 +02:00
|
|
|
mlClient: ILegacyScopedClusterClient;
|
2020-02-28 03:15:08 +01:00
|
|
|
};
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
[SIEM] Check ML Job status on ML Rule execution (#61715)
* Move isMlRule helper to a more general location
And use it during rule execution as well.
* Add error message back to rule error status
This was unintentionally removed in a previous merge commit.
* Expose mlClient as part of ML's Setup contract
This allows dependent plugins to leverage the exposed services without
having to define their own ml paths, e.g. "ml.jobs"
* Move ML Job predicates to common folder
These are pure functions and used on both the client and server.
* WIP: Check ML Job status on ML Rule execution
This works, but unfortunately it pushes this executor function to a
complexity of 25. We're gonna refactor this next.
* Move isMlRule and RuleType to common
These are used on both the frontend and the backend, and can be shared.
* Refactor Signal Rule executor to use RuleStatusService
RuleStatusService holds the logic for updating the current status as
well as adding an error status. It leverages a simple
RuleStatusSavedObjectClient to handle the communication with
SavedObjects.
This removes the need for our specialized 'writeError', 'writeGap', and
'writeSuccess' functions, which duplicated much of the rule status
logic and code. It also fixes a bug with gap failures, with should have
been treated the same as other failures.
NB that an error does not necessarily prevent the rule from running, as
in the case of a gap or an ML Job not running.
This also adds a buildRuleMessage helper to reduce the noise of
generating logs/messages, and to make them more consistent.
* Remove unneeded 'async' keywords
We're not awaiting here, so we can just return the promise.
* Make buildRuleStatusAttributes synchronous
We weren't doing anything async here, and in fact the returning of a
promise was causing a bug when we tried to spread it into our attributes
object.
* Fix incorrectly-named RuleStatus attributes
This mapping could be done within the ruleStatusService, but it
lives outside it for now.
Also renames the object holding these values to the more general
'result,' as creationSuccess implies it always succeeds.
* Move our rule message helpers to a separate file
Adds some tests, as well.
* Refactor how rule status objects interact
Only ruleStatusSavedObjectsClient receives a savedObjectsClient, the
other functions receive the ruleStatusSavedObjectsClient
* pluralizes savedObjects in ruleStatusSavedObjectsClient
* Backfills tests
* Handle adding multiple errors during a single rule execution
We were storing state in our RuleStatusClient, and consequently could
get into a situation where that state did not reflect reality, and we
would incorrectly try to delete a SavedObject that had already been
deleted.
Rather than try to store the _correct_ state in the service, we remove
state entirely and just fetch our statuses on each action.
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
2020-03-30 23:35:38 +02:00
|
|
|
export interface MlPluginSetup extends SharedServices {
|
2020-06-29 17:43:31 +02:00
|
|
|
mlClient: ILegacyCustomClusterClient;
|
[SIEM] Check ML Job status on ML Rule execution (#61715)
* Move isMlRule helper to a more general location
And use it during rule execution as well.
* Add error message back to rule error status
This was unintentionally removed in a previous merge commit.
* Expose mlClient as part of ML's Setup contract
This allows dependent plugins to leverage the exposed services without
having to define their own ml paths, e.g. "ml.jobs"
* Move ML Job predicates to common folder
These are pure functions and used on both the client and server.
* WIP: Check ML Job status on ML Rule execution
This works, but unfortunately it pushes this executor function to a
complexity of 25. We're gonna refactor this next.
* Move isMlRule and RuleType to common
These are used on both the frontend and the backend, and can be shared.
* Refactor Signal Rule executor to use RuleStatusService
RuleStatusService holds the logic for updating the current status as
well as adding an error status. It leverages a simple
RuleStatusSavedObjectClient to handle the communication with
SavedObjects.
This removes the need for our specialized 'writeError', 'writeGap', and
'writeSuccess' functions, which duplicated much of the rule status
logic and code. It also fixes a bug with gap failures, with should have
been treated the same as other failures.
NB that an error does not necessarily prevent the rule from running, as
in the case of a gap or an ML Job not running.
This also adds a buildRuleMessage helper to reduce the noise of
generating logs/messages, and to make them more consistent.
* Remove unneeded 'async' keywords
We're not awaiting here, so we can just return the promise.
* Make buildRuleStatusAttributes synchronous
We weren't doing anything async here, and in fact the returning of a
promise was causing a bug when we tried to spread it into our attributes
object.
* Fix incorrectly-named RuleStatus attributes
This mapping could be done within the ruleStatusService, but it
lives outside it for now.
Also renames the object holding these values to the more general
'result,' as creationSuccess implies it always succeeds.
* Move our rule message helpers to a separate file
Adds some tests, as well.
* Refactor how rule status objects interact
Only ruleStatusSavedObjectsClient receives a savedObjectsClient, the
other functions receive the ruleStatusSavedObjectsClient
* pluralizes savedObjects in ruleStatusSavedObjectsClient
* Backfills tests
* Handle adding multiple errors during a single rule execution
We were storing state in our RuleStatusClient, and consequently could
get into a situation where that state did not reflect reality, and we
would incorrectly try to delete a SavedObject that had already been
deleted.
Rather than try to store the _correct_ state in the service, we remove
state entirely and just fetch our statuses on each action.
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
2020-03-30 23:35:38 +02:00
|
|
|
}
|
2020-03-24 15:25:59 +01:00
|
|
|
export type MlPluginStart = void;
|
2020-03-12 11:04:40 +01:00
|
|
|
|
2020-03-24 15:25:59 +01:00
|
|
|
export class MlServerPlugin implements Plugin<MlPluginSetup, MlPluginStart, PluginsSetup> {
|
2020-02-28 03:15:08 +01:00
|
|
|
private log: Logger;
|
|
|
|
private version: string;
|
2020-03-04 11:14:11 +01:00
|
|
|
private mlLicense: MlServerLicense;
|
2020-04-24 13:08:19 +02:00
|
|
|
private capabilities: CapabilitiesStart | null = null;
|
2020-02-28 03:15:08 +01:00
|
|
|
|
|
|
|
constructor(ctx: PluginInitializerContext) {
|
|
|
|
this.log = ctx.logger.get();
|
|
|
|
this.version = ctx.env.packageInfo.branch;
|
2020-03-04 11:14:11 +01:00
|
|
|
this.mlLicense = new MlServerLicense();
|
2020-02-28 03:15:08 +01:00
|
|
|
}
|
|
|
|
|
2020-03-24 15:25:59 +01:00
|
|
|
public setup(coreSetup: CoreSetup, plugins: PluginsSetup): MlPluginSetup {
|
2020-04-29 19:25:48 +02:00
|
|
|
const { user, admin } = getPluginPrivileges();
|
2020-04-24 13:08:19 +02:00
|
|
|
|
2020-02-28 03:15:08 +01:00
|
|
|
plugins.features.registerFeature({
|
|
|
|
id: PLUGIN_ID,
|
|
|
|
name: i18n.translate('xpack.ml.featureRegistry.mlFeatureName', {
|
|
|
|
defaultMessage: 'Machine Learning',
|
|
|
|
}),
|
2020-03-13 20:16:41 +01:00
|
|
|
icon: PLUGIN_ICON,
|
2020-03-24 16:12:49 +01:00
|
|
|
order: 500,
|
2020-02-28 03:15:08 +01:00
|
|
|
navLinkId: PLUGIN_ID,
|
|
|
|
app: [PLUGIN_ID, 'kibana'],
|
|
|
|
catalogue: [PLUGIN_ID],
|
2020-03-24 16:12:49 +01:00
|
|
|
privileges: null,
|
2020-02-28 03:15:08 +01:00
|
|
|
reserved: {
|
|
|
|
description: i18n.translate('xpack.ml.feature.reserved.description', {
|
|
|
|
defaultMessage:
|
|
|
|
'To grant users access, you should also assign either the machine_learning_user or machine_learning_admin role.',
|
|
|
|
}),
|
2020-04-08 19:03:15 +02:00
|
|
|
privileges: [
|
|
|
|
{
|
|
|
|
id: 'ml_user',
|
|
|
|
privilege: {
|
2020-04-29 19:25:48 +02:00
|
|
|
api: user.api,
|
2020-04-08 19:03:15 +02:00
|
|
|
app: [PLUGIN_ID, 'kibana'],
|
|
|
|
catalogue: [PLUGIN_ID],
|
|
|
|
savedObject: {
|
|
|
|
all: [],
|
|
|
|
read: [],
|
|
|
|
},
|
2020-04-29 19:25:48 +02:00
|
|
|
ui: user.ui,
|
2020-04-08 19:03:15 +02:00
|
|
|
},
|
|
|
|
},
|
|
|
|
{
|
|
|
|
id: 'ml_admin',
|
|
|
|
privilege: {
|
2020-04-29 19:25:48 +02:00
|
|
|
api: admin.api,
|
2020-04-08 19:03:15 +02:00
|
|
|
app: [PLUGIN_ID, 'kibana'],
|
|
|
|
catalogue: [PLUGIN_ID],
|
|
|
|
savedObject: {
|
|
|
|
all: [],
|
|
|
|
read: [],
|
|
|
|
},
|
2020-04-29 19:25:48 +02:00
|
|
|
ui: admin.ui,
|
2020-04-08 19:03:15 +02:00
|
|
|
},
|
|
|
|
},
|
|
|
|
],
|
2020-02-28 03:15:08 +01:00
|
|
|
},
|
|
|
|
});
|
2020-04-28 10:22:46 +02:00
|
|
|
registerKibanaSettings(coreSetup);
|
|
|
|
|
2020-03-04 11:14:11 +01:00
|
|
|
this.mlLicense.setup(plugins.licensing.license$, [
|
|
|
|
(mlLicense: MlLicense) => initSampleDataSets(mlLicense, plugins),
|
|
|
|
]);
|
|
|
|
|
2020-04-24 13:08:19 +02:00
|
|
|
// initialize capabilities switcher to add license filter to ml capabilities
|
|
|
|
setupCapabilitiesSwitcher(coreSetup, plugins.licensing.license$, this.log);
|
|
|
|
|
2020-02-28 03:15:08 +01:00
|
|
|
// Can access via router's handler function 'context' parameter - context.ml.mlClient
|
2020-06-01 16:16:44 +02:00
|
|
|
const mlClient = coreSetup.elasticsearch.legacy.createClient(PLUGIN_ID, {
|
2020-02-28 03:15:08 +01:00
|
|
|
plugins: [elasticsearchJsPlugin],
|
|
|
|
});
|
|
|
|
|
|
|
|
coreSetup.http.registerRouteHandlerContext(PLUGIN_ID, (context, request) => {
|
|
|
|
return {
|
|
|
|
mlClient: mlClient.asScoped(request),
|
|
|
|
};
|
|
|
|
});
|
|
|
|
|
|
|
|
const routeInit: RouteInitialization = {
|
|
|
|
router: coreSetup.http.createRouter(),
|
2020-03-04 11:14:11 +01:00
|
|
|
mlLicense: this.mlLicense,
|
2020-02-28 03:15:08 +01:00
|
|
|
};
|
|
|
|
|
2020-04-24 13:08:19 +02:00
|
|
|
const resolveMlCapabilities = async (request: KibanaRequest) => {
|
|
|
|
if (this.capabilities === null) {
|
|
|
|
return null;
|
|
|
|
}
|
|
|
|
const capabilities = await this.capabilities.resolveCapabilities(request);
|
|
|
|
return capabilities.ml as MlCapabilities;
|
|
|
|
};
|
|
|
|
|
2020-02-28 03:15:08 +01:00
|
|
|
annotationRoutes(routeInit, plugins.security);
|
|
|
|
calendars(routeInit);
|
|
|
|
dataFeedRoutes(routeInit);
|
|
|
|
dataFrameAnalyticsRoutes(routeInit);
|
|
|
|
dataRecognizer(routeInit);
|
|
|
|
dataVisualizerRoutes(routeInit);
|
|
|
|
fieldsService(routeInit);
|
|
|
|
fileDataVisualizerRoutes(routeInit);
|
|
|
|
filtersRoutes(routeInit);
|
|
|
|
indicesRoutes(routeInit);
|
|
|
|
jobAuditMessagesRoutes(routeInit);
|
|
|
|
jobRoutes(routeInit);
|
2020-04-29 19:25:48 +02:00
|
|
|
jobServiceRoutes(routeInit);
|
2020-02-28 03:15:08 +01:00
|
|
|
notificationRoutes(routeInit);
|
|
|
|
resultsServiceRoutes(routeInit);
|
|
|
|
jobValidationRoutes(routeInit, this.version);
|
|
|
|
systemRoutes(routeInit, {
|
2020-03-05 16:06:36 +01:00
|
|
|
spaces: plugins.spaces,
|
2020-02-28 03:15:08 +01:00
|
|
|
cloud: plugins.cloud,
|
2020-04-24 13:08:19 +02:00
|
|
|
resolveMlCapabilities,
|
2020-02-28 03:15:08 +01:00
|
|
|
});
|
|
|
|
initMlServerLog({ log: this.log });
|
2020-03-18 09:33:53 +01:00
|
|
|
initMlTelemetry(coreSetup, plugins.usageCollection);
|
2020-03-12 11:04:40 +01:00
|
|
|
|
[SIEM] Check ML Job status on ML Rule execution (#61715)
* Move isMlRule helper to a more general location
And use it during rule execution as well.
* Add error message back to rule error status
This was unintentionally removed in a previous merge commit.
* Expose mlClient as part of ML's Setup contract
This allows dependent plugins to leverage the exposed services without
having to define their own ml paths, e.g. "ml.jobs"
* Move ML Job predicates to common folder
These are pure functions and used on both the client and server.
* WIP: Check ML Job status on ML Rule execution
This works, but unfortunately it pushes this executor function to a
complexity of 25. We're gonna refactor this next.
* Move isMlRule and RuleType to common
These are used on both the frontend and the backend, and can be shared.
* Refactor Signal Rule executor to use RuleStatusService
RuleStatusService holds the logic for updating the current status as
well as adding an error status. It leverages a simple
RuleStatusSavedObjectClient to handle the communication with
SavedObjects.
This removes the need for our specialized 'writeError', 'writeGap', and
'writeSuccess' functions, which duplicated much of the rule status
logic and code. It also fixes a bug with gap failures, with should have
been treated the same as other failures.
NB that an error does not necessarily prevent the rule from running, as
in the case of a gap or an ML Job not running.
This also adds a buildRuleMessage helper to reduce the noise of
generating logs/messages, and to make them more consistent.
* Remove unneeded 'async' keywords
We're not awaiting here, so we can just return the promise.
* Make buildRuleStatusAttributes synchronous
We weren't doing anything async here, and in fact the returning of a
promise was causing a bug when we tried to spread it into our attributes
object.
* Fix incorrectly-named RuleStatus attributes
This mapping could be done within the ruleStatusService, but it
lives outside it for now.
Also renames the object holding these values to the more general
'result,' as creationSuccess implies it always succeeds.
* Move our rule message helpers to a separate file
Adds some tests, as well.
* Refactor how rule status objects interact
Only ruleStatusSavedObjectsClient receives a savedObjectsClient, the
other functions receive the ruleStatusSavedObjectsClient
* pluralizes savedObjects in ruleStatusSavedObjectsClient
* Backfills tests
* Handle adding multiple errors during a single rule execution
We were storing state in our RuleStatusClient, and consequently could
get into a situation where that state did not reflect reality, and we
would incorrectly try to delete a SavedObject that had already been
deleted.
Rather than try to store the _correct_ state in the service, we remove
state entirely and just fetch our statuses on each action.
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
2020-03-30 23:35:38 +02:00
|
|
|
return {
|
2020-04-24 13:08:19 +02:00
|
|
|
...createSharedServices(this.mlLicense, plugins.spaces, plugins.cloud, resolveMlCapabilities),
|
[SIEM] Check ML Job status on ML Rule execution (#61715)
* Move isMlRule helper to a more general location
And use it during rule execution as well.
* Add error message back to rule error status
This was unintentionally removed in a previous merge commit.
* Expose mlClient as part of ML's Setup contract
This allows dependent plugins to leverage the exposed services without
having to define their own ml paths, e.g. "ml.jobs"
* Move ML Job predicates to common folder
These are pure functions and used on both the client and server.
* WIP: Check ML Job status on ML Rule execution
This works, but unfortunately it pushes this executor function to a
complexity of 25. We're gonna refactor this next.
* Move isMlRule and RuleType to common
These are used on both the frontend and the backend, and can be shared.
* Refactor Signal Rule executor to use RuleStatusService
RuleStatusService holds the logic for updating the current status as
well as adding an error status. It leverages a simple
RuleStatusSavedObjectClient to handle the communication with
SavedObjects.
This removes the need for our specialized 'writeError', 'writeGap', and
'writeSuccess' functions, which duplicated much of the rule status
logic and code. It also fixes a bug with gap failures, with should have
been treated the same as other failures.
NB that an error does not necessarily prevent the rule from running, as
in the case of a gap or an ML Job not running.
This also adds a buildRuleMessage helper to reduce the noise of
generating logs/messages, and to make them more consistent.
* Remove unneeded 'async' keywords
We're not awaiting here, so we can just return the promise.
* Make buildRuleStatusAttributes synchronous
We weren't doing anything async here, and in fact the returning of a
promise was causing a bug when we tried to spread it into our attributes
object.
* Fix incorrectly-named RuleStatus attributes
This mapping could be done within the ruleStatusService, but it
lives outside it for now.
Also renames the object holding these values to the more general
'result,' as creationSuccess implies it always succeeds.
* Move our rule message helpers to a separate file
Adds some tests, as well.
* Refactor how rule status objects interact
Only ruleStatusSavedObjectsClient receives a savedObjectsClient, the
other functions receive the ruleStatusSavedObjectsClient
* pluralizes savedObjects in ruleStatusSavedObjectsClient
* Backfills tests
* Handle adding multiple errors during a single rule execution
We were storing state in our RuleStatusClient, and consequently could
get into a situation where that state did not reflect reality, and we
would incorrectly try to delete a SavedObject that had already been
deleted.
Rather than try to store the _correct_ state in the service, we remove
state entirely and just fetch our statuses on each action.
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
2020-03-30 23:35:38 +02:00
|
|
|
mlClient,
|
|
|
|
};
|
2020-02-28 03:15:08 +01:00
|
|
|
}
|
|
|
|
|
2020-04-24 13:08:19 +02:00
|
|
|
public start(coreStart: CoreStart): MlPluginStart {
|
|
|
|
this.capabilities = coreStart.capabilities;
|
|
|
|
}
|
2020-02-28 03:15:08 +01:00
|
|
|
|
2020-03-04 11:14:11 +01:00
|
|
|
public stop() {
|
|
|
|
this.mlLicense.unsubscribe();
|
|
|
|
}
|
2020-02-28 03:15:08 +01:00
|
|
|
}
|