[ML] Adds API integration tests for data viz and fields endpoints (#64165)
* [ML] Adds API integration tests for data viz and fields endpoints * [ML] Fix review comments and errors from settings endpoints Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
This commit is contained in:
Pete Harverson
GitHub
parent
bc6291349c
commit
09c2727d78
33 changed files with 1399 additions and 154 deletions
|
|
@ -5,7 +5,6 @@
|
|||
*/
|
||||
|
||||
import { difference } from 'lodash';
|
||||
import Boom from 'boom';
|
||||
import { IScopedClusterClient } from 'kibana/server';
|
||||
import { EventManager, CalendarEvent } from './event_manager';
|
||||
|
||||
|
|
@ -33,43 +32,31 @@ export class CalendarManager {
|
|||
}
|
||||
|
||||
async getCalendar(calendarId: string) {
|
||||
try {
|
||||
const resp = await this._client('ml.calendars', {
|
||||
calendarId,
|
||||
});
|
||||
const resp = await this._client('ml.calendars', {
|
||||
calendarId,
|
||||
});
|
||||
|
||||
const calendars = resp.calendars;
|
||||
if (calendars.length) {
|
||||
const calendar = calendars[0];
|
||||
calendar.events = await this._eventManager.getCalendarEvents(calendarId);
|
||||
return calendar;
|
||||
} else {
|
||||
throw Boom.notFound(`Calendar with the id "${calendarId}" not found`);
|
||||
}
|
||||
} catch (error) {
|
||||
throw Boom.badRequest(error);
|
||||
}
|
||||
const calendars = resp.calendars;
|
||||
const calendar = calendars[0]; // Endpoint throws a 404 if calendar is not found.
|
||||
calendar.events = await this._eventManager.getCalendarEvents(calendarId);
|
||||
return calendar;
|
||||
}
|
||||
|
||||
async getAllCalendars() {
|
||||
try {
|
||||
const calendarsResp = await this._client('ml.calendars');
|
||||
const calendarsResp = await this._client('ml.calendars');
|
||||
|
||||
const events: CalendarEvent[] = await this._eventManager.getAllEvents();
|
||||
const calendars: Calendar[] = calendarsResp.calendars;
|
||||
calendars.forEach(cal => (cal.events = []));
|
||||
const events: CalendarEvent[] = await this._eventManager.getAllEvents();
|
||||
const calendars: Calendar[] = calendarsResp.calendars;
|
||||
calendars.forEach(cal => (cal.events = []));
|
||||
|
||||
// loop events and combine with related calendars
|
||||
events.forEach(event => {
|
||||
const calendar = calendars.find(cal => cal.calendar_id === event.calendar_id);
|
||||
if (calendar) {
|
||||
calendar.events.push(event);
|
||||
}
|
||||
});
|
||||
return calendars;
|
||||
} catch (error) {
|
||||
throw Boom.badRequest(error);
|
||||
}
|
||||
// loop events and combine with related calendars
|
||||
events.forEach(event => {
|
||||
const calendar = calendars.find(cal => cal.calendar_id === event.calendar_id);
|
||||
if (calendar) {
|
||||
calendar.events.push(event);
|
||||
}
|
||||
});
|
||||
return calendars;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -78,12 +65,8 @@ export class CalendarManager {
|
|||
* @returns {Promise<*>}
|
||||
*/
|
||||
async getCalendarsByIds(calendarIds: string) {
|
||||
try {
|
||||
const calendars: Calendar[] = await this.getAllCalendars();
|
||||
return calendars.filter(calendar => calendarIds.includes(calendar.calendar_id));
|
||||
} catch (error) {
|
||||
throw Boom.badRequest(error);
|
||||
}
|
||||
const calendars: Calendar[] = await this.getAllCalendars();
|
||||
return calendars.filter(calendar => calendarIds.includes(calendar.calendar_id));
|
||||
}
|
||||
|
||||
async newCalendar(calendar: FormCalendar) {
|
||||
|
|
@ -91,75 +74,67 @@ export class CalendarManager {
|
|||
const events = calendar.events;
|
||||
delete calendar.calendarId;
|
||||
delete calendar.events;
|
||||
try {
|
||||
await this._client('ml.addCalendar', {
|
||||
calendarId,
|
||||
body: calendar,
|
||||
});
|
||||
await this._client('ml.addCalendar', {
|
||||
calendarId,
|
||||
body: calendar,
|
||||
});
|
||||
|
||||
if (events.length) {
|
||||
await this._eventManager.addEvents(calendarId, events);
|
||||
}
|
||||
|
||||
// return the newly created calendar
|
||||
return await this.getCalendar(calendarId);
|
||||
} catch (error) {
|
||||
throw Boom.badRequest(error);
|
||||
if (events.length) {
|
||||
await this._eventManager.addEvents(calendarId, events);
|
||||
}
|
||||
|
||||
// return the newly created calendar
|
||||
return await this.getCalendar(calendarId);
|
||||
}
|
||||
|
||||
async updateCalendar(calendarId: string, calendar: Calendar) {
|
||||
const origCalendar: Calendar = await this.getCalendar(calendarId);
|
||||
try {
|
||||
// update job_ids
|
||||
const jobsToAdd = difference(calendar.job_ids, origCalendar.job_ids);
|
||||
const jobsToRemove = difference(origCalendar.job_ids, calendar.job_ids);
|
||||
// update job_ids
|
||||
const jobsToAdd = difference(calendar.job_ids, origCalendar.job_ids);
|
||||
const jobsToRemove = difference(origCalendar.job_ids, calendar.job_ids);
|
||||
|
||||
// workout the differences between the original events list and the new one
|
||||
// if an event has no event_id, it must be new
|
||||
const eventsToAdd = calendar.events.filter(
|
||||
event => origCalendar.events.find(e => this._eventManager.isEqual(e, event)) === undefined
|
||||
);
|
||||
// workout the differences between the original events list and the new one
|
||||
// if an event has no event_id, it must be new
|
||||
const eventsToAdd = calendar.events.filter(
|
||||
event => origCalendar.events.find(e => this._eventManager.isEqual(e, event)) === undefined
|
||||
);
|
||||
|
||||
// if an event in the original calendar cannot be found, it must have been deleted
|
||||
const eventsToRemove: CalendarEvent[] = origCalendar.events.filter(
|
||||
event => calendar.events.find(e => this._eventManager.isEqual(e, event)) === undefined
|
||||
);
|
||||
// if an event in the original calendar cannot be found, it must have been deleted
|
||||
const eventsToRemove: CalendarEvent[] = origCalendar.events.filter(
|
||||
event => calendar.events.find(e => this._eventManager.isEqual(e, event)) === undefined
|
||||
);
|
||||
|
||||
// note, both of the loops below could be removed if the add and delete endpoints
|
||||
// allowed multiple job_ids
|
||||
// note, both of the loops below could be removed if the add and delete endpoints
|
||||
// allowed multiple job_ids
|
||||
|
||||
// add all new jobs
|
||||
if (jobsToAdd.length) {
|
||||
await this._client('ml.addJobToCalendar', {
|
||||
calendarId,
|
||||
jobId: jobsToAdd.join(','),
|
||||
});
|
||||
}
|
||||
|
||||
// remove all removed jobs
|
||||
if (jobsToRemove.length) {
|
||||
await this._client('ml.removeJobFromCalendar', {
|
||||
calendarId,
|
||||
jobId: jobsToRemove.join(','),
|
||||
});
|
||||
}
|
||||
|
||||
// add all new events
|
||||
if (eventsToAdd.length !== 0) {
|
||||
await this._eventManager.addEvents(calendarId, eventsToAdd);
|
||||
}
|
||||
|
||||
// remove all removed events
|
||||
await Promise.all(
|
||||
eventsToRemove.map(async event => {
|
||||
await this._eventManager.deleteEvent(calendarId, event.event_id);
|
||||
})
|
||||
);
|
||||
} catch (error) {
|
||||
throw Boom.badRequest(error);
|
||||
// add all new jobs
|
||||
if (jobsToAdd.length) {
|
||||
await this._client('ml.addJobToCalendar', {
|
||||
calendarId,
|
||||
jobId: jobsToAdd.join(','),
|
||||
});
|
||||
}
|
||||
|
||||
// remove all removed jobs
|
||||
if (jobsToRemove.length) {
|
||||
await this._client('ml.removeJobFromCalendar', {
|
||||
calendarId,
|
||||
jobId: jobsToRemove.join(','),
|
||||
});
|
||||
}
|
||||
|
||||
// add all new events
|
||||
if (eventsToAdd.length !== 0) {
|
||||
await this._eventManager.addEvents(calendarId, eventsToAdd);
|
||||
}
|
||||
|
||||
// remove all removed events
|
||||
await Promise.all(
|
||||
eventsToRemove.map(async event => {
|
||||
await this._eventManager.deleteEvent(calendarId, event.event_id);
|
||||
})
|
||||
);
|
||||
|
||||
// return the updated calendar
|
||||
return await this.getCalendar(calendarId);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,8 +4,6 @@
|
|||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import Boom from 'boom';
|
||||
|
||||
import { GLOBAL_CALENDAR } from '../../../common/constants/calendars';
|
||||
|
||||
export interface CalendarEvent {
|
||||
|
|
@ -23,41 +21,29 @@ export class EventManager {
|
|||
}
|
||||
|
||||
async getCalendarEvents(calendarId: string) {
|
||||
try {
|
||||
const resp = await this._client('ml.events', { calendarId });
|
||||
const resp = await this._client('ml.events', { calendarId });
|
||||
|
||||
return resp.events;
|
||||
} catch (error) {
|
||||
throw Boom.badRequest(error);
|
||||
}
|
||||
return resp.events;
|
||||
}
|
||||
|
||||
// jobId is optional
|
||||
async getAllEvents(jobId?: string) {
|
||||
const calendarId = GLOBAL_CALENDAR;
|
||||
try {
|
||||
const resp = await this._client('ml.events', {
|
||||
calendarId,
|
||||
jobId,
|
||||
});
|
||||
const resp = await this._client('ml.events', {
|
||||
calendarId,
|
||||
jobId,
|
||||
});
|
||||
|
||||
return resp.events;
|
||||
} catch (error) {
|
||||
throw Boom.badRequest(error);
|
||||
}
|
||||
return resp.events;
|
||||
}
|
||||
|
||||
async addEvents(calendarId: string, events: CalendarEvent[]) {
|
||||
const body = { events };
|
||||
|
||||
try {
|
||||
return await this._client('ml.addEvent', {
|
||||
calendarId,
|
||||
body,
|
||||
});
|
||||
} catch (error) {
|
||||
throw Boom.badRequest(error);
|
||||
}
|
||||
return await this._client('ml.addEvent', {
|
||||
calendarId,
|
||||
body,
|
||||
});
|
||||
}
|
||||
|
||||
async deleteEvent(calendarId: string, eventId: string) {
|
||||
|
|
|
|||
|
|
@ -138,12 +138,14 @@ export function jobRoutes({ router, mlLicense }: RouteInitialization) {
|
|||
/**
|
||||
* @apiGroup AnomalyDetectors
|
||||
*
|
||||
* @api {put} /api/ml/anomaly_detectors/:jobId Instantiate an anomaly detection job
|
||||
* @api {put} /api/ml/anomaly_detectors/:jobId Create an anomaly detection job
|
||||
* @apiName CreateAnomalyDetectors
|
||||
* @apiDescription Creates an anomaly detection job.
|
||||
*
|
||||
* @apiSchema (params) jobIdSchema
|
||||
* @apiSchema (body) anomalyDetectionJobSchema
|
||||
*
|
||||
* @apiSuccess {Object} job the configuration of the job that has been created.
|
||||
*/
|
||||
router.put(
|
||||
{
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@
|
|||
"GetCategoryExamples",
|
||||
"GetPartitionFieldsValues",
|
||||
|
||||
"DataRecognizer",
|
||||
"Modules",
|
||||
"RecognizeIndex",
|
||||
"GetModule",
|
||||
"SetupModule",
|
||||
|
|
|
|||
|
|
@ -74,10 +74,12 @@ export function dataVisualizerRoutes({ router, mlLicense }: RouteInitialization)
|
|||
*
|
||||
* @api {post} /api/ml/data_visualizer/get_field_stats/:indexPatternTitle Get stats for fields
|
||||
* @apiName GetStatsForFields
|
||||
* @apiDescription Returns fields stats of the index pattern.
|
||||
* @apiDescription Returns the stats on individual fields in the specified index pattern.
|
||||
*
|
||||
* @apiSchema (params) indexPatternTitleSchema
|
||||
* @apiSchema (body) dataVisualizerFieldStatsSchema
|
||||
*
|
||||
* @apiSuccess {Object} fieldName stats by field, keyed on the name of the field.
|
||||
*/
|
||||
router.post(
|
||||
{
|
||||
|
|
@ -130,10 +132,16 @@ export function dataVisualizerRoutes({ router, mlLicense }: RouteInitialization)
|
|||
*
|
||||
* @api {post} /api/ml/data_visualizer/get_overall_stats/:indexPatternTitle Get overall stats
|
||||
* @apiName GetOverallStats
|
||||
* @apiDescription Returns overall stats of the index pattern.
|
||||
* @apiDescription Returns the top level overall stats for the specified index pattern.
|
||||
*
|
||||
* @apiSchema (params) indexPatternTitleSchema
|
||||
* @apiSchema (body) dataVisualizerOverallStatsSchema
|
||||
*
|
||||
* @apiSuccess {number} totalCount total count of documents.
|
||||
* @apiSuccess {Object} aggregatableExistsFields stats on aggregatable fields that exist in documents.
|
||||
* @apiSuccess {Object} aggregatableNotExistsFields stats on aggregatable fields that do not exist in documents.
|
||||
* @apiSuccess {Object} nonAggregatableExistsFields stats on non-aggregatable fields that exist in documents.
|
||||
* @apiSuccess {Object} nonAggregatableNotExistsFields stats on non-aggregatable fields that do not exist in documents.
|
||||
*/
|
||||
router.post(
|
||||
{
|
||||
|
|
|
|||
|
|
@ -37,6 +37,8 @@ export function fieldsService({ router, mlLicense }: RouteInitialization) {
|
|||
* @apiDescription Returns the cardinality of one or more fields. Returns an Object whose keys are the names of the fields, with values equal to the cardinality of the field
|
||||
*
|
||||
* @apiSchema (body) getCardinalityOfFieldsSchema
|
||||
*
|
||||
* @apiSuccess {number} fieldName cardinality of the field.
|
||||
*/
|
||||
router.post(
|
||||
{
|
||||
|
|
@ -64,9 +66,12 @@ export function fieldsService({ router, mlLicense }: RouteInitialization) {
|
|||
*
|
||||
* @api {post} /api/ml/fields_service/time_field_range Get time field range
|
||||
* @apiName GetTimeFieldRange
|
||||
* @apiDescription Returns the timefield range for the given index
|
||||
* @apiDescription Returns the time range for the given index and query using the specified time range.
|
||||
*
|
||||
* @apiSchema (body) getTimeFieldRangeSchema
|
||||
*
|
||||
* @apiSuccess {Object} start start of time range with epoch and string properties.
|
||||
* @apiSuccess {Object} end end of time range with epoch and string properties.
|
||||
*/
|
||||
router.post(
|
||||
{
|
||||
|
|
|
|||
|
|
@ -7,7 +7,10 @@
|
|||
import { wrapError } from '../client/error_wrapper';
|
||||
import { RouteInitialization } from '../types';
|
||||
import { jobAuditMessagesProvider } from '../models/job_audit_messages';
|
||||
import { jobAuditMessagesQuerySchema, jobIdSchema } from './schemas/job_audit_messages_schema';
|
||||
import {
|
||||
jobAuditMessagesQuerySchema,
|
||||
jobAuditMessagesJobIdSchema,
|
||||
} from './schemas/job_audit_messages_schema';
|
||||
|
||||
/**
|
||||
* Routes for job audit message routes
|
||||
|
|
@ -20,14 +23,14 @@ export function jobAuditMessagesRoutes({ router, mlLicense }: RouteInitializatio
|
|||
* @apiName GetJobAuditMessages
|
||||
* @apiDescription Returns audit messages for specified job ID
|
||||
*
|
||||
* @apiSchema (params) jobIdSchema
|
||||
* @apiSchema (params) jobAuditMessagesJobIdSchema
|
||||
* @apiSchema (query) jobAuditMessagesQuerySchema
|
||||
*/
|
||||
router.get(
|
||||
{
|
||||
path: '/api/ml/job_audit_messages/messages/{jobId}',
|
||||
validate: {
|
||||
params: jobIdSchema,
|
||||
params: jobAuditMessagesJobIdSchema,
|
||||
query: jobAuditMessagesQuerySchema,
|
||||
},
|
||||
},
|
||||
|
|
|
|||
|
|
@ -176,9 +176,14 @@ export function jobServiceRoutes({ router, mlLicense }: RouteInitialization) {
|
|||
*
|
||||
* @api {post} /api/ml/jobs/jobs_summary Jobs summary
|
||||
* @apiName JobsSummary
|
||||
* @apiDescription Creates a summary jobs list. Jobs include job stats, datafeed stats, and calendars.
|
||||
* @apiDescription Returns a list of anomaly detection jobs, with summary level information for every job.
|
||||
* For any supplied job IDs, full job information will be returned, which include the analysis configuration,
|
||||
* job stats, datafeed stats, and calendars.
|
||||
*
|
||||
* @apiSchema (body) jobIdsSchema
|
||||
*
|
||||
* @apiSuccess {Array} jobsList list of jobs. For any supplied job IDs, the job object will contain a fullJob property
|
||||
* which includes the full configuration and stats for the job.
|
||||
*/
|
||||
router.post(
|
||||
{
|
||||
|
|
|
|||
|
|
@ -81,7 +81,7 @@ function dataRecognizerJobsExist(context: RequestHandlerContext, moduleId: strin
|
|||
*/
|
||||
export function dataRecognizer({ router, mlLicense }: RouteInitialization) {
|
||||
/**
|
||||
* @apiGroup DataRecognizer
|
||||
* @apiGroup Modules
|
||||
*
|
||||
* @api {get} /api/ml/modules/recognize/:indexPatternTitle Recognize index pattern
|
||||
* @apiName RecognizeIndex
|
||||
|
|
@ -111,7 +111,7 @@ export function dataRecognizer({ router, mlLicense }: RouteInitialization) {
|
|||
);
|
||||
|
||||
/**
|
||||
* @apiGroup DataRecognizer
|
||||
* @apiGroup Modules
|
||||
*
|
||||
* @api {get} /api/ml/modules/get_module/:moduleId Get module
|
||||
* @apiName GetModule
|
||||
|
|
@ -146,7 +146,7 @@ export function dataRecognizer({ router, mlLicense }: RouteInitialization) {
|
|||
);
|
||||
|
||||
/**
|
||||
* @apiGroup DataRecognizer
|
||||
* @apiGroup Modules
|
||||
*
|
||||
* @api {post} /api/ml/modules/setup/:moduleId Setup module
|
||||
* @apiName SetupModule
|
||||
|
|
@ -204,7 +204,7 @@ export function dataRecognizer({ router, mlLicense }: RouteInitialization) {
|
|||
);
|
||||
|
||||
/**
|
||||
* @apiGroup DataRecognizer
|
||||
* @apiGroup Modules
|
||||
*
|
||||
* @api {post} /api/ml/modules/jobs_exist/:moduleId Check if module jobs exist
|
||||
* @apiName CheckExistingModuleJobs
|
||||
|
|
|
|||
|
|
@ -119,7 +119,7 @@ export const anomalyDetectionJobSchema = {
|
|||
};
|
||||
|
||||
export const jobIdSchema = schema.object({
|
||||
/** Job id */
|
||||
/** Job ID. */
|
||||
jobId: schema.string(),
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -7,26 +7,41 @@
|
|||
import { schema } from '@kbn/config-schema';
|
||||
|
||||
export const indexPatternTitleSchema = schema.object({
|
||||
/** Title of the index pattern for which to return stats. */
|
||||
indexPatternTitle: schema.string(),
|
||||
});
|
||||
|
||||
export const dataVisualizerFieldStatsSchema = schema.object({
|
||||
/** Query to match documents in the index. */
|
||||
query: schema.any(),
|
||||
fields: schema.arrayOf(schema.any()),
|
||||
/** Number of documents to be collected in the sample processed on each shard, or -1 for no sampling. */
|
||||
samplerShardSize: schema.number(),
|
||||
/** Name of the time field in the index (optional). */
|
||||
timeFieldName: schema.maybe(schema.string()),
|
||||
/** Earliest timestamp for search, as epoch ms (optional). */
|
||||
earliest: schema.maybe(schema.number()),
|
||||
/** Latest timestamp for search, as epoch ms (optional). */
|
||||
latest: schema.maybe(schema.number()),
|
||||
/** Aggregation interval to use for obtaining document counts over time (optional). */
|
||||
interval: schema.maybe(schema.string()),
|
||||
/** Maximum number of examples to return for text type fields. */
|
||||
maxExamples: schema.number(),
|
||||
});
|
||||
|
||||
export const dataVisualizerOverallStatsSchema = schema.object({
|
||||
/** Query to match documents in the index. */
|
||||
query: schema.any(),
|
||||
/** Names of aggregatable fields for which to return stats. */
|
||||
aggregatableFields: schema.arrayOf(schema.string()),
|
||||
/** Names of non-aggregatable fields for which to return stats. */
|
||||
nonAggregatableFields: schema.arrayOf(schema.string()),
|
||||
/** Number of documents to be collected in the sample processed on each shard, or -1 for no sampling. */
|
||||
samplerShardSize: schema.number(),
|
||||
/** Name of the time field in the index (optional). */
|
||||
timeFieldName: schema.maybe(schema.string()),
|
||||
/** Earliest timestamp for search, as epoch ms (optional). */
|
||||
earliest: schema.maybe(schema.number()),
|
||||
/** Latest timestamp for search, as epoch ms (optional). */
|
||||
latest: schema.maybe(schema.number()),
|
||||
});
|
||||
|
|
|
|||
|
|
@ -7,16 +7,25 @@
|
|||
import { schema } from '@kbn/config-schema';
|
||||
|
||||
export const getCardinalityOfFieldsSchema = schema.object({
|
||||
/** Index or indexes for which to return the time range. */
|
||||
index: schema.oneOf([schema.string(), schema.arrayOf(schema.string())]),
|
||||
/** Name(s) of the field(s) to return cardinality information. */
|
||||
fieldNames: schema.maybe(schema.arrayOf(schema.string())),
|
||||
/** Query to match documents in the index(es) (optional). */
|
||||
query: schema.maybe(schema.any()),
|
||||
/** Name of the time field in the index. */
|
||||
timeFieldName: schema.maybe(schema.string()),
|
||||
/** Earliest timestamp for search, as epoch ms (optional). */
|
||||
earliestMs: schema.maybe(schema.oneOf([schema.number(), schema.string()])),
|
||||
/** Latest timestamp for search, as epoch ms (optional). */
|
||||
latestMs: schema.maybe(schema.oneOf([schema.number(), schema.string()])),
|
||||
});
|
||||
|
||||
export const getTimeFieldRangeSchema = schema.object({
|
||||
/** Index or indexes for which to return the time range. */
|
||||
index: schema.oneOf([schema.string(), schema.arrayOf(schema.string())]),
|
||||
/** Name of the time field in the index. */
|
||||
timeFieldName: schema.maybe(schema.string()),
|
||||
/** Query to match documents in the index(es). */
|
||||
query: schema.maybe(schema.any()),
|
||||
});
|
||||
|
|
|
|||
|
|
@ -6,7 +6,10 @@
|
|||
|
||||
import { schema } from '@kbn/config-schema';
|
||||
|
||||
export const jobIdSchema = schema.object({ jobId: schema.maybe(schema.string()) });
|
||||
export const jobAuditMessagesJobIdSchema = schema.object({
|
||||
/** Job ID. */
|
||||
jobId: schema.maybe(schema.string()),
|
||||
});
|
||||
|
||||
export const jobAuditMessagesQuerySchema = schema.maybe(
|
||||
schema.object({ from: schema.maybe(schema.any()) })
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ export const forceStartDatafeedSchema = schema.object({
|
|||
});
|
||||
|
||||
export const jobIdsSchema = schema.object({
|
||||
/** Optional list of job ID(s). */
|
||||
jobIds: schema.maybe(
|
||||
schema.oneOf([schema.string(), schema.arrayOf(schema.maybe(schema.string()))])
|
||||
),
|
||||
|
|
|
|||
145
x-pack/test/api_integration/apis/ml/anomaly_detectors/create.ts
Normal file
145
x-pack/test/api_integration/apis/ml/anomaly_detectors/create.ts
Normal file
|
|
@ -0,0 +1,145 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import expect from '@kbn/expect';
|
||||
|
||||
import { FtrProviderContext } from '../../../ftr_provider_context';
|
||||
import { USER } from '../../../../functional/services/machine_learning/security_common';
|
||||
|
||||
const COMMON_HEADERS = {
|
||||
'kbn-xsrf': 'some-xsrf-token',
|
||||
};
|
||||
|
||||
// eslint-disable-next-line import/no-default-export
|
||||
export default ({ getService }: FtrProviderContext) => {
|
||||
const esArchiver = getService('esArchiver');
|
||||
const supertest = getService('supertestWithoutAuth');
|
||||
const ml = getService('ml');
|
||||
|
||||
const jobId = `fq_single_${Date.now()}`;
|
||||
|
||||
const testDataList = [
|
||||
{
|
||||
testTitle: 'ML Poweruser creates a single metric job',
|
||||
user: USER.ML_POWERUSER,
|
||||
jobId: `${jobId}_1`,
|
||||
requestBody: {
|
||||
job_id: `${jobId}_1`,
|
||||
description:
|
||||
'Single metric job based on the farequote dataset with 30m bucketspan and mean(responsetime)',
|
||||
groups: ['automated', 'farequote', 'single-metric'],
|
||||
analysis_config: {
|
||||
bucket_span: '30m',
|
||||
detectors: [{ function: 'mean', field_name: 'responsetime' }],
|
||||
influencers: [],
|
||||
summary_count_field_name: 'doc_count',
|
||||
},
|
||||
data_description: { time_field: '@timestamp' },
|
||||
analysis_limits: { model_memory_limit: '11MB' },
|
||||
model_plot_config: { enabled: true },
|
||||
},
|
||||
expected: {
|
||||
responseCode: 200,
|
||||
responseBody: {
|
||||
job_id: `${jobId}_1`,
|
||||
job_type: 'anomaly_detector',
|
||||
groups: ['automated', 'farequote', 'single-metric'],
|
||||
description:
|
||||
'Single metric job based on the farequote dataset with 30m bucketspan and mean(responsetime)',
|
||||
analysis_config: {
|
||||
bucket_span: '30m',
|
||||
summary_count_field_name: 'doc_count',
|
||||
detectors: [
|
||||
{
|
||||
detector_description: 'mean(responsetime)',
|
||||
function: 'mean',
|
||||
field_name: 'responsetime',
|
||||
detector_index: 0,
|
||||
},
|
||||
],
|
||||
influencers: [],
|
||||
},
|
||||
analysis_limits: { model_memory_limit: '11mb', categorization_examples_limit: 4 },
|
||||
data_description: { time_field: '@timestamp', time_format: 'epoch_ms' },
|
||||
model_plot_config: { enabled: true },
|
||||
model_snapshot_retention_days: 1,
|
||||
results_index_name: 'shared',
|
||||
allow_lazy_open: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
testTitle: 'ML viewer cannot create a job',
|
||||
user: USER.ML_VIEWER,
|
||||
jobId: `${jobId}_2`,
|
||||
requestBody: {
|
||||
job_id: `${jobId}_2`,
|
||||
description:
|
||||
'Single metric job based on the farequote dataset with 30m bucketspan and mean(responsetime)',
|
||||
groups: ['automated', 'farequote', 'single-metric'],
|
||||
analysis_config: {
|
||||
bucket_span: '30m',
|
||||
detectors: [{ function: 'mean', field_name: 'responsetime' }],
|
||||
influencers: [],
|
||||
summary_count_field_name: 'doc_count',
|
||||
},
|
||||
data_description: { time_field: '@timestamp' },
|
||||
analysis_limits: { model_memory_limit: '11MB' },
|
||||
model_plot_config: { enabled: true },
|
||||
},
|
||||
expected: {
|
||||
responseCode: 403,
|
||||
responseBody: {
|
||||
statusCode: 403,
|
||||
error: 'Forbidden',
|
||||
message:
|
||||
'[security_exception] action [cluster:admin/xpack/ml/job/put] is unauthorized for user [ml_viewer]',
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
describe('create', function() {
|
||||
before(async () => {
|
||||
await esArchiver.loadIfNeeded('ml/farequote');
|
||||
await ml.testResources.setKibanaTimeZoneToUTC();
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
await ml.api.cleanMlIndices();
|
||||
});
|
||||
|
||||
for (const testData of testDataList) {
|
||||
it(`${testData.testTitle}`, async () => {
|
||||
const { body } = await supertest
|
||||
.put(`/api/ml/anomaly_detectors/${testData.jobId}`)
|
||||
.auth(testData.user, ml.securityCommon.getPasswordForUser(testData.user))
|
||||
.set(COMMON_HEADERS)
|
||||
.send(testData.requestBody)
|
||||
.expect(testData.expected.responseCode);
|
||||
|
||||
if (body.error === undefined) {
|
||||
// Validate the important parts of the response.
|
||||
const expectedResponse = testData.expected.responseBody;
|
||||
expect(body.job_id).to.eql(expectedResponse.job_id);
|
||||
expect(body.groups).to.eql(expectedResponse.groups);
|
||||
expect(body.analysis_config!.bucket_span).to.eql(
|
||||
expectedResponse.analysis_config!.bucket_span
|
||||
);
|
||||
expect(body.analysis_config.detectors).to.have.length(
|
||||
expectedResponse.analysis_config!.detectors.length
|
||||
);
|
||||
expect(body.analysis_config.detectors[0]).to.eql(
|
||||
expectedResponse.analysis_config!.detectors[0]
|
||||
);
|
||||
} else {
|
||||
expect(body.error).to.eql(testData.expected.responseBody.error);
|
||||
expect(body.message).to.eql(testData.expected.responseBody.message);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
};
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
import { FtrProviderContext } from '../../../ftr_provider_context';
|
||||
|
||||
export default function({ loadTestFile }: FtrProviderContext) {
|
||||
describe('anomaly detectors', function() {
|
||||
loadTestFile(require.resolve('./create'));
|
||||
});
|
||||
}
|
||||
|
|
@ -0,0 +1,248 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import expect from '@kbn/expect';
|
||||
|
||||
import { FtrProviderContext } from '../../../ftr_provider_context';
|
||||
import { USER } from '../../../../functional/services/machine_learning/security_common';
|
||||
|
||||
const COMMON_HEADERS = {
|
||||
'kbn-xsrf': 'some-xsrf-token',
|
||||
};
|
||||
|
||||
// eslint-disable-next-line import/no-default-export
|
||||
export default ({ getService }: FtrProviderContext) => {
|
||||
const esArchiver = getService('esArchiver');
|
||||
const supertest = getService('supertestWithoutAuth');
|
||||
const ml = getService('ml');
|
||||
|
||||
const metricFieldsTestData = {
|
||||
testTitle: 'returns stats for metric fields over all time',
|
||||
index: 'ft_farequote',
|
||||
user: USER.ML_POWERUSER,
|
||||
requestBody: {
|
||||
query: {
|
||||
bool: {
|
||||
must: {
|
||||
term: { airline: 'JZA' }, // Only use one airline to ensure no sampling.
|
||||
},
|
||||
},
|
||||
},
|
||||
fields: [
|
||||
{ type: 'number', cardinality: 0 },
|
||||
{ fieldName: 'responsetime', type: 'number', cardinality: 4249 },
|
||||
],
|
||||
samplerShardSize: -1, // No sampling, as otherwise counts could vary on each run.
|
||||
timeFieldName: '@timestamp',
|
||||
interval: '1d',
|
||||
maxExamples: 10,
|
||||
},
|
||||
expected: {
|
||||
responseCode: 200,
|
||||
responseBody: [
|
||||
{
|
||||
documentCounts: {
|
||||
interval: '1d',
|
||||
buckets: {
|
||||
'1454803200000': 846,
|
||||
'1454889600000': 846,
|
||||
'1454976000000': 859,
|
||||
'1455062400000': 851,
|
||||
'1455148800000': 858,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
// Cannot verify median and percentiles responses as the ES percentiles agg is non-deterministic.
|
||||
fieldName: 'responsetime',
|
||||
count: 4260,
|
||||
min: 963.4293212890625,
|
||||
max: 1042.13525390625,
|
||||
avg: 1000.0378077547315,
|
||||
isTopValuesSampled: false,
|
||||
topValues: [
|
||||
{ key: 980.0411987304688, doc_count: 2 },
|
||||
{ key: 989.278076171875, doc_count: 2 },
|
||||
{ key: 989.763916015625, doc_count: 2 },
|
||||
{ key: 991.290771484375, doc_count: 2 },
|
||||
{ key: 992.0765991210938, doc_count: 2 },
|
||||
{ key: 993.8115844726562, doc_count: 2 },
|
||||
{ key: 993.8973999023438, doc_count: 2 },
|
||||
{ key: 994.0230102539062, doc_count: 2 },
|
||||
{ key: 994.364990234375, doc_count: 2 },
|
||||
{ key: 994.916015625, doc_count: 2 },
|
||||
],
|
||||
topValuesSampleSize: 4260,
|
||||
topValuesSamplerShardSize: -1,
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
const nonMetricFieldsTestData = {
|
||||
testTitle: 'returns stats for non-metric fields specifying query and time range',
|
||||
index: 'ft_farequote',
|
||||
user: USER.ML_POWERUSER,
|
||||
requestBody: {
|
||||
query: {
|
||||
bool: {
|
||||
must: {
|
||||
term: { airline: 'AAL' },
|
||||
},
|
||||
},
|
||||
},
|
||||
fields: [
|
||||
{ fieldName: '@timestamp', type: 'date', cardinality: 4751 },
|
||||
{ fieldName: '@version.keyword', type: 'keyword', cardinality: 1 },
|
||||
{ fieldName: 'airline', type: 'keyword', cardinality: 19 },
|
||||
{ fieldName: 'type', type: 'text', cardinality: 0 },
|
||||
{ fieldName: 'type.keyword', type: 'keyword', cardinality: 1 },
|
||||
],
|
||||
samplerShardSize: -1, // No sampling, as otherwise counts would vary on each run.
|
||||
timeFieldName: '@timestamp',
|
||||
earliest: 1454889600000, // February 8, 2016 12:00:00 AM GMT
|
||||
latest: 1454976000000, // February 9, 2016 12:00:00 AM GMT
|
||||
maxExamples: 10,
|
||||
},
|
||||
expected: {
|
||||
responseCode: 200,
|
||||
responseBody: [
|
||||
{ fieldName: '@timestamp', count: 1733, earliest: 1454889602000, latest: 1454975948000 },
|
||||
{
|
||||
fieldName: '@version.keyword',
|
||||
isTopValuesSampled: false,
|
||||
topValues: [{ key: '1', doc_count: 1733 }],
|
||||
topValuesSampleSize: 1733,
|
||||
topValuesSamplerShardSize: -1,
|
||||
},
|
||||
{
|
||||
fieldName: 'airline',
|
||||
isTopValuesSampled: false,
|
||||
topValues: [{ key: 'AAL', doc_count: 1733 }],
|
||||
topValuesSampleSize: 1733,
|
||||
topValuesSamplerShardSize: -1,
|
||||
},
|
||||
{
|
||||
fieldName: 'type.keyword',
|
||||
isTopValuesSampled: false,
|
||||
topValues: [{ key: 'farequote', doc_count: 1733 }],
|
||||
topValuesSampleSize: 1733,
|
||||
topValuesSamplerShardSize: -1,
|
||||
},
|
||||
{ fieldName: 'type', examples: ['farequote'] },
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
const errorTestData = {
|
||||
testTitle: 'returns error for index which does not exist',
|
||||
index: 'ft_farequote_not_exists',
|
||||
user: USER.ML_POWERUSER,
|
||||
requestBody: {
|
||||
query: { bool: { must: [{ match_all: {} }] } },
|
||||
fields: [
|
||||
{ type: 'number', cardinality: 0 },
|
||||
{ fieldName: 'responsetime', type: 'number', cardinality: 4249 },
|
||||
],
|
||||
samplerShardSize: -1, // No sampling, as otherwise counts could vary on each run.
|
||||
timeFieldName: '@timestamp',
|
||||
maxExamples: 10,
|
||||
},
|
||||
expected: {
|
||||
responseCode: 404,
|
||||
responseBody: {
|
||||
statusCode: 404,
|
||||
error: 'Not Found',
|
||||
message:
|
||||
'[index_not_found_exception] no such index [ft_farequote_not_exists], with { resource.type="index_or_alias" & resource.id="ft_farequote_not_exists" & index_uuid="_na_" & index="ft_farequote_not_exists" }',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
async function runGetFieldStatsRequest(
|
||||
index: string,
|
||||
user: USER,
|
||||
requestBody: object,
|
||||
expectedResponsecode: number
|
||||
): Promise<any> {
|
||||
const { body } = await supertest
|
||||
.post(`/api/ml/data_visualizer/get_field_stats/${index}`)
|
||||
.auth(user, ml.securityCommon.getPasswordForUser(user))
|
||||
.set(COMMON_HEADERS)
|
||||
.send(requestBody)
|
||||
.expect(expectedResponsecode);
|
||||
|
||||
return body;
|
||||
}
|
||||
|
||||
function compareByFieldName(a: { fieldName: string }, b: { fieldName: string }) {
|
||||
if (a.fieldName < b.fieldName) {
|
||||
return -1;
|
||||
}
|
||||
if (a.fieldName > b.fieldName) {
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
describe('get_field_stats', function() {
|
||||
before(async () => {
|
||||
await esArchiver.loadIfNeeded('ml/farequote');
|
||||
await ml.testResources.setKibanaTimeZoneToUTC();
|
||||
});
|
||||
|
||||
it(`${metricFieldsTestData.testTitle}`, async () => {
|
||||
const body = await runGetFieldStatsRequest(
|
||||
metricFieldsTestData.index,
|
||||
metricFieldsTestData.user,
|
||||
metricFieldsTestData.requestBody,
|
||||
metricFieldsTestData.expected.responseCode
|
||||
);
|
||||
|
||||
// Cannot verify median and percentiles responses as the ES percentiles agg is non-deterministic.
|
||||
const expected = metricFieldsTestData.expected;
|
||||
expect(body).to.have.length(expected.responseBody.length);
|
||||
|
||||
const actualDocCounts = body[0];
|
||||
const expectedDocCounts = expected.responseBody[0];
|
||||
expect(actualDocCounts).to.eql(expectedDocCounts);
|
||||
|
||||
const actualFieldData = { ...body[1] };
|
||||
delete actualFieldData.median;
|
||||
delete actualFieldData.distribution;
|
||||
|
||||
expect(actualFieldData).to.eql(expected.responseBody[1]);
|
||||
});
|
||||
|
||||
it(`${nonMetricFieldsTestData.testTitle}`, async () => {
|
||||
const body = await runGetFieldStatsRequest(
|
||||
nonMetricFieldsTestData.index,
|
||||
nonMetricFieldsTestData.user,
|
||||
nonMetricFieldsTestData.requestBody,
|
||||
nonMetricFieldsTestData.expected.responseCode
|
||||
);
|
||||
|
||||
// Sort the fields in the response before validating.
|
||||
const expectedRspFields = nonMetricFieldsTestData.expected.responseBody.sort(
|
||||
compareByFieldName
|
||||
);
|
||||
const actualRspFields = body.sort(compareByFieldName);
|
||||
expect(actualRspFields).to.eql(expectedRspFields);
|
||||
});
|
||||
|
||||
it(`${errorTestData.testTitle}`, async () => {
|
||||
const body = await runGetFieldStatsRequest(
|
||||
errorTestData.index,
|
||||
errorTestData.user,
|
||||
errorTestData.requestBody,
|
||||
errorTestData.expected.responseCode
|
||||
);
|
||||
|
||||
expect(body.error).to.eql(errorTestData.expected.responseBody.error);
|
||||
expect(body.message).to.eql(errorTestData.expected.responseBody.message);
|
||||
});
|
||||
});
|
||||
};
|
||||
|
|
@ -0,0 +1,154 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import expect from '@kbn/expect';
|
||||
|
||||
import { FtrProviderContext } from '../../../ftr_provider_context';
|
||||
import { USER } from '../../../../functional/services/machine_learning/security_common';
|
||||
|
||||
const COMMON_HEADERS = {
|
||||
'kbn-xsrf': 'some-xsrf-token',
|
||||
};
|
||||
|
||||
// eslint-disable-next-line import/no-default-export
|
||||
export default ({ getService }: FtrProviderContext) => {
|
||||
const esArchiver = getService('esArchiver');
|
||||
const supertest = getService('supertestWithoutAuth');
|
||||
const ml = getService('ml');
|
||||
|
||||
const testDataList = [
|
||||
{
|
||||
testTitle: 'returns stats over all time',
|
||||
index: 'ft_farequote',
|
||||
user: USER.ML_POWERUSER,
|
||||
requestBody: {
|
||||
query: { bool: { must: [{ match_all: {} }] } },
|
||||
aggregatableFields: ['@timestamp', 'airline', 'responsetime', 'sourcetype'],
|
||||
nonAggregatableFields: ['type'],
|
||||
samplerShardSize: -1, // No sampling, as otherwise counts would vary on each run.
|
||||
timeFieldName: '@timestamp',
|
||||
},
|
||||
expected: {
|
||||
responseCode: 200,
|
||||
responseBody: {
|
||||
totalCount: 86274,
|
||||
aggregatableExistsFields: [
|
||||
{
|
||||
fieldName: '@timestamp',
|
||||
existsInDocs: true,
|
||||
stats: { sampleCount: 86274, count: 86274, cardinality: 78580 },
|
||||
},
|
||||
{
|
||||
fieldName: 'airline',
|
||||
existsInDocs: true,
|
||||
stats: { sampleCount: 86274, count: 86274, cardinality: 19 },
|
||||
},
|
||||
{
|
||||
fieldName: 'responsetime',
|
||||
existsInDocs: true,
|
||||
stats: { sampleCount: 86274, count: 86274, cardinality: 83346 },
|
||||
},
|
||||
],
|
||||
aggregatableNotExistsFields: [{ fieldName: 'sourcetype', existsInDocs: false }],
|
||||
nonAggregatableExistsFields: [{ fieldName: 'type', existsInDocs: true, stats: {} }],
|
||||
nonAggregatableNotExistsFields: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
testTitle: 'returns stats when specifying query and time range',
|
||||
index: 'ft_farequote',
|
||||
user: USER.ML_POWERUSER,
|
||||
requestBody: {
|
||||
query: {
|
||||
bool: {
|
||||
must: {
|
||||
term: { airline: 'AAL' },
|
||||
},
|
||||
},
|
||||
},
|
||||
aggregatableFields: ['@timestamp', 'airline', 'responsetime', 'sourcetype'],
|
||||
nonAggregatableFields: ['type'],
|
||||
samplerShardSize: -1, // No sampling, as otherwise counts would vary on each run.
|
||||
timeFieldName: '@timestamp',
|
||||
earliest: 1454889600000, // February 8, 2016 12:00:00 AM GMT
|
||||
latest: 1454976000000, // February 9, 2016 12:00:00 AM GMT
|
||||
},
|
||||
expected: {
|
||||
responseCode: 200,
|
||||
responseBody: {
|
||||
totalCount: 1733,
|
||||
aggregatableExistsFields: [
|
||||
{
|
||||
fieldName: '@timestamp',
|
||||
existsInDocs: true,
|
||||
stats: { sampleCount: 1733, count: 1733, cardinality: 1713 },
|
||||
},
|
||||
{
|
||||
fieldName: 'airline',
|
||||
existsInDocs: true,
|
||||
stats: { sampleCount: 1733, count: 1733, cardinality: 1 },
|
||||
},
|
||||
{
|
||||
fieldName: 'responsetime',
|
||||
existsInDocs: true,
|
||||
stats: { sampleCount: 1733, count: 1733, cardinality: 1730 },
|
||||
},
|
||||
],
|
||||
aggregatableNotExistsFields: [{ fieldName: 'sourcetype', existsInDocs: false }],
|
||||
nonAggregatableExistsFields: [{ fieldName: 'type', existsInDocs: true, stats: {} }],
|
||||
nonAggregatableNotExistsFields: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
testTitle: 'returns error for index which does not exist',
|
||||
index: 'ft_farequote_not_exist',
|
||||
user: USER.ML_POWERUSER,
|
||||
requestBody: {
|
||||
query: { bool: { must: [{ match_all: {} }] } },
|
||||
aggregatableFields: ['@timestamp', 'airline', 'responsetime', 'sourcetype'],
|
||||
nonAggregatableFields: ['@version', 'type'],
|
||||
samplerShardSize: 1000,
|
||||
timeFieldName: '@timestamp',
|
||||
},
|
||||
expected: {
|
||||
responseCode: 404,
|
||||
responseBody: {
|
||||
statusCode: 404,
|
||||
error: 'Not Found',
|
||||
message:
|
||||
'[index_not_found_exception] no such index [ft_farequote_not_exist], with { resource.type="index_or_alias" & resource.id="ft_farequote_not_exist" & index_uuid="_na_" & index="ft_farequote_not_exist" }',
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
describe('get_overall_stats', function() {
|
||||
before(async () => {
|
||||
await esArchiver.loadIfNeeded('ml/farequote');
|
||||
await ml.testResources.setKibanaTimeZoneToUTC();
|
||||
});
|
||||
|
||||
for (const testData of testDataList) {
|
||||
it(`${testData.testTitle}`, async () => {
|
||||
const { body } = await supertest
|
||||
.post(`/api/ml/data_visualizer/get_overall_stats/${testData.index}`)
|
||||
.auth(testData.user, ml.securityCommon.getPasswordForUser(testData.user))
|
||||
.set(COMMON_HEADERS)
|
||||
.send(testData.requestBody)
|
||||
.expect(testData.expected.responseCode);
|
||||
|
||||
if (body.error === undefined) {
|
||||
expect(body).to.eql(testData.expected.responseBody);
|
||||
} else {
|
||||
expect(body.error).to.eql(testData.expected.responseBody.error);
|
||||
expect(body.message).to.eql(testData.expected.responseBody.message);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
};
|
||||
13
x-pack/test/api_integration/apis/ml/data_visualizer/index.ts
Normal file
13
x-pack/test/api_integration/apis/ml/data_visualizer/index.ts
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
import { FtrProviderContext } from '../../../ftr_provider_context';
|
||||
|
||||
export default function({ loadTestFile }: FtrProviderContext) {
|
||||
describe('data visualizer', function() {
|
||||
loadTestFile(require.resolve('./get_field_stats'));
|
||||
loadTestFile(require.resolve('./get_overall_stats'));
|
||||
});
|
||||
}
|
||||
|
|
@ -0,0 +1,115 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import expect from '@kbn/expect';
|
||||
|
||||
import { FtrProviderContext } from '../../../ftr_provider_context';
|
||||
import { USER } from '../../../../functional/services/machine_learning/security_common';
|
||||
|
||||
const COMMON_HEADERS = {
|
||||
'kbn-xsrf': 'some-xsrf-token',
|
||||
};
|
||||
|
||||
// eslint-disable-next-line import/no-default-export
|
||||
export default ({ getService }: FtrProviderContext) => {
|
||||
const esArchiver = getService('esArchiver');
|
||||
const supertest = getService('supertestWithoutAuth');
|
||||
const ml = getService('ml');
|
||||
|
||||
const testDataList = [
|
||||
{
|
||||
testTitle: 'returns cardinality of customer name fields over full time range',
|
||||
user: USER.ML_POWERUSER,
|
||||
requestBody: {
|
||||
index: 'ft_ecommerce',
|
||||
fieldNames: ['customer_first_name.keyword', 'customer_last_name.keyword'],
|
||||
query: { bool: { must: [{ match_all: {} }] } },
|
||||
timeFieldName: 'order_date',
|
||||
},
|
||||
expected: {
|
||||
responseBody: {
|
||||
'customer_first_name.keyword': 46,
|
||||
'customer_last_name.keyword': 183,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
testTitle: 'returns cardinality of geoip fields over specified range',
|
||||
user: USER.ML_POWERUSER,
|
||||
requestBody: {
|
||||
index: 'ft_ecommerce',
|
||||
fieldNames: ['geoip.city_name', 'geoip.continent_name', 'geoip.country_iso_code'],
|
||||
query: { bool: { must: [{ match_all: {} }] } },
|
||||
timeFieldName: 'order_date',
|
||||
earliestMs: 1560556800000, // June 15, 2019 12:00:00 AM GMT
|
||||
latestMs: 1560643199000, // June 15, 2019 11:59:59 PM GMT
|
||||
},
|
||||
expected: {
|
||||
responseBody: {
|
||||
'geoip.city_name': 10,
|
||||
'geoip.continent_name': 5,
|
||||
'geoip.country_iso_code': 9,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
testTitle: 'returns empty response for non aggregatable field',
|
||||
user: USER.ML_POWERUSER,
|
||||
requestBody: {
|
||||
index: 'ft_ecommerce',
|
||||
fieldNames: ['manufacturer'],
|
||||
query: { bool: { must: [{ match_all: {} }] } },
|
||||
timeFieldName: 'order_date',
|
||||
earliestMs: 1560556800000, // June 15, 2019 12:00:00 AM GMT
|
||||
latestMs: 1560643199000, // June 15, 2019 11:59:59 PM GMT
|
||||
},
|
||||
expected: {
|
||||
responseBody: {},
|
||||
},
|
||||
},
|
||||
{
|
||||
testTitle: 'returns error for index which does not exist',
|
||||
user: USER.ML_POWERUSER,
|
||||
requestBody: {
|
||||
index: 'ft_ecommerce_not_exist',
|
||||
fieldNames: ['customer_first_name.keyword', 'customer_last_name.keyword'],
|
||||
timeFieldName: 'order_date',
|
||||
},
|
||||
expected: {
|
||||
responseBody: {
|
||||
statusCode: 404,
|
||||
error: 'Not Found',
|
||||
message:
|
||||
'[index_not_found_exception] no such index [ft_ecommerce_not_exist], with { resource.type="index_or_alias" & resource.id="ft_ecommerce_not_exist" & index_uuid="_na_" & index="ft_ecommerce_not_exist" }',
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
describe('field_cardinality', function() {
|
||||
before(async () => {
|
||||
await esArchiver.loadIfNeeded('ml/ecommerce');
|
||||
await ml.testResources.setKibanaTimeZoneToUTC();
|
||||
});
|
||||
|
||||
for (const testData of testDataList) {
|
||||
it(`${testData.testTitle}`, async () => {
|
||||
const { body } = await supertest
|
||||
.post('/api/ml/fields_service/field_cardinality')
|
||||
.auth(testData.user, ml.securityCommon.getPasswordForUser(testData.user))
|
||||
.set(COMMON_HEADERS)
|
||||
.send(testData.requestBody);
|
||||
|
||||
if (body.error === undefined) {
|
||||
expect(body).to.eql(testData.expected.responseBody);
|
||||
} else {
|
||||
expect(body.error).to.eql(testData.expected.responseBody.error);
|
||||
expect(body.message).to.eql(testData.expected.responseBody.message);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
};
|
||||
13
x-pack/test/api_integration/apis/ml/fields_service/index.ts
Normal file
13
x-pack/test/api_integration/apis/ml/fields_service/index.ts
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
import { FtrProviderContext } from '../../../ftr_provider_context';
|
||||
|
||||
export default function({ loadTestFile }: FtrProviderContext) {
|
||||
describe('fields service', function() {
|
||||
loadTestFile(require.resolve('./field_cardinality'));
|
||||
loadTestFile(require.resolve('./time_field_range'));
|
||||
});
|
||||
}
|
||||
|
|
@ -0,0 +1,119 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import expect from '@kbn/expect';
|
||||
|
||||
import { FtrProviderContext } from '../../../ftr_provider_context';
|
||||
import { USER } from '../../../../functional/services/machine_learning/security_common';
|
||||
|
||||
const COMMON_HEADERS = {
|
||||
'kbn-xsrf': 'some-xsrf-token',
|
||||
};
|
||||
|
||||
// eslint-disable-next-line import/no-default-export
|
||||
export default ({ getService }: FtrProviderContext) => {
|
||||
const esArchiver = getService('esArchiver');
|
||||
const supertest = getService('supertestWithoutAuth');
|
||||
const ml = getService('ml');
|
||||
|
||||
const testDataList = [
|
||||
{
|
||||
testTitle: 'returns expected time range with index and match_all query',
|
||||
user: USER.ML_POWERUSER,
|
||||
requestBody: {
|
||||
index: 'ft_ecommerce',
|
||||
query: { bool: { must: [{ match_all: {} }] } },
|
||||
timeFieldName: 'order_date',
|
||||
},
|
||||
expected: {
|
||||
responseCode: 200,
|
||||
responseBody: {
|
||||
start: {
|
||||
epoch: 1560297859000,
|
||||
string: '2019-06-12T00:04:19.000Z',
|
||||
},
|
||||
end: {
|
||||
epoch: 1562975136000,
|
||||
string: '2019-07-12T23:45:36.000Z',
|
||||
},
|
||||
success: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
testTitle: 'returns expected time range with index and query',
|
||||
user: USER.ML_POWERUSER,
|
||||
requestBody: {
|
||||
index: 'ft_ecommerce',
|
||||
query: {
|
||||
term: {
|
||||
'customer_first_name.keyword': {
|
||||
value: 'Brigitte',
|
||||
},
|
||||
},
|
||||
},
|
||||
timeFieldName: 'order_date',
|
||||
},
|
||||
expected: {
|
||||
responseCode: 200,
|
||||
responseBody: {
|
||||
start: {
|
||||
epoch: 1560298982000,
|
||||
string: '2019-06-12T00:23:02.000Z',
|
||||
},
|
||||
end: {
|
||||
epoch: 1562973754000,
|
||||
string: '2019-07-12T23:22:34.000Z',
|
||||
},
|
||||
success: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
testTitle: 'returns error for index which does not exist',
|
||||
user: USER.ML_POWERUSER,
|
||||
requestBody: {
|
||||
index: 'ft_ecommerce_not_exist',
|
||||
query: { bool: { must: [{ match_all: {} }] } },
|
||||
timeFieldName: 'order_date',
|
||||
},
|
||||
expected: {
|
||||
responseCode: 404,
|
||||
responseBody: {
|
||||
statusCode: 404,
|
||||
error: 'Not Found',
|
||||
message:
|
||||
'[index_not_found_exception] no such index [ft_ecommerce_not_exist], with { resource.type="index_or_alias" & resource.id="ft_ecommerce_not_exist" & index_uuid="_na_" & index="ft_ecommerce_not_exist" }',
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
describe('time_field_range', function() {
|
||||
before(async () => {
|
||||
await esArchiver.loadIfNeeded('ml/ecommerce');
|
||||
await ml.testResources.setKibanaTimeZoneToUTC();
|
||||
});
|
||||
|
||||
for (const testData of testDataList) {
|
||||
it(`${testData.testTitle}`, async () => {
|
||||
const { body } = await supertest
|
||||
.post('/api/ml/fields_service/time_field_range')
|
||||
.auth(testData.user, ml.securityCommon.getPasswordForUser(testData.user))
|
||||
.set(COMMON_HEADERS)
|
||||
.send(testData.requestBody)
|
||||
.expect(testData.expected.responseCode);
|
||||
|
||||
if (body.error === undefined) {
|
||||
expect(body).to.eql(testData.expected.responseBody);
|
||||
} else {
|
||||
expect(body.error).to.eql(testData.expected.responseBody.error);
|
||||
expect(body.message).to.eql(testData.expected.responseBody.message);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
};
|
||||
|
|
@ -31,11 +31,11 @@ export default function({ getService, loadTestFile }: FtrProviderContext) {
|
|||
await ml.testResources.resetKibanaTimeZone();
|
||||
});
|
||||
|
||||
loadTestFile(require.resolve('./bucket_span_estimator'));
|
||||
loadTestFile(require.resolve('./calculate_model_memory_limit'));
|
||||
loadTestFile(require.resolve('./categorization_field_examples'));
|
||||
loadTestFile(require.resolve('./get_module'));
|
||||
loadTestFile(require.resolve('./recognize_module'));
|
||||
loadTestFile(require.resolve('./setup_module'));
|
||||
loadTestFile(require.resolve('./modules'));
|
||||
loadTestFile(require.resolve('./anomaly_detectors'));
|
||||
loadTestFile(require.resolve('./data_visualizer'));
|
||||
loadTestFile(require.resolve('./fields_service'));
|
||||
loadTestFile(require.resolve('./job_validation'));
|
||||
loadTestFile(require.resolve('./jobs'));
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,8 +6,8 @@
|
|||
|
||||
import expect from '@kbn/expect';
|
||||
|
||||
import { FtrProviderContext } from '../../ftr_provider_context';
|
||||
import { USER } from '../../../functional/services/machine_learning/security_common';
|
||||
import { FtrProviderContext } from '../../../ftr_provider_context';
|
||||
import { USER } from '../../../../functional/services/machine_learning/security_common';
|
||||
|
||||
const COMMON_HEADERS = {
|
||||
'kbn-xsrf': 'some-xsrf-token',
|
||||
|
|
@ -4,8 +4,8 @@
|
|||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { FtrProviderContext } from '../../ftr_provider_context';
|
||||
import { USER } from '../../../functional/services/machine_learning/security_common';
|
||||
import { FtrProviderContext } from '../../../ftr_provider_context';
|
||||
import { USER } from '../../../../functional/services/machine_learning/security_common';
|
||||
|
||||
const COMMON_HEADERS = {
|
||||
'kbn-xsrf': 'some-xsrf-token',
|
||||
13
x-pack/test/api_integration/apis/ml/job_validation/index.ts
Normal file
13
x-pack/test/api_integration/apis/ml/job_validation/index.ts
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
import { FtrProviderContext } from '../../../ftr_provider_context';
|
||||
|
||||
export default function({ loadTestFile }: FtrProviderContext) {
|
||||
describe('job validation', function() {
|
||||
loadTestFile(require.resolve('./bucket_span_estimator'));
|
||||
loadTestFile(require.resolve('./calculate_model_memory_limit'));
|
||||
});
|
||||
}
|
||||
|
|
@ -6,8 +6,8 @@
|
|||
|
||||
import expect from '@kbn/expect';
|
||||
|
||||
import { FtrProviderContext } from '../../ftr_provider_context';
|
||||
import { USER } from '../../../functional/services/machine_learning/security_common';
|
||||
import { FtrProviderContext } from '../../../ftr_provider_context';
|
||||
import { USER } from '../../../../functional/services/machine_learning/security_common';
|
||||
|
||||
const COMMON_HEADERS = {
|
||||
'kbn-xsrf': 'some-xsrf-token',
|
||||
13
x-pack/test/api_integration/apis/ml/jobs/index.ts
Normal file
13
x-pack/test/api_integration/apis/ml/jobs/index.ts
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
import { FtrProviderContext } from '../../../ftr_provider_context';
|
||||
|
||||
export default function({ loadTestFile }: FtrProviderContext) {
|
||||
describe('jobs', function() {
|
||||
loadTestFile(require.resolve('./categorization_field_examples'));
|
||||
loadTestFile(require.resolve('./jobs_summary'));
|
||||
});
|
||||
}
|
||||
374
x-pack/test/api_integration/apis/ml/jobs/jobs_summary.ts
Normal file
374
x-pack/test/api_integration/apis/ml/jobs/jobs_summary.ts
Normal file
|
|
@ -0,0 +1,374 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import expect from '@kbn/expect';
|
||||
|
||||
import { FtrProviderContext } from '../../../ftr_provider_context';
|
||||
import { USER } from '../../../../functional/services/machine_learning/security_common';
|
||||
import { Job } from '../../../../../plugins/ml/common/types/anomaly_detection_jobs';
|
||||
|
||||
const COMMON_HEADERS = {
|
||||
'kbn-xsrf': 'some-xsrf-token',
|
||||
};
|
||||
|
||||
const SINGLE_METRIC_JOB_CONFIG: Job = {
|
||||
job_id: `jobs_summary_fq_single_${Date.now()}`,
|
||||
description: 'mean(responsetime) on farequote dataset with 15m bucket span',
|
||||
groups: ['farequote', 'automated', 'single-metric'],
|
||||
analysis_config: {
|
||||
bucket_span: '15m',
|
||||
influencers: [],
|
||||
detectors: [
|
||||
{
|
||||
function: 'mean',
|
||||
field_name: 'responsetime',
|
||||
},
|
||||
],
|
||||
},
|
||||
data_description: { time_field: '@timestamp' },
|
||||
analysis_limits: { model_memory_limit: '10mb' },
|
||||
model_plot_config: { enabled: true },
|
||||
};
|
||||
|
||||
const MULTI_METRIC_JOB_CONFIG: Job = {
|
||||
job_id: `jobs_summary_fq_multi_${Date.now()}`,
|
||||
description: 'mean(responsetime) partition=airline on farequote dataset with 1h bucket span',
|
||||
groups: ['farequote', 'automated', 'multi-metric'],
|
||||
analysis_config: {
|
||||
bucket_span: '1h',
|
||||
influencers: ['airline'],
|
||||
detectors: [{ function: 'mean', field_name: 'responsetime', partition_field_name: 'airline' }],
|
||||
},
|
||||
data_description: { time_field: '@timestamp' },
|
||||
analysis_limits: { model_memory_limit: '20mb' },
|
||||
model_plot_config: { enabled: true },
|
||||
};
|
||||
|
||||
// eslint-disable-next-line import/no-default-export
|
||||
export default ({ getService }: FtrProviderContext) => {
|
||||
const esArchiver = getService('esArchiver');
|
||||
const supertest = getService('supertestWithoutAuth');
|
||||
const ml = getService('ml');
|
||||
|
||||
const testSetupJobConfigs = [SINGLE_METRIC_JOB_CONFIG, MULTI_METRIC_JOB_CONFIG];
|
||||
|
||||
const testDataListNoJobId = [
|
||||
{
|
||||
testTitle: 'as ML Poweruser',
|
||||
user: USER.ML_POWERUSER,
|
||||
requestBody: {},
|
||||
expected: {
|
||||
responseCode: 200,
|
||||
responseBody: [
|
||||
{
|
||||
id: SINGLE_METRIC_JOB_CONFIG.job_id,
|
||||
description: SINGLE_METRIC_JOB_CONFIG.description,
|
||||
groups: SINGLE_METRIC_JOB_CONFIG.groups,
|
||||
processed_record_count: 0,
|
||||
memory_status: 'ok',
|
||||
jobState: 'closed',
|
||||
hasDatafeed: false,
|
||||
datafeedId: '',
|
||||
datafeedIndices: [],
|
||||
datafeedState: '',
|
||||
isSingleMetricViewerJob: true,
|
||||
},
|
||||
{
|
||||
id: MULTI_METRIC_JOB_CONFIG.job_id,
|
||||
description: MULTI_METRIC_JOB_CONFIG.description,
|
||||
groups: MULTI_METRIC_JOB_CONFIG.groups,
|
||||
processed_record_count: 0,
|
||||
memory_status: 'ok',
|
||||
jobState: 'closed',
|
||||
hasDatafeed: false,
|
||||
datafeedId: '',
|
||||
datafeedIndices: [],
|
||||
datafeedState: '',
|
||||
isSingleMetricViewerJob: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
testTitle: 'as ML Viewer',
|
||||
user: USER.ML_VIEWER,
|
||||
requestBody: {},
|
||||
expected: {
|
||||
responseCode: 200,
|
||||
responseBody: [
|
||||
{
|
||||
id: SINGLE_METRIC_JOB_CONFIG.job_id,
|
||||
description: SINGLE_METRIC_JOB_CONFIG.description,
|
||||
groups: SINGLE_METRIC_JOB_CONFIG.groups,
|
||||
processed_record_count: 0,
|
||||
memory_status: 'ok',
|
||||
jobState: 'closed',
|
||||
hasDatafeed: false,
|
||||
datafeedId: '',
|
||||
datafeedIndices: [],
|
||||
datafeedState: '',
|
||||
isSingleMetricViewerJob: true,
|
||||
},
|
||||
{
|
||||
id: MULTI_METRIC_JOB_CONFIG.job_id,
|
||||
description: MULTI_METRIC_JOB_CONFIG.description,
|
||||
groups: MULTI_METRIC_JOB_CONFIG.groups,
|
||||
processed_record_count: 0,
|
||||
memory_status: 'ok',
|
||||
jobState: 'closed',
|
||||
hasDatafeed: false,
|
||||
datafeedId: '',
|
||||
datafeedIndices: [],
|
||||
datafeedState: '',
|
||||
isSingleMetricViewerJob: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const testDataListWithJobId = [
|
||||
{
|
||||
testTitle: 'as ML Poweruser',
|
||||
user: USER.ML_POWERUSER,
|
||||
requestBody: {
|
||||
jobIds: [SINGLE_METRIC_JOB_CONFIG.job_id],
|
||||
},
|
||||
expected: {
|
||||
responseCode: 200,
|
||||
responseBody: [
|
||||
{
|
||||
id: SINGLE_METRIC_JOB_CONFIG.job_id,
|
||||
description: SINGLE_METRIC_JOB_CONFIG.description,
|
||||
groups: SINGLE_METRIC_JOB_CONFIG.groups,
|
||||
processed_record_count: 0,
|
||||
memory_status: 'ok',
|
||||
jobState: 'closed',
|
||||
hasDatafeed: false,
|
||||
datafeedId: '',
|
||||
datafeedIndices: [],
|
||||
datafeedState: '',
|
||||
isSingleMetricViewerJob: true,
|
||||
fullJob: {
|
||||
// Only tests against some of the fields in the fullJob property.
|
||||
job_id: SINGLE_METRIC_JOB_CONFIG.job_id,
|
||||
job_type: 'anomaly_detector',
|
||||
description: SINGLE_METRIC_JOB_CONFIG.description,
|
||||
groups: SINGLE_METRIC_JOB_CONFIG.groups,
|
||||
analysis_config: {
|
||||
bucket_span: '15m',
|
||||
detectors: [
|
||||
{
|
||||
detector_description: 'mean(responsetime)',
|
||||
function: 'mean',
|
||||
field_name: 'responsetime',
|
||||
detector_index: 0,
|
||||
},
|
||||
],
|
||||
influencers: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
id: MULTI_METRIC_JOB_CONFIG.job_id,
|
||||
description: MULTI_METRIC_JOB_CONFIG.description,
|
||||
groups: MULTI_METRIC_JOB_CONFIG.groups,
|
||||
processed_record_count: 0,
|
||||
memory_status: 'ok',
|
||||
jobState: 'closed',
|
||||
hasDatafeed: false,
|
||||
datafeedId: '',
|
||||
datafeedIndices: [],
|
||||
datafeedState: '',
|
||||
isSingleMetricViewerJob: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const testDataListNegative = [
|
||||
{
|
||||
testTitle: 'as ML Unauthorized user',
|
||||
user: USER.ML_UNAUTHORIZED,
|
||||
requestBody: {},
|
||||
// Note that the jobs and datafeeds are loaded async so the actual error message is not deterministic.
|
||||
expected: {
|
||||
responseCode: 403,
|
||||
error: 'Forbidden',
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
async function runJobsSummaryRequest(
|
||||
user: USER,
|
||||
requestBody: object,
|
||||
expectedResponsecode: number
|
||||
): Promise<any> {
|
||||
const { body } = await supertest
|
||||
.post('/api/ml/jobs/jobs_summary')
|
||||
.auth(user, ml.securityCommon.getPasswordForUser(user))
|
||||
.set(COMMON_HEADERS)
|
||||
.send(requestBody)
|
||||
.expect(expectedResponsecode);
|
||||
|
||||
return body;
|
||||
}
|
||||
|
||||
function compareById(a: { id: string }, b: { id: string }) {
|
||||
if (a.id < b.id) {
|
||||
return -1;
|
||||
}
|
||||
if (a.id > b.id) {
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
function getGroups(jobs: Array<{ groups: string[] }>) {
|
||||
const groupIds: string[] = [];
|
||||
jobs.forEach(job => {
|
||||
const groups = job.groups;
|
||||
groups.forEach(group => {
|
||||
if (groupIds.indexOf(group) === -1) {
|
||||
groupIds.push(group);
|
||||
}
|
||||
});
|
||||
});
|
||||
return groupIds.sort();
|
||||
}
|
||||
|
||||
describe('jobs_summary', function() {
|
||||
before(async () => {
|
||||
await esArchiver.loadIfNeeded('ml/farequote');
|
||||
await ml.testResources.setKibanaTimeZoneToUTC();
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
await ml.api.cleanMlIndices();
|
||||
});
|
||||
|
||||
it('sets up jobs', async () => {
|
||||
for (const job of testSetupJobConfigs) {
|
||||
await ml.api.createAnomalyDetectionJob(job);
|
||||
}
|
||||
});
|
||||
|
||||
for (const testData of testDataListNoJobId) {
|
||||
describe('gets job summary with no job IDs supplied', function() {
|
||||
it(`${testData.testTitle}`, async () => {
|
||||
const body = await runJobsSummaryRequest(
|
||||
testData.user,
|
||||
testData.requestBody,
|
||||
testData.expected.responseCode
|
||||
);
|
||||
|
||||
// Validate the important parts of the response.
|
||||
const expectedResponse = testData.expected.responseBody;
|
||||
|
||||
// Validate job count.
|
||||
expect(body).to.have.length(expectedResponse.length);
|
||||
|
||||
// Validate job IDs.
|
||||
const expectedRspJobIds = expectedResponse
|
||||
.map((job: { id: string }) => {
|
||||
return { id: job.id };
|
||||
})
|
||||
.sort(compareById);
|
||||
const actualRspJobIds = body
|
||||
.map((job: { id: string }) => {
|
||||
return { id: job.id };
|
||||
})
|
||||
.sort(compareById);
|
||||
|
||||
expect(actualRspJobIds).to.eql(expectedRspJobIds);
|
||||
|
||||
// Validate created group IDs.
|
||||
const expectedRspGroupIds = getGroups(expectedResponse);
|
||||
const actualRspGroupsIds = getGroups(body);
|
||||
expect(actualRspGroupsIds).to.eql(expectedRspGroupIds);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
for (const testData of testDataListWithJobId) {
|
||||
describe('gets job summary with job ID supplied', function() {
|
||||
it(`${testData.testTitle}`, async () => {
|
||||
const body = await runJobsSummaryRequest(
|
||||
testData.user,
|
||||
testData.requestBody,
|
||||
testData.expected.responseCode
|
||||
);
|
||||
|
||||
// Validate the important parts of the response.
|
||||
const expectedResponse = testData.expected.responseBody;
|
||||
|
||||
// Validate job count.
|
||||
expect(body).to.have.length(expectedResponse.length);
|
||||
|
||||
// Validate job IDs.
|
||||
const expectedRspJobIds = expectedResponse
|
||||
.map((job: { id: string }) => {
|
||||
return { id: job.id };
|
||||
})
|
||||
.sort(compareById);
|
||||
const actualRspJobIds = body
|
||||
.map((job: { id: string }) => {
|
||||
return { id: job.id };
|
||||
})
|
||||
.sort(compareById);
|
||||
|
||||
expect(actualRspJobIds).to.eql(expectedRspJobIds);
|
||||
|
||||
// Validate created group IDs.
|
||||
const expectedRspGroupIds = getGroups(expectedResponse);
|
||||
const actualRspGroupsIds = getGroups(body);
|
||||
expect(actualRspGroupsIds).to.eql(expectedRspGroupIds);
|
||||
|
||||
// Validate the response for the specified job IDs contains a fullJob property.
|
||||
const requestedJobIds = testData.requestBody.jobIds;
|
||||
for (const job of body) {
|
||||
if (requestedJobIds.includes(job.id)) {
|
||||
expect(job).to.have.property('fullJob');
|
||||
} else {
|
||||
expect(job).not.to.have.property('fullJob');
|
||||
}
|
||||
}
|
||||
|
||||
for (const expectedJob of expectedResponse) {
|
||||
const expectedJobId = expectedJob.id;
|
||||
const actualJob = body.find((job: { id: string }) => job.id === expectedJobId);
|
||||
if (expectedJob.fullJob) {
|
||||
expect(actualJob).to.have.property('fullJob');
|
||||
expect(actualJob.fullJob).to.have.property('analysis_config');
|
||||
expect(actualJob.fullJob.analysis_config).to.eql(expectedJob.fullJob.analysis_config);
|
||||
} else {
|
||||
expect(actualJob).not.to.have.property('fullJob');
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
for (const testData of testDataListNegative) {
|
||||
describe('rejects request', function() {
|
||||
it(testData.testTitle, async () => {
|
||||
const body = await runJobsSummaryRequest(
|
||||
testData.user,
|
||||
testData.requestBody,
|
||||
testData.expected.responseCode
|
||||
);
|
||||
|
||||
expect(body)
|
||||
.to.have.property('error')
|
||||
.eql(testData.expected.error);
|
||||
|
||||
expect(body).to.have.property('message');
|
||||
});
|
||||
});
|
||||
}
|
||||
});
|
||||
};
|
||||
|
|
@ -6,8 +6,8 @@
|
|||
|
||||
import expect from '@kbn/expect';
|
||||
|
||||
import { FtrProviderContext } from '../../ftr_provider_context';
|
||||
import { USER } from '../../../functional/services/machine_learning/security_common';
|
||||
import { FtrProviderContext } from '../../../ftr_provider_context';
|
||||
import { USER } from '../../../../functional/services/machine_learning/security_common';
|
||||
|
||||
const COMMON_HEADERS = {
|
||||
'kbn-xsrf': 'some-xsrf-token',
|
||||
14
x-pack/test/api_integration/apis/ml/modules/index.ts
Normal file
14
x-pack/test/api_integration/apis/ml/modules/index.ts
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
import { FtrProviderContext } from '../../../ftr_provider_context';
|
||||
|
||||
export default function({ loadTestFile }: FtrProviderContext) {
|
||||
describe('modules', function() {
|
||||
loadTestFile(require.resolve('./get_module'));
|
||||
loadTestFile(require.resolve('./recognize_module'));
|
||||
loadTestFile(require.resolve('./setup_module'));
|
||||
});
|
||||
}
|
||||
|
|
@ -6,8 +6,8 @@
|
|||
|
||||
import expect from '@kbn/expect';
|
||||
|
||||
import { FtrProviderContext } from '../../ftr_provider_context';
|
||||
import { USER } from '../../../functional/services/machine_learning/security_common';
|
||||
import { FtrProviderContext } from '../../../ftr_provider_context';
|
||||
import { USER } from '../../../../functional/services/machine_learning/security_common';
|
||||
|
||||
const COMMON_HEADERS = {
|
||||
'kbn-xsrf': 'some-xsrf-token',
|
||||
|
|
@ -6,10 +6,10 @@
|
|||
|
||||
import expect from '@kbn/expect';
|
||||
|
||||
import { FtrProviderContext } from '../../ftr_provider_context';
|
||||
import { FtrProviderContext } from '../../../ftr_provider_context';
|
||||
|
||||
import { JOB_STATE, DATAFEED_STATE } from '../../../../plugins/ml/common/constants/states';
|
||||
import { USER } from '../../../functional/services/machine_learning/security_common';
|
||||
import { JOB_STATE, DATAFEED_STATE } from '../../../../../plugins/ml/common/constants/states';
|
||||
import { USER } from '../../../../functional/services/machine_learning/security_common';
|
||||
|
||||
const COMMON_HEADERS = {
|
||||
'kbn-xsrf': 'some-xsrf-token',
|
||||
Loading…
Reference in a new issue