From 3dcb98ec670e4b1b44febdf632523c9e5dbea955 Mon Sep 17 00:00:00 2001 From: Jean-Louis Leysens Date: Fri, 15 Jan 2021 13:32:16 +0100 Subject: [PATCH] [ILM] Absolute to relative time conversion (#87822) * cleaning up unused types and legacy logic * added new relative age logic with unit tests * export the calculate relative timing function that returns millisecond values * added exports to index.ts file * copy update and test update Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../common/types/policies.ts | 19 - .../data_tiers/determine_allocation_type.ts | 34 +- ...absolute_timing_to_relative_timing.test.ts | 507 ++++++++++++++++++ .../lib/absolute_timing_to_relative_timing.ts | 185 +++++++ .../sections/edit_policy/lib/index.ts | 12 + 5 files changed, 705 insertions(+), 52 deletions(-) create mode 100644 x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/lib/absolute_timing_to_relative_timing.test.ts create mode 100644 x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/lib/absolute_timing_to_relative_timing.ts create mode 100644 x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/lib/index.ts diff --git a/x-pack/plugins/index_lifecycle_management/common/types/policies.ts b/x-pack/plugins/index_lifecycle_management/common/types/policies.ts index 58468f06e3b2..1f4b06e80c49 100644 --- a/x-pack/plugins/index_lifecycle_management/common/types/policies.ts +++ b/x-pack/plugins/index_lifecycle_management/common/types/policies.ts @@ -154,25 +154,6 @@ export interface PhaseWithMinAge { selectedMinimumAgeUnits: string; } -/** - * Different types of allocation markers we use in deserialized policies. - * - * default - use data tier based data allocation based on node roles -- this is ES best practice mode. - * custom - use node_attrs to allocate data to specific nodes - * none - do not move data anywhere when entering a phase - */ -export type DataTierAllocationType = 'default' | 'custom' | 'none'; - -export interface PhaseWithAllocationAction { - selectedNodeAttrs: string; - selectedReplicaCount: string; - /** - * A string value indicating allocation type. If unspecified we assume the user - * wants to use default allocation. - */ - dataTierAllocationType: DataTierAllocationType; -} - export interface PhaseWithIndexPriority { phaseIndexPriority: string; } diff --git a/x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/determine_allocation_type.ts b/x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/determine_allocation_type.ts index 20ac439e9964..6dde03ec4593 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/determine_allocation_type.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/determine_allocation_type.ts @@ -4,39 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { DataTierAllocationType, AllocateAction, MigrateAction } from '../../../../common/types'; - -/** - * Determine what deserialized state the policy config represents. - * - * See {@DataTierAllocationType} for more information. - */ -export const determineDataTierAllocationTypeLegacy = ( - actions: { - allocate?: AllocateAction; - migrate?: MigrateAction; - } = {} -): DataTierAllocationType => { - const { allocate, migrate } = actions; - - if (migrate?.enabled === false) { - return 'none'; - } - - if (!allocate) { - return 'default'; - } - - if ( - (allocate.require && Object.keys(allocate.require).length) || - (allocate.include && Object.keys(allocate.include).length) || - (allocate.exclude && Object.keys(allocate.exclude).length) - ) { - return 'custom'; - } - - return 'default'; -}; +import { AllocateAction, MigrateAction } from '../../../../common/types'; export const determineDataTierAllocationType = ( actions: { diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/lib/absolute_timing_to_relative_timing.test.ts b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/lib/absolute_timing_to_relative_timing.test.ts new file mode 100644 index 000000000000..28910871fa33 --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/lib/absolute_timing_to_relative_timing.test.ts @@ -0,0 +1,507 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { deserializer } from '../form'; + +import { + absoluteTimingToRelativeTiming, + calculateRelativeTimingMs, +} from './absolute_timing_to_relative_timing'; + +describe('Conversion of absolute policy timing to relative timing', () => { + describe('calculateRelativeTimingMs', () => { + describe('policy that never deletes data (keep forever)', () => { + test('always hot', () => { + expect( + calculateRelativeTimingMs( + deserializer({ + name: 'test', + phases: { + hot: { + min_age: '0ms', + actions: {}, + }, + }, + }) + ) + ).toEqual({ total: Infinity, phases: { hot: Infinity, warm: undefined, cold: undefined } }); + }); + + test('hot, then always warm', () => { + expect( + calculateRelativeTimingMs( + deserializer({ + name: 'test', + phases: { + hot: { + min_age: '0ms', + actions: {}, + }, + warm: { + actions: {}, + }, + }, + }) + ) + ).toEqual({ total: Infinity, phases: { hot: 0, warm: Infinity, cold: undefined } }); + }); + + test('hot, then warm, then always cold', () => { + expect( + calculateRelativeTimingMs( + deserializer({ + name: 'test', + phases: { + hot: { + min_age: '0ms', + actions: {}, + }, + warm: { + min_age: '1M', + actions: {}, + }, + cold: { + min_age: '34d', + actions: {}, + }, + }, + }) + ) + ).toEqual({ + total: Infinity, + phases: { + hot: 2592000000, + warm: 345600000, + cold: Infinity, + }, + }); + }); + + test('hot, then always cold', () => { + expect( + calculateRelativeTimingMs( + deserializer({ + name: 'test', + phases: { + hot: { + min_age: '0ms', + actions: {}, + }, + cold: { + min_age: '34d', + actions: {}, + }, + }, + }) + ) + ).toEqual({ + total: Infinity, + phases: { hot: 2937600000, warm: undefined, cold: Infinity }, + }); + }); + }); + + describe('policy that deletes data', () => { + test('hot, then delete', () => { + expect( + calculateRelativeTimingMs( + deserializer({ + name: 'test', + phases: { + hot: { + min_age: '0ms', + actions: {}, + }, + delete: { + min_age: '1M', + actions: {}, + }, + }, + }) + ) + ).toEqual({ + total: 2592000000, + phases: { + hot: 2592000000, + warm: undefined, + cold: undefined, + }, + }); + }); + + test('hot, then warm, then delete', () => { + expect( + calculateRelativeTimingMs( + deserializer({ + name: 'test', + phases: { + hot: { + min_age: '0ms', + actions: {}, + }, + warm: { + min_age: '24d', + actions: {}, + }, + delete: { + min_age: '1M', + actions: {}, + }, + }, + }) + ) + ).toEqual({ + total: 2592000000, + phases: { + hot: 2073600000, + warm: 518400000, + cold: undefined, + }, + }); + }); + + test('hot, then warm, then cold, then delete', () => { + expect( + calculateRelativeTimingMs( + deserializer({ + name: 'test', + phases: { + hot: { + min_age: '0ms', + actions: {}, + }, + warm: { + min_age: '24d', + actions: {}, + }, + cold: { + min_age: '2M', + actions: {}, + }, + delete: { + min_age: '2d', + actions: {}, + }, + }, + }) + ) + ).toEqual({ + total: 5270400000, + phases: { + hot: 2073600000, + warm: 3196800000, + cold: 0, + }, + }); + }); + + test('hot, then cold, then delete', () => { + expect( + calculateRelativeTimingMs( + deserializer({ + name: 'test', + phases: { + hot: { + min_age: '0ms', + actions: {}, + }, + cold: { + min_age: '2M', + actions: {}, + }, + delete: { + min_age: '2d', + actions: {}, + }, + }, + }) + ) + ).toEqual({ + total: 5270400000, + phases: { + hot: 5270400000, + warm: undefined, + cold: 0, + }, + }); + }); + + test('hot, then long warm, then short cold, then delete', () => { + expect( + calculateRelativeTimingMs( + deserializer({ + name: 'test', + phases: { + hot: { + min_age: '0ms', + actions: {}, + }, + warm: { + min_age: '2M', + actions: {}, + }, + cold: { + min_age: '1d', + actions: {}, + }, + delete: { + min_age: '2d', + actions: {}, + }, + }, + }) + ) + ).toEqual({ + total: 5270400000, + phases: { + hot: 5270400000, + warm: 0, + cold: 0, + }, + }); + }); + }); + }); + + describe('absoluteTimingToRelativeTiming', () => { + describe('policy that never deletes data (keep forever)', () => { + test('always hot', () => { + expect( + absoluteTimingToRelativeTiming( + deserializer({ + name: 'test', + phases: { + hot: { + min_age: '0ms', + actions: {}, + }, + }, + }) + ) + ).toEqual({ total: 'Forever', hot: 'Forever', warm: undefined, cold: undefined }); + }); + + test('hot, then always warm', () => { + expect( + absoluteTimingToRelativeTiming( + deserializer({ + name: 'test', + phases: { + hot: { + min_age: '0ms', + actions: {}, + }, + warm: { + actions: {}, + }, + }, + }) + ) + ).toEqual({ total: 'Forever', hot: 'Less than a day', warm: 'Forever', cold: undefined }); + }); + + test('hot, then warm, then always cold', () => { + expect( + absoluteTimingToRelativeTiming( + deserializer({ + name: 'test', + phases: { + hot: { + min_age: '0ms', + actions: {}, + }, + warm: { + min_age: '1M', + actions: {}, + }, + cold: { + min_age: '34d', + actions: {}, + }, + }, + }) + ) + ).toEqual({ + total: 'Forever', + hot: '30 days', + warm: '4 days', + cold: 'Forever', + }); + }); + + test('hot, then always cold', () => { + expect( + absoluteTimingToRelativeTiming( + deserializer({ + name: 'test', + phases: { + hot: { + min_age: '0ms', + actions: {}, + }, + cold: { + min_age: '34d', + actions: {}, + }, + }, + }) + ) + ).toEqual({ total: 'Forever', hot: '34 days', warm: undefined, cold: 'Forever' }); + }); + }); + + describe('policy that deletes data', () => { + test('hot, then delete', () => { + expect( + absoluteTimingToRelativeTiming( + deserializer({ + name: 'test', + phases: { + hot: { + min_age: '0ms', + actions: {}, + }, + delete: { + min_age: '1M', + actions: {}, + }, + }, + }) + ) + ).toEqual({ + total: '30 days', + hot: '30 days', + warm: undefined, + cold: undefined, + }); + }); + + test('hot, then warm, then delete', () => { + expect( + absoluteTimingToRelativeTiming( + deserializer({ + name: 'test', + phases: { + hot: { + min_age: '0ms', + actions: {}, + }, + warm: { + min_age: '24d', + actions: {}, + }, + delete: { + min_age: '1M', + actions: {}, + }, + }, + }) + ) + ).toEqual({ + total: '30 days', + hot: '24 days', + warm: '6 days', + cold: undefined, + }); + }); + + test('hot, then warm, then cold, then delete', () => { + expect( + absoluteTimingToRelativeTiming( + deserializer({ + name: 'test', + phases: { + hot: { + min_age: '0ms', + actions: {}, + }, + warm: { + min_age: '24d', + actions: {}, + }, + cold: { + min_age: '2M', + actions: {}, + }, + delete: { + min_age: '2d', + actions: {}, + }, + }, + }) + ) + ).toEqual({ + total: '61 days', + hot: '24 days', + warm: '37 days', + cold: 'Less than a day', + }); + }); + + test('hot, then cold, then delete', () => { + expect( + absoluteTimingToRelativeTiming( + deserializer({ + name: 'test', + phases: { + hot: { + min_age: '0ms', + actions: {}, + }, + cold: { + min_age: '2M', + actions: {}, + }, + delete: { + min_age: '2d', + actions: {}, + }, + }, + }) + ) + ).toEqual({ + total: '61 days', + hot: '61 days', + warm: undefined, + cold: 'Less than a day', + }); + }); + + test('hot, then long warm, then short cold, then delete', () => { + expect( + absoluteTimingToRelativeTiming( + deserializer({ + name: 'test', + phases: { + hot: { + min_age: '0ms', + actions: {}, + }, + warm: { + min_age: '2M', + actions: {}, + }, + cold: { + min_age: '1d', + actions: {}, + }, + delete: { + min_age: '2d', + actions: {}, + }, + }, + }) + ) + ).toEqual({ + total: '61 days', + hot: '61 days', + warm: 'Less than a day', + cold: 'Less than a day', + }); + }); + }); + }); +}); diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/lib/absolute_timing_to_relative_timing.ts b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/lib/absolute_timing_to_relative_timing.ts new file mode 100644 index 000000000000..c77b171a56be --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/lib/absolute_timing_to_relative_timing.ts @@ -0,0 +1,185 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/** + * READ ME: + * + * ILM policies express data age thresholds as minimum age from an absolute point of reference. + * The absolute point of reference could be when data was created, but it could also be when + * rollover has occurred. This is useful for configuring a policy, but when trying to understand + * how long data will be in a specific phase, when thinking of data tiers, it is not as useful. + * + * This code converts the absolute timings to _relative_ timings of the form: 30 days in hot phase, + * 40 days in warm phase then forever in cold phase. + */ + +import moment from 'moment'; +import { flow } from 'fp-ts/lib/function'; +import { i18n } from '@kbn/i18n'; + +import { splitSizeAndUnits } from '../../../lib/policies'; + +import { FormInternal } from '../types'; + +type MinAgePhase = 'warm' | 'cold' | 'delete'; + +type Phase = 'hot' | MinAgePhase; + +const i18nTexts = { + forever: i18n.translate('xpack.indexLifecycleMgmt.relativeTiming.Forever', { + defaultMessage: 'Forever', + }), + lessThanADay: i18n.translate('xpack.indexLifecycleMgmt.relativeTiming.lessThanADay', { + defaultMessage: 'Less than a day', + }), + day: i18n.translate('xpack.indexLifecycleMgmt.relativeTiming.day', { + defaultMessage: 'day', + }), + days: i18n.translate('xpack.indexLifecycleMgmt.relativeTiming.days', { + defaultMessage: 'days', + }), +}; + +interface AbsoluteTimings { + hot: { + min_age: undefined; + }; + warm?: { + min_age: string; + }; + cold?: { + min_age: string; + }; + delete?: { + min_age: string; + }; +} + +export interface PhaseAgeInMilliseconds { + total: number; + phases: { + hot: number; + warm?: number; + cold?: number; + }; +} + +const phaseOrder: Phase[] = ['hot', 'warm', 'cold', 'delete']; + +const getMinAge = (phase: MinAgePhase, formData: FormInternal) => ({ + min_age: formData.phases[phase]?.min_age + ? formData.phases[phase]!.min_age! + formData._meta[phase].minAgeUnit + : '0ms', +}); + +const formDataToAbsoluteTimings = (formData: FormInternal): AbsoluteTimings => { + const { _meta } = formData; + if (!_meta) { + return { hot: { min_age: undefined } }; + } + return { + hot: { min_age: undefined }, + warm: _meta.warm.enabled ? getMinAge('warm', formData) : undefined, + cold: _meta.cold.enabled ? getMinAge('cold', formData) : undefined, + delete: _meta.delete.enabled ? getMinAge('delete', formData) : undefined, + }; +}; + +/** + * See https://www.elastic.co/guide/en/elasticsearch/reference/current/common-options.html#date-math + * for all date math values. ILM policies also support "micros" and "nanos". + */ +const getPhaseMinAgeInMilliseconds = (phase: { min_age: string }): number => { + let milliseconds: number; + const { units, size } = splitSizeAndUnits(phase.min_age); + if (units === 'micros') { + milliseconds = parseInt(size, 10) / 1e3; + } else if (units === 'nanos') { + milliseconds = parseInt(size, 10) / 1e6; + } else { + milliseconds = moment.duration(size, units as any).asMilliseconds(); + } + return milliseconds; +}; + +/** + * Given a set of phase minimum age absolute timings, like hot phase 0ms and warm phase 3d, work out + * the number of milliseconds data will reside in phase. + */ +const calculateMilliseconds = (inputs: AbsoluteTimings): PhaseAgeInMilliseconds => { + return phaseOrder.reduce( + (acc, phaseName, idx) => { + // Delete does not have an age associated with it + if (phaseName === 'delete') { + return acc; + } + const phase = inputs[phaseName]; + if (!phase) { + return acc; + } + const nextPhase = phaseOrder + .slice(idx + 1) + .find((nextPhaseName) => Boolean(inputs[nextPhaseName])); // find the next existing phase + + let nextPhaseMinAge = Infinity; + + // If we have a next phase, calculate the timing between this phase and the next + if (nextPhase && inputs[nextPhase]?.min_age) { + nextPhaseMinAge = getPhaseMinAgeInMilliseconds(inputs[nextPhase] as { min_age: string }); + } + + return { + // data will be the age of the phase with the highest min age requirement + total: Math.max(acc.total, nextPhaseMinAge), + phases: { + ...acc.phases, + [phaseName]: Math.max(nextPhaseMinAge - acc.total, 0), // get the max age for the current phase, take 0 if negative number + }, + }; + }, + { + total: 0, + phases: { + hot: 0, + warm: inputs.warm ? 0 : undefined, + cold: inputs.cold ? 0 : undefined, + }, + } + ); +}; + +const millisecondsToDays = (milliseconds?: number): string | undefined => { + if (milliseconds == null) { + return; + } + if (!isFinite(milliseconds)) { + return i18nTexts.forever; + } + const days = milliseconds / 8.64e7; + return days < 1 + ? i18nTexts.lessThanADay + : `${Math.floor(days)} ${days === 1 ? i18nTexts.day : i18nTexts.days}`; +}; + +export const normalizeTimingsToHumanReadable = ({ + total, + phases, +}: PhaseAgeInMilliseconds): { total?: string; hot?: string; warm?: string; cold?: string } => { + return { + total: millisecondsToDays(total), + hot: millisecondsToDays(phases.hot), + warm: millisecondsToDays(phases.warm), + cold: millisecondsToDays(phases.cold), + }; +}; + +export const calculateRelativeTimingMs = flow(formDataToAbsoluteTimings, calculateMilliseconds); + +export const absoluteTimingToRelativeTiming = flow( + formDataToAbsoluteTimings, + calculateMilliseconds, + normalizeTimingsToHumanReadable +); diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/lib/index.ts b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/lib/index.ts new file mode 100644 index 000000000000..9593fcc810a6 --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/lib/index.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { + absoluteTimingToRelativeTiming, + calculateRelativeTimingMs, + normalizeTimingsToHumanReadable, + PhaseAgeInMilliseconds, +} from './absolute_timing_to_relative_timing';