maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[ILM] Absolute to relative time conversion (#87822)

Browse source 
* cleaning up unused types and legacy logic

* added new relative age logic with unit tests

* export the calculate relative timing function that returns millisecond values

* added exports to index.ts file

* copy update and test update

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Jean-Louis Leysens 2021-01-15 13:32:16 +01:00 committed by GitHub
parent abfd8bb9d3
commit 3dcb98ec67
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 705 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
x-pack/plugins/index_lifecycle_management/common/types/policies.ts
View file

@ -154,25 +154,6 @@ export interface PhaseWithMinAge {
selectedMinimumAgeUnits: string;
}
/**
* Different types of allocation markers we use in deserialized policies.
*
* default - use data tier based data allocation based on node roles -- this is ES best practice mode.
* custom - use node_attrs to allocate data to specific nodes
* none - do not move data anywhere when entering a phase
*/
export type DataTierAllocationType = 'default' | 'custom' | 'none';
export interface PhaseWithAllocationAction {
selectedNodeAttrs: string;
selectedReplicaCount: string;
/**
* A string value indicating allocation type. If unspecified we assume the user
* wants to use default allocation.
*/
dataTierAllocationType: DataTierAllocationType;
}
export interface PhaseWithIndexPriority {
phaseIndexPriority: string;
}

34
x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/determine_allocation_type.ts
View file

@ -4,39 +4,7 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { DataTierAllocationType, AllocateAction, MigrateAction } from '../../../../common/types';
/**
* Determine what deserialized state the policy config represents.
*
* See {@DataTierAllocationType} for more information.
*/
export const determineDataTierAllocationTypeLegacy = (
actions: {
allocate?: AllocateAction;
migrate?: MigrateAction;
} = {}
): DataTierAllocationType => {
const { allocate, migrate } = actions;
if (migrate?.enabled === false) {
return 'none';
}
if (!allocate) {
return 'default';
}
if (
(allocate.require && Object.keys(allocate.require).length) ||
(allocate.include && Object.keys(allocate.include).length) ||
(allocate.exclude && Object.keys(allocate.exclude).length)
) {
return 'custom';
}
return 'default';
};
import { AllocateAction, MigrateAction } from '../../../../common/types';
export const determineDataTierAllocationType = (
actions: {

507
x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/lib/absolute_timing_to_relative_timing.test.ts Normal file
View file

@ -0,0 +1,507 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { deserializer } from '../form';
import {
absoluteTimingToRelativeTiming,
calculateRelativeTimingMs,
} from './absolute_timing_to_relative_timing';
describe('Conversion of absolute policy timing to relative timing', () => {
describe('calculateRelativeTimingMs', () => {
describe('policy that never deletes data (keep forever)', () => {
test('always hot', () => {
expect(
calculateRelativeTimingMs(
deserializer({
name: 'test',
phases: {
hot: {
min_age: '0ms',
actions: {},
},
},
})
)
).toEqual({ total: Infinity, phases: { hot: Infinity, warm: undefined, cold: undefined } });
});
test('hot, then always warm', () => {
expect(
calculateRelativeTimingMs(
deserializer({
name: 'test',
phases: {
hot: {
min_age: '0ms',
actions: {},
},
warm: {
actions: {},
},
},
})
)
).toEqual({ total: Infinity, phases: { hot: 0, warm: Infinity, cold: undefined } });
});
test('hot, then warm, then always cold', () => {
expect(
calculateRelativeTimingMs(
deserializer({
name: 'test',
phases: {
hot: {
min_age: '0ms',
actions: {},
},
warm: {
min_age: '1M',
actions: {},
},
cold: {
min_age: '34d',
actions: {},
},
},
})
)
).toEqual({
total: Infinity,
phases: {
hot: 2592000000,
warm: 345600000,
cold: Infinity,
},
});
});
test('hot, then always cold', () => {
expect(
calculateRelativeTimingMs(
deserializer({
name: 'test',
phases: {
hot: {
min_age: '0ms',
actions: {},
},
cold: {
min_age: '34d',
actions: {},
},
},
})
)
).toEqual({
total: Infinity,
phases: { hot: 2937600000, warm: undefined, cold: Infinity },
});
});
});
describe('policy that deletes data', () => {
test('hot, then delete', () => {
expect(
calculateRelativeTimingMs(
deserializer({
name: 'test',
phases: {
hot: {
min_age: '0ms',
actions: {},
},
delete: {
min_age: '1M',
actions: {},
},
},
})
)
).toEqual({
total: 2592000000,
phases: {
hot: 2592000000,
warm: undefined,
cold: undefined,
},
});
});
test('hot, then warm, then delete', () => {
expect(
calculateRelativeTimingMs(
deserializer({
name: 'test',
phases: {
hot: {
min_age: '0ms',
actions: {},
},
warm: {
min_age: '24d',
actions: {},
},
delete: {
min_age: '1M',
actions: {},
},
},
})
)
).toEqual({
total: 2592000000,
phases: {
hot: 2073600000,
warm: 518400000,
cold: undefined,
},
});
});
test('hot, then warm, then cold, then delete', () => {
expect(
calculateRelativeTimingMs(
deserializer({
name: 'test',
phases: {
hot: {
min_age: '0ms',
actions: {},
},
warm: {
min_age: '24d',
actions: {},
},
cold: {
min_age: '2M',
actions: {},
},
delete: {
min_age: '2d',
actions: {},
},
},
})
)
).toEqual({
total: 5270400000,
phases: {
hot: 2073600000,
warm: 3196800000,
cold: 0,
},
});
});
test('hot, then cold, then delete', () => {
expect(
calculateRelativeTimingMs(
deserializer({
name: 'test',
phases: {
hot: {
min_age: '0ms',
actions: {},
},
cold: {
min_age: '2M',
actions: {},
},
delete: {
min_age: '2d',
actions: {},
},
},
})
)
).toEqual({
total: 5270400000,
phases: {
hot: 5270400000,
warm: undefined,
cold: 0,
},
});
});
test('hot, then long warm, then short cold, then delete', () => {
expect(
calculateRelativeTimingMs(
deserializer({
name: 'test',
phases: {
hot: {
min_age: '0ms',
actions: {},
},
warm: {
min_age: '2M',
actions: {},
},
cold: {
min_age: '1d',
actions: {},
},
delete: {
min_age: '2d',
actions: {},
},
},
})
)
).toEqual({
total: 5270400000,
phases: {
hot: 5270400000,
warm: 0,
cold: 0,
},
});
});
});
});
describe('absoluteTimingToRelativeTiming', () => {
describe('policy that never deletes data (keep forever)', () => {
test('always hot', () => {
expect(
absoluteTimingToRelativeTiming(
deserializer({
name: 'test',
phases: {
hot: {
min_age: '0ms',
actions: {},
},
},
})
)
).toEqual({ total: 'Forever', hot: 'Forever', warm: undefined, cold: undefined });
});
test('hot, then always warm', () => {
expect(
absoluteTimingToRelativeTiming(
deserializer({
name: 'test',
phases: {
hot: {
min_age: '0ms',
actions: {},
},
warm: {
actions: {},
},
},
})
)
).toEqual({ total: 'Forever', hot: 'Less than a day', warm: 'Forever', cold: undefined });
});
test('hot, then warm, then always cold', () => {
expect(
absoluteTimingToRelativeTiming(
deserializer({
name: 'test',
phases: {
hot: {
min_age: '0ms',
actions: {},
},
warm: {
min_age: '1M',
actions: {},
},
cold: {
min_age: '34d',
actions: {},
},
},
})
)
).toEqual({
total: 'Forever',
hot: '30 days',
warm: '4 days',
cold: 'Forever',
});
});
test('hot, then always cold', () => {
expect(
absoluteTimingToRelativeTiming(
deserializer({
name: 'test',
phases: {
hot: {
min_age: '0ms',
actions: {},
},
cold: {
min_age: '34d',
actions: {},
},
},
})
)
).toEqual({ total: 'Forever', hot: '34 days', warm: undefined, cold: 'Forever' });
});
});
describe('policy that deletes data', () => {
test('hot, then delete', () => {
expect(
absoluteTimingToRelativeTiming(
deserializer({
name: 'test',
phases: {
hot: {
min_age: '0ms',
actions: {},
},
delete: {
min_age: '1M',
actions: {},
},
},
})
)
).toEqual({
total: '30 days',
hot: '30 days',
warm: undefined,
cold: undefined,
});
});
test('hot, then warm, then delete', () => {
expect(
absoluteTimingToRelativeTiming(
deserializer({
name: 'test',
phases: {
hot: {
min_age: '0ms',
actions: {},
},
warm: {
min_age: '24d',
actions: {},
},
delete: {
min_age: '1M',
actions: {},
},
},
})
)
).toEqual({
total: '30 days',
hot: '24 days',
warm: '6 days',
cold: undefined,
});
});
test('hot, then warm, then cold, then delete', () => {
expect(
absoluteTimingToRelativeTiming(
deserializer({
name: 'test',
phases: {
hot: {
min_age: '0ms',
actions: {},
},
warm: {
min_age: '24d',
actions: {},
},
cold: {
min_age: '2M',
actions: {},
},
delete: {
min_age: '2d',
actions: {},
},
},
})
)
).toEqual({
total: '61 days',
hot: '24 days',
warm: '37 days',
cold: 'Less than a day',
});
});
test('hot, then cold, then delete', () => {
expect(
absoluteTimingToRelativeTiming(
deserializer({
name: 'test',
phases: {
hot: {
min_age: '0ms',
actions: {},
},
cold: {
min_age: '2M',
actions: {},
},
delete: {
min_age: '2d',
actions: {},
},
},
})
)
).toEqual({
total: '61 days',
hot: '61 days',
warm: undefined,
cold: 'Less than a day',
});
});
test('hot, then long warm, then short cold, then delete', () => {
expect(
absoluteTimingToRelativeTiming(
deserializer({
name: 'test',
phases: {
hot: {
min_age: '0ms',
actions: {},
},
warm: {
min_age: '2M',
actions: {},
},
cold: {
min_age: '1d',
actions: {},
},
delete: {
min_age: '2d',
actions: {},
},
},
})
)
).toEqual({
total: '61 days',
hot: '61 days',
warm: 'Less than a day',
cold: 'Less than a day',
});
});
});
});
});

185
x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/lib/absolute_timing_to_relative_timing.ts Normal file
View file

@ -0,0 +1,185 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
/**
* READ ME:
*
* ILM policies express data age thresholds as minimum age from an absolute point of reference.
* The absolute point of reference could be when data was created, but it could also be when
* rollover has occurred. This is useful for configuring a policy, but when trying to understand
* how long data will be in a specific phase, when thinking of data tiers, it is not as useful.
*
* This code converts the absolute timings to _relative_ timings of the form: 30 days in hot phase,
* 40 days in warm phase then forever in cold phase.
*/
import moment from 'moment';
import { flow } from 'fp-ts/lib/function';
import { i18n } from '@kbn/i18n';
import { splitSizeAndUnits } from '../../../lib/policies';
import { FormInternal } from '../types';
type MinAgePhase = 'warm' | 'cold' | 'delete';
type Phase = 'hot' | MinAgePhase;
const i18nTexts = {
forever: i18n.translate('xpack.indexLifecycleMgmt.relativeTiming.Forever', {
defaultMessage: 'Forever',
}),
lessThanADay: i18n.translate('xpack.indexLifecycleMgmt.relativeTiming.lessThanADay', {
defaultMessage: 'Less than a day',
}),
day: i18n.translate('xpack.indexLifecycleMgmt.relativeTiming.day', {
defaultMessage: 'day',
}),
days: i18n.translate('xpack.indexLifecycleMgmt.relativeTiming.days', {
defaultMessage: 'days',
}),
};
interface AbsoluteTimings {
hot: {
min_age: undefined;
};
warm?: {
min_age: string;
};
cold?: {
min_age: string;
};
delete?: {
min_age: string;
};
}
export interface PhaseAgeInMilliseconds {
total: number;
phases: {
hot: number;
warm?: number;
cold?: number;
};
}
const phaseOrder: Phase[] = ['hot', 'warm', 'cold', 'delete'];
const getMinAge = (phase: MinAgePhase, formData: FormInternal) => ({
min_age: formData.phases[phase]?.min_age
? formData.phases[phase]!.min_age! + formData._meta[phase].minAgeUnit
: '0ms',
});
const formDataToAbsoluteTimings = (formData: FormInternal): AbsoluteTimings => {
const { _meta } = formData;
if (!_meta) {
return { hot: { min_age: undefined } };
}
return {
hot: { min_age: undefined },
warm: _meta.warm.enabled ? getMinAge('warm', formData) : undefined,
cold: _meta.cold.enabled ? getMinAge('cold', formData) : undefined,
delete: _meta.delete.enabled ? getMinAge('delete', formData) : undefined,
};
};
/**
* See https://www.elastic.co/guide/en/elasticsearch/reference/current/common-options.html#date-math
* for all date math values. ILM policies also support "micros" and "nanos".
*/
const getPhaseMinAgeInMilliseconds = (phase: { min_age: string }): number => {
let milliseconds: number;
const { units, size } = splitSizeAndUnits(phase.min_age);
if (units === 'micros') {
milliseconds = parseInt(size, 10) / 1e3;
} else if (units === 'nanos') {
milliseconds = parseInt(size, 10) / 1e6;
} else {
milliseconds = moment.duration(size, units as any).asMilliseconds();
}
return milliseconds;
};
/**
* Given a set of phase minimum age absolute timings, like hot phase 0ms and warm phase 3d, work out
* the number of milliseconds data will reside in phase.
*/
const calculateMilliseconds = (inputs: AbsoluteTimings): PhaseAgeInMilliseconds => {
return phaseOrder.reduce<PhaseAgeInMilliseconds>(
(acc, phaseName, idx) => {
// Delete does not have an age associated with it
if (phaseName === 'delete') {
return acc;
}
const phase = inputs[phaseName];
if (!phase) {
return acc;
}
const nextPhase = phaseOrder
.slice(idx + 1)
.find((nextPhaseName) => Boolean(inputs[nextPhaseName])); // find the next existing phase
let nextPhaseMinAge = Infinity;
// If we have a next phase, calculate the timing between this phase and the next
if (nextPhase && inputs[nextPhase]?.min_age) {
nextPhaseMinAge = getPhaseMinAgeInMilliseconds(inputs[nextPhase] as { min_age: string });
}
return {
// data will be the age of the phase with the highest min age requirement
total: Math.max(acc.total, nextPhaseMinAge),
phases: {
...acc.phases,
[phaseName]: Math.max(nextPhaseMinAge - acc.total, 0), // get the max age for the current phase, take 0 if negative number
},
};
},
{
total: 0,
phases: {
hot: 0,
warm: inputs.warm ? 0 : undefined,
cold: inputs.cold ? 0 : undefined,
},
}
);
};
const millisecondsToDays = (milliseconds?: number): string | undefined => {
if (milliseconds == null) {
return;
}
if (!isFinite(milliseconds)) {
return i18nTexts.forever;
}
const days = milliseconds / 8.64e7;
return days < 1
? i18nTexts.lessThanADay
: `${Math.floor(days)} ${days === 1 ? i18nTexts.day : i18nTexts.days}`;
};
export const normalizeTimingsToHumanReadable = ({
total,
phases,
}: PhaseAgeInMilliseconds): { total?: string; hot?: string; warm?: string; cold?: string } => {
return {
total: millisecondsToDays(total),
hot: millisecondsToDays(phases.hot),
warm: millisecondsToDays(phases.warm),
cold: millisecondsToDays(phases.cold),
};
};
export const calculateRelativeTimingMs = flow(formDataToAbsoluteTimings, calculateMilliseconds);
export const absoluteTimingToRelativeTiming = flow(
formDataToAbsoluteTimings,
calculateMilliseconds,
normalizeTimingsToHumanReadable
);

12
x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/lib/index.ts Normal file
View file

@ -0,0 +1,12 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
export {
absoluteTimingToRelativeTiming,
calculateRelativeTimingMs,
normalizeTimingsToHumanReadable,
PhaseAgeInMilliseconds,
} from './absolute_timing_to_relative_timing';