[UA] Support previously re-indexed ML/Watcher indices (#31046)
* [UA] Support previously re-indexed ML/Watcher indices Previously we were comparing the start of the indices to determine if ML/Watcher should be stopped before re-indexing. Since we are now pre-pending the index name we should compare the index names without the re-indexed portion. Signed-off-by: Tyler Smalley <tyler.smalley@elastic.co>
This commit is contained in:
parent
61c2754cca
commit
cadec2326c
|
@ -8,7 +8,12 @@ import {
|
|||
CURRENT_MAJOR_VERSION,
|
||||
PREV_MAJOR_VERSION,
|
||||
} from 'x-pack/plugins/upgrade_assistant/common/version';
|
||||
import { getReindexWarnings, parseIndexName, transformFlatSettings } from './index_settings';
|
||||
import {
|
||||
generateNewIndexName,
|
||||
getReindexWarnings,
|
||||
sourceNameForIndex,
|
||||
transformFlatSettings,
|
||||
} from './index_settings';
|
||||
|
||||
describe('transformFlatSettings', () => {
|
||||
it('does not blow up for empty mappings', () => {
|
||||
|
@ -53,45 +58,57 @@ describe('transformFlatSettings', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('parseIndexName', () => {
|
||||
describe('sourceNameForIndex', () => {
|
||||
it('parses internal indices', () => {
|
||||
expect(parseIndexName('.watches').baseName).toBe('watches');
|
||||
expect(sourceNameForIndex('.myInternalIndex')).toEqual('.myInternalIndex');
|
||||
});
|
||||
|
||||
it('parses non-internal indices', () => {
|
||||
expect(parseIndexName('myIndex').baseName).toBe('myIndex');
|
||||
expect(sourceNameForIndex('myIndex')).toEqual('myIndex');
|
||||
});
|
||||
|
||||
it('excludes appended v5 reindexing string from newIndexName', () => {
|
||||
expect(parseIndexName('myIndex-reindexed-v5')).toEqual({
|
||||
baseName: 'myIndex-reindexed-v5',
|
||||
cleanBaseName: 'myIndex',
|
||||
cleanIndexName: 'myIndex',
|
||||
newIndexName: `reindexed-v${CURRENT_MAJOR_VERSION}-myIndex`,
|
||||
});
|
||||
|
||||
expect(parseIndexName('.myInternalIndex-reindexed-v5')).toEqual({
|
||||
baseName: 'myInternalIndex-reindexed-v5',
|
||||
cleanBaseName: 'myInternalIndex',
|
||||
cleanIndexName: '.myInternalIndex',
|
||||
newIndexName: `.reindexed-v${CURRENT_MAJOR_VERSION}-myInternalIndex`,
|
||||
});
|
||||
expect(sourceNameForIndex('myIndex-reindexed-v5')).toEqual('myIndex');
|
||||
expect(sourceNameForIndex('.myInternalIndex-reindexed-v5')).toEqual('.myInternalIndex');
|
||||
});
|
||||
|
||||
it('replaces reindexed-v${PREV_MAJOR_VERSION} with reindexed-v${CURRENT_MAJOR_VERSION} in newIndexName', () => {
|
||||
expect(parseIndexName(`reindexed-v${PREV_MAJOR_VERSION}-myIndex`)).toEqual({
|
||||
baseName: `reindexed-v${PREV_MAJOR_VERSION}-myIndex`,
|
||||
cleanBaseName: 'myIndex',
|
||||
cleanIndexName: 'myIndex',
|
||||
newIndexName: `reindexed-v${CURRENT_MAJOR_VERSION}-myIndex`,
|
||||
});
|
||||
expect(sourceNameForIndex(`reindexed-v${PREV_MAJOR_VERSION}-myIndex`)).toEqual('myIndex');
|
||||
expect(sourceNameForIndex(`.reindexed-v${PREV_MAJOR_VERSION}-myInternalIndex`)).toEqual(
|
||||
'.myInternalIndex'
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
expect(parseIndexName(`.reindexed-v${PREV_MAJOR_VERSION}-myInternalIndex`)).toEqual({
|
||||
baseName: `reindexed-v${PREV_MAJOR_VERSION}-myInternalIndex`,
|
||||
cleanBaseName: 'myInternalIndex',
|
||||
cleanIndexName: '.myInternalIndex',
|
||||
newIndexName: `.reindexed-v${CURRENT_MAJOR_VERSION}-myInternalIndex`,
|
||||
});
|
||||
describe('generateNewIndexName', () => {
|
||||
it('parses internal indices', () => {
|
||||
expect(generateNewIndexName('.myInternalIndex')).toEqual(
|
||||
`.reindexed-v${CURRENT_MAJOR_VERSION}-myInternalIndex`
|
||||
);
|
||||
});
|
||||
|
||||
it('parses non-internal indices', () => {
|
||||
expect(generateNewIndexName('myIndex')).toEqual(`reindexed-v${CURRENT_MAJOR_VERSION}-myIndex`);
|
||||
});
|
||||
|
||||
it('excludes appended v5 reindexing string from generateNewIndexName', () => {
|
||||
expect(generateNewIndexName('myIndex-reindexed-v5')).toEqual(
|
||||
`reindexed-v${CURRENT_MAJOR_VERSION}-myIndex`
|
||||
);
|
||||
|
||||
expect(generateNewIndexName('.myInternalIndex-reindexed-v5')).toEqual(
|
||||
`.reindexed-v${CURRENT_MAJOR_VERSION}-myInternalIndex`
|
||||
);
|
||||
});
|
||||
|
||||
it('replaces reindexed-v${PREV_MAJOR_VERSION} with reindexed-v${CURRENT_MAJOR_VERSION} in generateNewIndexName', () => {
|
||||
expect(generateNewIndexName(`reindexed-v${PREV_MAJOR_VERSION}-myIndex`)).toEqual(
|
||||
`reindexed-v${CURRENT_MAJOR_VERSION}-myIndex`
|
||||
);
|
||||
|
||||
expect(generateNewIndexName(`.reindexed-v${PREV_MAJOR_VERSION}-myInternalIndex`)).toEqual(
|
||||
`.reindexed-v${CURRENT_MAJOR_VERSION}-myInternalIndex`
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
|
|
|
@ -31,28 +31,41 @@ export const transformFlatSettings = (flatSettings: FlatSettings) => {
|
|||
};
|
||||
|
||||
/**
|
||||
* Parses an index name
|
||||
* Provides the assumed source of the index name stripping any prefixing
|
||||
* introduced by the upgrade assistant
|
||||
*
|
||||
* Examples:
|
||||
* .reindex-v7-foo => .foo
|
||||
* reindex-v7-foo => foo
|
||||
*
|
||||
* @param indexName
|
||||
*/
|
||||
export const parseIndexName = (indexName: string): ParsedIndexName => {
|
||||
export const sourceNameForIndex = (indexName: string): string => {
|
||||
const matches = indexName.match(/^([\.])?(.*)$/) || [];
|
||||
const internal = matches[1] || '';
|
||||
const baseName = matches[2];
|
||||
|
||||
const currentVersion = `reindexed-v${CURRENT_MAJOR_VERSION}`;
|
||||
|
||||
// in 5.6 the upgrade assistant appended to the index, in 6.7+ we prepend to
|
||||
// avoid conflicts with index patterns/templates/etc
|
||||
const reindexedMatcher = new RegExp(`(-reindexed-v5$|reindexed-v${PREV_MAJOR_VERSION}-)`, 'g');
|
||||
|
||||
const cleanBaseName = baseName.replace(reindexedMatcher, '');
|
||||
return `${internal}${cleanBaseName}`;
|
||||
};
|
||||
|
||||
return {
|
||||
cleanIndexName: `${internal}${cleanBaseName}`,
|
||||
baseName,
|
||||
cleanBaseName,
|
||||
newIndexName: `${internal}${currentVersion}-${cleanBaseName}`,
|
||||
};
|
||||
/**
|
||||
* Provides the index name to re-index into
|
||||
*
|
||||
* .foo -> .reindexed-v7-foo
|
||||
* foo => reindexed-v7-foo
|
||||
*/
|
||||
export const generateNewIndexName = (indexName: string): string => {
|
||||
const sourceName = sourceNameForIndex(indexName);
|
||||
const currentVersion = `reindexed-v${CURRENT_MAJOR_VERSION}`;
|
||||
|
||||
return indexName.startsWith('.')
|
||||
? `.${currentVersion}-${sourceName.substr(1)}`
|
||||
: `${currentVersion}-${sourceName}`;
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
|
@ -19,7 +19,7 @@ import {
|
|||
ReindexStatus,
|
||||
ReindexStep,
|
||||
} from '../../../common/types';
|
||||
import { parseIndexName } from './index_settings';
|
||||
import { generateNewIndexName } from './index_settings';
|
||||
import { FlatSettings } from './types';
|
||||
|
||||
// TODO: base on elasticsearch.requestTimeout?
|
||||
|
@ -157,7 +157,7 @@ export const reindexActionsFactory = (
|
|||
async createReindexOp(indexName: string) {
|
||||
return client.create<ReindexOperation>(REINDEX_OP_TYPE, {
|
||||
indexName,
|
||||
newIndexName: parseIndexName(indexName).newIndexName,
|
||||
newIndexName: generateNewIndexName(indexName),
|
||||
status: ReindexStatus.inProgress,
|
||||
lastCompletedStep: ReindexStep.created,
|
||||
locked: null,
|
||||
|
|
|
@ -16,7 +16,12 @@ import {
|
|||
ReindexStatus,
|
||||
ReindexStep,
|
||||
} from '../../../common/types';
|
||||
import { ReindexService, reindexServiceFactory } from './reindex_service';
|
||||
import {
|
||||
isMlIndex,
|
||||
isWatcherIndex,
|
||||
ReindexService,
|
||||
reindexServiceFactory,
|
||||
} from './reindex_service';
|
||||
|
||||
describe('reindexService', () => {
|
||||
let actions: jest.Mocked<any>;
|
||||
|
@ -451,6 +456,40 @@ describe('reindexService', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('isMlIndex', () => {
|
||||
it('is false for non-ml indices', () => {
|
||||
expect(isMlIndex('.literally-anything')).toBe(false);
|
||||
});
|
||||
|
||||
it('is true for ML indices', () => {
|
||||
expect(isMlIndex('.ml-state')).toBe(true);
|
||||
expect(isMlIndex('.ml-anomalies')).toBe(true);
|
||||
expect(isMlIndex('.ml-config')).toBe(true);
|
||||
});
|
||||
|
||||
it('is true for ML re-indexed indices', () => {
|
||||
expect(isMlIndex(`.reindexed-v${PREV_MAJOR_VERSION}-ml-state`)).toBe(true);
|
||||
expect(isMlIndex(`.reindexed-v${PREV_MAJOR_VERSION}-ml-anomalies`)).toBe(true);
|
||||
expect(isMlIndex(`.reindexed-v${PREV_MAJOR_VERSION}-ml-config`)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('isWatcherIndex', () => {
|
||||
it('is false for non-watcher indices', () => {
|
||||
expect(isWatcherIndex('.literally-anything')).toBe(false);
|
||||
});
|
||||
|
||||
it('is true for watcher indices', () => {
|
||||
expect(isWatcherIndex('.watches')).toBe(true);
|
||||
expect(isWatcherIndex('.triggered-watches')).toBe(true);
|
||||
});
|
||||
|
||||
it('is true for watcher re-indexed indices', () => {
|
||||
expect(isWatcherIndex(`.reindexed-v${PREV_MAJOR_VERSION}-watches`)).toBe(true);
|
||||
expect(isWatcherIndex(`.reindexed-v${PREV_MAJOR_VERSION}-triggered-watches`)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('state machine, lastCompletedStep ===', () => {
|
||||
const defaultAttributes = {
|
||||
indexName: 'myIndex',
|
||||
|
@ -484,6 +523,34 @@ describe('reindexService', () => {
|
|||
expect(callCluster).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('supports an already migrated ML index', async () => {
|
||||
actions.incrementIndexGroupReindexes.mockResolvedValueOnce();
|
||||
actions.runWhileIndexGroupLocked.mockImplementationOnce(async (group: string, f: any) =>
|
||||
f()
|
||||
);
|
||||
callCluster
|
||||
// Mock call to /_nodes for version check
|
||||
.mockResolvedValueOnce({ nodes: { nodeX: { version: '6.7.0-alpha' } } })
|
||||
// Mock call to /_ml/set_upgrade_mode?enabled=true
|
||||
.mockResolvedValueOnce({ acknowledged: true });
|
||||
|
||||
const mlReindexedOp = {
|
||||
id: '2',
|
||||
attributes: { ...reindexOp.attributes, indexName: '.reindexed-v7-ml-anomalies' },
|
||||
} as ReindexSavedObject;
|
||||
const updatedOp = await service.processNextStep(mlReindexedOp);
|
||||
|
||||
expect(updatedOp.attributes.lastCompletedStep).toEqual(
|
||||
ReindexStep.indexGroupServicesStopped
|
||||
);
|
||||
expect(actions.incrementIndexGroupReindexes).toHaveBeenCalled();
|
||||
expect(actions.runWhileIndexGroupLocked).toHaveBeenCalled();
|
||||
expect(callCluster).toHaveBeenCalledWith('transport.request', {
|
||||
path: '/_ml/set_upgrade_mode?enabled=true',
|
||||
method: 'POST',
|
||||
});
|
||||
});
|
||||
|
||||
it('increments ML reindexes and calls ML stop endpoint', async () => {
|
||||
actions.incrementIndexGroupReindexes.mockResolvedValueOnce();
|
||||
actions.runWhileIndexGroupLocked.mockImplementationOnce(async (group: string, f: any) =>
|
||||
|
|
|
@ -15,10 +15,17 @@ import {
|
|||
ReindexStep,
|
||||
ReindexWarning,
|
||||
} from '../../../common/types';
|
||||
import { getReindexWarnings, parseIndexName, transformFlatSettings } from './index_settings';
|
||||
import {
|
||||
generateNewIndexName,
|
||||
getReindexWarnings,
|
||||
sourceNameForIndex,
|
||||
transformFlatSettings,
|
||||
} from './index_settings';
|
||||
import { ReindexActions } from './reindex_actions';
|
||||
|
||||
const VERSION_REGEX = new RegExp(/^([1-9]+)\.([0-9]+)\.([0-9]+)/);
|
||||
const ML_INDICES = ['.ml-state', '.ml-anomalies', '.ml-config'];
|
||||
const WATCHER_INDICES = ['.watches', '.triggered-watches'];
|
||||
|
||||
export interface ReindexService {
|
||||
/**
|
||||
|
@ -442,13 +449,13 @@ export const reindexServiceFactory = (
|
|||
return true;
|
||||
}
|
||||
|
||||
const index = parseIndexName(indexName);
|
||||
const names = [indexName, index.newIndexName];
|
||||
const names = [indexName, generateNewIndexName(indexName)];
|
||||
const sourceName = sourceNameForIndex(indexName);
|
||||
|
||||
// if we have re-indexed this in the past, there will be an
|
||||
// underlying alias we will also need to update.
|
||||
if (index.cleanIndexName !== indexName) {
|
||||
names.push(index.cleanIndexName);
|
||||
if (sourceName !== indexName) {
|
||||
names.push(sourceName);
|
||||
}
|
||||
|
||||
// Otherwise, query for required privileges for this index.
|
||||
|
@ -647,8 +654,12 @@ export const reindexServiceFactory = (
|
|||
};
|
||||
};
|
||||
|
||||
const isMlIndex = (indexName: string) =>
|
||||
indexName.startsWith('.ml-state') || indexName.startsWith('.ml-anomalies');
|
||||
export const isMlIndex = (indexName: string) => {
|
||||
const sourceName = sourceNameForIndex(indexName);
|
||||
return ML_INDICES.indexOf(sourceName) >= 0;
|
||||
};
|
||||
|
||||
const isWatcherIndex = (indexName: string) =>
|
||||
indexName.startsWith('.watches') || indexName.startsWith('.triggered-watches');
|
||||
export const isWatcherIndex = (indexName: string) => {
|
||||
const sourceName = sourceNameForIndex(indexName);
|
||||
return WATCHER_INDICES.indexOf(sourceName) >= 0;
|
||||
};
|
||||
|
|
Loading…
Reference in a new issue