maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Rename REINDEX_SOURCE_TO_TEMP_INDEX to REINDEX_SOURCE_TO_TEMP_TRANSFORM (#112727)

Browse source
This commit is contained in:
Rudolf Meijering 2021-09-22 15:12:18 +02:00 committed by GitHub
parent 93cc4fcd9b
commit 68fb9f6d84
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 32 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
src/core/server/saved_objects/migrationsv2/README.md
View file

@ -36,7 +36,7 @@
- [REINDEX_SOURCE_TO_TEMP_READ](#reindex_source_to_temp_read)
- [Next action](#next-action-11)
- [New control state](#new-control-state-11)
- [REINDEX_SOURCE_TO_TEMP_INDEX](#reindex_source_to_temp_index)
- [REINDEX_SOURCE_TO_TEMP_TRANSFORM](#REINDEX_SOURCE_TO_TEMP_TRANSFORM)
- [Next action](#next-action-12)
- [New control state](#new-control-state-12)
- [REINDEX_SOURCE_TO_TEMP_INDEX_BULK](#reindex_source_to_temp_index_bulk)
@ -284,11 +284,11 @@ Read the next batch of outdated documents from the source index by using search
### New control state
1. If the batch contained > 0 documents
→ `REINDEX_SOURCE_TO_TEMP_INDEX`
→ `REINDEX_SOURCE_TO_TEMP_TRANSFORM`
2. If there are no more documents returned
→ `REINDEX_SOURCE_TO_TEMP_CLOSE_PIT`
## REINDEX_SOURCE_TO_TEMP_INDEX
## REINDEX_SOURCE_TO_TEMP_TRANSFORM
### Next action
`transformRawDocs`
@ -357,7 +357,7 @@ documents.
If another instance has a disabled plugin it will reindex that plugin's
documents without transforming them. Because this instance doesn't know which
plugins were disabled by the instance that performed the
`REINDEX_SOURCE_TO_TEMP_INDEX` step, we need to search for outdated documents
`REINDEX_SOURCE_TO_TEMP_TRANSFORM` step, we need to search for outdated documents
and transform them to ensure that everything is up to date.
### New control state

6
src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.ts
View file

@ -13,7 +13,7 @@ import type { ElasticsearchClient } from '../../elasticsearch';
import { getErrorMessage, getRequestDebugMeta } from '../../elasticsearch';
import { Model, Next, stateActionMachine } from './state_action_machine';
import { cleanup } from './migrations_state_machine_cleanup';
import { ReindexSourceToTempIndex, ReindexSourceToTempIndexBulk, State } from './types';
import { ReindexSourceToTempTransform, ReindexSourceToTempIndexBulk, State } from './types';
import { SavedObjectsRawDoc } from '../serialization';
interface StateTransitionLogMeta extends LogMeta {
@ -115,7 +115,9 @@ export async function migrationStateActionMachine({
const redactedNewState = {
...newState,
...{
outdatedDocuments: ((newState as ReindexSourceToTempIndex).outdatedDocuments ?? []).map(
outdatedDocuments: (
(newState as ReindexSourceToTempTransform).outdatedDocuments ?? []
).map(
(doc) =>
({
_id: doc._id,

32
src/core/server/saved_objects/migrationsv2/model/model.test.ts
View file

@ -20,7 +20,7 @@ import type {
ReindexSourceToTempOpenPit,
ReindexSourceToTempRead,
ReindexSourceToTempClosePit,
ReindexSourceToTempIndex,
ReindexSourceToTempTransform,
RefreshTarget,
UpdateTargetMappingsState,
UpdateTargetMappingsWaitForTaskState,
@ -962,7 +962,7 @@ describe('migrations v2 model', () => {
progress: createInitialProgress(),
};
it('REINDEX_SOURCE_TO_TEMP_READ -> REINDEX_SOURCE_TO_TEMP_INDEX if the index has outdated documents to reindex', () => {
it('REINDEX_SOURCE_TO_TEMP_READ -> REINDEX_SOURCE_TO_TEMP_TRANSFORM if the index has outdated documents to reindex', () => {
const outdatedDocuments = [{ _id: '1', _source: { type: 'vis' } }];
const lastHitSortValue = [123456];
const res: ResponseType<'REINDEX_SOURCE_TO_TEMP_READ'> = Either.right({
@ -970,8 +970,8 @@ describe('migrations v2 model', () => {
lastHitSortValue,
totalHits: 1,
});
const newState = model(state, res) as ReindexSourceToTempIndex;
expect(newState.controlState).toBe('REINDEX_SOURCE_TO_TEMP_INDEX');
const newState = model(state, res) as ReindexSourceToTempTransform;
expect(newState.controlState).toBe('REINDEX_SOURCE_TO_TEMP_TRANSFORM');
expect(newState.outdatedDocuments).toBe(outdatedDocuments);
expect(newState.lastHitSortValue).toBe(lastHitSortValue);
expect(newState.progress.processed).toBe(undefined);
@ -1032,16 +1032,16 @@ describe('migrations v2 model', () => {
it('REINDEX_SOURCE_TO_TEMP_CLOSE_PIT -> SET_TEMP_WRITE_BLOCK if action succeeded', () => {
const res: ResponseType<'REINDEX_SOURCE_TO_TEMP_CLOSE_PIT'> = Either.right({});
const newState = model(state, res) as ReindexSourceToTempIndex;
const newState = model(state, res) as ReindexSourceToTempTransform;
expect(newState.controlState).toBe('SET_TEMP_WRITE_BLOCK');
expect(newState.sourceIndex).toEqual(state.sourceIndex);
});
});
describe('REINDEX_SOURCE_TO_TEMP_INDEX', () => {
const state: ReindexSourceToTempIndex = {
describe('REINDEX_SOURCE_TO_TEMP_TRANSFORM', () => {
const state: ReindexSourceToTempTransform = {
...baseState,
controlState: 'REINDEX_SOURCE_TO_TEMP_INDEX',
controlState: 'REINDEX_SOURCE_TO_TEMP_TRANSFORM',
outdatedDocuments: [],
versionIndexReadyActions: Option.none,
sourceIndex: Option.some('.kibana') as Option.Some<string>,
@ -1059,8 +1059,8 @@ describe('migrations v2 model', () => {
},
] as SavedObjectsRawDoc[];
it('REINDEX_SOURCE_TO_TEMP_INDEX -> REINDEX_SOURCE_TO_TEMP_INDEX_BULK if action succeeded', () => {
const res: ResponseType<'REINDEX_SOURCE_TO_TEMP_INDEX'> = Either.right({
it('REINDEX_SOURCE_TO_TEMP_TRANSFORM -> REINDEX_SOURCE_TO_TEMP_INDEX_BULK if action succeeded', () => {
const res: ResponseType<'REINDEX_SOURCE_TO_TEMP_TRANSFORM'> = Either.right({
processedDocs,
});
const newState = model(state, res) as ReindexSourceToTempIndexBulk;
@ -1071,7 +1071,7 @@ describe('migrations v2 model', () => {
});
it('increments the progress.processed counter', () => {
const res: ResponseType<'REINDEX_SOURCE_TO_TEMP_INDEX'> = Either.right({
const res: ResponseType<'REINDEX_SOURCE_TO_TEMP_TRANSFORM'> = Either.right({
processedDocs,
});
@ -1089,8 +1089,8 @@ describe('migrations v2 model', () => {
expect(newState.progress.processed).toBe(2);
});
it('REINDEX_SOURCE_TO_TEMP_INDEX -> REINDEX_SOURCE_TO_TEMP_READ if action succeeded but we have carried through previous failures', () => {
const res: ResponseType<'REINDEX_SOURCE_TO_TEMP_INDEX'> = Either.right({
it('REINDEX_SOURCE_TO_TEMP_TRANSFORM -> REINDEX_SOURCE_TO_TEMP_READ if action succeeded but we have carried through previous failures', () => {
const res: ResponseType<'REINDEX_SOURCE_TO_TEMP_TRANSFORM'> = Either.right({
processedDocs,
});
const testState = {
@ -1098,15 +1098,15 @@ describe('migrations v2 model', () => {
corruptDocumentIds: ['a:b'],
transformErrors: [],
};
const newState = model(testState, res) as ReindexSourceToTempIndex;
const newState = model(testState, res) as ReindexSourceToTempTransform;
expect(newState.controlState).toEqual('REINDEX_SOURCE_TO_TEMP_READ');
expect(newState.corruptDocumentIds.length).toEqual(1);
expect(newState.transformErrors.length).toEqual(0);
expect(newState.progress.processed).toBe(0);
});
it('REINDEX_SOURCE_TO_TEMP_INDEX -> REINDEX_SOURCE_TO_TEMP_READ when response is left documents_transform_failed', () => {
const res: ResponseType<'REINDEX_SOURCE_TO_TEMP_INDEX'> = Either.left({
it('REINDEX_SOURCE_TO_TEMP_TRANSFORM -> REINDEX_SOURCE_TO_TEMP_READ when response is left documents_transform_failed', () => {
const res: ResponseType<'REINDEX_SOURCE_TO_TEMP_TRANSFORM'> = Either.left({
type: 'documents_transform_failed',
corruptDocumentIds: ['a:b'],
transformErrors: [],

6
src/core/server/saved_objects/migrationsv2/model/model.ts
View file

@ -446,7 +446,7 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
if (res.right.outdatedDocuments.length > 0) {
return {
...stateP,
controlState: 'REINDEX_SOURCE_TO_TEMP_INDEX',
controlState: 'REINDEX_SOURCE_TO_TEMP_TRANSFORM',
outdatedDocuments: res.right.outdatedDocuments,
lastHitSortValue: res.right.lastHitSortValue,
progress,
@ -489,11 +489,11 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
} else {
throwBadResponse(stateP, res);
}
} else if (stateP.controlState === 'REINDEX_SOURCE_TO_TEMP_INDEX') {
} else if (stateP.controlState === 'REINDEX_SOURCE_TO_TEMP_TRANSFORM') {
// We follow a similar control flow as for
// outdated document search -> outdated document transform -> transform documents bulk index
// collecting issues along the way rather than failing
// REINDEX_SOURCE_TO_TEMP_INDEX handles the document transforms
// REINDEX_SOURCE_TO_TEMP_TRANSFORM handles the document transforms
const res = resW as ExcludeRetryableEsError<ResponseType<typeof stateP.controlState>>;
// Increment the processed documents, no matter what the results are.

4
src/core/server/saved_objects/migrationsv2/next.ts
View file

@ -12,7 +12,7 @@ import type {
ReindexSourceToTempOpenPit,
ReindexSourceToTempRead,
ReindexSourceToTempClosePit,
ReindexSourceToTempIndex,
ReindexSourceToTempTransform,
MarkVersionIndexReady,
InitState,
LegacyCreateReindexTargetState,
@ -105,7 +105,7 @@ export const nextActionMap = (client: ElasticsearchClient, transformRawDocs: Tra
}),
REINDEX_SOURCE_TO_TEMP_CLOSE_PIT: (state: ReindexSourceToTempClosePit) =>
Actions.closePit({ client, pitId: state.sourceIndexPitId }),
REINDEX_SOURCE_TO_TEMP_INDEX: (state: ReindexSourceToTempIndex) =>
REINDEX_SOURCE_TO_TEMP_TRANSFORM: (state: ReindexSourceToTempTransform) =>
Actions.transformDocs({ transformRawDocs, outdatedDocuments: state.outdatedDocuments }),
REINDEX_SOURCE_TO_TEMP_INDEX_BULK: (state: ReindexSourceToTempIndexBulk) =>
Actions.bulkOverwriteTransformedDocuments({

6
src/core/server/saved_objects/migrationsv2/types.ts
View file

@ -233,8 +233,8 @@ export interface ReindexSourceToTempClosePit extends PostInitState {
readonly sourceIndexPitId: string;
}
export interface ReindexSourceToTempIndex extends PostInitState {
readonly controlState: 'REINDEX_SOURCE_TO_TEMP_INDEX';
export interface ReindexSourceToTempTransform extends PostInitState {
readonly controlState: 'REINDEX_SOURCE_TO_TEMP_TRANSFORM';
readonly outdatedDocuments: SavedObjectsRawDoc[];
readonly sourceIndexPitId: string;
readonly lastHitSortValue: number[] | undefined;
@ -434,7 +434,7 @@ export type State = Readonly<
| ReindexSourceToTempOpenPit
| ReindexSourceToTempRead
| ReindexSourceToTempClosePit
| ReindexSourceToTempIndex
| ReindexSourceToTempTransform
| ReindexSourceToTempIndexBulk
| SetTempWriteBlock
| CloneTempToSource