maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[RAC][Rule Registry] Implement versioning and backing indices (#109276)

Browse source 
**Ticket:** https://github.com/elastic/kibana/issues/109293

🚨 **This PR is critical for Observability 7.15** 🚨

## Summary

This PR fixes the indexing implementation in `rule_registry`. It implements the suggestions for backwards compatibility described in the ticket:

- changes the naming scheme and introduces the concept of "backing indices", so that names of the concrete ("backing") indices != names of their aliases
- adds versioning based on the current Kibana version

TODO:

- [x] Change index naming (implement the concept of backing indices)
- [x] Include Kibana version into the index template metadata
- [x] Include Kibana version into the document fields
- [x] Remove `version` from `IndexOptions` (parameters provided by solutions/plugins when initializing alerts-as-data indices)
- [x] Fix CI

### Checklist

- [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials
- [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios
This commit is contained in:
Georgii Gorbachev 2021-08-25 19:54:25 +02:00 committed by GitHub
parent 406df4d986
commit a299604c58
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
25 changed files with 301 additions and 284 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
x-pack/plugins/apm/server/lib/alerts/test_utils/index.ts
View file

@ -8,7 +8,8 @@
import { Logger } from 'kibana/server';
import { of } from 'rxjs';
import { elasticsearchServiceMock } from 'src/core/server/mocks';
import type { IRuleDataClient } from '../../../../../rule_registry/server';
import { IRuleDataClient } from '../../../../../rule_registry/server';
import { ruleRegistryMocks } from '../../../../../rule_registry/server/mocks';
import { PluginSetupContract as AlertingPluginSetupContract } from '../../../../../alerting/server';
import { APMConfig, APM_SERVER_FEATURE_ID } from '../../..';
@ -51,20 +52,9 @@ export const createRuleTypeMocks = () => {
alerting,
config$: mockedConfig$,
logger: loggerMock,
ruleDataClient: ({
getReader: () => {
return {
search: jest.fn(),
};
},
getWriter: () => {
return {
bulk: jest.fn(),
};
},
isWriteEnabled: jest.fn(() => true),
indexName: '.alerts-observability.apm.alerts',
} as unknown) as IRuleDataClient,
ruleDataClient: ruleRegistryMocks.createRuleDataClient(
'.alerts-observability.apm.alerts'
) as IRuleDataClient,
},
services,
scheduleActions,

4
x-pack/plugins/apm/server/plugin.ts
View file

@ -122,7 +122,6 @@ export class APMPlugin
componentTemplates: [
{
name: 'mappings',
version: 0,
mappings: mappingFromFieldMap(
{
[SERVICE_NAME]: {
@ -142,9 +141,6 @@ export class APMPlugin
),
},
],
indexTemplate: {
version: 0,
},
});
const resourcePlugins = mapValues(plugins, (value, key) => {

4
x-pack/plugins/infra/server/services/rules/rule_data_client.ts
View file

@ -31,12 +31,8 @@ export const createRuleDataClient = ({
componentTemplates: [
{
name: 'mappings',
version: 0,
mappings: {},
},
],
indexTemplate: {
version: 0,
},
});
};

2
x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.ts
View file

@ -30,7 +30,7 @@ export const technicalRuleFieldMap = {
[Fields.ALERT_EVALUATION_THRESHOLD]: { type: 'scaled_float', scaling_factor: 100 },
[Fields.ALERT_EVALUATION_VALUE]: { type: 'scaled_float', scaling_factor: 100 },
[Fields.VERSION]: {
type: 'keyword',
type: 'version',
array: false,
required: false,
},

23
x-pack/plugins/rule_registry/common/field_map/es_field_type_map.ts
View file

@ -1,23 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import * as t from 'io-ts';
export const esFieldTypeMap = {
keyword: t.string,
text: t.string,
date: t.string,
boolean: t.boolean,
byte: t.number,
long: t.number,
integer: t.number,
short: t.number,
double: t.number,
float: t.number,
scaled_float: t.number,
unsigned_long: t.number,
flattened: t.record(t.string, t.array(t.string)),
};

1
x-pack/plugins/rule_registry/common/field_map/runtime_type_from_fieldmap.ts
View file

@ -45,6 +45,7 @@ const BooleanFromString = new t.Type(
const esFieldTypeMap = {
keyword: t.string,
version: t.string,
text: t.string,
date: t.string,
boolean: t.union([t.number, BooleanFromString]),

1
x-pack/plugins/rule_registry/server/config.ts
View file

@ -24,3 +24,4 @@ export const config = {
export type RuleRegistryPluginConfig = TypeOf<typeof config.schema>;
export const INDEX_PREFIX = '.alerts' as const;
export const INDEX_PREFIX_FOR_BACKING_INDICES = '.internal.alerts' as const;

2
x-pack/plugins/rule_registry/server/mocks.ts
View file

@ -6,11 +6,13 @@
*/
import { alertsClientMock } from './alert_data_client/alerts_client.mock';
import { createRuleDataClientMock } from './rule_data_client/rule_data_client.mock';
import { ruleDataPluginServiceMock } from './rule_data_plugin_service/rule_data_plugin_service.mock';
import { createLifecycleAlertServicesMock } from './utils/lifecycle_alert_services_mock';
export const ruleRegistryMocks = {
createLifecycleAlertServices: createLifecycleAlertServicesMock,
createRuleDataPluginService: ruleDataPluginServiceMock.create,
createRuleDataClient: createRuleDataClientMock,
createAlertsClientMock: alertsClientMock,
};

8
x-pack/plugins/rule_registry/server/plugin.ts
View file

@ -19,7 +19,7 @@ import {
import { PluginStartContract as AlertingStart } from '../../alerting/server';
import { SecurityPluginSetup } from '../../security/server';
import { INDEX_PREFIX, RuleRegistryPluginConfig } from './config';
import { RuleRegistryPluginConfig } from './config';
import { RuleDataPluginService } from './rule_data_plugin_service';
import { AlertsClientFactory } from './alert_data_client/alerts_client_factory';
import { AlertsClient } from './alert_data_client/alerts_client';
@ -54,6 +54,7 @@ export class RuleRegistryPlugin
private readonly config: RuleRegistryPluginConfig;
private readonly legacyConfig: SharedGlobalConfig;
private readonly logger: Logger;
private readonly kibanaVersion: string;
private readonly alertsClientFactory: AlertsClientFactory;
private ruleDataService: RuleDataPluginService | null;
private security: SecurityPluginSetup | undefined;
@ -63,6 +64,7 @@ export class RuleRegistryPlugin
// TODO: Can be removed in 8.0.0. Exists to work around multi-tenancy users.
this.legacyConfig = initContext.config.legacy.get();
this.logger = initContext.logger.get();
this.kibanaVersion = initContext.env.packageInfo.version;
this.ruleDataService = null;
this.alertsClientFactory = new AlertsClientFactory();
}
@ -71,7 +73,7 @@ export class RuleRegistryPlugin
core: CoreSetup<RuleRegistryPluginStartDependencies, RuleRegistryPluginStartContract>,
plugins: RuleRegistryPluginSetupDependencies
): RuleRegistryPluginSetupContract {
const { logger } = this;
const { logger, kibanaVersion } = this;
const startDependencies = core.getStartServices().then(([coreStart, pluginStart]) => {
return {
@ -99,8 +101,8 @@ export class RuleRegistryPlugin
this.ruleDataService = new RuleDataPluginService({
logger,
kibanaVersion,
isWriteEnabled: isWriteEnabled(this.config, this.legacyConfig),
index: INDEX_PREFIX,
getClusterClient: async () => {
const deps = await startDependencies;
return deps.core.elasticsearch.client.asInternalUser;

12
x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.mock.ts
View file

@ -18,23 +18,25 @@ type RuleDataClientMock = jest.Mocked<Omit<IRuleDataClient, 'getWriter' | 'getRe
getWriter: (...args: Parameters<IRuleDataClient['getWriter']>) => MockInstances<IRuleDataWriter>;
};
export function createRuleDataClientMock(): RuleDataClientMock {
export const createRuleDataClientMock = (
indexName: string = '.alerts-security.alerts'
): RuleDataClientMock => {
const bulk = jest.fn();
const search = jest.fn();
const getDynamicIndexPattern = jest.fn();
return {
indexName: '.alerts-security.alerts',
indexName,
kibanaVersion: '7.16.0',
isWriteEnabled: jest.fn(() => true),
getReader: jest.fn((_options?: { namespace?: string }) => ({
getDynamicIndexPattern,
search,
getDynamicIndexPattern,
})),
getWriter: jest.fn(() => ({
bulk,
})),
};
}
};

4
x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.ts
View file

@ -33,6 +33,10 @@ export class RuleDataClient implements IRuleDataClient {
return this.options.indexInfo.baseName;
}
public get kibanaVersion(): string {
return this.options.indexInfo.kibanaVersion;
}
public isWriteEnabled(): boolean {
return this.options.isWriteEnabled;
}

1
x-pack/plugins/rule_registry/server/rule_data_client/types.ts
View file

@ -14,6 +14,7 @@ import { TechnicalRuleDataFieldName } from '../../common/technical_rule_data_fie
export interface IRuleDataClient {
indexName: string;
kibanaVersion: string;
isWriteEnabled(): boolean;
getReader(options?: { namespace?: string }): IRuleDataReader;
getWriter(options?: { namespace?: string }): IRuleDataWriter;

72
x-pack/plugins/rule_registry/server/rule_data_plugin_service/index_info.ts
View file

@ -5,22 +5,13 @@
* 2.0.
*/
import { INDEX_PREFIX, INDEX_PREFIX_FOR_BACKING_INDICES } from '../config';
import { IndexOptions } from './index_options';
import { joinWithDash } from './utils';
interface ConstructorOptions {
/**
* Prepends a relative resource name (defined in the code) with
* a full resource prefix, which starts with '.alerts' and can
* optionally include a user-defined part in it.
* @example 'security.alerts' => '.alerts-security.alerts'
*/
getResourceName(relativeName: string): string;
/**
* Options provided by the plugin/solution defining the index.
*/
indexOptions: IndexOptions;
kibanaVersion: string;
}
/**
@ -31,12 +22,17 @@ interface ConstructorOptions {
*/
export class IndexInfo {
constructor(options: ConstructorOptions) {
const { getResourceName, indexOptions } = options;
const { indexOptions, kibanaVersion } = options;
const { registrationContext, dataset } = indexOptions;
this.indexOptions = indexOptions;
this.baseName = getResourceName(`${registrationContext}.${dataset}`);
this.kibanaVersion = kibanaVersion;
this.baseName = joinWithDash(INDEX_PREFIX, `${registrationContext}.${dataset}`);
this.basePattern = joinWithDash(this.baseName, '*');
this.baseNameForBackingIndices = joinWithDash(
INDEX_PREFIX_FOR_BACKING_INDICES,
`${registrationContext}.${dataset}`
);
}
/**
@ -45,7 +41,13 @@ export class IndexInfo {
public readonly indexOptions: IndexOptions;
/**
* Base index name, prefixed with the full resource prefix.
* Current version of Kibana. We version our index resources and documents based on it.
* @example '7.16.0'
*/
public readonly kibanaVersion: string;
/**
* Base index name, prefixed with the resource prefix.
* @example '.alerts-security.alerts'
*/
public readonly baseName: string;
@ -56,6 +58,12 @@ export class IndexInfo {
*/
public readonly basePattern: string;
/**
* Base name for internal backing indices, prefixed with a special prefix.
* @example '.internal.alerts-security.alerts'
*/
private readonly baseNameForBackingIndices: string;
/**
* Primary index alias. Includes a namespace.
* Used as a write target when writing documents to the index.
@ -65,14 +73,6 @@ export class IndexInfo {
return joinWithDash(this.baseName, namespace);
}
/**
* Index pattern based on the primary alias.
* @example '.alerts-security.alerts-default-*'
*/
public getPrimaryAliasPattern(namespace: string): string {
return joinWithDash(this.baseName, namespace, '*');
}
/**
* Optional secondary alias that can be applied to concrete indices in
* addition to the primary one.
@ -83,6 +83,26 @@ export class IndexInfo {
return secondaryAlias ? joinWithDash(secondaryAlias, namespace) : null;
}
/**
* Name of the initial concrete index, with the namespace and the ILM suffix.
* @example '.internal.alerts-security.alerts-default-000001'
*/
public getConcreteIndexInitialName(namespace: string): string {
return joinWithDash(this.baseNameForBackingIndices, namespace, '000001');
}
/**
* Index pattern for internal backing indices. Used in the index bootstrapping logic.
* Can include or exclude the namespace.
*
* WARNING: Must not be used for reading documents! If you use it, you should know what you're doing.
*
* @example '.internal.alerts-security.alerts-default-*', '.internal.alerts-security.alerts-*'
*/
public getPatternForBackingIndices(namespace?: string): string {
return joinWithDash(this.baseNameForBackingIndices, namespace, '*');
}
/**
* Index pattern that should be used when reading documents from the index.
* Can include or exclude the namespace.
@ -100,14 +120,6 @@ export class IndexInfo {
return `${joinWithDash(this.baseName, namespace)}*`;
}
/**
* Name of the initial concrete index, with the namespace and the ILM suffix.
* @example '.alerts-security.alerts-default-000001'
*/
public getConcreteIndexInitialName(namespace: string): string {
return joinWithDash(this.baseName, namespace, '000001');
}
/**
* Name of the custom ILM policy (if it's provided by the plugin/solution).
* Specific to the index. Shared between all namespaces of the index.

4
x-pack/plugins/rule_registry/server/rule_data_plugin_service/index_options.ts
View file

@ -75,7 +75,7 @@ export interface IndexOptions {
/**
* Additional properties for the namespaced index template.
*/
indexTemplate: IndexTemplateOptions;
indexTemplate?: IndexTemplateOptions;
/**
* Optional custom ILM policy for the index.
@ -120,7 +120,6 @@ export type Meta = estypes.Metadata;
*/
export interface ComponentTemplateOptions {
name: string;
version: Version; // TODO: encapsulate versioning (base on Kibana version)
mappings?: Mappings;
settings?: Settings;
_meta?: Meta;
@ -140,7 +139,6 @@ export interface ComponentTemplateOptions {
* https://www.elastic.co/guide/en/elasticsearch/reference/current/index-templates.html
*/
export interface IndexTemplateOptions {
version: Version; // TODO: encapsulate versioning (base on Kibana version)
_meta?: Meta;
}

350
x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.ts
View file

@ -132,7 +132,6 @@ export class ResourceInstaller {
settings: ct.settings ?? {},
mappings: ct.mappings,
},
version: ct.version,
_meta: ct._meta,
},
});
@ -146,29 +145,22 @@ export class ResourceInstaller {
}
private async updateIndexMappings(indexInfo: IndexInfo) {
const { logger, getClusterClient } = this.options;
const clusterClient = await getClusterClient();
const { logger } = this.options;
const aliases = indexInfo.basePattern;
const backingIndices = indexInfo.getPatternForBackingIndices();
logger.debug(`Updating mappings of existing concrete indices for ${indexInfo.baseName}`);
const { body: aliasesResponse } = await clusterClient.indices.getAlias({
index: indexInfo.basePattern,
});
// Find all concrete indices for all namespaces of the index.
const concreteIndices = await this.fetchConcreteIndices(aliases, backingIndices);
const concreteWriteIndices = concreteIndices.filter((item) => item.isWriteIndex);
const writeIndicesAndAliases = Object.entries(aliasesResponse).flatMap(([index, { aliases }]) =>
Object.entries(aliases)
.filter(([, aliasProperties]) => aliasProperties.is_write_index)
.map(([aliasName]) => ({ index, alias: aliasName }))
);
await Promise.all(
writeIndicesAndAliases.map((indexAndAlias) =>
this.updateAliasWriteIndexMapping(indexAndAlias)
)
);
// Update mappings of the found write indices.
await Promise.all(concreteWriteIndices.map((item) => this.updateAliasWriteIndexMapping(item)));
}
private async updateAliasWriteIndexMapping({ index, alias }: { index: string; alias: string }) {
private async updateAliasWriteIndexMapping({ index, alias }: ConcreteIndexInfo) {
const { logger, getClusterClient } = this.options;
const clusterClient = await getClusterClient();
@ -228,57 +220,159 @@ export class ResourceInstaller {
indexInfo: IndexInfo,
namespace: string
): Promise<void> {
await this.createWriteTargetIfNeeded(indexInfo, namespace);
const { logger } = this.options;
const alias = indexInfo.getPrimaryAlias(namespace);
logger.info(`Installing namespace-level resources and creating concrete index for ${alias}`);
// If we find a concrete backing index which is the write index for the alias here, we shouldn't
// be making a new concrete index. We return early because we don't need a new write target.
const indexExists = await this.checkIfConcreteWriteIndexExists(indexInfo, namespace);
if (indexExists) {
return;
}
await this.installNamespacedIndexTemplate(indexInfo, namespace);
await this.createConcreteWriteIndex(indexInfo, namespace);
}
private async createWriteTargetIfNeeded(indexInfo: IndexInfo, namespace: string) {
private async checkIfConcreteWriteIndexExists(
indexInfo: IndexInfo,
namespace: string
): Promise<boolean> {
const { logger } = this.options;
const primaryNamespacedAlias = indexInfo.getPrimaryAlias(namespace);
const indexPatternForBackingIndices = indexInfo.getPatternForBackingIndices(namespace);
logger.debug(`Checking if concrete write index exists for ${primaryNamespacedAlias}`);
const concreteIndices = await this.fetchConcreteIndices(
primaryNamespacedAlias,
indexPatternForBackingIndices
);
const concreteIndicesExist = concreteIndices.some(
(item) => item.alias === primaryNamespacedAlias
);
const concreteWriteIndicesExist = concreteIndices.some(
(item) => item.alias === primaryNamespacedAlias && item.isWriteIndex
);
// If we find backing indices for the alias here, we shouldn't be making a new concrete index -
// either one of the indices is the write index so we return early because we don't need a new write target,
// or none of them are the write index so we'll throw an error because one of the existing indices should have
// been the write target
// If there are some concrete indices but none of them are the write index, we'll throw an error
// because one of the existing indices should have been the write target.
if (concreteIndicesExist && !concreteWriteIndicesExist) {
throw new Error(
`Indices matching pattern ${indexPatternForBackingIndices} exist but none are set as the write index for alias ${primaryNamespacedAlias}`
);
}
return concreteWriteIndicesExist;
}
private async installNamespacedIndexTemplate(indexInfo: IndexInfo, namespace: string) {
const { logger, getResourceName } = this.options;
const {
componentTemplateRefs,
componentTemplates,
indexTemplate = {},
ilmPolicy,
} = indexInfo.indexOptions;
const primaryNamespacedAlias = indexInfo.getPrimaryAlias(namespace);
const secondaryNamespacedAlias = indexInfo.getSecondaryAlias(namespace);
const indexPatternForBackingIndices = indexInfo.getPatternForBackingIndices(namespace);
logger.debug(`Installing index template for ${primaryNamespacedAlias}`);
const technicalComponentNames = [getResourceName(TECHNICAL_COMPONENT_TEMPLATE_NAME)];
const referencedComponentNames = componentTemplateRefs.map((ref) => getResourceName(ref));
const ownComponentNames = componentTemplates.map((template) =>
indexInfo.getComponentTemplateName(template.name)
);
const ilmPolicyName = ilmPolicy
? indexInfo.getIlmPolicyName()
: getResourceName(DEFAULT_ILM_POLICY_ID);
const indexMetadata: estypes.Metadata = {
...indexTemplate._meta,
kibana: {
...indexTemplate._meta?.kibana,
version: indexInfo.kibanaVersion,
},
namespace,
};
// TODO: need a way to update this template if/when we decide to make changes to the
// built in index template. Probably do it as part of updateIndexMappingsForAsset?
// (Before upgrading any indices, find and upgrade all namespaced index templates - component templates
// will already have been upgraded by solutions or rule registry, in the case of technical/ECS templates)
// With the current structure, it's tricky because the index template creation
// depends on both the namespace and secondary alias, both of which are not currently available
// to updateIndexMappingsForAsset. We can make the secondary alias available since
// it's known at plugin startup time, but
// the namespace values can really only come from the existing templates that we're trying to update
// - maybe we want to store the namespace as a _meta field on the index template for easy retrieval
await this.createOrUpdateIndexTemplate({
name: indexInfo.getIndexTemplateName(namespace),
body: {
index_patterns: [indexPatternForBackingIndices],
// Order matters:
// - first go external component templates referenced by this index (e.g. the common full ECS template)
// - then we include own component templates registered with this index
// - finally, we include technical component templates to make sure the index gets all the
// mappings and settings required by all Kibana plugins using rule registry to work properly
composed_of: [
...referencedComponentNames,
...ownComponentNames,
...technicalComponentNames,
],
template: {
settings: {
'index.lifecycle': {
name: ilmPolicyName,
// TODO: fix the types in the ES package, they don't include rollover_alias???
// @ts-expect-error
rollover_alias: primaryNamespacedAlias,
},
},
mappings: {
_meta: indexMetadata,
},
aliases:
secondaryNamespacedAlias != null
? {
[secondaryNamespacedAlias]: {
is_write_index: false,
},
}
: undefined,
},
_meta: indexMetadata,
// By setting the priority to namespace.length, we ensure that if one namespace is a prefix of another namespace
// then newly created indices will use the matching template with the *longest* namespace
priority: namespace.length,
},
});
}
private async createConcreteWriteIndex(indexInfo: IndexInfo, namespace: string) {
const { logger, getClusterClient } = this.options;
const clusterClient = await getClusterClient();
const primaryNamespacedAlias = indexInfo.getPrimaryAlias(namespace);
const primaryNamespacedPattern = indexInfo.getPrimaryAliasPattern(namespace);
const initialIndexName = indexInfo.getConcreteIndexInitialName(namespace);
logger.debug(`Creating write target for ${primaryNamespacedAlias}`);
try {
// When a new namespace is created we expect getAlias to return a 404 error,
// we'll catch it below and continue on. A non-404 error is a real problem so we throw.
// It's critical that we specify *both* the index pattern and alias in this request. The alias prevents the
// request from finding other namespaces that could match the -* part of the index pattern
// (see https://github.com/elastic/kibana/issues/107704). The index pattern prevents the request from
// finding legacy .siem-signals indices that we add the alias to for backwards compatibility reasons. Together,
// the index pattern and alias should ensure that we retrieve only the "new" backing indices for this
// particular alias.
const { body: aliases } = await clusterClient.indices.getAlias({
index: primaryNamespacedPattern,
name: primaryNamespacedAlias,
});
// If we find backing indices for the alias here, we shouldn't be making a new concrete index -
// either one of the indices is the write index so we return early because we don't need a new write target,
// or none of them are the write index so we'll throw an error because one of the existing indices should have
// been the write target
if (
Object.values(aliases).some(
(aliasesObject) => aliasesObject.aliases[primaryNamespacedAlias].is_write_index
)
) {
return;
} else {
throw new Error(
`Indices matching pattern ${primaryNamespacedPattern} exist but none are set as the write index for alias ${primaryNamespacedAlias}`
);
}
} catch (err) {
// 404 is expected if the alerts-as-data index hasn't been created yet
if (err.statusCode !== 404) {
throw err;
}
}
await this.installNamespacedIndexTemplate(indexInfo, namespace);
logger.debug(`Creating concrete write index for ${primaryNamespacedAlias}`);
try {
await clusterClient.indices.create({
@ -310,89 +404,6 @@ export class ResourceInstaller {
}
}
private async installNamespacedIndexTemplate(indexInfo: IndexInfo, namespace: string) {
const { logger, getResourceName } = this.options;
const {
componentTemplateRefs,
componentTemplates,
indexTemplate,
ilmPolicy,
} = indexInfo.indexOptions;
const primaryNamespacedAlias = indexInfo.getPrimaryAlias(namespace);
const primaryNamespacedPattern = indexInfo.getPrimaryAliasPattern(namespace);
const secondaryNamespacedAlias = indexInfo.getSecondaryAlias(namespace);
logger.debug(`Installing index template for ${primaryNamespacedAlias}`);
const technicalComponentNames = [getResourceName(TECHNICAL_COMPONENT_TEMPLATE_NAME)];
const referencedComponentNames = componentTemplateRefs.map((ref) => getResourceName(ref));
const ownComponentNames = componentTemplates.map((template) =>
indexInfo.getComponentTemplateName(template.name)
);
const ilmPolicyName = ilmPolicy
? indexInfo.getIlmPolicyName()
: getResourceName(DEFAULT_ILM_POLICY_ID);
// TODO: need a way to update this template if/when we decide to make changes to the
// built in index template. Probably do it as part of updateIndexMappingsForAsset?
// (Before upgrading any indices, find and upgrade all namespaced index templates - component templates
// will already have been upgraded by solutions or rule registry, in the case of technical/ECS templates)
// With the current structure, it's tricky because the index template creation
// depends on both the namespace and secondary alias, both of which are not currently available
// to updateIndexMappingsForAsset. We can make the secondary alias available since
// it's known at plugin startup time, but
// the namespace values can really only come from the existing templates that we're trying to update
// - maybe we want to store the namespace as a _meta field on the index template for easy retrieval
await this.createOrUpdateIndexTemplate({
name: indexInfo.getIndexTemplateName(namespace),
body: {
index_patterns: [primaryNamespacedPattern],
// Order matters:
// - first go external component templates referenced by this index (e.g. the common full ECS template)
// - then we include own component templates registered with this index
// - finally, we include technical component templates to make sure the index gets all the
// mappings and settings required by all Kibana plugins using rule registry to work properly
composed_of: [
...referencedComponentNames,
...ownComponentNames,
...technicalComponentNames,
],
template: {
settings: {
'index.lifecycle': {
name: ilmPolicyName,
// TODO: fix the types in the ES package, they don't include rollover_alias???
// @ts-expect-error
rollover_alias: primaryNamespacedAlias,
},
},
aliases:
secondaryNamespacedAlias != null
? {
[secondaryNamespacedAlias]: {
is_write_index: false,
},
}
: undefined,
},
_meta: {
...indexTemplate._meta,
namespace,
},
version: indexTemplate.version,
// By setting the priority to namespace.length, we ensure that if one namespace is a prefix of another namespace
// then newly created indices will use the matching template with the *longest* namespace
priority: namespace.length,
},
});
}
// -----------------------------------------------------------------------------------------------
// Helpers
@ -431,4 +442,55 @@ export class ResourceInstaller {
return clusterClient.indices.putIndexTemplate(template);
}
private async fetchConcreteIndices(
aliasOrPatternForAliases: string,
indexPatternForBackingIndices: string
): Promise<ConcreteIndexInfo[]> {
const { logger, getClusterClient } = this.options;
const clusterClient = await getClusterClient();
logger.debug(`Fetching concrete indices for ${indexPatternForBackingIndices}`);
try {
// It's critical that we specify *both* the index pattern for backing indices and their alias(es) in this request.
// The alias prevents the request from finding other namespaces that could match the -* part of the index pattern
// (see https://github.com/elastic/kibana/issues/107704). The backing index pattern prevents the request from
// finding legacy .siem-signals indices that we add the alias to for backwards compatibility reasons. Together,
// the index pattern and alias should ensure that we retrieve only the "new" backing indices for this
// particular alias.
const { body: response } = await clusterClient.indices.getAlias({
index: indexPatternForBackingIndices,
name: aliasOrPatternForAliases,
});
return createConcreteIndexInfo(response);
} catch (err) {
// 404 is expected if the alerts-as-data indices haven't been created yet
if (err.statusCode === 404) {
return createConcreteIndexInfo({});
}
// A non-404 error is a real problem so we re-throw.
throw err;
}
}
}
interface ConcreteIndexInfo {
index: string;
alias: string;
isWriteIndex: boolean;
}
const createConcreteIndexInfo = (
response: estypes.IndicesGetAliasResponse
): ConcreteIndexInfo[] => {
return Object.entries(response).flatMap(([index, { aliases }]) =>
Object.entries(aliases).map(([aliasName, aliasProperties]) => ({
index,
alias: aliasName,
isWriteIndex: aliasProperties.is_write_index ?? false,
}))
);
};

20
x-pack/plugins/rule_registry/server/rule_data_plugin_service/rule_data_plugin_service.ts
View file

@ -10,6 +10,7 @@ import { ValidFeatureId } from '@kbn/rule-data-utils';
import { ElasticsearchClient, Logger } from 'kibana/server';
import { INDEX_PREFIX } from '../config';
import { IRuleDataClient, RuleDataClient, WaitResult } from '../rule_data_client';
import { IndexInfo } from './index_info';
import { Dataset, IndexOptions } from './index_options';
@ -19,8 +20,8 @@ import { joinWithDash } from './utils';
interface ConstructorOptions {
getClusterClient: () => Promise<ElasticsearchClient>;
logger: Logger;
kibanaVersion: string;
isWriteEnabled: boolean;
index: string;
}
/**
@ -49,18 +50,16 @@ export class RuleDataPluginService {
}
/**
* Returns a full resource prefix.
* - it's '.alerts' by default
* - it can be adjusted by the user via Kibana config
* Returns a prefix used in the naming scheme of index aliases, templates
* and other Elasticsearch resources that this service creates
* for alerts-as-data indices.
*/
public getResourcePrefix(): string {
// TODO: https://github.com/elastic/kibana/issues/106432
return this.options.index;
return INDEX_PREFIX;
}
/**
* Prepends a relative resource name with a full resource prefix, which
* starts with '.alerts' and can optionally include a user-defined part in it.
* Prepends a relative resource name with the resource prefix.
* @returns Full name of the resource.
* @example 'security.alerts' => '.alerts-security.alerts'
*/
@ -106,10 +105,7 @@ export class RuleDataPluginService {
);
}
const indexInfo = new IndexInfo({
getResourceName: (name) => this.getResourceName(name),
indexOptions,
});
const indexInfo = new IndexInfo({ indexOptions, kibanaVersion: this.options.kibanaVersion });
const indicesAssociatedWithFeature = this.indicesByFeatureId.get(indexOptions.feature) ?? [];
this.indicesByFeatureId.set(indexOptions.feature, [...indicesAssociatedWithFeature, indexInfo]);

2
x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts
View file

@ -35,6 +35,7 @@ import {
EVENT_KIND,
SPACE_IDS,
TIMESTAMP,
VERSION,
} from '../../common/technical_rule_data_field_names';
import { IRuleDataClient } from '../rule_data_client';
import { AlertExecutorOptionsWithExtraServices } from '../types';
@ -250,6 +251,7 @@ export const createLifecycleExecutor = (
[EVENT_KIND]: 'signal',
[ALERT_RULE_CONSUMER]: rule.consumer,
[ALERT_ID]: alertId,
[VERSION]: ruleDataClient.kibanaVersion,
} as ParsedTechnicalFields;
const isNew = !state.trackedAlerts[alertId];

2
x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type.test.ts
View file

@ -203,6 +203,7 @@ describe('createLifecycleRuleTypeFactory', () => {
"kibana.space_ids": Array [
"spaceId",
],
"kibana.version": "7.16.0",
"service.name": "opbeans-java",
"tags": Array [
"tags",
@ -226,6 +227,7 @@ describe('createLifecycleRuleTypeFactory', () => {
"kibana.space_ids": Array [
"spaceId",
],
"kibana.version": "7.16.0",
"service.name": "opbeans-node",
"tags": Array [
"tags",

5
x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_factory.ts
View file

@ -5,7 +5,7 @@
* 2.0.
*/
import { ALERT_ID } from '@kbn/rule-data-utils';
import { ALERT_ID, VERSION } from '@kbn/rule-data-utils';
import { CreatePersistenceRuleTypeFactory } from './persistence_types';
export const createPersistenceRuleTypeFactory: CreatePersistenceRuleTypeFactory = ({
@ -28,8 +28,9 @@ export const createPersistenceRuleTypeFactory: CreatePersistenceRuleTypeFactory
body: alerts.flatMap((event) => [
{ index: {} },
{
[ALERT_ID]: event.id,
...event.fields,
[ALERT_ID]: event.id,
[VERSION]: ruleDataClient.kibanaVersion,
},
]),
refresh,

4
x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_adapter/rule_registry_log_client/rule_registry_log_client.ts
View file

@ -82,13 +82,9 @@ export class RuleRegistryLogClient implements IRuleRegistryLogClient {
componentTemplates: [
{
name: 'mappings',
version: 0,
mappings: mappingFromFieldMap(ruleExecutionFieldMap, 'strict'),
},
],
indexTemplate: {
version: 0,
},
});
}

15
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/rule_type.ts
View file

@ -12,12 +12,12 @@ import { Logger, SavedObject } from 'kibana/server';
import { elasticsearchServiceMock, savedObjectsClientMock } from 'src/core/server/mocks';
import type { IRuleDataClient } from '../../../../../../rule_registry/server';
import { ruleRegistryMocks } from '../../../../../../rule_registry/server/mocks';
import { PluginSetupContract as AlertingPluginSetupContract } from '../../../../../../alerting/server';
import { ConfigType } from '../../../../config';
import { AlertAttributes } from '../../signals/types';
import { createRuleMock } from './rule';
import { listMock } from '../../../../../../lists/server/mocks';
import { ruleRegistryMocks } from '../../../../../../rule_registry/server/mocks';
import { RuleParams } from '../../schemas/rule_schemas';
export const createRuleTypeMocks = (
@ -81,16 +81,9 @@ export const createRuleTypeMocks = (
config$: mockedConfig$,
lists: listMock.createSetup(),
logger: loggerMock,
ruleDataClient: ({
getReader: jest.fn(() => ({
search: jest.fn(),
})),
getWriter: jest.fn(() => ({
bulk: jest.fn(),
})),
isWriteEnabled: jest.fn(() => true),
indexName: '.alerts-security.alerts',
} as unknown) as IRuleDataClient,
ruleDataClient: ruleRegistryMocks.createRuleDataClient(
'.alerts-security.alerts'
) as IRuleDataClient,
ruleDataService: ruleRegistryMocks.createRuleDataPluginService(),
},
services,

4
x-pack/plugins/security_solution/server/plugin.ts
View file

@ -233,16 +233,12 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
componentTemplates: [
{
name: 'mappings',
version: 0,
mappings: mappingFromFieldMap(
{ ...alertsFieldMap, ...rulesFieldMap, ...ctiFieldMap },
false
),
},
],
indexTemplate: {
version: 0,
},
secondaryAlias: config.signalsIndex,
});

18
x-pack/plugins/uptime/server/lib/alerts/test_utils/index.ts
View file

@ -10,6 +10,7 @@ import { UMServerLibs } from '../../lib';
import { UptimeCorePlugins, UptimeCoreSetup } from '../../adapters';
import type { UptimeRouter } from '../../../types';
import type { IRuleDataClient } from '../../../../../rule_registry/server';
import { ruleRegistryMocks } from '../../../../../rule_registry/server/mocks';
import { getUptimeESMockClient } from '../../requests/helper';
import { alertsMock } from '../../../../../alerting/server/mocks';
import { DynamicSettings } from '../../../../common/runtime_types';
@ -61,20 +62,9 @@ export const createRuleTypeMocks = (
return {
dependencies: {
logger: loggerMock,
ruleDataClient: ({
getReader: () => {
return {
search: jest.fn(),
};
},
getWriter: () => {
return {
bulk: jest.fn(),
};
},
isWriteEnabled: jest.fn(() => true),
indexName: '.alerts-observability.uptime.alerts',
} as unknown) as IRuleDataClient,
ruleDataClient: ruleRegistryMocks.createRuleDataClient(
'.alerts-observability.uptime.alerts'
) as IRuleDataClient,
},
services,
scheduleActions,

4
x-pack/plugins/uptime/server/plugin.ts
View file

@ -43,13 +43,9 @@ export class Plugin implements PluginType {
componentTemplates: [
{
name: 'mappings',
version: 0,
mappings: mappingFromFieldMap(uptimeRuleFieldMap, 'strict'),
},
],
indexTemplate: {
version: 0,
},
});
initServerWithKibana(

3
x-pack/test/apm_api_integration/tests/alerts/rule_registry.ts
View file

@ -14,6 +14,7 @@ import {
ALERT_STATUS,
ALERT_UUID,
EVENT_KIND,
VERSION,
} from '@kbn/rule-data-utils';
import { merge, omit } from 'lodash';
import { FtrProviderContext } from '../../common/ftr_provider_context';
@ -376,7 +377,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
any
>;
const exclude = ['@timestamp', ALERT_START, ALERT_UUID, ALERT_RULE_UUID];
const exclude = ['@timestamp', ALERT_START, ALERT_UUID, ALERT_RULE_UUID, VERSION];
const toCompare = omit(alertEvent, exclude);