maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[Fleet] Setup fleet server indices in Kibana without packages (#90658)

Browse source
This commit is contained in:
Nicolas Chaulet 2021-02-16 13:54:54 -05:00 committed by GitHub
parent ea9f1e3dfb
commit a99ccc27d7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 730 additions and 75 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
x-pack/plugins/fleet/common/constants/index.ts
View file

@ -22,6 +22,8 @@ export * from './settings';
// setting in the future?
export const SO_SEARCH_LIMIT = 10000;
export const FLEET_SERVER_INDICES_VERSION = 1;
export const FLEET_SERVER_INDICES = [
'.fleet-actions',
'.fleet-agents',

3
x-pack/plugins/fleet/server/collectors/agent_collectors.ts
View file

@ -7,7 +7,8 @@
import { ElasticsearchClient, SavedObjectsClient } from 'kibana/server';
import * as AgentService from '../services/agents';
import { isFleetServerSetup } from '../services/fleet_server_migration';
import { isFleetServerSetup } from '../services/fleet_server';
export interface AgentUsage {
total: number;
online: number;

17
x-pack/plugins/fleet/server/plugin.ts
View file

@ -83,7 +83,7 @@ import { agentCheckinState } from './services/agents/checkin/state';
import { registerFleetUsageCollector } from './collectors/register';
import { getInstallation } from './services/epm/packages';
import { makeRouterEnforcingSuperuser } from './routes/security';
import { isFleetServerSetup } from './services/fleet_server_migration';
import { startFleetServerSetup } from './services/fleet_server';
export interface FleetSetupDeps {
licensing: LicensingPluginSetup;
@ -297,18 +297,9 @@ export class FleetPlugin
licenseService.start(this.licensing$);
agentCheckinState.start();
const fleetServerEnabled = appContextService.getConfig()?.agents?.fleetServerEnabled;
if (fleetServerEnabled) {
// We need licence to be initialized before using the SO service.
await this.licensing$.pipe(first()).toPromise();
const fleetSetup = await isFleetServerSetup();
if (!fleetSetup) {
this.logger?.warn(
'Extra setup is needed to be able to use central management for agent, please visit the Fleet app in Kibana.'
);
}
if (appContextService.getConfig()?.agents?.fleetServerEnabled) {
// Break the promise chain, the error handling is done in startFleetServerSetup
startFleetServerSetup();
}
return {

4
x-pack/plugins/fleet/server/services/app_context.ts
View file

@ -80,6 +80,10 @@ class AppContextService {
return this.security;
}
public hasSecurity() {
return !!this.security;
}
public getCloud() {
return this.cloud;
}

156
x-pack/plugins/fleet/server/services/fleet_server/elastic_index.test.ts Normal file
View file

@ -0,0 +1,156 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { elasticsearchServiceMock } from 'src/core/server/mocks';
import hash from 'object-hash';
import { setupFleetServerIndexes } from './elastic_index';
import ESFleetAgentIndex from './elasticsearch/fleet_agents.json';
import ESFleetPoliciesIndex from './elasticsearch/fleet_policies.json';
import ESFleetPoliciesLeaderIndex from './elasticsearch/fleet_policies_leader.json';
import ESFleetServersIndex from './elasticsearch/fleet_servers.json';
import ESFleetEnrollmentApiKeysIndex from './elasticsearch/fleet_enrollment_api_keys.json';
import EsFleetActionsIndex from './elasticsearch/fleet_actions.json';
const FLEET_INDEXES_MIGRATION_HASH = {
'.fleet-actions': hash(EsFleetActionsIndex),
'.fleet-agents': hash(ESFleetAgentIndex),
'.fleet-enrollment-apy-keys': hash(ESFleetEnrollmentApiKeysIndex),
'.fleet-policies': hash(ESFleetPoliciesIndex),
'.fleet-policies-leader': hash(ESFleetPoliciesLeaderIndex),
'.fleet-servers': hash(ESFleetServersIndex),
};
describe('setupFleetServerIndexes ', () => {
it('should create all the indices and aliases if nothings exists', async () => {
const esMock = elasticsearchServiceMock.createInternalClient();
await setupFleetServerIndexes(esMock);
const indexesCreated = esMock.indices.create.mock.calls.map((call) => call[0].index).sort();
expect(indexesCreated).toEqual([
'.fleet-actions_1',
'.fleet-agents_1',
'.fleet-enrollment-api-keys_1',
'.fleet-policies-leader_1',
'.fleet-policies_1',
'.fleet-servers_1',
]);
const aliasesCreated = esMock.indices.updateAliases.mock.calls
.map((call) => (call[0].body as any)?.actions[0].add.alias)
.sort();
expect(aliasesCreated).toEqual([
'.fleet-actions',
'.fleet-agents',
'.fleet-enrollment-api-keys',
'.fleet-policies',
'.fleet-policies-leader',
'.fleet-servers',
]);
});
it('should not create any indices and create aliases if indices exists but not the aliases', async () => {
const esMock = elasticsearchServiceMock.createInternalClient();
// @ts-expect-error
esMock.indices.exists.mockResolvedValue({ body: true });
// @ts-expect-error
esMock.indices.getMapping.mockImplementation((params: { index: string }) => {
return {
body: {
[params.index]: {
mappings: {
_meta: {
// @ts-expect-error
migrationHash: FLEET_INDEXES_MIGRATION_HASH[params.index.replace(/_1$/, '')],
},
},
},
},
};
});
await setupFleetServerIndexes(esMock);
expect(esMock.indices.create).not.toBeCalled();
const aliasesCreated = esMock.indices.updateAliases.mock.calls
.map((call) => (call[0].body as any)?.actions[0].add.alias)
.sort();
expect(aliasesCreated).toEqual([
'.fleet-actions',
'.fleet-agents',
'.fleet-enrollment-api-keys',
'.fleet-policies',
'.fleet-policies-leader',
'.fleet-servers',
]);
});
it('should put new indices mapping if the mapping has been updated ', async () => {
const esMock = elasticsearchServiceMock.createInternalClient();
// @ts-expect-error
esMock.indices.exists.mockResolvedValue({ body: true });
// @ts-expect-error
esMock.indices.getMapping.mockImplementation((params: { index: string }) => {
return {
body: {
[params.index]: {
mappings: {
_meta: {
migrationHash: 'NOT_VALID_HASH',
},
},
},
},
};
});
await setupFleetServerIndexes(esMock);
expect(esMock.indices.create).not.toBeCalled();
const indexesMappingUpdated = esMock.indices.putMapping.mock.calls
.map((call) => call[0].index)
.sort();
expect(indexesMappingUpdated).toEqual([
'.fleet-actions_1',
'.fleet-agents_1',
'.fleet-enrollment-api-keys_1',
'.fleet-policies-leader_1',
'.fleet-policies_1',
'.fleet-servers_1',
]);
});
it('should not create any indices or aliases if indices and aliases already exists', async () => {
const esMock = elasticsearchServiceMock.createInternalClient();
// @ts-expect-error
esMock.indices.exists.mockResolvedValue({ body: true });
// @ts-expect-error
esMock.indices.getMapping.mockImplementation((params: { index: string }) => {
return {
body: {
[params.index]: {
mappings: {
_meta: {
// @ts-expect-error
migrationHash: FLEET_INDEXES_MIGRATION_HASH[params.index.replace(/_1$/, '')],
},
},
},
},
};
});
// @ts-expect-error
esMock.indices.existsAlias.mockResolvedValue({ body: true });
await setupFleetServerIndexes(esMock);
expect(esMock.indices.create).not.toBeCalled();
expect(esMock.indices.updateAliases).not.toBeCalled();
});
});

117
x-pack/plugins/fleet/server/services/fleet_server/elastic_index.ts Normal file
View file

@ -0,0 +1,117 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ElasticsearchClient } from 'kibana/server';
import hash from 'object-hash';
import { FLEET_SERVER_INDICES, FLEET_SERVER_INDICES_VERSION } from '../../../common';
import { appContextService } from '../app_context';
import ESFleetAgentIndex from './elasticsearch/fleet_agents.json';
import ESFleetPoliciesIndex from './elasticsearch/fleet_policies.json';
import ESFleetPoliciesLeaderIndex from './elasticsearch/fleet_policies_leader.json';
import ESFleetServersIndex from './elasticsearch/fleet_servers.json';
import ESFleetEnrollmentApiKeysIndex from './elasticsearch/fleet_enrollment_api_keys.json';
import EsFleetActionsIndex from './elasticsearch/fleet_actions.json';
const FLEET_INDEXES: Array<[typeof FLEET_SERVER_INDICES[number], any]> = [
['.fleet-actions', EsFleetActionsIndex],
['.fleet-agents', ESFleetAgentIndex],
['.fleet-enrollment-api-keys', ESFleetEnrollmentApiKeysIndex],
['.fleet-policies', ESFleetPoliciesIndex],
['.fleet-policies-leader', ESFleetPoliciesLeaderIndex],
['.fleet-servers', ESFleetServersIndex],
];
export async function setupFleetServerIndexes(
esClient = appContextService.getInternalUserESClient()
) {
await Promise.all(
FLEET_INDEXES.map(async ([indexAlias, indexData]) => {
const index = `${indexAlias}_${FLEET_SERVER_INDICES_VERSION}`;
await createOrUpdateIndex(esClient, index, indexData);
await createAliasIfDoNotExists(esClient, indexAlias, index);
})
);
}
export async function createAliasIfDoNotExists(
esClient: ElasticsearchClient,
alias: string,
index: string
) {
const { body: exists } = await esClient.indices.existsAlias({
name: alias,
});
if (exists === true) {
return;
}
await esClient.indices.updateAliases({
body: {
actions: [
{
add: { index, alias },
},
],
},
});
}
async function createOrUpdateIndex(
esClient: ElasticsearchClient,
indexName: string,
indexData: any
) {
const resExists = await esClient.indices.exists({
index: indexName,
});
// Support non destructive migration only (adding new field)
if (resExists.body === true) {
return updateIndex(esClient, indexName, indexData);
}
return createIndex(esClient, indexName, indexData);
}
async function updateIndex(esClient: ElasticsearchClient, indexName: string, indexData: any) {
const res = await esClient.indices.getMapping({
index: indexName,
});
const migrationHash = hash(indexData);
if (res.body[indexName].mappings?._meta?.migrationHash !== migrationHash) {
await esClient.indices.putMapping({
index: indexName,
body: Object.assign({
...indexData.mappings,
_meta: { ...(indexData.mappings._meta || {}), migrationHash },
}),
});
}
}
async function createIndex(esClient: ElasticsearchClient, indexName: string, indexData: any) {
try {
const migrationHash = hash(indexData);
await esClient.indices.create({
index: indexName,
body: {
...indexData,
mappings: Object.assign({
...indexData.mappings,
_meta: { ...(indexData.mappings._meta || {}), migrationHash },
}),
},
});
} catch (err) {
// Swallow already exists errors as concurent Kibana can try to create that indice
if (err?.body?.error?.type !== 'resource_already_exists_exception') {
throw err;
}
}
}

30
x-pack/plugins/fleet/server/services/fleet_server/elasticsearch/fleet_actions.json Normal file
View file

@ -0,0 +1,30 @@
{
"settings": {},
"mappings": {
"dynamic": false,
"properties": {
"action_id": {
"type": "keyword"
},
"agents": {
"type": "keyword"
},
"data": {
"enabled": false,
"type": "object"
},
"expiration": {
"type": "date"
},
"input_type": {
"type": "keyword"
},
"@timestamp": {
"type": "date"
},
"type": {
"type": "keyword"
}
}
}
}

220
x-pack/plugins/fleet/server/services/fleet_server/elasticsearch/fleet_agents.json Normal file
View file

@ -0,0 +1,220 @@
{
"settings": {},
"mappings": {
"dynamic": false,
"properties": {
"access_api_key_id": {
"type": "keyword"
},
"action_seq_no": {
"type": "integer"
},
"active": {
"type": "boolean"
},
"agent": {
"properties": {
"id": {
"type": "keyword"
},
"version": {
"type": "keyword"
}
}
},
"default_api_key": {
"type": "keyword"
},
"default_api_key_id": {
"type": "keyword"
},
"enrolled_at": {
"type": "date"
},
"last_checkin": {
"type": "date"
},
"last_checkin_status": {
"type": "keyword"
},
"last_updated": {
"type": "date"
},
"local_metadata": {
"properties": {
"elastic": {
"properties": {
"agent": {
"properties": {
"build": {
"properties": {
"original": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"id": {
"type": "keyword"
},
"log_level": {
"type": "keyword"
},
"snapshot": {
"type": "boolean"
},
"upgradeable": {
"type": "boolean"
},
"version": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 16
}
}
}
}
}
}
},
"host": {
"properties": {
"architecture": {
"type": "keyword"
},
"hostname": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"id": {
"type": "keyword"
},
"ip": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 64
}
}
},
"mac": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 17
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"os": {
"properties": {
"family": {
"type": "keyword"
},
"full": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 128
}
}
},
"kernel": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 128
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"platform": {
"type": "keyword"
},
"version": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 32
}
}
}
}
}
}
},
"packages": {
"type": "keyword"
},
"policy_coordinator_idx": {
"type": "integer"
},
"policy_id": {
"type": "keyword"
},
"policy_revision_idx": {
"type": "integer"
},
"shared_id": {
"type": "keyword"
},
"type": {
"type": "keyword"
},
"unenrolled_at": {
"type": "date"
},
"unenrollment_started_at": {
"type": "date"
},
"updated_at": {
"type": "date"
},
"upgrade_started_at": {
"type": "date"
},
"upgraded_at": {
"type": "date"
},
"user_provided_metadata": {
"type": "object",
"enabled": false
}
}
}
}

32
x-pack/plugins/fleet/server/services/fleet_server/elasticsearch/fleet_enrollment_api_keys.json Normal file
View file

@ -0,0 +1,32 @@
{
"settings": {},
"mappings": {
"dynamic": false,
"properties": {
"active": {
"type": "boolean"
},
"api_key": {
"type": "keyword"
},
"api_key_id": {
"type": "keyword"
},
"created_at": {
"type": "date"
},
"expire_at": {
"type": "date"
},
"name": {
"type": "keyword"
},
"policy_id": {
"type": "keyword"
},
"updated_at": {
"type": "date"
}
}
}
}

27
x-pack/plugins/fleet/server/services/fleet_server/elasticsearch/fleet_policies.json Normal file
View file

@ -0,0 +1,27 @@
{
"settings": {},
"mappings": {
"dynamic": false,
"properties": {
"coordinator_idx": {
"type": "integer"
},
"data": {
"enabled": false,
"type": "object"
},
"default_fleet_server": {
"type": "boolean"
},
"policy_id": {
"type": "keyword"
},
"revision_idx": {
"type": "integer"
},
"@timestamp": {
"type": "date"
}
}
}
}

21
x-pack/plugins/fleet/server/services/fleet_server/elasticsearch/fleet_policies_leader.json Normal file
View file

@ -0,0 +1,21 @@
{
"settings": {},
"mappings": {
"dynamic": false,
"properties": {
"server": {
"properties": {
"id": {
"type": "keyword"
},
"version": {
"type": "keyword"
}
}
},
"@timestamp": {
"type": "date"
}
}
}
}

47
x-pack/plugins/fleet/server/services/fleet_server/elasticsearch/fleet_servers.json Normal file
View file

@ -0,0 +1,47 @@
{
"settings": {},
"mappings": {
"dynamic": false,
"properties": {
"agent": {
"properties": {
"id": {
"type": "keyword"
},
"version": {
"type": "keyword"
}
}
},
"host": {
"properties": {
"architecture": {
"type": "keyword"
},
"id": {
"type": "keyword"
},
"ip": {
"type": "keyword"
},
"name": {
"type": "keyword"
}
}
},
"server": {
"properties": {
"id": {
"type": "keyword"
},
"version": {
"type": "keyword"
}
}
},
"@timestamp": {
"type": "date"
}
}
}
}

57
x-pack/plugins/fleet/server/services/fleet_server/index.ts Normal file
View file

@ -0,0 +1,57 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { first } from 'rxjs/operators';
import { appContextService } from '../app_context';
import { licenseService } from '../license';
import { setupFleetServerIndexes } from './elastic_index';
import { runFleetServerMigration } from './saved_object_migrations';
let _isFleetServerSetup = false;
let _isPending = false;
let _status: Promise<any> | undefined;
let _onResolve: (arg?: any) => void;
export function isFleetServerSetup() {
return _isFleetServerSetup;
}
export function awaitIfFleetServerSetupPending() {
if (!_isPending) {
return;
}
return _status;
}
export async function startFleetServerSetup() {
_isPending = true;
_status = new Promise((resolve) => {
_onResolve = resolve;
});
const logger = appContextService.getLogger();
if (!appContextService.hasSecurity()) {
// Fleet will not work if security is not enabled
logger?.warn('Fleet requires the security plugin to be enabled.');
return;
}
try {
// We need licence to be initialized before using the SO service.
await licenseService.getLicenseInformation$()?.pipe(first())?.toPromise();
await setupFleetServerIndexes();
await runFleetServerMigration();
_isFleetServerSetup = true;
} catch (err) {
logger?.error('Setup for central management of agents failed.');
logger?.error(err);
}
_isPending = false;
if (_onResolve) {
_onResolve();
}
}

36
x-pack/plugins/fleet/server/services/fleet_server_migration.ts → x-pack/plugins/fleet/server/services/fleet_server/saved_object_migrations.ts
View file

@ -17,38 +17,12 @@ import {
AgentSOAttributes,
FleetServerAgent,
SO_SEARCH_LIMIT,
FLEET_SERVER_PACKAGE,
FLEET_SERVER_INDICES,
} from '../../common';
import { listEnrollmentApiKeys, getEnrollmentAPIKey } from './api_keys/enrollment_api_key_so';
import { appContextService } from './app_context';
import { getInstallation } from './epm/packages';
} from '../../../common';
import { listEnrollmentApiKeys, getEnrollmentAPIKey } from '../api_keys/enrollment_api_key_so';
import { appContextService } from '../app_context';
import { isAgentsSetup } from './agents';
import { agentPolicyService } from './agent_policy';
export async function isFleetServerSetup() {
const pkgInstall = await getInstallation({
savedObjectsClient: getInternalUserSOClient(),
pkgName: FLEET_SERVER_PACKAGE,
});
if (!pkgInstall) {
return false;
}
const esClient = appContextService.getInternalUserESClient();
const exists = await Promise.all(
FLEET_SERVER_INDICES.map(async (index) => {
const res = await esClient.indices.exists({
index,
});
return res.statusCode !== 404;
})
);
return exists.every((exist) => exist === true);
}
import { isAgentsSetup } from '../agents';
import { agentPolicyService } from '../agent_policy';
export async function runFleetServerMigration() {
// If Agents are not setup skip as there is nothing to migrate

33
x-pack/plugins/fleet/server/services/setup.ts
View file

@ -23,7 +23,6 @@ import {
Output,
DEFAULT_AGENT_POLICIES_PACKAGES,
FLEET_SERVER_PACKAGE,
FLEET_SERVER_INDICES,
} from '../../common';
import { SO_SEARCH_LIMIT } from '../constants';
import { getPackageInfo } from './epm/packages';
@ -34,7 +33,7 @@ import { awaitIfPending } from './setup_utils';
import { createDefaultSettings } from './settings';
import { ensureAgentActionPolicyChangeExists } from './agents';
import { appContextService } from './app_context';
import { runFleetServerMigration } from './fleet_server_migration';
import { awaitIfFleetServerSetupPending } from './fleet_server';
const FLEET_ENROLL_USERNAME = 'fleet_enroll';
const FLEET_ENROLL_ROLE = 'fleet_enroll';
@ -88,24 +87,15 @@ async function createSetupSideEffects(
// By moving this outside of the Promise.all, the upgrade will occur first, and then we'll attempt to reinstall any
// packages that are stuck in the installing state.
await ensurePackagesCompletedInstall(soClient, callCluster);
if (isFleetServerEnabled) {
await ensureInstalledPackage({
savedObjectsClient: soClient,
pkgName: FLEET_SERVER_PACKAGE,
callCluster,
});
await ensureFleetServerIndicesCreated(esClient);
await runFleetServerMigration();
}
if (appContextService.getConfig()?.agents?.fleetServerEnabled) {
if (isFleetServerEnabled) {
await awaitIfFleetServerSetupPending();
const fleetServerPackage = await ensureInstalledPackage({
savedObjectsClient: soClient,
pkgName: FLEET_SERVER_PACKAGE,
callCluster,
});
await ensureFleetServerIndicesCreated(esClient);
await runFleetServerMigration();
if (defaultFleetServerPolicyCreated) {
await addPackageToAgentPolicy(
@ -187,21 +177,6 @@ async function updateFleetRoleIfExists(callCluster: CallESAsCurrentUser) {
return putFleetRole(callCluster);
}
async function ensureFleetServerIndicesCreated(esClient: ElasticsearchClient) {
await Promise.all(
FLEET_SERVER_INDICES.map(async (index) => {
const res = await esClient.indices.exists({
index,
});
if (res.statusCode === 404) {
await esClient.indices.create({
index,
});
}
})
);
}
async function putFleetRole(callCluster: CallESAsCurrentUser) {
return callCluster('transport.request', {
method: 'PUT',

3
x-pack/plugins/fleet/tsconfig.json
View file

@ -5,13 +5,14 @@
"outDir": "./target/types",
"emitDeclarationOnly": true,
"declaration": true,
"declarationMap": true
"declarationMap": true,
},
"include": [
// add all the folders containg files to be compiled
"common/**/*",
"public/**/*",
"server/**/*",
"server/**/*.json",
"scripts/**/*",
"package.json",
"../../typings/**/*"