maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[Endpoint] Deleting data streams and indices in generator (#67168)

Browse source 
* Deleting data streams and indices

* Posting metadata template

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
This commit is contained in:
Jonathan Buttner 2020-05-21 16:02:09 -04:00 committed by GitHub
parent 204fd5b5a2
commit 70b92c8f42
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 182 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

146
x-pack/plugins/siem/scripts/endpoint/metadata_mapping.json Normal file
View file

@ -0,0 +1,146 @@
{
"mappings": {
"_meta": {
"version": "1.5.0"
},
"date_detection": false,
"dynamic_templates": [
{
"strings_as_keyword": {
"mapping": {
"ignore_above": 1024,
"type": "keyword"
},
"match_mapping_type": "string"
}
}
],
"properties": {
"@timestamp": {
"type": "date"
},
"agent": {
"properties": {
"id": {
"ignore_above": 1024,
"type": "keyword"
},
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"ecs": {
"properties": {
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"elastic": {
"properties": {
"agent": {
"properties": {
"id": {
"ignore_above": 1024,
"type": "keyword"
}
},
"type": "object"
}
}
},
"endpoint": {
"properties": {
"policy": {
"properties": {
"id": {
"ignore_above": 1024,
"type": "keyword"
}
},
"type": "object"
}
}
},
"event": {
"properties": {
"created": {
"type": "date"
}
}
},
"host": {
"properties": {
"architecture": {
"ignore_above": 1024,
"type": "keyword"
},
"hostname": {
"ignore_above": 1024,
"type": "keyword"
},
"id": {
"ignore_above": 1024,
"type": "keyword"
},
"ip": {
"type": "ip"
},
"mac": {
"ignore_above": 1024,
"type": "keyword"
},
"os": {
"properties": {
"full": {
"fields": {
"text": {
"norms": false,
"type": "text"
}
},
"ignore_above": 1024,
"type": "keyword"
},
"name": {
"fields": {
"text": {
"norms": false,
"type": "text"
}
},
"ignore_above": 1024,
"type": "keyword"
},
"variant": {
"ignore_above": 1024,
"type": "keyword"
},
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
}
}
}
}
},
"settings": {
"index": {
"mapping": {
"total_fields": {
"limit": 10000
}
},
"refresh_interval": "5s"
}
}
}

45
x-pack/plugins/siem/scripts/endpoint/resolver_generator.ts
View file

@ -11,9 +11,37 @@ import { EndpointDocGenerator, Event } from '../../common/endpoint/generate_data
import { default as eventMapping } from './event_mapping.json';
import { default as alertMapping } from './alert_mapping.json';
import { default as policyMapping } from './policy_mapping.json';
import { default as metadataMapping } from './metadata_mapping.json';
main();
async function deleteIndices(indices: string[], client: Client) {
const handleErr = (err: unknown) => {
if (err instanceof ResponseError && err.statusCode !== 404) {
// eslint-disable-next-line no-console
console.log(JSON.stringify(err, null, 2));
// eslint-disable-next-line no-process-exit
process.exit(1);
}
};
for (const index of indices) {
try {
// The index could be a data stream so let's try deleting that first
// The ES client in Kibana doesn't support data streams yet so we need to make a raw request to the ES route
await client.transport.request({ method: 'DELETE', path: `_data_stream/${index}` });
} catch (err) {
handleErr(err);
}
try {
await client.indices.delete({ index });
} catch (err) {
handleErr(err);
}
}
}
async function main() {
const argv = yargs.help().options({
seed: {
@ -134,18 +162,10 @@ async function main() {
}
const client = new Client(clientOptions);
if (argv.delete) {
try {
await client.indices.delete({
index: [argv.eventIndex, argv.metadataIndex, argv.alertIndex, argv.policyIndex],
});
} catch (err) {
if (err instanceof ResponseError && err.statusCode !== 404) {
// eslint-disable-next-line no-console
console.log(err);
// eslint-disable-next-line no-process-exit
process.exit(1);
}
}
await deleteIndices(
[argv.eventIndex, argv.metadataIndex, argv.alertIndex, argv.policyIndex],
client
);
}
const pipeline = {
@ -181,6 +201,7 @@ async function main() {
await createIndex(client, argv.alertIndex, alertMapping);
await createIndex(client, argv.eventIndex, eventMapping);
await createIndex(client, argv.policyIndex, policyMapping);
await createIndex(client, argv.metadataIndex, metadataMapping);
if (argv.setupOnly) {
// eslint-disable-next-line no-process-exit
process.exit(0);

3
x-pack/plugins/siem/server/endpoint/routes/metadata/index.ts
View file

@ -33,6 +33,7 @@ const HOST_STATUS_MAPPING = new Map<AgentStatus, HostStatus>([
]);
export function registerEndpointRoutes(router: IRouter, endpointAppContext: EndpointAppContext) {
const logger = endpointAppContext.logFactory.get('metadata');
router.post(
{
path: '/api/endpoint/metadata',
@ -85,6 +86,7 @@ export function registerEndpointRoutes(router: IRouter, endpointAppContext: Endp
}),
});
} catch (err) {
logger.warn(JSON.stringify(err, null, 2));
return res.internalError({ body: err });
}
}
@ -112,6 +114,7 @@ export function registerEndpointRoutes(router: IRouter, endpointAppContext: Endp
}
return res.notFound({ body: 'Endpoint Not Found' });
} catch (err) {
logger.warn(JSON.stringify(err, null, 2));
return res.internalError({ body: err });
}
}