[Endpoint] Deleting data streams and indices in generator (#67168)
* Deleting data streams and indices * Posting metadata template Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
This commit is contained in:
parent
204fd5b5a2
commit
70b92c8f42
146
x-pack/plugins/siem/scripts/endpoint/metadata_mapping.json
Normal file
146
x-pack/plugins/siem/scripts/endpoint/metadata_mapping.json
Normal file
|
@ -0,0 +1,146 @@
|
|||
{
|
||||
"mappings": {
|
||||
"_meta": {
|
||||
"version": "1.5.0"
|
||||
},
|
||||
"date_detection": false,
|
||||
"dynamic_templates": [
|
||||
{
|
||||
"strings_as_keyword": {
|
||||
"mapping": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"match_mapping_type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"@timestamp": {
|
||||
"type": "date"
|
||||
},
|
||||
"agent": {
|
||||
"properties": {
|
||||
"id": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"name": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"version": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
}
|
||||
}
|
||||
},
|
||||
"ecs": {
|
||||
"properties": {
|
||||
"version": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
}
|
||||
}
|
||||
},
|
||||
"elastic": {
|
||||
"properties": {
|
||||
"agent": {
|
||||
"properties": {
|
||||
"id": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
}
|
||||
},
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
},
|
||||
"endpoint": {
|
||||
"properties": {
|
||||
"policy": {
|
||||
"properties": {
|
||||
"id": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
}
|
||||
},
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
},
|
||||
"event": {
|
||||
"properties": {
|
||||
"created": {
|
||||
"type": "date"
|
||||
}
|
||||
}
|
||||
},
|
||||
"host": {
|
||||
"properties": {
|
||||
"architecture": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"hostname": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"id": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"ip": {
|
||||
"type": "ip"
|
||||
},
|
||||
"mac": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"os": {
|
||||
"properties": {
|
||||
"full": {
|
||||
"fields": {
|
||||
"text": {
|
||||
"norms": false,
|
||||
"type": "text"
|
||||
}
|
||||
},
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"name": {
|
||||
"fields": {
|
||||
"text": {
|
||||
"norms": false,
|
||||
"type": "text"
|
||||
}
|
||||
},
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"variant": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"version": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"settings": {
|
||||
"index": {
|
||||
"mapping": {
|
||||
"total_fields": {
|
||||
"limit": 10000
|
||||
}
|
||||
},
|
||||
"refresh_interval": "5s"
|
||||
}
|
||||
}
|
||||
}
|
|
@ -11,9 +11,37 @@ import { EndpointDocGenerator, Event } from '../../common/endpoint/generate_data
|
|||
import { default as eventMapping } from './event_mapping.json';
|
||||
import { default as alertMapping } from './alert_mapping.json';
|
||||
import { default as policyMapping } from './policy_mapping.json';
|
||||
import { default as metadataMapping } from './metadata_mapping.json';
|
||||
|
||||
main();
|
||||
|
||||
async function deleteIndices(indices: string[], client: Client) {
|
||||
const handleErr = (err: unknown) => {
|
||||
if (err instanceof ResponseError && err.statusCode !== 404) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(JSON.stringify(err, null, 2));
|
||||
// eslint-disable-next-line no-process-exit
|
||||
process.exit(1);
|
||||
}
|
||||
};
|
||||
|
||||
for (const index of indices) {
|
||||
try {
|
||||
// The index could be a data stream so let's try deleting that first
|
||||
// The ES client in Kibana doesn't support data streams yet so we need to make a raw request to the ES route
|
||||
await client.transport.request({ method: 'DELETE', path: `_data_stream/${index}` });
|
||||
} catch (err) {
|
||||
handleErr(err);
|
||||
}
|
||||
|
||||
try {
|
||||
await client.indices.delete({ index });
|
||||
} catch (err) {
|
||||
handleErr(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const argv = yargs.help().options({
|
||||
seed: {
|
||||
|
@ -134,18 +162,10 @@ async function main() {
|
|||
}
|
||||
const client = new Client(clientOptions);
|
||||
if (argv.delete) {
|
||||
try {
|
||||
await client.indices.delete({
|
||||
index: [argv.eventIndex, argv.metadataIndex, argv.alertIndex, argv.policyIndex],
|
||||
});
|
||||
} catch (err) {
|
||||
if (err instanceof ResponseError && err.statusCode !== 404) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(err);
|
||||
// eslint-disable-next-line no-process-exit
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
await deleteIndices(
|
||||
[argv.eventIndex, argv.metadataIndex, argv.alertIndex, argv.policyIndex],
|
||||
client
|
||||
);
|
||||
}
|
||||
|
||||
const pipeline = {
|
||||
|
@ -181,6 +201,7 @@ async function main() {
|
|||
await createIndex(client, argv.alertIndex, alertMapping);
|
||||
await createIndex(client, argv.eventIndex, eventMapping);
|
||||
await createIndex(client, argv.policyIndex, policyMapping);
|
||||
await createIndex(client, argv.metadataIndex, metadataMapping);
|
||||
if (argv.setupOnly) {
|
||||
// eslint-disable-next-line no-process-exit
|
||||
process.exit(0);
|
||||
|
|
|
@ -33,6 +33,7 @@ const HOST_STATUS_MAPPING = new Map<AgentStatus, HostStatus>([
|
|||
]);
|
||||
|
||||
export function registerEndpointRoutes(router: IRouter, endpointAppContext: EndpointAppContext) {
|
||||
const logger = endpointAppContext.logFactory.get('metadata');
|
||||
router.post(
|
||||
{
|
||||
path: '/api/endpoint/metadata',
|
||||
|
@ -85,6 +86,7 @@ export function registerEndpointRoutes(router: IRouter, endpointAppContext: Endp
|
|||
}),
|
||||
});
|
||||
} catch (err) {
|
||||
logger.warn(JSON.stringify(err, null, 2));
|
||||
return res.internalError({ body: err });
|
||||
}
|
||||
}
|
||||
|
@ -112,6 +114,7 @@ export function registerEndpointRoutes(router: IRouter, endpointAppContext: Endp
|
|||
}
|
||||
return res.notFound({ body: 'Endpoint Not Found' });
|
||||
} catch (err) {
|
||||
logger.warn(JSON.stringify(err, null, 2));
|
||||
return res.internalError({ body: err });
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue