From ce7a0bb8fcd2729354f6382e2dc65de6a4a4da01 Mon Sep 17 00:00:00 2001 From: Bryan Clement Date: Mon, 15 Mar 2021 08:18:36 -0700 Subject: [PATCH] [Asset Management] Osquery autocomplete (#94255) * added osquery mode to autocomplete * clean up and formatting * arm wrestling with the compiler * more fighting with ace types * Delete v4.5.0.json removed unused schema file * playing the hokey pokey with import statements * lazy load the schema file * remove include rule now that we are lazy loading schema json * update out of date comment * reduce schema file to what is currently being used, add script for formatting generated api files * added a readme, and points the compiler at the scripts directory * swip-swapped the argument order, fixed linting complaints Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../osquery/public/editor/ace_types.ts | 19 ++ .../plugins/osquery/public/editor/index.tsx | 6 +- .../public/editor/osquery_highlight_rules.ts | 184 ++++++++++++++++++ .../osquery/public/editor/osquery_mode.ts | 34 ++++ .../public/editor/osquery_schema/v4.6.0.json | 1 + .../osquery/public/editor/osquery_tables.ts | 30 +++ x-pack/plugins/osquery/scripts/readme.md | 10 + .../osquery/scripts/schema_formatter/index.js | 9 + .../scripts/schema_formatter/script.ts | 55 ++++++ x-pack/plugins/osquery/tsconfig.json | 1 + 10 files changed, 346 insertions(+), 3 deletions(-) create mode 100644 x-pack/plugins/osquery/public/editor/ace_types.ts create mode 100644 x-pack/plugins/osquery/public/editor/osquery_highlight_rules.ts create mode 100644 x-pack/plugins/osquery/public/editor/osquery_mode.ts create mode 100644 x-pack/plugins/osquery/public/editor/osquery_schema/v4.6.0.json create mode 100644 x-pack/plugins/osquery/public/editor/osquery_tables.ts create mode 100644 x-pack/plugins/osquery/scripts/readme.md create mode 100644 x-pack/plugins/osquery/scripts/schema_formatter/index.js create mode 100644 x-pack/plugins/osquery/scripts/schema_formatter/script.ts diff --git a/x-pack/plugins/osquery/public/editor/ace_types.ts b/x-pack/plugins/osquery/public/editor/ace_types.ts new file mode 100644 index 000000000000..93d8b4c5e2d2 --- /dev/null +++ b/x-pack/plugins/osquery/public/editor/ace_types.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Ace#define is not defined in the published types, so we define our own + * interface. + */ +export interface AceInterface { + define: ( + name: string, + deps: string[], + // eslint-disable-next-line @typescript-eslint/no-explicit-any + cb: (acequire: (name: string) => any, exports: any) => void + ) => void; +} diff --git a/x-pack/plugins/osquery/public/editor/index.tsx b/x-pack/plugins/osquery/public/editor/index.tsx index 2a8a52d06239..4cdb027480f9 100644 --- a/x-pack/plugins/osquery/public/editor/index.tsx +++ b/x-pack/plugins/osquery/public/editor/index.tsx @@ -7,9 +7,9 @@ import React, { useCallback } from 'react'; import { EuiCodeEditor } from '@elastic/eui'; -import 'brace/mode/sql'; import 'brace/theme/tomorrow'; -import 'brace/ext/language_tools'; + +import './osquery_mode.ts'; const EDITOR_SET_OPTIONS = { enableBasicAutocompletion: true, @@ -36,7 +36,7 @@ const OsqueryEditorComponent: React.FC = ({ defaultValue, on return ( |@>|<@|&|\\^|~|<|>|<=|=>|==|!=|<>|=', + }, + { + token: 'paren.lparen', + regex: '[\\(]', + }, + { + token: 'paren.rparen', + regex: '[\\)]', + }, + { + token: 'text', + regex: '\\s+', + }, + ], + }; + + this.normalizeRules(); + } + } + + exports.OsqueryHighlightRules = OsqueryHighlightRules; + } +); diff --git a/x-pack/plugins/osquery/public/editor/osquery_mode.ts b/x-pack/plugins/osquery/public/editor/osquery_mode.ts new file mode 100644 index 000000000000..6f526280ea87 --- /dev/null +++ b/x-pack/plugins/osquery/public/editor/osquery_mode.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import ace from 'brace'; +import 'brace/mode/sql'; +import 'brace/ext/language_tools'; +import { AceInterface } from './ace_types'; +import './osquery_highlight_rules'; + +((ace as unknown) as AceInterface).define( + 'ace/mode/osquery', + ['require', 'exports', 'ace/mode/sql', 'ace/mode/osquery_highlight_rules'], + function (acequire, exports) { + const TextMode = acequire('./sql').Mode; + const OsqueryHighlightRules = acequire('./osquery_highlight_rules').OsqueryHighlightRules; + + class Mode extends TextMode { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + constructor(...args: any[]) { + super(...args); + this.HighlightRules = OsqueryHighlightRules; + } + } + + Mode.prototype.lineCommentStart = '--'; + Mode.prototype.$id = 'ace/mode/osquery'; + + exports.Mode = Mode; + } +); diff --git a/x-pack/plugins/osquery/public/editor/osquery_schema/v4.6.0.json b/x-pack/plugins/osquery/public/editor/osquery_schema/v4.6.0.json new file mode 100644 index 000000000000..f42e63b35fbf --- /dev/null +++ b/x-pack/plugins/osquery/public/editor/osquery_schema/v4.6.0.json @@ -0,0 +1 @@ +[{"name":"account_policy_data"},{"name":"acpi_tables"},{"name":"ad_config"},{"name":"alf"},{"name":"alf_exceptions"},{"name":"alf_explicit_auths"},{"name":"app_schemes"},{"name":"apparmor_events"},{"name":"apparmor_profiles"},{"name":"appcompat_shims"},{"name":"apps"},{"name":"apt_sources"},{"name":"arp_cache"},{"name":"asl"},{"name":"atom_packages"},{"name":"augeas"},{"name":"authenticode"},{"name":"authorization_mechanisms"},{"name":"authorizations"},{"name":"authorized_keys"},{"name":"autoexec"},{"name":"azure_instance_metadata"},{"name":"azure_instance_tags"},{"name":"background_activities_moderator"},{"name":"battery"},{"name":"bitlocker_info"},{"name":"block_devices"},{"name":"bpf_process_events"},{"name":"bpf_socket_events"},{"name":"browser_plugins"},{"name":"carbon_black_info"},{"name":"carves"},{"name":"certificates"},{"name":"chassis_info"},{"name":"chocolatey_packages"},{"name":"chrome_extension_content_scripts"},{"name":"chrome_extensions"},{"name":"connectivity"},{"name":"cpu_info"},{"name":"cpu_time"},{"name":"cpuid"},{"name":"crashes"},{"name":"crontab"},{"name":"cups_destinations"},{"name":"cups_jobs"},{"name":"curl"},{"name":"curl_certificate"},{"name":"deb_packages"},{"name":"default_environment"},{"name":"device_file"},{"name":"device_firmware"},{"name":"device_hash"},{"name":"device_partitions"},{"name":"disk_encryption"},{"name":"disk_events"},{"name":"disk_info"},{"name":"dns_cache"},{"name":"dns_resolvers"},{"name":"docker_container_fs_changes"},{"name":"docker_container_labels"},{"name":"docker_container_mounts"},{"name":"docker_container_networks"},{"name":"docker_container_ports"},{"name":"docker_container_processes"},{"name":"docker_container_stats"},{"name":"docker_containers"},{"name":"docker_image_labels"},{"name":"docker_image_layers"},{"name":"docker_images"},{"name":"docker_info"},{"name":"docker_network_labels"},{"name":"docker_networks"},{"name":"docker_version"},{"name":"docker_volume_labels"},{"name":"docker_volumes"},{"name":"drivers"},{"name":"ec2_instance_metadata"},{"name":"ec2_instance_tags"},{"name":"elf_dynamic"},{"name":"elf_info"},{"name":"elf_sections"},{"name":"elf_segments"},{"name":"elf_symbols"},{"name":"etc_hosts"},{"name":"etc_protocols"},{"name":"etc_services"},{"name":"event_taps"},{"name":"example"},{"name":"extended_attributes"},{"name":"fan_speed_sensors"},{"name":"fbsd_kmods"},{"name":"file"},{"name":"file_events"},{"name":"firefox_addons"},{"name":"gatekeeper"},{"name":"gatekeeper_approved_apps"},{"name":"groups"},{"name":"hardware_events"},{"name":"hash"},{"name":"homebrew_packages"},{"name":"hvci_status"},{"name":"ibridge_info"},{"name":"ie_extensions"},{"name":"intel_me_info"},{"name":"interface_addresses"},{"name":"interface_details"},{"name":"interface_ipv6"},{"name":"iokit_devicetree"},{"name":"iokit_registry"},{"name":"iptables"},{"name":"kernel_extensions"},{"name":"kernel_info"},{"name":"kernel_modules"},{"name":"kernel_panics"},{"name":"keychain_acls"},{"name":"keychain_items"},{"name":"known_hosts"},{"name":"kva_speculative_info"},{"name":"last"},{"name":"launchd"},{"name":"launchd_overrides"},{"name":"listening_ports"},{"name":"lldp_neighbors"},{"name":"load_average"},{"name":"logged_in_users"},{"name":"logical_drives"},{"name":"logon_sessions"},{"name":"lxd_certificates"},{"name":"lxd_cluster"},{"name":"lxd_cluster_members"},{"name":"lxd_images"},{"name":"lxd_instance_config"},{"name":"lxd_instance_devices"},{"name":"lxd_instances"},{"name":"lxd_networks"},{"name":"lxd_storage_pools"},{"name":"magic"},{"name":"managed_policies"},{"name":"md_devices"},{"name":"md_drives"},{"name":"md_personalities"},{"name":"mdfind"},{"name":"mdls"},{"name":"memory_array_mapped_addresses"},{"name":"memory_arrays"},{"name":"memory_device_mapped_addresses"},{"name":"memory_devices"},{"name":"memory_error_info"},{"name":"memory_info"},{"name":"memory_map"},{"name":"mounts"},{"name":"msr"},{"name":"nfs_shares"},{"name":"npm_packages"},{"name":"ntdomains"},{"name":"ntfs_acl_permissions"},{"name":"ntfs_journal_events"},{"name":"nvram"},{"name":"oem_strings"},{"name":"office_mru"},{"name":"opera_extensions"},{"name":"os_version"},{"name":"osquery_events"},{"name":"osquery_extensions"},{"name":"osquery_flags"},{"name":"osquery_info"},{"name":"osquery_packs"},{"name":"osquery_registry"},{"name":"osquery_schedule"},{"name":"package_bom"},{"name":"package_install_history"},{"name":"package_receipts"},{"name":"patches"},{"name":"pci_devices"},{"name":"physical_disk_performance"},{"name":"pipes"},{"name":"pkg_packages"},{"name":"platform_info"},{"name":"plist"},{"name":"portage_keywords"},{"name":"portage_packages"},{"name":"portage_use"},{"name":"power_sensors"},{"name":"powershell_events"},{"name":"preferences"},{"name":"process_envs"},{"name":"process_events"},{"name":"process_file_events"},{"name":"process_memory_map"},{"name":"process_namespaces"},{"name":"process_open_files"},{"name":"process_open_pipes"},{"name":"process_open_sockets"},{"name":"processes"},{"name":"programs"},{"name":"prometheus_metrics"},{"name":"python_packages"},{"name":"quicklook_cache"},{"name":"registry"},{"name":"routes"},{"name":"rpm_package_files"},{"name":"rpm_packages"},{"name":"running_apps"},{"name":"safari_extensions"},{"name":"sandboxes"},{"name":"scheduled_tasks"},{"name":"screenlock"},{"name":"selinux_events"},{"name":"selinux_settings"},{"name":"services"},{"name":"shadow"},{"name":"shared_folders"},{"name":"shared_memory"},{"name":"shared_resources"},{"name":"sharing_preferences"},{"name":"shell_history"},{"name":"shimcache"},{"name":"signature"},{"name":"sip_config"},{"name":"smart_drive_info"},{"name":"smbios_tables"},{"name":"smc_keys"},{"name":"socket_events"},{"name":"ssh_configs"},{"name":"startup_items"},{"name":"sudoers"},{"name":"suid_bin"},{"name":"syslog_events"},{"name":"system_controls"},{"name":"system_info"},{"name":"temperature_sensors"},{"name":"time"},{"name":"time_machine_backups"},{"name":"time_machine_destinations"},{"name":"ulimit_info"},{"name":"uptime"},{"name":"usb_devices"},{"name":"user_events"},{"name":"user_groups"},{"name":"user_interaction_events"},{"name":"user_ssh_keys"},{"name":"userassist"},{"name":"users"},{"name":"video_info"},{"name":"virtual_memory_info"},{"name":"wifi_networks"},{"name":"wifi_status"},{"name":"wifi_survey"},{"name":"winbaseobj"},{"name":"windows_crashes"},{"name":"windows_eventlog"},{"name":"windows_events"},{"name":"windows_optional_features"},{"name":"windows_security_center"},{"name":"windows_security_products"},{"name":"wmi_bios_info"},{"name":"wmi_cli_event_consumers"},{"name":"wmi_event_filters"},{"name":"wmi_filter_consumer_binding"},{"name":"wmi_script_event_consumers"},{"name":"xprotect_entries"},{"name":"xprotect_meta"},{"name":"xprotect_reports"},{"name":"yara"},{"name":"yara_events"},{"name":"yum_sources"}] \ No newline at end of file diff --git a/x-pack/plugins/osquery/public/editor/osquery_tables.ts b/x-pack/plugins/osquery/public/editor/osquery_tables.ts new file mode 100644 index 000000000000..8fdacbdf3574 --- /dev/null +++ b/x-pack/plugins/osquery/public/editor/osquery_tables.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { flatMap, sortBy } from 'lodash'; + +type TablesJSON = Array<{ + name: string; +}>; +export const normalizeTables = (tablesJSON: TablesJSON) => { + return sortBy(tablesJSON, (table) => { + return table.name; + }); +}; + +let osqueryTables: TablesJSON | null = null; +export const getOsqueryTables = () => { + if (!osqueryTables) { + // eslint-disable-next-line @typescript-eslint/no-var-requires + osqueryTables = normalizeTables(require('./osquery_schema/v4.6.0.json')); + } + return osqueryTables; +}; +export const getOsqueryTableNames = () => + flatMap(getOsqueryTables(), (table) => { + return table.name; + }); diff --git a/x-pack/plugins/osquery/scripts/readme.md b/x-pack/plugins/osquery/scripts/readme.md new file mode 100644 index 000000000000..d52876f5f633 --- /dev/null +++ b/x-pack/plugins/osquery/scripts/readme.md @@ -0,0 +1,10 @@ +### Schema formatter + +In order to manage the size of the osquery schema files, there is a script +available to extract only the currently used fields (this selection is +currently manually curated). This assumes the targeted schema files will be in +`public/editor/osquery_schema`. + +``` +node scripts/schema_formatter --schema_version=v4.6.0 +``` diff --git a/x-pack/plugins/osquery/scripts/schema_formatter/index.js b/x-pack/plugins/osquery/scripts/schema_formatter/index.js new file mode 100644 index 000000000000..28004b679e59 --- /dev/null +++ b/x-pack/plugins/osquery/scripts/schema_formatter/index.js @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +require('../../../../../src/setup_node_env'); +require('./script'); diff --git a/x-pack/plugins/osquery/scripts/schema_formatter/script.ts b/x-pack/plugins/osquery/scripts/schema_formatter/script.ts new file mode 100644 index 000000000000..146bd4a9d49d --- /dev/null +++ b/x-pack/plugins/osquery/scripts/schema_formatter/script.ts @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { promises as fs } from 'fs'; +import path from 'path'; + +import { run } from '@kbn/dev-utils'; +interface DestField { + [key: string]: boolean | DestField; +} + +run( + async ({ flags }) => { + const schemaPath = path.resolve('../../public/editor/osquery_schema/'); + const schemaFile = path.join(schemaPath, flags.schema_version as string); + const schemaData = await require(schemaFile); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + function pullFields(destSchema: DestField, source: { [key: string]: any }) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const dest: { [key: string]: any } = {}; + Object.keys(destSchema).forEach((key) => { + switch (typeof source[key]) { + case 'object': + dest[key] = pullFields(destSchema[key] as DestField, source[key]); + break; + default: + dest[key] = source[key]; + } + }); + return dest; + } + + const mapFunc = pullFields.bind(null, { name: true }); + const formattedSchema = schemaData.map(mapFunc); + await fs.writeFile( + path.join(schemaPath, `${flags.schema_version}-formatted`), + JSON.stringify(formattedSchema) + ); + }, + { + description: ` + Script for formatting generated osquery API schema JSON file. + `, + flags: { + string: ['schema_version'], + help: ` + --schema_version The semver string for the schema file located in public/editor/osquery_schema + `, + }, + } +); diff --git a/x-pack/plugins/osquery/tsconfig.json b/x-pack/plugins/osquery/tsconfig.json index 407830d6a6c2..291b0f7c607c 100644 --- a/x-pack/plugins/osquery/tsconfig.json +++ b/x-pack/plugins/osquery/tsconfig.json @@ -11,6 +11,7 @@ // add all the folders contains files to be compiled "common/**/*", "public/**/*", + "scripts/**/*", "server/**/*" ], "references": [