[Asset Management] Osquery autocomplete (#94255)

* added osquery mode to autocomplete

* clean up and formatting

* arm wrestling with the compiler

* more fighting with ace types

* Delete v4.5.0.json

removed unused schema file

* playing the hokey pokey with import statements

* lazy load the schema file

* remove include rule now that we are lazy loading schema json

* update out of date comment

* reduce schema file to what is currently being used, add script for formatting generated api files

* added a readme, and points the compiler at the scripts directory

* swip-swapped the argument order, fixed linting complaints

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/osquery/public/editor/ace_types.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/**
+ * Ace#define is not defined in the published types, so we define our own
+ * interface.
+ */
+export interface AceInterface {
+  define: (
+    name: string,
+    deps: string[],
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    cb: (acequire: (name: string) => any, exports: any) => void
+  ) => void;
+}

x-pack/plugins/osquery/public/editor/index.tsx

@@ -7,9 +7,9 @@
 
 import React, { useCallback } from 'react';
 import { EuiCodeEditor } from '@elastic/eui';
-import 'brace/mode/sql';
 import 'brace/theme/tomorrow';
-import 'brace/ext/language_tools';
+
+import './osquery_mode.ts';
 
 const EDITOR_SET_OPTIONS = {
   enableBasicAutocompletion: true,
@@ -36,7 +36,7 @@ const OsqueryEditorComponent: React.FC<OsqueryEditorProps> = ({ defaultValue, on
   return (
     <EuiCodeEditor
       value={defaultValue}
-      mode="sql"
+      mode="osquery"
       theme="tomorrow"
       onChange={handleChange}
       name="osquery_editor"

x-pack/plugins/osquery/public/editor/osquery_highlight_rules.ts

@@ -0,0 +1,184 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import ace from 'brace';
+import 'brace/ext/language_tools';
+import { AceInterface } from './ace_types';
+import { getOsqueryTableNames } from './osquery_tables';
+
+const osqueryTables = getOsqueryTableNames().join('|');
+
+const keywords = [
+  'select',
+  'insert',
+  'update',
+  'delete',
+  'from',
+  'where',
+  'and',
+  'or',
+  'group',
+  'by',
+  'order',
+  'limit',
+  'offset',
+  'having',
+  'as',
+  'case',
+  'when',
+  'else',
+  'end',
+  'type',
+  'left',
+  'right',
+  'join',
+  'on',
+  'outer',
+  'desc',
+  'asc',
+  'union',
+  'create',
+  'table',
+  'primary',
+  'key',
+  'if',
+  'foreign',
+  'not',
+  'references',
+  'default',
+  'null',
+  'inner',
+  'cross',
+  'natural',
+  'database',
+  'drop',
+  'grant',
+].join('|');
+
+const builtinConstants = ['true', 'false'].join('|');
+
+const builtinFunctions = [
+  'avg',
+  'count',
+  'first',
+  'last',
+  'max',
+  'min',
+  'sum',
+  'ucase',
+  'lcase',
+  'mid',
+  'len',
+  'round',
+  'rank',
+  'now',
+  'format',
+  'coalesce',
+  'ifnull',
+  'isnull',
+  'nvl',
+].join('|');
+
+const dataTypes = [
+  'int',
+  'numeric',
+  'decimal',
+  'date',
+  'varchar',
+  'char',
+  'bigint',
+  'float',
+  'double',
+  'bit',
+  'binary',
+  'text',
+  'set',
+  'timestamp',
+  'money',
+  'real',
+  'number',
+  'integer',
+].join('|');
+
+// This is gross, but the types exported by brace are lagging and incorrect: https://github.com/thlorenz/brace/issues/182
+((ace as unknown) as AceInterface).define(
+  'ace/mode/osquery_highlight_rules',
+  ['require', 'exports', 'ace/mode/sql_highlight_rules'],
+  function (acequire, exports) {
+    'use strict';
+
+    const SqlHighlightRules = acequire('./sql_highlight_rules').SqlHighlightRules;
+
+    class OsqueryHighlightRules extends SqlHighlightRules {
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      constructor(...args: any) {
+        super(...args);
+        const keywordMapper = this.createKeywordMapper(
+          {
+            'osquery-token': osqueryTables,
+            'support.function': builtinFunctions,
+            keyword: keywords,
+            'constant.language': builtinConstants,
+            'storage.type': dataTypes,
+          },
+          'identifier',
+          true
+        );
+
+        this.$rules = {
+          start: [
+            {
+              token: 'comment',
+              regex: '--.*$',
+            },
+            {
+              token: 'comment',
+              start: '/\\*',
+              end: '\\*/',
+            },
+            {
+              token: 'string', // " string
+              regex: '".*?"',
+            },
+            {
+              token: 'string', // ' string
+              regex: "'.*?'",
+            },
+            {
+              token: 'constant.numeric', // float
+              regex: '[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b',
+            },
+            {
+              token: keywordMapper,
+              regex: '[a-zA-Z_$][a-zA-Z0-9_$]*\\b',
+            },
+            {
+              token: 'keyword.operator',
+              regex: '\\+|\\-|\\/|\\/\\/|%|<@>|@>|<@|&|\\^|~|<|>|<=|=>|==|!=|<>|=',
+            },
+            {
+              token: 'paren.lparen',
+              regex: '[\\(]',
+            },
+            {
+              token: 'paren.rparen',
+              regex: '[\\)]',
+            },
+            {
+              token: 'text',
+              regex: '\\s+',
+            },
+          ],
+        };
+
+        this.normalizeRules();
+      }
+    }
+
+    exports.OsqueryHighlightRules = OsqueryHighlightRules;
+  }
+);

x-pack/plugins/osquery/public/editor/osquery_mode.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import ace from 'brace';
+import 'brace/mode/sql';
+import 'brace/ext/language_tools';
+import { AceInterface } from './ace_types';
+import './osquery_highlight_rules';
+
+((ace as unknown) as AceInterface).define(
+  'ace/mode/osquery',
+  ['require', 'exports', 'ace/mode/sql', 'ace/mode/osquery_highlight_rules'],
+  function (acequire, exports) {
+    const TextMode = acequire('./sql').Mode;
+    const OsqueryHighlightRules = acequire('./osquery_highlight_rules').OsqueryHighlightRules;
+
+    class Mode extends TextMode {
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      constructor(...args: any[]) {
+        super(...args);
+        this.HighlightRules = OsqueryHighlightRules;
+      }
+    }
+
+    Mode.prototype.lineCommentStart = '--';
+    Mode.prototype.$id = 'ace/mode/osquery';
+
+    exports.Mode = Mode;
+  }
+);

x-pack/plugins/osquery/public/editor/osquery_tables.ts

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { flatMap, sortBy } from 'lodash';
+
+type TablesJSON = Array<{
+  name: string;
+}>;
+export const normalizeTables = (tablesJSON: TablesJSON) => {
+  return sortBy(tablesJSON, (table) => {
+    return table.name;
+  });
+};
+
+let osqueryTables: TablesJSON | null = null;
+export const getOsqueryTables = () => {
+  if (!osqueryTables) {
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    osqueryTables = normalizeTables(require('./osquery_schema/v4.6.0.json'));
+  }
+  return osqueryTables;
+};
+export const getOsqueryTableNames = () =>
+  flatMap(getOsqueryTables(), (table) => {
+    return table.name;
+  });

x-pack/plugins/osquery/scripts/readme.md

@@ -0,0 +1,10 @@
+### Schema formatter 
+
+In order to manage the size of the osquery schema files, there is a script
+available to extract only the currently used fields (this selection is
+currently manually curated). This assumes the targeted schema files will be in
+`public/editor/osquery_schema`.
+
+```
+node scripts/schema_formatter --schema_version=v4.6.0
+```

x-pack/plugins/osquery/scripts/schema_formatter/index.js

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+require('../../../../../src/setup_node_env');
+require('./script');

x-pack/plugins/osquery/scripts/schema_formatter/script.ts

@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { promises as fs } from 'fs';
+import path from 'path';
+
+import { run } from '@kbn/dev-utils';
+interface DestField {
+  [key: string]: boolean | DestField;
+}
+
+run(
+  async ({ flags }) => {
+    const schemaPath = path.resolve('../../public/editor/osquery_schema/');
+    const schemaFile = path.join(schemaPath, flags.schema_version as string);
+    const schemaData = await require(schemaFile);
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    function pullFields(destSchema: DestField, source: { [key: string]: any }) {
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      const dest: { [key: string]: any } = {};
+      Object.keys(destSchema).forEach((key) => {
+        switch (typeof source[key]) {
+          case 'object':
+            dest[key] = pullFields(destSchema[key] as DestField, source[key]);
+            break;
+          default:
+            dest[key] = source[key];
+        }
+      });
+      return dest;
+    }
+
+    const mapFunc = pullFields.bind(null, { name: true });
+    const formattedSchema = schemaData.map(mapFunc);
+    await fs.writeFile(
+      path.join(schemaPath, `${flags.schema_version}-formatted`),
+      JSON.stringify(formattedSchema)
+    );
+  },
+  {
+    description: `
+      Script for formatting generated osquery API schema JSON file.
+    `,
+    flags: {
+      string: ['schema_version'],
+      help: `
+        --schema_version The semver string for the schema file located in public/editor/osquery_schema
+      `,
+    },
+  }
+);

x-pack/plugins/osquery/tsconfig.json

@@ -11,6 +11,7 @@
     // add all the folders contains files to be compiled
     "common/**/*",
     "public/**/*",
+    "scripts/**/*",
     "server/**/*"
   ],
   "references": [