[Asset Management] Osquery telemetry updates (#100754)

* first pass of basic osquery usage stats collection

* updates, linting

* updated exported metrics

* clean up comments, add description fields to metric fields

* reworked types

* actually use the updated types

* added tests around the route usage recoder functions

* review comments

* update aggregate types

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/osquery/common/types.ts

@@ -9,7 +9,8 @@ import { PackagePolicy, PackagePolicyInput, PackagePolicyInputStream } from '../
 
 export const savedQuerySavedObjectType = 'osquery-saved-query';
 export const packSavedObjectType = 'osquery-pack';
-export type SavedObjectType = 'osquery-saved-query' | 'osquery-pack';
+export const usageMetricSavedObjectType = 'osquery-usage-metric';
+export type SavedObjectType = 'osquery-saved-query' | 'osquery-pack' | 'osquery-usage-metric';
 
 /**
  * This makes any optional property the same as Required<T> would but also has the

x-pack/plugins/osquery/kibana.json

@@ -10,6 +10,7 @@
   "kibanaVersion": "kibana",
   "optionalPlugins": [
     "home",
+    "usageCollection",
     "lens"
   ],
   "requiredBundles": [

x-pack/plugins/osquery/server/plugin.ts

@@ -18,6 +18,7 @@ import { OsqueryPluginSetup, OsqueryPluginStart, SetupPlugins, StartPlugins } fr
 import { defineRoutes } from './routes';
 import { osquerySearchStrategyProvider } from './search_strategy/osquery';
 import { initSavedObjects } from './saved_objects';
+import { initUsageCollectors } from './usage';
 import { OsqueryAppContext, OsqueryAppContextService } from './lib/osquery_app_context_services';
 import { ConfigType } from './config';
 
@@ -48,6 +49,11 @@ export class OsqueryPlugin implements Plugin<OsqueryPluginSetup, OsqueryPluginSt
     };
 
     initSavedObjects(core.savedObjects, osqueryContext);
+    initUsageCollectors({
+      core,
+      osqueryContext,
+      usageCollection: plugins.usageCollection,
+    });
     defineRoutes(router, osqueryContext);
 
     core.getStartServices().then(([, depsStart]) => {

x-pack/plugins/osquery/server/routes/action/create_action_route.ts

@@ -18,6 +18,8 @@ import {
   CreateActionRequestBodySchema,
 } from '../../../common/schemas/routes/action/create_action_request_body_schema';
 
+import { incrementCount } from '../usage';
+
 export const createActionRoute = (router: IRouter, osqueryContext: OsqueryAppContext) => {
   router.post(
     {
@@ -39,34 +41,43 @@ export const createActionRoute = (router: IRouter, osqueryContext: OsqueryAppCon
         osqueryContext,
         agentSelection
       );
-
+      incrementCount(soClient, 'live_query');
       if (!selectedAgents.length) {
+        incrementCount(soClient, 'live_query', 'errors');
         return response.badRequest({ body: new Error('No agents found for selection') });
       }
 
-      const action = {
-        action_id: uuid.v4(),
-        '@timestamp': moment().toISOString(),
-        expiration: moment().add(1, 'days').toISOString(),
-        type: 'INPUT_ACTION',
-        input_type: 'osquery',
-        agents: selectedAgents,
-        data: {
-          id: uuid.v4(),
-          query: request.body.query,
-        },
-      };
-      const actionResponse = await esClient.index<{}, {}>({
-        index: '.fleet-actions',
-        body: action,
-      });
+      try {
+        const action = {
+          action_id: uuid.v4(),
+          '@timestamp': moment().toISOString(),
+          expiration: moment().add(1, 'days').toISOString(),
+          type: 'INPUT_ACTION',
+          input_type: 'osquery',
+          agents: selectedAgents,
+          data: {
+            id: uuid.v4(),
+            query: request.body.query,
+          },
+        };
+        const actionResponse = await esClient.index<{}, {}>({
+          index: '.fleet-actions',
+          body: action,
+        });
 
-      return response.ok({
-        body: {
-          response: actionResponse,
-          actions: [action],
-        },
-      });
+        return response.ok({
+          body: {
+            response: actionResponse,
+            actions: [action],
+          },
+        });
+      } catch (error) {
+        incrementCount(soClient, 'live_query', 'errors');
+        return response.customError({
+          statusCode: 500,
+          body: new Error(`Error occurred whlie processing ${error}`),
+        });
+      }
     }
   );
 };

x-pack/plugins/osquery/server/routes/usage/index.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export * from './recorder';

x-pack/plugins/osquery/server/routes/usage/recorder.test.ts

@@ -0,0 +1,135 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { savedObjectsClientMock } from '../../../../../../src/core/server/mocks';
+
+import { usageMetricSavedObjectType } from '../../../common/types';
+
+import {
+  CounterValue,
+  createMetricObjects,
+  getRouteMetric,
+  incrementCount,
+  RouteString,
+  routeStrings,
+} from './recorder';
+
+const savedObjectsClient = savedObjectsClientMock.create();
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function checkGetCalls(calls: any[]) {
+  expect(calls.length).toEqual(routeStrings.length);
+  for (let i = 0; i < routeStrings.length; ++i) {
+    expect(calls[i]).toEqual([usageMetricSavedObjectType, routeStrings[i]]);
+  }
+}
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function checkCreateCalls(calls: any[], expectedCallRoutes: string[] = routeStrings) {
+  expect(calls.length).toEqual(expectedCallRoutes.length);
+  for (let i = 0; i < expectedCallRoutes.length; ++i) {
+    expect(calls[i][0]).toEqual(usageMetricSavedObjectType);
+    expect(calls[i][2].id).toEqual(expectedCallRoutes[i]);
+  }
+}
+
+describe('Usage metric recorder', () => {
+  describe('Metric initalizer', () => {
+    const get = savedObjectsClient.get as jest.Mock;
+    const create = savedObjectsClient.create as jest.Mock;
+    afterEach(() => {
+      get.mockClear();
+      create.mockClear();
+    });
+    it('should seed route metrics objects', async () => {
+      get.mockRejectedValueOnce('stub value');
+      create.mockReturnValueOnce('stub value');
+      const result = await createMetricObjects(savedObjectsClient);
+      checkGetCalls(get.mock.calls);
+      checkCreateCalls(create.mock.calls);
+      expect(result).toBe(true);
+    });
+
+    it('should handle previously seeded objects properly', async () => {
+      get.mockReturnValueOnce('stub value');
+      create.mockRejectedValueOnce('stub value');
+      const result = await createMetricObjects(savedObjectsClient);
+      checkGetCalls(get.mock.calls);
+      checkCreateCalls(create.mock.calls, []);
+      expect(result).toBe(true);
+    });
+
+    it('should report failure to create the metrics object', async () => {
+      get.mockRejectedValueOnce('stub value');
+      create.mockRejectedValueOnce('stub value');
+      const result = await createMetricObjects(savedObjectsClient);
+      checkGetCalls(get.mock.calls);
+      checkCreateCalls(create.mock.calls);
+      expect(result).toBe(false);
+    });
+  });
+
+  describe('Incrementation', () => {
+    let counterMap: { [key: string]: CounterValue };
+    const get = savedObjectsClient.get as jest.Mock;
+    const update = savedObjectsClient.update as jest.Mock;
+    update.mockImplementation(
+      async (objectType: string, route: RouteString, newVal: CounterValue) => {
+        counterMap[`${objectType}-${route}`] = newVal;
+      }
+    );
+    get.mockImplementation(async (objectType: string, route: RouteString) => ({
+      attributes: counterMap[`${objectType}-${route}`],
+    }));
+    beforeEach(() => {
+      counterMap = routeStrings.reduce((acc, route) => {
+        acc[`${usageMetricSavedObjectType}-${route}`] = {
+          count: 0,
+          errors: 0,
+        };
+        return acc;
+      }, {} as { [key: string]: CounterValue });
+      get.mockClear();
+      update.mockClear();
+    });
+    it('should increment the route counter', async () => {
+      expect(await getRouteMetric(savedObjectsClient, 'live_query')).toEqual({
+        count: 0,
+        errors: 0,
+      });
+      await incrementCount(savedObjectsClient, 'live_query');
+      expect(await getRouteMetric(savedObjectsClient, 'live_query')).toEqual({
+        count: 1,
+        errors: 0,
+      });
+    });
+
+    it('should allow incrementing the error counter', async () => {
+      expect(await getRouteMetric(savedObjectsClient, 'live_query')).toEqual({
+        count: 0,
+        errors: 0,
+      });
+      await incrementCount(savedObjectsClient, 'live_query', 'errors');
+      expect(await getRouteMetric(savedObjectsClient, 'live_query')).toEqual({
+        count: 0,
+        errors: 1,
+      });
+    });
+
+    it('should allow adjustment of the increment', async () => {
+      expect(await getRouteMetric(savedObjectsClient, 'live_query')).toEqual({
+        count: 0,
+        errors: 0,
+      });
+      await incrementCount(savedObjectsClient, 'live_query', 'count', 2);
+      expect(await getRouteMetric(savedObjectsClient, 'live_query')).toEqual({
+        count: 2,
+        errors: 0,
+      });
+    });
+  });
+});

x-pack/plugins/osquery/server/routes/usage/recorder.ts

@@ -0,0 +1,65 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { SavedObjectsClientContract } from 'kibana/server';
+import { usageMetricSavedObjectType } from '../../../common/types';
+import { LiveQuerySessionUsage } from '../../usage/types';
+
+export interface RouteUsageMetric {
+  queries: number;
+  errors: number;
+}
+
+export type RouteString = 'live_query';
+
+export const routeStrings: RouteString[] = ['live_query'];
+
+export async function createMetricObjects(soClient: SavedObjectsClientContract) {
+  const res = await Promise.allSettled(
+    routeStrings.map(async (route) => {
+      try {
+        await soClient.get(usageMetricSavedObjectType, route);
+      } catch (e) {
+        await soClient.create(
+          usageMetricSavedObjectType,
+          {
+            errors: 0,
+            count: 0,
+          },
+          {
+            id: route,
+          }
+        );
+      }
+    })
+  );
+  return !res.some((e) => e.status === 'rejected');
+}
+
+export async function getCount(soClient: SavedObjectsClientContract, route: RouteString) {
+  return await soClient.get<LiveQuerySessionUsage>(usageMetricSavedObjectType, route);
+}
+
+export interface CounterValue {
+  count: number;
+  errors: number;
+}
+
+export async function incrementCount(
+  soClient: SavedObjectsClientContract,
+  route: RouteString,
+  key: keyof CounterValue = 'count',
+  increment = 1
+) {
+  const metric = await soClient.get<CounterValue>(usageMetricSavedObjectType, route);
+  metric.attributes[key] += increment;
+  await soClient.update(usageMetricSavedObjectType, route, metric.attributes);
+}
+
+export async function getRouteMetric(soClient: SavedObjectsClientContract, route: RouteString) {
+  return (await getCount(soClient, route)).attributes;
+}

x-pack/plugins/osquery/server/routes/usage/saved_object_mappings.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { SavedObjectsType } from '../../../../../../src/core/server';
+
+import { usageMetricSavedObjectType } from '../../../common/types';
+
+export const usageMetricSavedObjectMappings: SavedObjectsType['mappings'] = {
+  properties: {
+    count: {
+      type: 'long',
+    },
+    errors: {
+      type: 'long',
+    },
+  },
+};
+
+export const usageMetricType: SavedObjectsType = {
+  name: usageMetricSavedObjectType,
+  hidden: false,
+  namespaceType: 'single',
+  mappings: usageMetricSavedObjectMappings,
+};

x-pack/plugins/osquery/server/saved_objects.ts

@@ -9,6 +9,7 @@ import { CoreSetup } from '../../../../src/core/server';
 
 import { OsqueryAppContext } from './lib/osquery_app_context_services';
 import { savedQueryType, packType } from './lib/saved_query/saved_object_mappings';
+import { usageMetricType } from './routes/usage/saved_object_mappings';
 
 const types = [savedQueryType, packType];
 
@@ -20,6 +21,8 @@ export const initSavedObjects = (
 ) => {
   const config = osqueryContext.config();
 
+  savedObjects.registerType(usageMetricType);
+
   if (config.savedQueries) {
     savedObjects.registerType(savedQueryType);
   }

x-pack/plugins/osquery/server/types.ts

@@ -11,6 +11,7 @@ import {
   PluginStart as DataPluginStart,
 } from '../../../../src/plugins/data/server';
 import { FleetStartContract } from '../../fleet/server';
+import { UsageCollectionSetup } from '../../../../src/plugins/usage_collection/server';
 import { PluginSetupContract } from '../../features/server';
 
 // eslint-disable-next-line @typescript-eslint/no-empty-interface
@@ -19,6 +20,7 @@ export interface OsqueryPluginSetup {}
 export interface OsqueryPluginStart {}
 
 export interface SetupPlugins {
+  usageCollection?: UsageCollectionSetup;
   actions: ActionsPlugin['setup'];
   data: DataPluginSetup;
   features: PluginSetupContract;

x-pack/plugins/osquery/server/usage/collector.ts

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { CoreSetup, SavedObjectsClient } from '../../../../../src/core/server';
+import { CollectorFetchContext } from '../../../../../src/plugins/usage_collection/server';
+import { createMetricObjects } from '../routes/usage';
+import { getBeatUsage, getLiveQueryUsage, getPolicyLevelUsage } from './fetchers';
+import { CollectorDependencies, usageSchema, UsageData } from './types';
+
+export type RegisterCollector = (deps: CollectorDependencies) => void;
+export async function getInternalSavedObjectsClient(core: CoreSetup) {
+  return core.getStartServices().then(async ([coreStart]) => {
+    return coreStart.savedObjects.createInternalRepository();
+  });
+}
+
+export const registerCollector: RegisterCollector = ({ core, osqueryContext, usageCollection }) => {
+  if (!usageCollection) {
+    return;
+  }
+  const collector = usageCollection.makeUsageCollector<UsageData>({
+    type: 'osquery',
+    schema: usageSchema,
+    isReady: async () => {
+      const savedObjectsClient = new SavedObjectsClient(await getInternalSavedObjectsClient(core));
+      return await createMetricObjects(savedObjectsClient);
+    },
+    fetch: async ({ esClient }: CollectorFetchContext): Promise<UsageData> => {
+      const savedObjectsClient = new SavedObjectsClient(await getInternalSavedObjectsClient(core));
+      return {
+        beat_metrics: {
+          usage: await getBeatUsage(esClient),
+        },
+        live_query_usage: await getLiveQueryUsage(savedObjectsClient, esClient),
+        ...(await getPolicyLevelUsage(
+          esClient,
+          savedObjectsClient,
+          osqueryContext.service.getPackagePolicyService()
+        )),
+      };
+    },
+  });
+
+  usageCollection.registerCollector(collector);
+};

x-pack/plugins/osquery/server/usage/constants.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const METRICS_INDICES = 'logs-elastic_agent.osquerybeat*';

x-pack/plugins/osquery/server/usage/fetchers.ts

@@ -0,0 +1,223 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  AggregationsSingleBucketAggregate,
+  AggregationsTopHitsAggregate,
+  AggregationsValueAggregate,
+} from '@elastic/elasticsearch/api/types';
+import { PackagePolicyServiceInterface } from '../../../fleet/server';
+import { getRouteMetric } from '../routes/usage';
+import { ElasticsearchClient, SavedObjectsClientContract } from '../../../../../src/core/server';
+import { ListResult, PackagePolicy, PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '../../../fleet/common';
+import { OSQUERY_INTEGRATION_NAME } from '../../common';
+import { METRICS_INDICES } from './constants';
+import { AgentInfo, BeatMetricsUsage, LiveQueryUsage } from './types';
+
+interface PolicyLevelUsage {
+  scheduled_queries?: ScheduledQueryUsageMetrics;
+  agent_info?: AgentInfo;
+}
+
+export async function getPolicyLevelUsage(
+  esClient: ElasticsearchClient,
+  soClient: SavedObjectsClientContract,
+  packagePolicyService?: PackagePolicyServiceInterface
+): Promise<PolicyLevelUsage> {
+  if (!packagePolicyService) {
+    return {};
+  }
+  const packagePolicies = await packagePolicyService.list(soClient, {
+    kuery: `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.name:${OSQUERY_INTEGRATION_NAME}`,
+    perPage: 10_000,
+  });
+
+  const result: PolicyLevelUsage = {
+    scheduled_queries: getScheduledQueryUsage(packagePolicies),
+    // TODO: figure out how to support dynamic keys in metrics
+    // packageVersions: getPackageVersions(packagePolicies),
+  };
+  const agentResponse = await esClient.search({
+    body: {
+      size: 0,
+      aggs: {
+        policied: {
+          filter: {
+            terms: {
+              policy_id: packagePolicies.items.map((p) => p.policy_id),
+            },
+          },
+        },
+      },
+    },
+    index: '.fleet-agents',
+  });
+  const policied = agentResponse.body.aggregations?.policied as AggregationsSingleBucketAggregate;
+  if (policied && typeof policied.doc_count === 'number') {
+    result.agent_info = {
+      enrolled: policied.doc_count,
+    };
+  }
+  return result;
+}
+
+export function getPackageVersions(packagePolicies: ListResult<PackagePolicy>) {
+  return packagePolicies.items.reduce((acc, item) => {
+    if (item.package) {
+      acc[item.package.version] = (acc[item.package.version] ?? 0) + 1;
+    }
+    return acc;
+  }, {} as { [version: string]: number });
+}
+
+interface ScheduledQueryUsageMetrics {
+  queryGroups: {
+    total: number;
+    empty: number;
+  };
+}
+
+export function getScheduledQueryUsage(packagePolicies: ListResult<PackagePolicy>) {
+  return packagePolicies.items.reduce(
+    (acc, item) => {
+      ++acc.queryGroups.total;
+      if (item.inputs.length === 0) {
+        ++acc.queryGroups.empty;
+      }
+      return acc;
+    },
+    {
+      queryGroups: {
+        total: 0,
+        empty: 0,
+      },
+    } as ScheduledQueryUsageMetrics
+  );
+}
+
+export async function getLiveQueryUsage(
+  soClient: SavedObjectsClientContract,
+  esClient: ElasticsearchClient
+) {
+  const { body: metricResponse } = await esClient.search({
+    body: {
+      size: 0,
+      aggs: {
+        queries: {
+          filter: {
+            term: {
+              input_type: 'osquery',
+            },
+          },
+        },
+      },
+    },
+    index: '.fleet-actions',
+  });
+  const result: LiveQueryUsage = {
+    session: await getRouteMetric(soClient, 'live_query'),
+  };
+  const esQueries = metricResponse.aggregations?.queries as AggregationsSingleBucketAggregate;
+  if (esQueries && typeof esQueries.doc_count === 'number') {
+    // getting error stats out of ES is difficult due to a lack of error info on .fleet-actions
+    // and a lack of indexable osquery specific info on .fleet-actions-results
+    result.cumulative = {
+      queries: esQueries.doc_count,
+    };
+  }
+
+  return result;
+}
+
+export async function getBeatUsage(esClient: ElasticsearchClient) {
+  const { body: metricResponse } = await esClient.search({
+    body: {
+      size: 0,
+      aggs: {
+        lastDay: {
+          filter: {
+            range: {
+              '@timestamp': {
+                gte: 'now-24h',
+                lte: 'now',
+              },
+            },
+          },
+          aggs: {
+            latest: {
+              top_hits: {
+                sort: [
+                  {
+                    '@timestamp': {
+                      order: 'desc',
+                    },
+                  },
+                ],
+                size: 1,
+              },
+            },
+            max_rss: {
+              max: {
+                field: 'monitoring.metrics.beat.memstats.rss',
+              },
+            },
+            avg_rss: {
+              avg: {
+                field: 'monitoring.metrics.beat.memstats.rss',
+              },
+            },
+            max_cpu: {
+              max: {
+                field: 'monitoring.metrics.beat.cpu.total.time.ms',
+              },
+            },
+            avg_cpu: {
+              avg: {
+                field: 'monitoring.metrics.beat.cpu.total.time.ms',
+              },
+            },
+          },
+        },
+      },
+    },
+    index: METRICS_INDICES,
+  });
+  const lastDayAggs = metricResponse.aggregations?.lastDay as AggregationsSingleBucketAggregate;
+  const result: BeatMetricsUsage = {
+    memory: {
+      rss: {},
+    },
+    cpu: {},
+  };
+
+  if ('max_rss' in lastDayAggs) {
+    result.memory.rss.max = (lastDayAggs.max_rss as AggregationsValueAggregate).value;
+  }
+
+  if ('avg_rss' in lastDayAggs) {
+    result.memory.rss.avg = (lastDayAggs.max_rss as AggregationsValueAggregate).value;
+  }
+
+  if ('max_cpu' in lastDayAggs) {
+    result.cpu.max = (lastDayAggs.max_cpu as AggregationsValueAggregate).value;
+  }
+
+  if ('avg_cpu' in lastDayAggs) {
+    result.cpu.avg = (lastDayAggs.max_cpu as AggregationsValueAggregate).value;
+  }
+
+  if ('latest' in lastDayAggs) {
+    const latest = (lastDayAggs.latest as AggregationsTopHitsAggregate).hits.hits[0]?._source
+      ?.monitoring.metrics.beat;
+    if (latest) {
+      result.cpu.latest = latest.cpu.total.time.ms;
+      result.memory.rss.latest = latest.memstats.rss;
+    }
+  }
+
+  return result;
+}

x-pack/plugins/osquery/server/usage/index.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { CollectorDependencies } from './types';
+import { registerCollector } from './collector';
+
+export type InitUsageCollectors = (deps: CollectorDependencies) => void;
+
+export const initUsageCollectors: InitUsageCollectors = (dependencies) => {
+  registerCollector(dependencies);
+};

x-pack/plugins/osquery/server/usage/types.ts

@@ -0,0 +1,160 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { CoreSetup } from 'src/core/server';
+import { OsqueryAppContext } from '../lib/osquery_app_context_services';
+import { MakeSchemaFrom } from '../../../../../src/plugins/usage_collection/server';
+import { SetupPlugins } from '../types';
+
+export type CollectorDependencies = {
+  osqueryContext: OsqueryAppContext;
+  core: CoreSetup;
+} & Pick<SetupPlugins, 'usageCollection'>;
+
+export interface LiveQuerySessionUsage {
+  count: number;
+  errors: number;
+}
+export interface LiveQueryCumulativeUsage {
+  queries: number;
+}
+
+export interface LiveQueryUsage {
+  session: LiveQuerySessionUsage;
+  cumulative?: LiveQueryCumulativeUsage;
+}
+
+export interface ScheduledQueryUsage {
+  queryGroups: {
+    total: number;
+    empty: number;
+  };
+}
+export interface AgentInfo {
+  enrolled: number;
+}
+
+export interface MetricEntry {
+  max?: number;
+  latest?: number;
+  avg?: number;
+}
+
+export interface BeatMetricsUsage {
+  cpu: MetricEntry;
+  memory: {
+    rss: MetricEntry;
+  };
+}
+
+export interface BeatMetrics {
+  usage: BeatMetricsUsage;
+}
+
+export interface UsageData {
+  live_query_usage?: LiveQueryUsage;
+  scheduled_queries?: ScheduledQueryUsage;
+  agent_info?: AgentInfo;
+  beat_metrics?: BeatMetrics;
+}
+
+export const usageSchema: MakeSchemaFrom<UsageData> = {
+  live_query_usage: {
+    session: {
+      count: {
+        type: 'long',
+        _meta: {
+          description: 'Number of osquery action requests',
+        },
+      },
+      errors: {
+        type: 'long',
+        _meta: {
+          description: 'Number of osquery action requests that resulted in errors',
+        },
+      },
+    },
+    cumulative: {
+      queries: {
+        type: 'long',
+        _meta: {
+          description: 'Number of osquery actions stored in Elasticsearch',
+        },
+      },
+    },
+  },
+  scheduled_queries: {
+    queryGroups: {
+      total: {
+        type: 'long',
+        _meta: {
+          description: 'Number of osquery policies/query groups',
+        },
+      },
+      empty: {
+        type: 'long',
+        _meta: {
+          description: 'Number of empty osquery policies/query groups',
+        },
+      },
+    },
+  },
+  agent_info: {
+    enrolled: {
+      type: 'long',
+      _meta: {
+        description: 'Number of agents enrolled in a policy with an osquery integration',
+      },
+    },
+  },
+  beat_metrics: {
+    usage: {
+      cpu: {
+        latest: {
+          type: 'long',
+          _meta: {
+            description: 'Latest cpu usage sample in ms',
+          },
+        },
+        max: {
+          type: 'long',
+          _meta: {
+            description: 'Max cpu usage sample over 24 hours in ms',
+          },
+        },
+        avg: {
+          type: 'long',
+          _meta: {
+            description: 'Mean cpu usage over 24 hours in ms',
+          },
+        },
+      },
+      memory: {
+        rss: {
+          latest: {
+            type: 'long',
+            _meta: {
+              description: 'Latest resident set size sample',
+            },
+          },
+          max: {
+            type: 'long',
+            _meta: {
+              description: 'Max resident set size sample over 24 hours',
+            },
+          },
+          avg: {
+            type: 'long',
+            _meta: {
+              description: 'Mean resident set size sample over 24 hours',
+            },
+          },
+        },
+      },
+    },
+  },
+};

x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json

@@ -3773,6 +3773,126 @@
         }
       }
     },
+    "osquery": {
+      "properties": {
+        "live_query_usage": {
+          "properties": {
+            "session": {
+              "properties": {
+                "count": {
+                  "type": "long",
+                  "_meta": {
+                    "description": "Number of osquery action requests"
+                  }
+                },
+                "errors": {
+                  "type": "long",
+                  "_meta": {
+                    "description": "Number of osquery action requests that resulted in errors"
+                  }
+                }
+              }
+            },
+            "cumulative": {
+              "properties": {
+                "queries": {
+                  "type": "long",
+                  "_meta": {
+                    "description": "Number of osquery actions stored in Elasticsearch"
+                  }
+                }
+              }
+            }
+          }
+        },
+        "scheduled_queries": {
+          "properties": {
+            "queryGroups": {
+              "properties": {
+                "total": {
+                  "type": "long",
+                  "_meta": {
+                    "description": "Number of osquery policies/query groups"
+                  }
+                },
+                "empty": {
+                  "type": "long",
+                  "_meta": {
+                    "description": "Number of empty osquery policies/query groups"
+                  }
+                }
+              }
+            }
+          }
+        },
+        "agent_info": {
+          "properties": {
+            "enrolled": {
+              "type": "long",
+              "_meta": {
+                "description": "Number of agents enrolled in a policy with an osquery integration"
+              }
+            }
+          }
+        },
+        "beat_metrics": {
+          "properties": {
+            "usage": {
+              "properties": {
+                "cpu": {
+                  "properties": {
+                    "latest": {
+                      "type": "long",
+                      "_meta": {
+                        "description": "Latest cpu usage sample in ms"
+                      }
+                    },
+                    "max": {
+                      "type": "long",
+                      "_meta": {
+                        "description": "Max cpu usage sample over 24 hours in ms"
+                      }
+                    },
+                    "avg": {
+                      "type": "long",
+                      "_meta": {
+                        "description": "Mean cpu usage over 24 hours in ms"
+                      }
+                    }
+                  }
+                },
+                "memory": {
+                  "properties": {
+                    "rss": {
+                      "properties": {
+                        "latest": {
+                          "type": "long",
+                          "_meta": {
+                            "description": "Latest resident set size sample"
+                          }
+                        },
+                        "max": {
+                          "type": "long",
+                          "_meta": {
+                            "description": "Max resident set size sample over 24 hours"
+                          }
+                        },
+                        "avg": {
+                          "type": "long",
+                          "_meta": {
+                            "description": "Mean resident set size sample over 24 hours"
+                          }
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    },
     "reporting": {
       "properties": {
         "csv": {