[Logs UI] Use log message field configured in the yaml file (#32502)

This fixes the message reconstruction rules to actually use the `xpack.infra.sources.default.fields.message` field configuration from the Kibana config file.

fixes elastic/kibana#31355

x-pack/plugins/infra/common/graphql/types.ts

@@ -60,6 +60,8 @@ export interface InfraSourceFields {
   container: string;
   /** The fields to identify a host by */
   host: string;
+  /** The fields to use as the log message */
+  message: string[];
   /** The field to identify a pod by */
   pod: string;
   /** The field to use as a tiebreaker for log events that have identical timestamps */
@@ -944,6 +946,8 @@ export namespace SourceFields {
 
     host: string;
 
+    message: string[];
+
     pod: string;
 
     tiebreaker: string;

x-pack/plugins/infra/common/source_configuration.test.ts

@@ -15,6 +15,7 @@ const initialConfiguration: InfraSourceConfiguration = {
   fields: {
     container: 'INITIAL_CONTAINER_FIELD',
     host: 'INITIAL_HOST_FIELD',
+    message: ['INITIAL_MESSAGE_FIELD'],
     pod: 'INITIAL_POD_FIELD',
     tiebreaker: 'INITIAL_TIEBREAKER_FIELD',
     timestamp: 'INITIAL_TIMESTAMP_FIELD',
@@ -163,6 +164,7 @@ describe('infrastructure source configuration', () => {
         expect(updateConfiguration(initialConfiguration)).toEqual({
           ...initialConfiguration,
           fields: {
+            ...initialConfiguration.fields,
             container: 'CHANGED_CONTAINER',
             host: 'CHANGED_HOST',
             pod: 'CHANGED_POD',

x-pack/plugins/infra/common/source_configuration.ts

@@ -6,10 +6,12 @@
 
 import { InfraSourceConfiguration, UpdateSourceInput } from './graphql/types';
 
-export const convertChangeToUpdater = (change: UpdateSourceInput) => (
-  configuration: InfraSourceConfiguration
-): InfraSourceConfiguration => {
-  const updaters: Array<(c: InfraSourceConfiguration) => InfraSourceConfiguration> = [
+export const convertChangeToUpdater = (change: UpdateSourceInput) => <
+  C extends InfraSourceConfiguration
+>(
+  configuration: C
+): C => {
+  const updaters: Array<(c: C) => C> = [
     c => (change.setName ? { ...c, name: change.setName.name } : c),
     c => (change.setDescription ? { ...c, description: change.setDescription.description } : c),
     c =>
@@ -27,6 +29,7 @@ export const convertChangeToUpdater = (change: UpdateSourceInput) => (
             fields: {
               container: defaultTo(c.fields.container, change.setFields.container),
               host: defaultTo(c.fields.host, change.setFields.host),
+              message: c.fields.message,
               pod: defaultTo(c.fields.pod, change.setFields.pod),
               tiebreaker: defaultTo(c.fields.tiebreaker, change.setFields.tiebreaker),
               timestamp: defaultTo(c.fields.timestamp, change.setFields.timestamp),

x-pack/plugins/infra/index.ts

@@ -8,12 +8,7 @@ import { i18n } from '@kbn/i18n';
 import JoiNamespace from 'joi';
 import { resolve } from 'path';
 
-import {
-  getConfigSchema,
-  // getDeprecations,
-  initServerWithKibana,
-  KbnServer,
-} from './server/kibana.index';
+import { getConfigSchema, initServerWithKibana, KbnServer } from './server/kibana.index';
 import { savedObjectMappings } from './server/saved_objects';
 
 const APP_ID = 'infra';
@@ -72,9 +67,6 @@ export function infra(kibana: any) {
     config(Joi: typeof JoiNamespace) {
       return getConfigSchema(Joi);
     },
-    // deprecations(helpers: any) {
-    //   return getDeprecations(helpers);
-    // },
     init(server: KbnServer) {
       initServerWithKibana(server);
     },

x-pack/plugins/infra/public/components/source_configuration/source_configuration_flyout.tsx

@@ -46,6 +46,7 @@ export const SourceConfigurationFlyout = injectI18n(({ intl }: SourceConfigurati
                   fields: {
                     container: configuration.fields.container,
                     host: configuration.fields.host,
+                    message: configuration.fields.message,
                     pod: configuration.fields.pod,
                     tiebreaker: configuration.fields.tiebreaker,
                     timestamp: configuration.fields.timestamp,

x-pack/plugins/infra/public/components/source_configuration/source_configuration_form_state.tsx

@@ -27,6 +27,8 @@ export interface InputFieldProps<
 
 type FieldErrorMessage = string | JSX.Element;
 
+type EditableFieldName = 'container' | 'host' | 'pod' | 'tiebreaker' | 'timestamp';
+
 interface FormState {
   name: string;
   description: string;
@@ -35,6 +37,7 @@ interface FormState {
   fields: {
     container: string;
     host: string;
+    message: string[];
     pod: string;
     tiebreaker: string;
     timestamp: string;
@@ -50,7 +53,7 @@ interface Actions {
   updateName: (name: string) => void;
   updateLogAlias: (value: string) => void;
   updateMetricAlias: (value: string) => void;
-  updateField: (field: keyof FormState['fields'], value: string) => void;
+  updateField: (field: EditableFieldName, value: string) => void;
 }
 
 interface Selectors {
@@ -58,7 +61,7 @@ interface Selectors {
   getNameFieldValidationErrors: () => FieldErrorMessage[];
   getLogAliasFieldValidationErrors: () => FieldErrorMessage[];
   getMetricAliasFieldValidationErrors: () => FieldErrorMessage[];
-  getFieldFieldValidationErrors: (field: keyof FormState['fields']) => FieldErrorMessage[];
+  getFieldFieldValidationErrors: (field: EditableFieldName) => FieldErrorMessage[];
   isFormValid: () => boolean;
 }
 
@@ -128,7 +131,7 @@ interface WithSourceConfigurationFormStateProps {
     State &
       Actions &
       Selectors & {
-        getFieldFieldProps: (field: keyof FormState['fields']) => InputFieldProps;
+        getFieldFieldProps: (field: EditableFieldName) => InputFieldProps;
         getLogAliasFieldProps: () => InputFieldProps;
         getMetricAliasFieldProps: () => InputFieldProps;
         getNameFieldProps: () => InputFieldProps;

x-pack/plugins/infra/public/containers/with_source/source_fields_fragment.gql_query.ts

@@ -19,6 +19,7 @@ export const sourceFieldsFragment = gql`
       fields {
         container
         host
+        message
         pod
         tiebreaker
         timestamp

x-pack/plugins/infra/public/graphql/introspection.json

@@ -550,6 +550,26 @@
             "isDeprecated": false,
             "deprecationReason": null
           },
+          {
+            "name": "message",
+            "description": "The fields to use as the log message",
+            "args": [],
+            "type": {
+              "kind": "NON_NULL",
+              "name": null,
+              "ofType": {
+                "kind": "LIST",
+                "name": null,
+                "ofType": {
+                  "kind": "NON_NULL",
+                  "name": null,
+                  "ofType": { "kind": "SCALAR", "name": "String", "ofType": null }
+                }
+              }
+            },
+            "isDeprecated": false,
+            "deprecationReason": null
+          },
           {
             "name": "pod",
             "description": "The field to identify a pod by",

x-pack/plugins/infra/public/graphql/types.ts

@@ -60,6 +60,8 @@ export interface InfraSourceFields {
   container: string;
   /** The fields to identify a host by */
   host: string;
+  /** The fields to use as the log message */
+  message: string[];
   /** The field to identify a pod by */
   pod: string;
   /** The field to use as a tiebreaker for log events that have identical timestamps */
@@ -944,6 +946,8 @@ export namespace SourceFields {
 
     host: string;
 
+    message: string[];
+
     pod: string;
 
     tiebreaker: string;

x-pack/plugins/infra/server/graphql/sources/schema.gql.ts

@@ -44,6 +44,8 @@ export const sourcesSchema = gql`
     container: String!
     "The fields to identify a host by"
     host: String!
+    "The fields to use as the log message"
+    message: [String!]!
     "The field to identify a pod by"
     pod: String!
     "The field to use as a tiebreaker for log events that have identical timestamps"

x-pack/plugins/infra/server/graphql/types.ts

@@ -88,6 +88,8 @@ export interface InfraSourceFields {
   container: string;
   /** The fields to identify a host by */
   host: string;
+  /** The fields to use as the log message */
+  message: string[];
   /** The field to identify a pod by */
   pod: string;
   /** The field to use as a tiebreaker for log events that have identical timestamps */
@@ -798,6 +800,8 @@ export namespace InfraSourceFieldsResolvers {
     container?: ContainerResolver<string, TypeParent, Context>;
     /** The fields to identify a host by */
     host?: HostResolver<string, TypeParent, Context>;
+    /** The fields to use as the log message */
+    message?: MessageResolver<string[], TypeParent, Context>;
     /** The field to identify a pod by */
     pod?: PodResolver<string, TypeParent, Context>;
     /** The field to use as a tiebreaker for log events that have identical timestamps */
@@ -816,6 +820,11 @@ export namespace InfraSourceFieldsResolvers {
     Parent = InfraSourceFields,
     Context = InfraContext
   > = Resolver<R, Parent, Context>;
+  export type MessageResolver<
+    R = string[],
+    Parent = InfraSourceFields,
+    Context = InfraContext
+  > = Resolver<R, Parent, Context>;
   export type PodResolver<
     R = string,
     Parent = InfraSourceFields,

x-pack/plugins/infra/server/kibana.index.ts

@@ -29,6 +29,9 @@ export const getConfigSchema = (Joi: typeof JoiNamespace) => {
     fields: Joi.object({
       container: Joi.string(),
       host: Joi.string(),
+      message: Joi.array()
+        .items(Joi.string())
+        .single(),
       pod: Joi.string(),
       tiebreaker: Joi.string(),
       timestamp: Joi.string(),
@@ -50,10 +53,3 @@ export const getConfigSchema = (Joi: typeof JoiNamespace) => {
 
   return InfraRootConfigSchema;
 };
-
-export const getDeprecations = () => [];
-
-// interface DeprecationHelpers {
-//   rename(oldKey: string, newKey: string): (settings: unknown, log: unknown) => void;
-//   unused(oldKey: string): (settings: unknown, log: unknown) => void;
-// }

x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_apache2.test.ts

@@ -4,10 +4,10 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { builtinRules } from '.';
+import { getBuiltinRules } from '.';
 import { compileFormattingRules } from '../message';
 
-const { format } = compileFormattingRules(builtinRules);
+const { format } = compileFormattingRules(getBuiltinRules([]));
 
 describe('Filebeat Rules', () => {
   describe('in ECS format', () => {

x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_iis.test.ts

@@ -4,10 +4,10 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { builtinRules } from '.';
+import { getBuiltinRules } from '.';
 import { compileFormattingRules } from '../message';
 
-const { format } = compileFormattingRules(builtinRules);
+const { format } = compileFormattingRules(getBuiltinRules([]));
 
 describe('Filebeat Rules', () => {
   describe('in ECS format', () => {

x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_kafka.test.ts

@@ -4,10 +4,10 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { builtinRules } from '.';
+import { getBuiltinRules } from '.';
 import { compileFormattingRules } from '../message';
 
-const { format } = compileFormattingRules(builtinRules);
+const { format } = compileFormattingRules(getBuiltinRules([]));
 
 describe('Filebeat Rules', () => {
   describe('in ECS format', () => {

x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_logstash.test.ts

@@ -4,10 +4,10 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { builtinRules } from '.';
+import { getBuiltinRules } from '.';
 import { compileFormattingRules } from '../message';
 
-const { format } = compileFormattingRules(builtinRules);
+const { format } = compileFormattingRules(getBuiltinRules([]));
 
 describe('Filebeat Rules', () => {
   describe('in ECS format', () => {

x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_mysql.test.ts

@@ -4,10 +4,10 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { builtinRules } from '.';
+import { getBuiltinRules } from '.';
 import { compileFormattingRules } from '../message';
 
-const { format } = compileFormattingRules(builtinRules);
+const { format } = compileFormattingRules(getBuiltinRules([]));
 
 describe('Filebeat Rules', () => {
   describe('in ECS format', () => {

x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_nginx.test.ts

@@ -4,10 +4,10 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { builtinRules } from '.';
+import { getBuiltinRules } from '.';
 import { compileFormattingRules } from '../message';
 
-const { format } = compileFormattingRules(builtinRules);
+const { format } = compileFormattingRules(getBuiltinRules([]));
 
 describe('Filebeat Rules', () => {
   describe('in ECS format', () => {

x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/generic.test.ts

@@ -0,0 +1,163 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { getBuiltinRules } from '.';
+import { compileFormattingRules } from '../message';
+
+const { format } = compileFormattingRules(
+  getBuiltinRules(['first_generic_message', 'second_generic_message'])
+);
+
+describe('Generic Rules', () => {
+  describe('configurable message rules', () => {
+    test('includes the event.dataset and log.level if present', () => {
+      const flattenedDocument = {
+        '@timestamp': '2016-12-26T16:22:13.000Z',
+        'event.dataset': 'generic.test',
+        'log.level': 'TEST_LEVEL',
+        first_generic_message: 'TEST_MESSAGE',
+      };
+
+      expect(format(flattenedDocument)).toMatchInlineSnapshot(`
+Array [
+  Object {
+    "constant": "[",
+  },
+  Object {
+    "field": "event.dataset",
+    "highlights": Array [],
+    "value": "generic.test",
+  },
+  Object {
+    "constant": "][",
+  },
+  Object {
+    "field": "log.level",
+    "highlights": Array [],
+    "value": "TEST_LEVEL",
+  },
+  Object {
+    "constant": "] ",
+  },
+  Object {
+    "field": "first_generic_message",
+    "highlights": Array [],
+    "value": "TEST_MESSAGE",
+  },
+]
+`);
+    });
+
+    test('includes the log.level if present', () => {
+      const flattenedDocument = {
+        '@timestamp': '2016-12-26T16:22:13.000Z',
+        'log.level': 'TEST_LEVEL',
+        first_generic_message: 'TEST_MESSAGE',
+      };
+
+      expect(format(flattenedDocument)).toMatchInlineSnapshot(`
+Array [
+  Object {
+    "constant": "[",
+  },
+  Object {
+    "field": "log.level",
+    "highlights": Array [],
+    "value": "TEST_LEVEL",
+  },
+  Object {
+    "constant": "] ",
+  },
+  Object {
+    "field": "first_generic_message",
+    "highlights": Array [],
+    "value": "TEST_MESSAGE",
+  },
+]
+`);
+    });
+
+    test('includes the message', () => {
+      const firstFlattenedDocument = {
+        '@timestamp': '2016-12-26T16:22:13.000Z',
+        first_generic_message: 'FIRST_TEST_MESSAGE',
+      };
+
+      expect(format(firstFlattenedDocument)).toMatchInlineSnapshot(`
+Array [
+  Object {
+    "field": "first_generic_message",
+    "highlights": Array [],
+    "value": "FIRST_TEST_MESSAGE",
+  },
+]
+`);
+
+      const secondFlattenedDocument = {
+        '@timestamp': '2016-12-26T16:22:13.000Z',
+        second_generic_message: 'SECOND_TEST_MESSAGE',
+      };
+
+      expect(format(secondFlattenedDocument)).toMatchInlineSnapshot(`
+Array [
+  Object {
+    "field": "second_generic_message",
+    "highlights": Array [],
+    "value": "SECOND_TEST_MESSAGE",
+  },
+]
+`);
+    });
+  });
+
+  describe('log.original fallback', () => {
+    test('includes the event.dataset if present', () => {
+      const flattenedDocument = {
+        '@timestamp': '2016-12-26T16:22:13.000Z',
+        'event.dataset': 'generic.test',
+        'log.original': 'TEST_MESSAGE',
+      };
+
+      expect(format(flattenedDocument)).toMatchInlineSnapshot(`
+Array [
+  Object {
+    "constant": "[",
+  },
+  Object {
+    "field": "event.dataset",
+    "highlights": Array [],
+    "value": "generic.test",
+  },
+  Object {
+    "constant": "] ",
+  },
+  Object {
+    "field": "log.original",
+    "highlights": Array [],
+    "value": "TEST_MESSAGE",
+  },
+]
+`);
+    });
+
+    test('includes the original message', () => {
+      const flattenedDocument = {
+        '@timestamp': '2016-12-26T16:22:13.000Z',
+        'log.original': 'TEST_MESSAGE',
+      };
+
+      expect(format(flattenedDocument)).toMatchInlineSnapshot(`
+Array [
+  Object {
+    "field": "log.original",
+    "highlights": Array [],
+    "value": "TEST_MESSAGE",
+  },
+]
+`);
+    });
+  });
+});

x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/generic.ts

@@ -4,71 +4,14 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-export const genericRules = [
-  {
-    when: {
-      exists: ['event.dataset', 'log.level', 'message'],
-    },
-    format: [
-      {
-        constant: '[',
-      },
-      {
-        field: 'event.dataset',
-      },
-      {
-        constant: '][',
-      },
-      {
-        field: 'log.level',
-      },
-      {
-        constant: '] ',
-      },
-      {
-        field: 'message',
-      },
-    ],
-  },
-  {
-    when: {
-      exists: ['log.level', 'message'],
-    },
-    format: [
-      {
-        constant: '[',
-      },
-      {
-        field: 'log.level',
-      },
-      {
-        constant: '] ',
-      },
-      {
-        field: 'message',
-      },
-    ],
-  },
-  {
-    when: {
-      exists: ['message'],
-    },
-    format: [
-      {
-        field: 'message',
-      },
-    ],
-  },
-  {
-    when: {
-      exists: ['@message'],
-    },
-    format: [
-      {
-        field: '@message',
-      },
-    ],
-  },
+import { LogMessageFormattingRule } from '../rule_types';
+
+const BUILTIN_GENERIC_MESSAGE_FIELDS = ['message', '@message'];
+
+export const getGenericRules = (genericMessageFields: string[]) => [
+  ...Array.from(new Set([...genericMessageFields, ...BUILTIN_GENERIC_MESSAGE_FIELDS])).reduce<
+    LogMessageFormattingRule[]
+  >((genericRules, fieldName) => [...genericRules, ...createGenericRulesForField(fieldName)], []),
   {
     when: {
       exists: ['event.dataset', 'log.original'],
@@ -99,3 +42,60 @@ export const genericRules = [
     ],
   },
 ];
+
+const createGenericRulesForField = (fieldName: string) => [
+  {
+    when: {
+      exists: ['event.dataset', 'log.level', fieldName],
+    },
+    format: [
+      {
+        constant: '[',
+      },
+      {
+        field: 'event.dataset',
+      },
+      {
+        constant: '][',
+      },
+      {
+        field: 'log.level',
+      },
+      {
+        constant: '] ',
+      },
+      {
+        field: fieldName,
+      },
+    ],
+  },
+  {
+    when: {
+      exists: ['log.level', fieldName],
+    },
+    format: [
+      {
+        constant: '[',
+      },
+      {
+        field: 'log.level',
+      },
+      {
+        constant: '] ',
+      },
+      {
+        field: fieldName,
+      },
+    ],
+  },
+  {
+    when: {
+      exists: [fieldName],
+    },
+    format: [
+      {
+        field: fieldName,
+      },
+    ],
+  },
+];

x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/index.ts

@@ -18,10 +18,10 @@ import { filebeatRedisRules } from './filebeat_redis';
 import { filebeatSystemRules } from './filebeat_system';
 import { filebeatTraefikRules } from './filebeat_traefik';
 
-import { genericRules } from './generic';
+import { getGenericRules } from './generic';
 import { genericWebserverRules } from './generic_webserver';
 
-export const builtinRules = [
+export const getBuiltinRules = (genericMessageFields: string[]) => [
   ...filebeatApache2Rules,
   ...filebeatNginxRules,
   ...filebeatRedisRules,
@@ -36,7 +36,7 @@ export const builtinRules = [
   ...filebeatOsqueryRules,
   ...filebeatTraefikRules,
   ...genericWebserverRules,
-  ...genericRules,
+  ...getGenericRules(genericMessageFields),
   {
     when: {
       exists: ['log.path'],

x-pack/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts

@@ -15,7 +15,7 @@ import {
 } from '../../../graphql/types';
 import { InfraDateRangeAggregationBucket, InfraFrameworkRequest } from '../../adapters/framework';
 import { InfraSourceConfiguration, InfraSources } from '../../sources';
-import { builtinRules } from './builtin_rules';
+import { getBuiltinRules } from './builtin_rules';
 import { convertDocumentSourceToLogItemFields } from './convert_document_source_to_log_item_fields';
 import { compileFormattingRules } from './message';
 
@@ -42,7 +42,7 @@ export class InfraLogEntriesDomain {
     }
 
     const { configuration } = await this.libs.sources.getSourceConfiguration(request, sourceId);
-    const formattingRules = compileFormattingRules(builtinRules);
+    const formattingRules = compileFormattingRules(getBuiltinRules(configuration.fields.message));
 
     const documentsBefore = await this.adapter.getAdjacentLogEntryDocuments(
       request,
@@ -90,7 +90,7 @@ export class InfraLogEntriesDomain {
     highlightQuery?: string
   ): Promise<InfraLogEntry[]> {
     const { configuration } = await this.libs.sources.getSourceConfiguration(request, sourceId);
-    const formattingRules = compileFormattingRules(builtinRules);
+    const formattingRules = compileFormattingRules(getBuiltinRules(configuration.fields.message));
     const documents = await this.adapter.getContainedLogEntryDocuments(
       request,
       configuration,

x-pack/plugins/infra/server/lib/domains/log_entries_domain/message.ts

@@ -7,6 +7,11 @@
 import stringify from 'json-stable-stringify';
 
 import { InfraLogMessageSegment } from '../../../graphql/types';
+import {
+  LogMessageFormattingCondition,
+  LogMessageFormattingInstruction,
+  LogMessageFormattingRule,
+} from './rule_types';
 
 export function compileFormattingRules(rules: LogMessageFormattingRule[]) {
   const compiledRules = rules.map(compileRule);
@@ -158,37 +163,6 @@ interface Fields {
   [fieldName: string]: string | number | object | boolean | null;
 }
 
-interface LogMessageFormattingRule {
-  when: LogMessageFormattingCondition;
-  format: LogMessageFormattingInstruction[];
-}
-
-type LogMessageFormattingCondition =
-  | LogMessageFormattingExistsCondition
-  | LogMessageFormattingFieldValueCondition;
-
-interface LogMessageFormattingExistsCondition {
-  exists: string[];
-}
-
-interface LogMessageFormattingFieldValueCondition {
-  values: {
-    [fieldName: string]: string | number | boolean | null;
-  };
-}
-
-type LogMessageFormattingInstruction =
-  | LogMessageFormattingFieldReference
-  | LogMessageFormattingConstant;
-
-interface LogMessageFormattingFieldReference {
-  field: string;
-}
-
-interface LogMessageFormattingConstant {
-  constant: string;
-}
-
 interface CompiledLogMessageFormattingRule {
   requiredFields: string[];
   fulfillsCondition(fields: Fields): boolean;

x-pack/plugins/infra/server/lib/domains/log_entries_domain/rule_types.ts

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export interface LogMessageFormattingRule {
+  when: LogMessageFormattingCondition;
+  format: LogMessageFormattingInstruction[];
+}
+
+export type LogMessageFormattingCondition =
+  | LogMessageFormattingExistsCondition
+  | LogMessageFormattingFieldValueCondition;
+
+export interface LogMessageFormattingExistsCondition {
+  exists: string[];
+}
+
+export interface LogMessageFormattingFieldValueCondition {
+  values: {
+    [fieldName: string]: string | number | boolean | null;
+  };
+}
+
+export type LogMessageFormattingInstruction =
+  | LogMessageFormattingFieldReference
+  | LogMessageFormattingConstant;
+
+export interface LogMessageFormattingFieldReference {
+  field: string;
+}
+
+export interface LogMessageFormattingConstant {
+  constant: string;
+}

x-pack/plugins/infra/server/lib/sources/defaults.ts

@@ -12,6 +12,7 @@ export const defaultSourceConfiguration = {
   fields: {
     container: 'container.id',
     host: 'host.name',
+    message: ['message', '@message'],
     pod: 'kubernetes.pod.uid',
     tiebreaker: '_doc',
     timestamp: '@timestamp',

x-pack/plugins/infra/server/lib/sources/saved_object_mappings.ts

@@ -5,12 +5,12 @@
  */
 
 import { ElasticsearchMappingOf } from '../../utils/typed_elasticsearch_mappings';
-import { InfraSourceConfiguration } from './types';
+import { InfraSavedSourceConfiguration } from './types';
 
 export const infraSourceConfigurationSavedObjectType = 'infrastructure-ui-source';
 
 export const infraSourceConfigurationSavedObjectMappings: {
-  [infraSourceConfigurationSavedObjectType]: ElasticsearchMappingOf<InfraSourceConfiguration>;
+  [infraSourceConfigurationSavedObjectType]: ElasticsearchMappingOf<InfraSavedSourceConfiguration>;
 } = {
   [infraSourceConfigurationSavedObjectType]: {
     properties: {

x-pack/plugins/infra/server/lib/sources/sources.ts

@@ -14,10 +14,12 @@ import { InfraFrameworkRequest, internalInfraFrameworkRequest } from '../adapter
 import { defaultSourceConfiguration } from './defaults';
 import { infraSourceConfigurationSavedObjectType } from './saved_object_mappings';
 import {
-  InfraSavedSourceConfigurationRuntimeType,
+  InfraSavedSourceConfiguration,
   InfraSourceConfiguration,
-  PartialInfraSourceConfiguration,
-  PartialInfraSourceConfigurationRuntimeType,
+  InfraStaticSourceConfiguration,
+  pickSavedSourceConfiguration,
+  SourceConfigurationSavedObjectRuntimeType,
+  StaticSourceConfigurationRuntimeType,
 } from './types';
 
 export class InfraSources {
@@ -71,7 +73,7 @@ export class InfraSources {
   public async createSourceConfiguration(
     request: InfraFrameworkRequest,
     sourceId: string,
-    source: PartialInfraSourceConfiguration
+    source: InfraSavedSourceConfiguration
   ) {
     const staticDefaultSourceConfiguration = await this.getStaticDefaultSourceConfiguration();
 
@@ -85,7 +87,7 @@ export class InfraSources {
         .getScopedSavedObjectsClient(request[internalInfraFrameworkRequest])
         .create(
           infraSourceConfigurationSavedObjectType,
-          { ...newSourceConfiguration },
+          pickSavedSourceConfiguration(newSourceConfiguration),
           { id: sourceId }
         )
     );
@@ -125,7 +127,7 @@ export class InfraSources {
         .update(
           infraSourceConfigurationSavedObjectType,
           sourceId,
-          { ...updatedConfigurationAttributes },
+          pickSavedSourceConfiguration(updatedConfigurationAttributes),
           {
             version,
           }
@@ -146,7 +148,7 @@ export class InfraSources {
     const staticSourceConfiguration = runtimeTypes
       .type({
         sources: runtimeTypes.type({
-          default: PartialInfraSourceConfigurationRuntimeType,
+          default: StaticSourceConfigurationRuntimeType,
         }),
       })
       .decode(staticConfiguration)
@@ -184,7 +186,7 @@ export class InfraSources {
 
 const mergeSourceConfiguration = (
   first: InfraSourceConfiguration,
-  ...others: PartialInfraSourceConfiguration[]
+  ...others: InfraStaticSourceConfiguration[]
 ) =>
   others.reduce<InfraSourceConfiguration>(
     (previousSourceConfiguration, currentSourceConfiguration) => ({
@@ -199,7 +201,7 @@ const mergeSourceConfiguration = (
   );
 
 const convertSavedObjectToSavedSourceConfiguration = (savedObject: unknown) =>
-  InfraSavedSourceConfigurationRuntimeType.decode(savedObject)
+  SourceConfigurationSavedObjectRuntimeType.decode(savedObject)
     .map(savedSourceConfiguration => ({
       id: savedSourceConfiguration.id,
       version: savedSourceConfiguration.version,

x-pack/plugins/infra/server/lib/sources/types.ts

@@ -20,35 +20,82 @@ export const TimestampFromString = new runtimeTypes.Type<number, string>(
   output => new Date(output).toISOString()
 );
 
-export const InfraSourceConfigurationRuntimeType = runtimeTypes.type({
+/**
+ * Stored source configuration as read from and written to saved objects
+ */
+
+const SavedSourceConfigurationFieldsRuntimeType = runtimeTypes.partial({
+  container: runtimeTypes.string,
+  host: runtimeTypes.string,
+  pod: runtimeTypes.string,
+  tiebreaker: runtimeTypes.string,
+  timestamp: runtimeTypes.string,
+});
+
+export const SavedSourceConfigurationRuntimeType = runtimeTypes.partial({
   name: runtimeTypes.string,
   description: runtimeTypes.string,
   metricAlias: runtimeTypes.string,
   logAlias: runtimeTypes.string,
-  fields: runtimeTypes.type({
-    container: runtimeTypes.string,
-    host: runtimeTypes.string,
-    pod: runtimeTypes.string,
-    tiebreaker: runtimeTypes.string,
-    timestamp: runtimeTypes.string,
-  }),
+  fields: SavedSourceConfigurationFieldsRuntimeType,
+});
+
+export interface InfraSavedSourceConfiguration
+  extends runtimeTypes.TypeOf<typeof SavedSourceConfigurationRuntimeType> {}
+
+export const pickSavedSourceConfiguration = (value: InfraSourceConfiguration) => {
+  const { container, host, pod, tiebreaker, timestamp } = value.fields;
+
+  return {
+    ...value,
+    fields: { container, host, pod, tiebreaker, timestamp },
+  };
+};
+
+/**
+ * Static source configuration as read from the configuration file
+ */
+
+const StaticSourceConfigurationFieldsRuntimeType = runtimeTypes.partial({
+  ...SavedSourceConfigurationFieldsRuntimeType.props,
+  message: runtimeTypes.array(runtimeTypes.string),
+});
+
+export const StaticSourceConfigurationRuntimeType = runtimeTypes.partial({
+  name: runtimeTypes.string,
+  description: runtimeTypes.string,
+  metricAlias: runtimeTypes.string,
+  logAlias: runtimeTypes.string,
+  fields: StaticSourceConfigurationFieldsRuntimeType,
+});
+
+export interface InfraStaticSourceConfiguration
+  extends runtimeTypes.TypeOf<typeof StaticSourceConfigurationRuntimeType> {}
+
+/**
+ * Full source configuration type after all cleanup has been done at the edges
+ */
+
+const SourceConfigurationFieldsRuntimeType = runtimeTypes.type({
+  ...StaticSourceConfigurationFieldsRuntimeType.props,
+});
+
+export const SourceConfigurationRuntimeType = runtimeTypes.type({
+  ...SavedSourceConfigurationRuntimeType.props,
+  fields: SourceConfigurationFieldsRuntimeType,
 });
 
 export interface InfraSourceConfiguration
-  extends runtimeTypes.TypeOf<typeof InfraSourceConfigurationRuntimeType> {}
+  extends runtimeTypes.TypeOf<typeof SourceConfigurationRuntimeType> {}
 
-export const PartialInfraSourceConfigurationRuntimeType = runtimeTypes.partial({
-  ...InfraSourceConfigurationRuntimeType.props,
-  fields: runtimeTypes.partial(InfraSourceConfigurationRuntimeType.props.fields.props),
-});
+/**
+ * Saved object type with metadata
+ */
 
-export interface PartialInfraSourceConfiguration
-  extends runtimeTypes.TypeOf<typeof PartialInfraSourceConfigurationRuntimeType> {}
-
-export const InfraSavedSourceConfigurationRuntimeType = runtimeTypes.intersection([
+export const SourceConfigurationSavedObjectRuntimeType = runtimeTypes.intersection([
   runtimeTypes.type({
     id: runtimeTypes.string,
-    attributes: PartialInfraSourceConfigurationRuntimeType,
+    attributes: SavedSourceConfigurationRuntimeType,
   }),
   runtimeTypes.partial({
     version: runtimeTypes.string,
@@ -56,5 +103,5 @@ export const InfraSavedSourceConfigurationRuntimeType = runtimeTypes.intersectio
   }),
 ]);
 
-export interface InfraSavedSourceConfiguration
-  extends runtimeTypes.TypeOf<typeof InfraSavedSourceConfigurationRuntimeType> {}
+export interface SourceConfigurationSavedObject
+  extends runtimeTypes.TypeOf<typeof SourceConfigurationSavedObjectRuntimeType> {}