[kbn-es] Set password for native realm accounts (#35586)

Signed-off-by: Tyler Smalley <tyler.smalley@elastic.co>

packages/kbn-es/package.json

@@ -5,6 +5,7 @@
   "license": "Apache-2.0",
   "private": true,
   "dependencies": {
+    "@elastic/elasticsearch": "^7.0.0-rc.2",
     "@kbn/dev-utils": "1.0.0",
     "abort-controller": "^2.0.3",
     "chalk": "^2.4.1",

packages/kbn-es/src/cli_commands/archive.js

@@ -32,10 +32,11 @@ exports.help = (defaults = {}) => {
   return dedent`
     Options:
 
-      --base-path     Path containing cache/installations [default: ${basePath}]
-      --install-path  Installation path, defaults to 'source' within base-path
-      --password      Sets password for elastic user [default: ${password}]
-      -E              Additional key=value settings to pass to Elasticsearch
+      --base-path       Path containing cache/installations [default: ${basePath}]
+      --install-path    Installation path, defaults to 'source' within base-path
+      --password        Sets password for elastic user [default: ${password}]
+      --password.[user] Sets password for native realm user [default: ${password}]
+      -E                Additional key=value settings to pass to Elasticsearch
 
     Example:
 

packages/kbn-es/src/cli_commands/snapshot.js

@@ -29,14 +29,15 @@ exports.help = (defaults = {}) => {
   return dedent`
     Options:
 
-      --license       Run with a 'oss', 'basic', or 'trial' license [default: ${license}]
-      --version       Version of ES to download [default: ${defaults.version}]
-      --base-path     Path containing cache/installations [default: ${basePath}]
-      --install-path  Installation path, defaults to 'source' within base-path
-      --data-archive  Path to zip or tarball containing an ES data directory to seed the cluster with.
-      --password      Sets password for elastic user [default: ${password}]
-      -E              Additional key=value settings to pass to Elasticsearch
-      --download-only Download the snapshot but don't actually start it
+      --license         Run with a 'oss', 'basic', or 'trial' license [default: ${license}]
+      --version         Version of ES to download [default: ${defaults.version}]
+      --base-path       Path containing cache/installations [default: ${basePath}]
+      --install-path    Installation path, defaults to 'source' within base-path
+      --data-archive    Path to zip or tarball containing an ES data directory to seed the cluster with.
+      --password        Sets password for elastic user [default: ${password}]
+      --password.[user] Sets password for native realm user [default: ${password}]
+      -E                Additional key=value settings to pass to Elasticsearch
+      --download-only   Download the snapshot but don't actually start it
 
     Example:
 
@@ -69,6 +70,6 @@ exports.run = async (defaults = {}) => {
       await cluster.extractDataDirectory(installPath, options.dataArchive);
     }
 
-    await cluster.run(installPath, { esArgs: options.esArgs });
+    await cluster.run(installPath, options);
   }
 };

packages/kbn-es/src/cluster.js

@@ -22,7 +22,13 @@ const chalk = require('chalk');
 const path = require('path');
 const { downloadSnapshot, installSnapshot, installSource, installArchive } = require('./install');
 const { ES_BIN } = require('./paths');
-const { log: defaultLog, parseEsLog, extractConfigFiles, decompress } = require('./utils');
+const {
+  log: defaultLog,
+  parseEsLog,
+  extractConfigFiles,
+  decompress,
+  NativeRealm,
+} = require('./utils');
 const { createCliError } = require('./errors');
 const { promisify } = require('util');
 const treeKillAsync = promisify(require('tree-kill'));
@@ -215,7 +221,7 @@ exports.Cluster = class Cluster {
    * @property {Array} options.esArgs
    * @return {undefined}
    */
-  _exec(installPath, { esArgs = [] }) {
+  _exec(installPath, options = {}) {
     if (this._process || this._outcome) {
       throw new Error('ES has already been started');
     }
@@ -223,7 +229,7 @@ exports.Cluster = class Cluster {
     this._log.info(chalk.bold('Starting'));
     this._log.indent(4);
 
-    const args = extractConfigFiles(esArgs, installPath, {
+    const args = extractConfigFiles(options.esArgs || [], installPath, {
       log: this._log,
     }).reduce((acc, cur) => acc.concat(['-E', cur]), []);
 
@@ -236,7 +242,23 @@ exports.Cluster = class Cluster {
 
     this._process.stdout.on('data', data => {
       const lines = parseEsLog(data.toString());
-      lines.forEach(line => this._log.info(line.formattedMessage));
+      lines.forEach(line => {
+        this._log.info(line.formattedMessage);
+
+        // once we have the port we can stop checking for it
+        if (this.httpPort) {
+          return;
+        }
+
+        const httpAddressMatch = line.message.match(
+          /HttpServer.+publish_address {[0-9.]+:([0-9]+)/
+        );
+
+        if (httpAddressMatch) {
+          this.httpPort = httpAddressMatch[1];
+          new NativeRealm(options.password, this.httpPort, this._log).setPasswords(options);
+        }
+      });
     });
 
     this._process.stderr.on('data', data => this._log.error(chalk.red(data.toString())));

packages/kbn-es/src/utils/index.js

@@ -23,3 +23,4 @@ exports.parseEsLog = require('./parse_es_log').parseEsLog;
 exports.findMostRecentlyChanged = require('./find_most_recently_changed').findMostRecentlyChanged;
 exports.extractConfigFiles = require('./extract_config_files').extractConfigFiles;
 exports.decompress = require('./decompress').decompress;
+exports.NativeRealm = require('./native_realm').NativeRealm;

packages/kbn-es/src/utils/native_realm.js

@@ -0,0 +1,82 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+const { Client } = require('@elastic/elasticsearch');
+const chalk = require('chalk');
+
+const { log: defaultLog } = require('./log');
+
+exports.NativeRealm = class NativeRealm {
+  constructor(elasticPassword, port, log = defaultLog) {
+    this._client = new Client({ node: `http://elastic:${elasticPassword}@localhost:${port}` });
+    this._elasticPassword = elasticPassword;
+    this._log = log;
+  }
+
+  async setPassword(username, password = this._elasticPassword) {
+    this._log.info(`setting ${chalk.bold(username)} password to ${chalk.bold(password)}`);
+
+    try {
+      await this._client.security.changePassword({
+        username,
+        refresh: 'wait_for',
+        body: {
+          password,
+        },
+      });
+    } catch (e) {
+      this._log.error(
+        chalk.red(`unable to set password for ${chalk.bold(username)}: ${e.message}`)
+      );
+    }
+  }
+
+  async setPasswords(options) {
+    if (!(await this.isSecurityEnabled())) {
+      this._log.info('security is not enabled, unable to set native realm passwords');
+      return;
+    }
+
+    (await this.getReservedUsers()).forEach(user => {
+      this.setPassword(user, options[`password.${user}`]);
+    });
+  }
+
+  async getReservedUsers() {
+    const users = await this._client.security.getUser();
+
+    return Object.keys(users.body).reduce((acc, user) => {
+      if (users.body[user].metadata._reserved === true) {
+        acc.push(user);
+      }
+      return acc;
+    }, []);
+  }
+
+  async isSecurityEnabled() {
+    try {
+      const {
+        body: { features },
+      } = await this._client.xpack.info({ categories: 'features' });
+      return features.security && features.security.enabled && features.security.available;
+    } catch (e) {
+      return false;
+    }
+  }
+};

packages/kbn-es/src/utils/native_realm.test.js

@@ -0,0 +1,221 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+const { NativeRealm } = require('./native_realm');
+
+jest.genMockFromModule('@elastic/elasticsearch');
+jest.mock('@elastic/elasticsearch');
+
+const { Client } = require('@elastic/elasticsearch');
+
+const mockClient = {
+  xpack: {
+    info: jest.fn(),
+  },
+  security: {
+    changePassword: jest.fn(),
+    getUser: jest.fn(),
+  },
+};
+Client.mockImplementation(() => mockClient);
+
+const log = {
+  error: jest.fn(),
+  info: jest.fn(),
+};
+
+let nativeRealm;
+
+beforeEach(() => {
+  nativeRealm = new NativeRealm('changeme', '9200', log);
+});
+
+afterAll(() => {
+  jest.clearAllMocks();
+});
+
+function mockXPackInfo(available, enabled) {
+  mockClient.xpack.info.mockImplementation(() => ({
+    body: {
+      features: {
+        security: {
+          available,
+          enabled,
+        },
+      },
+    },
+  }));
+}
+
+describe('isSecurityEnabled', () => {
+  test('returns true if enabled and available', async () => {
+    mockXPackInfo(true, true);
+    expect(await nativeRealm.isSecurityEnabled()).toBe(true);
+  });
+
+  test('returns false if not available', async () => {
+    mockXPackInfo(false, true);
+    expect(await nativeRealm.isSecurityEnabled()).toBe(false);
+  });
+
+  test('returns false if not enabled', async () => {
+    mockXPackInfo(true, false);
+    expect(await nativeRealm.isSecurityEnabled()).toBe(false);
+  });
+
+  test('logs exception and returns false', async () => {
+    mockClient.xpack.info.mockImplementation(() => {
+      throw new Error('ResponseError');
+    });
+    expect(await nativeRealm.isSecurityEnabled()).toBe(false);
+  });
+});
+
+describe('setPasswords', () => {
+  it('uses provided passwords', async () => {
+    mockXPackInfo(true, true);
+
+    mockClient.security.getUser.mockImplementation(() => ({
+      body: {
+        kibana: {
+          metadata: {
+            _reserved: true,
+          },
+        },
+        non_native: {
+          metadata: {
+            _reserved: false,
+          },
+        },
+        logstash_system: {
+          metadata: {
+            _reserved: true,
+          },
+        },
+        elastic: {
+          metadata: {
+            _reserved: true,
+          },
+        },
+        beats_system: {
+          metadata: {
+            _reserved: true,
+          },
+        },
+      },
+    }));
+
+    await nativeRealm.setPasswords({
+      'password.kibana': 'bar',
+    });
+
+    expect(mockClient.security.changePassword.mock.calls).toMatchInlineSnapshot(`
+Array [
+  Array [
+    Object {
+      "body": Object {
+        "password": "bar",
+      },
+      "refresh": "wait_for",
+      "username": "kibana",
+    },
+  ],
+  Array [
+    Object {
+      "body": Object {
+        "password": "changeme",
+      },
+      "refresh": "wait_for",
+      "username": "logstash_system",
+    },
+  ],
+  Array [
+    Object {
+      "body": Object {
+        "password": "changeme",
+      },
+      "refresh": "wait_for",
+      "username": "elastic",
+    },
+  ],
+  Array [
+    Object {
+      "body": Object {
+        "password": "changeme",
+      },
+      "refresh": "wait_for",
+      "username": "beats_system",
+    },
+  ],
+]
+`);
+  });
+});
+
+describe('getReservedUsers', () => {
+  it('returns array of reserved usernames', async () => {
+    mockClient.security.getUser.mockImplementation(() => ({
+      body: {
+        kibana: {
+          metadata: {
+            _reserved: true,
+          },
+        },
+        non_native: {
+          metadata: {
+            _reserved: false,
+          },
+        },
+        logstash_system: {
+          metadata: {
+            _reserved: true,
+          },
+        },
+      },
+    }));
+
+    expect(await nativeRealm.getReservedUsers()).toEqual(['kibana', 'logstash_system']);
+  });
+});
+
+describe('setPassword', () => {
+  it('sets password for provided user', async () => {
+    await nativeRealm.setPassword('kibana', 'foo');
+    expect(mockClient.security.changePassword).toHaveBeenCalledWith({
+      body: { password: 'foo' },
+      refresh: 'wait_for',
+      username: 'kibana',
+    });
+  });
+
+  it('logs error', async () => {
+    mockClient.security.changePassword.mockImplementation(() => {
+      throw new Error('SomeError');
+    });
+
+    await nativeRealm.setPassword('kibana', 'foo');
+    expect(log.error.mock.calls).toMatchInlineSnapshot(`
+Array [
+  Array [
+    "unable to set password for kibana: SomeError",
+  ],
+]
+`);
+  });
+});

packages/kbn-test/src/functional_tests/lib/auth.js

@@ -22,7 +22,7 @@ import { format as formatUrl } from 'url';
 import request from 'request';
 import { delay } from 'bluebird';
 
-export const DEFAULT_SUPERUSER_PASS = 'iamsuperuser';
+export const DEFAULT_SUPERUSER_PASS = 'changeme';
 
 async function updateCredentials(port, auth, username, password, retries = 10) {
   const result = await new Promise((resolve, reject) =>

src/cli/serve/serve.js

@@ -75,7 +75,7 @@ function applyConfigOverrides(rawConfig, opts, extraCliOptions) {
     set('optimize.watch', true);
 
     if (!has('elasticsearch.username')) {
-      set('elasticsearch.username', 'elastic');
+      set('elasticsearch.username', 'kibana');
     }
 
     if (!has('elasticsearch.password')) {

x-pack/README.md

@@ -12,6 +12,8 @@ Elasticsearch will run with a basic license. To run with a trial license, includ
 
 Example: `yarn es snapshot --license trial --password changeme`
 
+By default, this will also set the password for native realm accounts to the password provided (`changeme` by default). This includes that of the `kibana` user which `elasticsearch.username` defaults to in development. If you wish to specific a password for a given native realm account, you can do that like so: `--password.kibana=notsecure`
+
 # Testing
 ## Running specific tests
 | Test runner  | Test location                                                                       | Runner command (working directory is kibana/x-pack)                                     |

x-pack/test/functional/config.js

@@ -179,7 +179,7 @@ export default async function ({ readConfigFile }) {
     esTestCluster: {
       license: 'trial',
       from: 'snapshot',
-      serverArgs: ['xpack.license.self_generated.type=trial', 'xpack.security.enabled=true'],
+      serverArgs: [],
     },
 
     kbnTestServer: {

yarn.lock

@@ -1317,6 +1317,18 @@
     lodash "^4.17.11"
     to-fast-properties "^2.0.0"
 
+"@elastic/elasticsearch@^7.0.0-rc.2":
+  version "7.0.0-rc.2"
+  resolved "https://registry.yarnpkg.com/@elastic/elasticsearch/-/elasticsearch-7.0.0-rc.2.tgz#2fb07978d647a257af3976b170e3f61704ba0a18"
+  integrity sha512-NAivETj4KDzNhN/x+nqcnz4K/0wqqT6UicZP0ezCu1oRgia8xHcD6KxrDAiElexD2/z6vY1BkNqYju5Uel14eA==
+  dependencies:
+    debug "^4.1.1"
+    decompress-response "^4.2.0"
+    into-stream "^5.1.0"
+    ms "^2.1.1"
+    once "^1.4.0"
+    pump "^3.0.0"
+
 "@elastic/eui@0.0.23":
   version "0.0.23"
   resolved "https://registry.yarnpkg.com/@elastic/eui/-/eui-0.0.23.tgz#01a3d88aeaff175da5d42b70d407d08a32783f3d"
@@ -9088,6 +9100,13 @@ decompress-response@^3.2.0, decompress-response@^3.3.0:
   dependencies:
     mimic-response "^1.0.0"
 
+decompress-response@^4.2.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/decompress-response/-/decompress-response-4.2.0.tgz#805ca9d1d3cdf17a03951475ad6cdc93115cec3f"
+  integrity sha512-MHebOkORCgLW1ramLri5vzfR4r7HgXXrVkVr/eaPVRCtYWFUp9hNAuqsBxhpABbpqd7zY2IrjxXfTuaVrW0Z2A==
+  dependencies:
+    mimic-response "^2.0.0"
+
 decompress-tar@^4.0.0, decompress-tar@^4.1.0, decompress-tar@^4.1.1:
   version "4.1.1"
   resolved "https://registry.yarnpkg.com/decompress-tar/-/decompress-tar-4.1.1.tgz#718cbd3fcb16209716e70a26b84e7ba4592e5af1"
@@ -11810,7 +11829,7 @@ fresh@0.5.2:
   resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.5.2.tgz#3d8cadd90d976569fa835ab1f8e4b23a105605a7"
   integrity sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=
 
-from2@^2.1.0, from2@^2.1.1:
+from2@^2.1.0, from2@^2.1.1, from2@^2.3.0:
   version "2.3.0"
   resolved "https://registry.yarnpkg.com/from2/-/from2-2.3.0.tgz#8bfb5502bde4a4d36cfdeea007fcca21d7e382af"
   integrity sha1-i/tVAr3kpNNs/e6gB/zKIdfjgq8=
@@ -14329,6 +14348,14 @@ into-stream@^3.1.0:
     from2 "^2.1.1"
     p-is-promise "^1.1.0"
 
+into-stream@^5.1.0:
+  version "5.1.0"
+  resolved "https://registry.yarnpkg.com/into-stream/-/into-stream-5.1.0.tgz#b05f37d8fed05c06a0b43b556d74e53e5af23878"
+  integrity sha512-cbDhb8qlxKMxPBk/QxTtYg1DQ4CwXmadu7quG3B7nrJsgSncEreF2kwWKZFdnjc/lSNNIkFPsjI7SM0Cx/QXPw==
+  dependencies:
+    from2 "^2.3.0"
+    p-is-promise "^2.0.0"
+
 invariant@^2.0.0, invariant@^2.2.1, invariant@^2.2.2:
   version "2.2.2"
   resolved "https://registry.yarnpkg.com/invariant/-/invariant-2.2.2.tgz#9e1f56ac0acdb6bf303306f338be3b204ae60360"
@@ -17741,6 +17768,11 @@ mimic-response@^1.0.0:
   resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-1.0.1.tgz#4923538878eef42063cb8a3e3b0798781487ab1b"
   integrity sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==
 
+mimic-response@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-2.0.0.tgz#996a51c60adf12cb8a87d7fb8ef24c2f3d5ebb46"
+  integrity sha512-8ilDoEapqA4uQ3TwS0jakGONKXVJqpy+RpM+3b7pLdOjghCrEiGp9SRkFbUHAmZW9vdnrENWHjaweIoTIJExSQ==
+
 mimos@4.x.x:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/mimos/-/mimos-4.0.0.tgz#76e3d27128431cb6482fd15b20475719ad626a5a"
@@ -19200,6 +19232,11 @@ p-is-promise@^1.1.0:
   resolved "https://registry.yarnpkg.com/p-is-promise/-/p-is-promise-1.1.0.tgz#9c9456989e9f6588017b0434d56097675c3da05e"
   integrity sha1-nJRWmJ6fZYgBewQ01WCXZ1w9oF4=
 
+p-is-promise@^2.0.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/p-is-promise/-/p-is-promise-2.1.0.tgz#918cebaea248a62cf7ffab8e3bca8c5f882fc42e"
+  integrity sha512-Y3W0wlRPK8ZMRbNq97l4M5otioeA5lm1z7bkNkxCka8HSPjR0xRWmpCmc9utiaLP9Jb1eD8BgeIxTW4AIF45Pg==
+
 p-limit@^1.0.0, p-limit@^1.1.0:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-1.3.0.tgz#b86bd5f0c25690911c7590fcbfc2010d54b3ccb8"