[7.x] [a11y] add initial accessibility functional tests (#4358… (#50086)
* [a11y] add initial accessibility functional tests * add accessibility jobs * fix config path * remove percy setup from scripts * disable color-contrast rule * apply changes from @myasonik * define aria-controls/owns props even when suggestions aren't visible * [ftr/a11y] only throw error when there are errors * adding tests for management page * add a11y test for management page * adding ignore rules' to a11y * accessibility test for kibana home * 7 passing tests, 0 failures * jest snapshot update * support a11y test in pipeline job * update a11y test script for pipelines * use oss compatible ci setup * fix exclude syntax * add default exclusion syntax
This commit is contained in:
parent
1d3c0fd46c
commit
9a6724c3ed
|
@ -14,6 +14,7 @@ JOB:
|
|||
- kibana-ciGroup10
|
||||
- kibana-ciGroup11
|
||||
- kibana-ciGroup12
|
||||
- kibana-accessibility
|
||||
- kibana-visualRegression
|
||||
|
||||
# make sure all x-pack-ciGroups are listed in test/scripts/jenkins_xpack_ci_group.sh
|
||||
|
@ -28,6 +29,7 @@ JOB:
|
|||
- x-pack-ciGroup8
|
||||
- x-pack-ciGroup9
|
||||
- x-pack-ciGroup10
|
||||
- x-pack-accessibility
|
||||
- x-pack-visualRegression
|
||||
|
||||
# `~` is yaml for `null`
|
||||
|
|
|
@ -21,6 +21,9 @@ kibana-ciGroup*)
|
|||
kibana-visualRegression*)
|
||||
./test/scripts/jenkins_visual_regression.sh
|
||||
;;
|
||||
kibana-accessibility*)
|
||||
./test/scripts/jenkins_accessibility.sh
|
||||
;;
|
||||
kibana-firefoxSmoke*)
|
||||
./test/scripts/jenkins_firefox_smoke.sh
|
||||
;;
|
||||
|
@ -34,6 +37,9 @@ x-pack-ciGroup*)
|
|||
x-pack-visualRegression*)
|
||||
./test/scripts/jenkins_xpack_visual_regression.sh
|
||||
;;
|
||||
x-pack-accessibility*)
|
||||
./test/scripts/jenkins_xpack_accessibility.sh
|
||||
;;
|
||||
x-pack-firefoxSmoke*)
|
||||
./test/scripts/jenkins_xpack_firefox_smoke.sh
|
||||
;;
|
||||
|
|
|
@ -394,6 +394,7 @@ module.exports = {
|
|||
'x-pack/test/functional/apps/**/*.js',
|
||||
'x-pack/legacy/plugins/apm/**/*.js',
|
||||
'test/*/config.ts',
|
||||
'test/*/{tests,test_suites,apis,apps}/**/*',
|
||||
'test/visual_regression/tests/**/*',
|
||||
'x-pack/test/*/{tests,test_suites,apis,apps}/**/*',
|
||||
'x-pack/test/*/*config.*ts',
|
||||
|
|
2
Jenkinsfile
vendored
2
Jenkinsfile
vendored
|
@ -25,6 +25,7 @@ stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a
|
|||
'oss-ciGroup11': kibanaPipeline.getOssCiGroupWorker(11),
|
||||
'oss-ciGroup12': kibanaPipeline.getOssCiGroupWorker(12),
|
||||
'oss-firefoxSmoke': kibanaPipeline.getPostBuildWorker('firefoxSmoke', { runbld('./test/scripts/jenkins_firefox_smoke.sh', 'Execute kibana-firefoxSmoke') }),
|
||||
'oss-accessibility': kibanaPipeline.getPostBuildWorker('accessibility', { runbld('./test/scripts/jenkins_accessibility.sh', 'Execute accessibility tests') }),
|
||||
'oss-visualRegression': kibanaPipeline.getPostBuildWorker('visualRegression', { runbld('./test/scripts/jenkins_visual_regression.sh', 'Execute kibana-visualRegression') }),
|
||||
]),
|
||||
'kibana-xpack-agent': kibanaPipeline.withWorkers('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [
|
||||
|
@ -39,6 +40,7 @@ stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a
|
|||
'xpack-ciGroup9': kibanaPipeline.getXpackCiGroupWorker(9),
|
||||
'xpack-ciGroup10': kibanaPipeline.getXpackCiGroupWorker(10),
|
||||
'xpack-firefoxSmoke': kibanaPipeline.getPostBuildWorker('xpack-firefoxSmoke', { runbld('./test/scripts/jenkins_xpack_firefox_smoke.sh', 'Execute xpack-firefoxSmoke') }),
|
||||
'xpack-accessibility': kibanaPipeline.getPostBuildWorker('xpack-accessibility', { runbld('./test/scripts/jenkins_xpack_accessibility.sh', 'Execute xpack-accessibility tests') }),
|
||||
'xpack-visualRegression': kibanaPipeline.getPostBuildWorker('xpack-visualRegression', { runbld('./test/scripts/jenkins_xpack_visual_regression.sh', 'Execute xpack-visualRegression') }),
|
||||
]),
|
||||
])
|
||||
|
|
|
@ -347,6 +347,7 @@
|
|||
"@typescript-eslint/parser": "^2.8.0",
|
||||
"angular-mocks": "^1.7.8",
|
||||
"archiver": "^3.1.1",
|
||||
"axe-core": "^3.3.2",
|
||||
"babel-eslint": "^10.0.3",
|
||||
"babel-jest": "^24.9.0",
|
||||
"babel-plugin-dynamic-import-node": "^2.2.0",
|
||||
|
|
|
@ -463,7 +463,7 @@ exports[`Field for boolean setting should render as read only if saving is disab
|
|||
display="row"
|
||||
error={null}
|
||||
fullWidth={false}
|
||||
hasChildLabel={true}
|
||||
hasChildLabel={false}
|
||||
hasEmptyLabelSpace={false}
|
||||
helpText={null}
|
||||
isInvalid={false}
|
||||
|
@ -557,7 +557,7 @@ exports[`Field for boolean setting should render as read only with help text if
|
|||
display="row"
|
||||
error={null}
|
||||
fullWidth={false}
|
||||
hasChildLabel={true}
|
||||
hasChildLabel={false}
|
||||
hasEmptyLabelSpace={false}
|
||||
helpText={
|
||||
<EuiText
|
||||
|
@ -650,7 +650,7 @@ exports[`Field for boolean setting should render custom setting icon if it is cu
|
|||
display="row"
|
||||
error={null}
|
||||
fullWidth={false}
|
||||
hasChildLabel={true}
|
||||
hasChildLabel={false}
|
||||
hasEmptyLabelSpace={false}
|
||||
helpText={null}
|
||||
isInvalid={false}
|
||||
|
@ -722,7 +722,7 @@ exports[`Field for boolean setting should render default value if there is no us
|
|||
display="row"
|
||||
error={null}
|
||||
fullWidth={false}
|
||||
hasChildLabel={true}
|
||||
hasChildLabel={false}
|
||||
hasEmptyLabelSpace={false}
|
||||
helpText={null}
|
||||
isInvalid={false}
|
||||
|
@ -816,7 +816,7 @@ exports[`Field for boolean setting should render user value if there is user val
|
|||
display="row"
|
||||
error={null}
|
||||
fullWidth={false}
|
||||
hasChildLabel={true}
|
||||
hasChildLabel={false}
|
||||
hasEmptyLabelSpace={false}
|
||||
helpText={
|
||||
<span>
|
||||
|
|
|
@ -195,7 +195,7 @@ export class Field extends PureComponent {
|
|||
this.setState({
|
||||
unsavedValue: newUnsavedValue,
|
||||
isInvalid,
|
||||
error
|
||||
error,
|
||||
});
|
||||
};
|
||||
|
||||
|
@ -764,6 +764,7 @@ export class Field extends PureComponent {
|
|||
helpText={this.renderHelpText(setting)}
|
||||
describedByIds={[`${setting.name}-aria`]}
|
||||
className="mgtAdvancedSettings__fieldRow"
|
||||
hasChildLabel={setting.type !== 'boolean'}
|
||||
>
|
||||
{this.renderField(setting)}
|
||||
</EuiFormRow>
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
<div class="app-wrapper-panel">
|
||||
<div class="app-wrapper-panel" data-test-subj="appA11yRoot">
|
||||
<div id="globalBannerList"></div>
|
||||
|
||||
<div
|
||||
|
|
46
test/accessibility/apps/discover.ts
Normal file
46
test/accessibility/apps/discover.ts
Normal file
|
@ -0,0 +1,46 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch B.V. under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch B.V. licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
import { FtrProviderContext } from '../ftr_provider_context';
|
||||
|
||||
const FROM_TIME = '2015-09-19 06:31:44.000';
|
||||
const TO_TIME = '2015-09-23 18:31:44.000';
|
||||
|
||||
export default function({ getService, getPageObjects }: FtrProviderContext) {
|
||||
const PageObjects = getPageObjects(['common', 'timePicker']);
|
||||
const a11y = getService('a11y');
|
||||
const esArchiver = getService('esArchiver');
|
||||
const kibanaServer = getService('kibanaServer');
|
||||
|
||||
describe('Discover', () => {
|
||||
before(async () => {
|
||||
await esArchiver.load('discover');
|
||||
await esArchiver.loadIfNeeded('logstash_functional');
|
||||
await kibanaServer.uiSettings.update({
|
||||
defaultIndex: 'logstash-*',
|
||||
});
|
||||
await PageObjects.common.navigateToApp('discover');
|
||||
await PageObjects.timePicker.setAbsoluteRange(FROM_TIME, TO_TIME);
|
||||
});
|
||||
|
||||
it('main view', async () => {
|
||||
await a11y.testAppSnapshot();
|
||||
});
|
||||
});
|
||||
}
|
35
test/accessibility/apps/home.ts
Normal file
35
test/accessibility/apps/home.ts
Normal file
|
@ -0,0 +1,35 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch B.V. under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch B.V. licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
import { FtrProviderContext } from '../ftr_provider_context';
|
||||
|
||||
export default function({ getService, getPageObjects }: FtrProviderContext) {
|
||||
const PageObjects = getPageObjects(['common']);
|
||||
const a11y = getService('a11y');
|
||||
|
||||
describe('Kibana Home', () => {
|
||||
before(async () => {
|
||||
await PageObjects.common.navigateToApp('home');
|
||||
});
|
||||
|
||||
it('Kibana Home view', async () => {
|
||||
await a11y.testAppSnapshot();
|
||||
});
|
||||
});
|
||||
}
|
55
test/accessibility/apps/management.ts
Normal file
55
test/accessibility/apps/management.ts
Normal file
|
@ -0,0 +1,55 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch B.V. under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch B.V. licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
import { FtrProviderContext } from '../ftr_provider_context';
|
||||
|
||||
export default function({ getService, getPageObjects }: FtrProviderContext) {
|
||||
const PageObjects = getPageObjects(['common', 'settings']);
|
||||
const a11y = getService('a11y');
|
||||
|
||||
describe('Management', () => {
|
||||
before(async () => {
|
||||
await PageObjects.common.navigateToApp('settings');
|
||||
});
|
||||
|
||||
it('main view', async () => {
|
||||
await a11y.testAppSnapshot();
|
||||
});
|
||||
|
||||
it('index pattern page', async () => {
|
||||
await PageObjects.settings.clickKibanaIndexPatterns();
|
||||
await a11y.testAppSnapshot();
|
||||
});
|
||||
|
||||
it('Single indexpattern view', async () => {
|
||||
await PageObjects.settings.clickIndexPatternLogstash();
|
||||
await a11y.testAppSnapshot();
|
||||
});
|
||||
|
||||
it('Saved objects view', async () => {
|
||||
await PageObjects.settings.clickKibanaSavedObjects();
|
||||
await a11y.testAppSnapshot();
|
||||
});
|
||||
|
||||
it('Advanced settings', async () => {
|
||||
await PageObjects.settings.clickKibanaSettings();
|
||||
await a11y.testAppSnapshot();
|
||||
});
|
||||
});
|
||||
}
|
42
test/accessibility/config.ts
Normal file
42
test/accessibility/config.ts
Normal file
|
@ -0,0 +1,42 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch B.V. under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch B.V. licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
import { FtrConfigProviderContext } from '@kbn/test/types/ftr';
|
||||
import { services } from './services';
|
||||
import { pageObjects } from './page_objects';
|
||||
|
||||
export default async function({ readConfigFile }: FtrConfigProviderContext) {
|
||||
const functionalConfig = await readConfigFile(require.resolve('../functional/config'));
|
||||
|
||||
return {
|
||||
...functionalConfig.getAll(),
|
||||
|
||||
testFiles: [
|
||||
require.resolve('./apps/discover'),
|
||||
require.resolve('./apps/management'),
|
||||
require.resolve('./apps/home'),
|
||||
],
|
||||
pageObjects,
|
||||
services,
|
||||
|
||||
junit: {
|
||||
reportName: 'Accessibility Tests',
|
||||
},
|
||||
};
|
||||
}
|
25
test/accessibility/ftr_provider_context.d.ts
vendored
Normal file
25
test/accessibility/ftr_provider_context.d.ts
vendored
Normal file
|
@ -0,0 +1,25 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch B.V. under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch B.V. licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
import { GenericFtrProviderContext } from '@kbn/test/types/ftr';
|
||||
|
||||
import { pageObjects } from './page_objects';
|
||||
import { services } from './services';
|
||||
|
||||
export type FtrProviderContext = GenericFtrProviderContext<typeof services, typeof pageObjects>;
|
20
test/accessibility/page_objects.ts
Normal file
20
test/accessibility/page_objects.ts
Normal file
|
@ -0,0 +1,20 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch B.V. under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch B.V. licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
export { pageObjects } from '../functional/page_objects';
|
130
test/accessibility/services/a11y/a11y.ts
Normal file
130
test/accessibility/services/a11y/a11y.ts
Normal file
|
@ -0,0 +1,130 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch B.V. under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch B.V. licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
import chalk from 'chalk';
|
||||
import testSubjectToCss from '@kbn/test-subj-selector';
|
||||
|
||||
import { FtrProviderContext } from '../../ftr_provider_context';
|
||||
import { AxeReport, printResult } from './axe_report';
|
||||
// @ts-ignore JS that is run in browser as is
|
||||
import { analyzeWithAxe, analyzeWithAxeWithClient } from './analyze_with_axe';
|
||||
|
||||
interface AxeContext {
|
||||
include?: string[];
|
||||
exclude?: string[][];
|
||||
}
|
||||
|
||||
interface TestOptions {
|
||||
excludeTestSubj?: string | string[];
|
||||
}
|
||||
|
||||
export const normalizeResult = (report: any) => {
|
||||
if (report.error) {
|
||||
throw report.error;
|
||||
}
|
||||
|
||||
return report.result as false | AxeReport;
|
||||
};
|
||||
|
||||
export function A11yProvider({ getService }: FtrProviderContext) {
|
||||
const browser = getService('browser');
|
||||
const Wd = getService('__webdriver__');
|
||||
const log = getService('log');
|
||||
|
||||
/**
|
||||
* Accessibility testing service using the Axe (https://www.deque.com/axe/)
|
||||
* toolset to validate a11y rules similar to ESLint. In order to test against
|
||||
* the rules we must load up the UI and feed a full HTML snapshot into Axe.
|
||||
*/
|
||||
return new (class Accessibility {
|
||||
public async testAppSnapshot(options: TestOptions = {}) {
|
||||
const context = this.getAxeContext(true, options.excludeTestSubj);
|
||||
const report = await this.captureAxeReport(context);
|
||||
await this.testAxeReport(report);
|
||||
}
|
||||
|
||||
public async testGlobalSnapshot(options: TestOptions = {}) {
|
||||
const context = this.getAxeContext(false, options.excludeTestSubj);
|
||||
const report = await this.captureAxeReport(context);
|
||||
await this.testAxeReport(report);
|
||||
}
|
||||
|
||||
private getAxeContext(global: boolean, excludeTestSubj?: string | string[]): AxeContext {
|
||||
return {
|
||||
include: global ? undefined : [testSubjectToCss('appA11yRoot')],
|
||||
exclude: ([] as string[])
|
||||
.concat(excludeTestSubj || [])
|
||||
.map(ts => [testSubjectToCss(ts)])
|
||||
.concat([['.ace_scrollbar']]),
|
||||
};
|
||||
}
|
||||
|
||||
private testAxeReport(report: AxeReport) {
|
||||
const errorMsgs = [];
|
||||
|
||||
for (const result of report.incomplete) {
|
||||
// these items require human review and can't be definitively validated
|
||||
log.warning(printResult(chalk.yellow('UNABLE TO VALIDATE'), result));
|
||||
}
|
||||
|
||||
for (const result of report.violations) {
|
||||
errorMsgs.push(printResult(chalk.red('VIOLATION'), result));
|
||||
}
|
||||
|
||||
if (errorMsgs.length) {
|
||||
throw new Error(`a11y report:\n${errorMsgs.join('\n')}`);
|
||||
}
|
||||
}
|
||||
|
||||
private async captureAxeReport(context: AxeContext): Promise<AxeReport> {
|
||||
const axeOptions = {
|
||||
reporter: 'v2',
|
||||
runOnly: ['wcag2a', 'wcag2aa'],
|
||||
rules: {
|
||||
'color-contrast': {
|
||||
enabled: false,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
await (Wd.driver.manage() as any).setTimeouts({
|
||||
...(await (Wd.driver.manage() as any).getTimeouts()),
|
||||
script: 600000,
|
||||
});
|
||||
|
||||
const report = normalizeResult(
|
||||
await browser.executeAsync(analyzeWithAxe, context, axeOptions)
|
||||
);
|
||||
|
||||
if (report !== false) {
|
||||
return report;
|
||||
}
|
||||
|
||||
const withClientReport = normalizeResult(
|
||||
await browser.executeAsync(analyzeWithAxeWithClient, context, axeOptions)
|
||||
);
|
||||
|
||||
if (withClientReport === false) {
|
||||
throw new Error('Attempted to analyze with axe but failed to load axe client');
|
||||
}
|
||||
|
||||
return withClientReport;
|
||||
}
|
||||
})();
|
||||
}
|
39
test/accessibility/services/a11y/analyze_with_axe.js
Normal file
39
test/accessibility/services/a11y/analyze_with_axe.js
Normal file
|
@ -0,0 +1,39 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch B.V. under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch B.V. licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
import { readFileSync } from 'fs';
|
||||
|
||||
export function analyzeWithAxe(context, options, callback) {
|
||||
Promise.resolve()
|
||||
.then(() => {
|
||||
if (window.axe) {
|
||||
return window.axe.run(context, options);
|
||||
}
|
||||
|
||||
// return a false report to trigger analyzeWithAxeWithClient
|
||||
return false;
|
||||
})
|
||||
.then(result => callback({ result }), error => callback({ error }));
|
||||
}
|
||||
|
||||
export const analyzeWithAxeWithClient = `
|
||||
${readFileSync(require.resolve('axe-core/axe.js'), 'utf8')}
|
||||
|
||||
return (${analyzeWithAxe.toString()}).apply(null, arguments);
|
||||
`;
|
69
test/accessibility/services/a11y/axe_report.ts
Normal file
69
test/accessibility/services/a11y/axe_report.ts
Normal file
|
@ -0,0 +1,69 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch B.V. under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch B.V. licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
type AxeImpact = 'minor' | 'moderate' | 'serious' | 'critical';
|
||||
|
||||
type AxeRelatedNodes = Array<{
|
||||
data: any;
|
||||
id: string;
|
||||
impact: AxeImpact;
|
||||
message: string;
|
||||
relatedNodes: [];
|
||||
}>;
|
||||
|
||||
export interface AxeResult {
|
||||
/* Rule description */
|
||||
description: string;
|
||||
/* rule title/error message */
|
||||
help: string;
|
||||
/* documentation url */
|
||||
helpUrl: string;
|
||||
/* rule id */
|
||||
id: string;
|
||||
/* severity level */
|
||||
impact?: AxeImpact;
|
||||
/* tags used to group rules */
|
||||
tags: string[];
|
||||
/* nodes grouped in this result */
|
||||
nodes: Array<{
|
||||
all: AxeRelatedNodes;
|
||||
any: AxeRelatedNodes;
|
||||
none: AxeRelatedNodes;
|
||||
|
||||
html: string;
|
||||
impact: AxeImpact;
|
||||
target: string[];
|
||||
}>;
|
||||
}
|
||||
|
||||
export type AxeResultGroup = AxeResult[];
|
||||
|
||||
export interface AxeReport {
|
||||
inapplicable: AxeResultGroup;
|
||||
passes: AxeResultGroup;
|
||||
incomplete: AxeResultGroup;
|
||||
violations: AxeResultGroup;
|
||||
}
|
||||
|
||||
export const printResult = (title: string, result: AxeResult) => `
|
||||
${title}
|
||||
[${result.id}]: ${result.description}
|
||||
Help: ${result.helpUrl}
|
||||
Elements:
|
||||
- ${result.nodes.map(node => node.target).join('\n - ')}`;
|
20
test/accessibility/services/a11y/index.ts
Normal file
20
test/accessibility/services/a11y/index.ts
Normal file
|
@ -0,0 +1,20 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch B.V. under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch B.V. licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
export { A11yProvider } from './a11y';
|
26
test/accessibility/services/index.ts
Normal file
26
test/accessibility/services/index.ts
Normal file
|
@ -0,0 +1,26 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch B.V. under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch B.V. licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
import { services as kibanaFunctionalServices } from '../../functional/services';
|
||||
import { A11yProvider } from './a11y';
|
||||
|
||||
export const services = {
|
||||
...kibanaFunctionalServices,
|
||||
a11y: A11yProvider,
|
||||
};
|
|
@ -470,10 +470,7 @@ export async function BrowserProvider({ getService }: FtrProviderContext) {
|
|||
);
|
||||
}
|
||||
|
||||
public async executeAsync<A extends any[], R>(
|
||||
fn: string | ((...args: A) => R),
|
||||
...args: A
|
||||
): Promise<R> {
|
||||
public async executeAsync<R>(fn: string | ((...args: any[]) => R), ...args: any[]): Promise<R> {
|
||||
return await driver.executeAsyncScript(
|
||||
fn,
|
||||
...cloneDeep<any>(args, arg => {
|
||||
|
|
26
test/scripts/jenkins_accessibility.sh
Executable file
26
test/scripts/jenkins_accessibility.sh
Executable file
|
@ -0,0 +1,26 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
if [[ -n "$IS_PIPELINE_JOB" ]] ; then
|
||||
source src/dev/ci_setup/setup_env.sh
|
||||
fi
|
||||
|
||||
export TEST_BROWSER_HEADLESS=1
|
||||
|
||||
if [[ -z "$IS_PIPELINE_JOB" ]] ; then
|
||||
yarn run grunt functionalTests:ensureAllTestsInCiGroup;
|
||||
node scripts/build --debug --oss;
|
||||
else
|
||||
installDir="$(realpath $PARENT_DIR/kibana/build/oss/kibana-*-SNAPSHOT-linux-x86_64)"
|
||||
destDir=${installDir}-${CI_WORKER_NUMBER}
|
||||
cp -R "$installDir" "$destDir"
|
||||
|
||||
export KIBANA_INSTALL_DIR="$destDir"
|
||||
fi
|
||||
|
||||
checks-reporter-with-killswitch "Kibana accessibility tests" \
|
||||
node scripts/functional_tests \
|
||||
--debug --bail \
|
||||
--kibana-install-dir "$installDir" \
|
||||
--config test/accessibility/config.ts;
|
35
test/scripts/jenkins_xpack_accessibility.sh
Executable file
35
test/scripts/jenkins_xpack_accessibility.sh
Executable file
|
@ -0,0 +1,35 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
if [[ -n "$IS_PIPELINE_JOB" ]] ; then
|
||||
source src/dev/ci_setup/setup_env.sh
|
||||
fi
|
||||
|
||||
if [[ -z "$IS_PIPELINE_JOB" ]] ; then
|
||||
echo " -> building and extracting default Kibana distributable for use in functional tests"
|
||||
node scripts/build --debug --no-oss
|
||||
|
||||
linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')"
|
||||
installDir="$PARENT_DIR/install/kibana"
|
||||
|
||||
mkdir -p "$installDir"
|
||||
tar -xzf "$linuxBuild" -C "$installDir" --strip=1
|
||||
|
||||
export KIBANA_INSTALL_DIR="$installDir"
|
||||
else
|
||||
installDir="$PARENT_DIR/install/kibana"
|
||||
destDir="${installDir}-${CI_WORKER_NUMBER}"
|
||||
cp -R "$installDir" "$destDir"
|
||||
|
||||
export KIBANA_INSTALL_DIR="$destDir"
|
||||
fi
|
||||
|
||||
export TEST_BROWSER_HEADLESS=1
|
||||
cd "$XPACK_DIR"
|
||||
|
||||
checks-reporter-with-killswitch "X-Pack accessibility tests" \
|
||||
node scripts/functional_tests \
|
||||
--debug --bail \
|
||||
--kibana-install-dir "$installDir" \
|
||||
--config test/accessibility/config.ts;
|
43
x-pack/test/accessibility/apps/login_page.ts
Normal file
43
x-pack/test/accessibility/apps/login_page.ts
Normal file
|
@ -0,0 +1,43 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { FtrProviderContext } from '../ftr_provider_context';
|
||||
|
||||
export default function({ getService, getPageObjects }: FtrProviderContext) {
|
||||
const esArchiver = getService('esArchiver');
|
||||
const a11y = getService('a11y');
|
||||
const testSubjects = getService('testSubjects');
|
||||
const retry = getService('retry');
|
||||
const PageObjects = getPageObjects(['common', 'security']);
|
||||
|
||||
describe('Security', () => {
|
||||
describe('Login Page', () => {
|
||||
before(async () => {
|
||||
await esArchiver.load('empty_kibana');
|
||||
await PageObjects.security.logout();
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
await esArchiver.unload('empty_kibana');
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await PageObjects.security.logout();
|
||||
});
|
||||
|
||||
it('meets a11y requirements', async () => {
|
||||
await PageObjects.common.navigateToApp('login');
|
||||
|
||||
await retry.waitFor(
|
||||
'login page visible',
|
||||
async () => await testSubjects.exists('loginSubmit')
|
||||
);
|
||||
|
||||
await a11y.testAppSnapshot();
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
26
x-pack/test/accessibility/config.ts
Normal file
26
x-pack/test/accessibility/config.ts
Normal file
|
@ -0,0 +1,26 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { FtrConfigProviderContext } from '@kbn/test/types/ftr';
|
||||
import { services } from './services';
|
||||
import { pageObjects } from './page_objects';
|
||||
|
||||
export default async function({ readConfigFile }: FtrConfigProviderContext) {
|
||||
const functionalConfig = await readConfigFile(require.resolve('../functional/config'));
|
||||
|
||||
return {
|
||||
...functionalConfig.getAll(),
|
||||
|
||||
testFiles: [require.resolve('./apps/login_page')],
|
||||
|
||||
pageObjects,
|
||||
services,
|
||||
|
||||
junit: {
|
||||
reportName: 'X-Pack Accessibility Tests',
|
||||
},
|
||||
};
|
||||
}
|
12
x-pack/test/accessibility/ftr_provider_context.d.ts
vendored
Normal file
12
x-pack/test/accessibility/ftr_provider_context.d.ts
vendored
Normal file
|
@ -0,0 +1,12 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { GenericFtrProviderContext } from '@kbn/test/types/ftr';
|
||||
|
||||
import { pageObjects } from './page_objects';
|
||||
import { services } from './services';
|
||||
|
||||
export type FtrProviderContext = GenericFtrProviderContext<typeof services, typeof pageObjects>;
|
7
x-pack/test/accessibility/page_objects.ts
Normal file
7
x-pack/test/accessibility/page_objects.ts
Normal file
|
@ -0,0 +1,7 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
export { pageObjects } from '../functional/page_objects';
|
13
x-pack/test/accessibility/services.ts
Normal file
13
x-pack/test/accessibility/services.ts
Normal file
|
@ -0,0 +1,13 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { services as kibanaA11yServices } from '../../../test/accessibility/services';
|
||||
import { services as functionalServices } from '../functional/services';
|
||||
|
||||
export const services = {
|
||||
...kibanaA11yServices,
|
||||
...functionalServices,
|
||||
};
|
|
@ -5802,6 +5802,11 @@ aws4@^1.6.0:
|
|||
resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.6.0.tgz#83ef5ca860b2b32e4a0deedee8c771b9db57471e"
|
||||
integrity sha1-g+9cqGCysy5KDe7e6MdxudtXRx4=
|
||||
|
||||
axe-core@^3.3.2:
|
||||
version "3.3.2"
|
||||
resolved "https://registry.yarnpkg.com/axe-core/-/axe-core-3.3.2.tgz#7baf3c55a5cf1621534a2c38735f5a1bf2f7e1a8"
|
||||
integrity sha512-lRdxsRt7yNhqpcXQk1ao1BL73OZDzmFCWOG0mC4tGR/r14ohH2payjHwCMQjHGbBKm924eDlmG7utAGHiX/A6g==
|
||||
|
||||
axios@^0.18.0, axios@^0.18.1:
|
||||
version "0.18.1"
|
||||
resolved "https://registry.yarnpkg.com/axios/-/axios-0.18.1.tgz#ff3f0de2e7b5d180e757ad98000f1081b87bcea3"
|
||||
|
|
Loading…
Reference in a new issue