Web - run smoke tests using playwright (#89918)
* playwright - initial version
* browser - use existing page and not create new context
* macOS: document how to remove the security flag
* smoke test - allow to run against server build with --build option
* do not rely on args
* fix path for windows
* smoke test - smoke 💄 and -ci option
This commit is contained in:
Benjamin Pasero
GitHub
parent
16c7551f36
commit
ec41f20c40
|
|
@ -22,18 +22,18 @@
|
|||
"devDependencies": {
|
||||
"@types/mkdirp": "0.5.1",
|
||||
"@types/ncp": "2.0.1",
|
||||
"@types/node": "8.0.33",
|
||||
"@types/puppeteer": "^1.19.0",
|
||||
"@types/debug": "4.1.5",
|
||||
"@types/node": "^12.11.7",
|
||||
"@types/tmp": "0.1.0",
|
||||
"concurrently": "^3.5.1",
|
||||
"cpx": "^1.5.0",
|
||||
"typescript": "2.9.2",
|
||||
"typescript": "3.7.5",
|
||||
"watch": "^1.0.2"
|
||||
},
|
||||
"dependencies": {
|
||||
"mkdirp": "^0.5.1",
|
||||
"ncp": "^2.0.0",
|
||||
"puppeteer": "^1.19.0",
|
||||
"playwright": "0.10.0",
|
||||
"tmp": "0.1.0",
|
||||
"vscode-uri": "^2.0.3"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -126,6 +126,7 @@ export class Application {
|
|||
extraArgs,
|
||||
remote: this.options.remote,
|
||||
web: this.options.web,
|
||||
browser: this.options.browser,
|
||||
headless: this.options.headless
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import * as fs from 'fs';
|
|||
import * as mkdirp from 'mkdirp';
|
||||
import { tmpName } from 'tmp';
|
||||
import { IDriver, connect as connectElectronDriver, IDisposable, IElement, Thenable } from './driver';
|
||||
import { connect as connectPuppeteerDriver, launch } from './puppeteerDriver';
|
||||
import { connect as connectPlaywrightDriver, launch } from './playwrightDriver';
|
||||
import { Logger } from './logger';
|
||||
import { ncp } from 'ncp';
|
||||
import { URI } from 'vscode-uri';
|
||||
|
|
@ -101,6 +101,8 @@ export interface SpawnOptions {
|
|||
remote?: boolean;
|
||||
/** Run in the web */
|
||||
web?: boolean;
|
||||
/** A specific browser to use (requires web: true) */
|
||||
browser?: 'chromium' | 'webkit' | 'firefox';
|
||||
/** Run in headless mode (only applies when web is true) */
|
||||
headless?: boolean;
|
||||
}
|
||||
|
|
@ -120,68 +122,69 @@ export async function spawn(options: SpawnOptions): Promise<Code> {
|
|||
const outPath = codePath ? getBuildOutPath(codePath) : getDevOutPath();
|
||||
const handle = await createDriverHandle();
|
||||
|
||||
const args = [
|
||||
options.workspacePath,
|
||||
'--skip-getting-started',
|
||||
'--skip-release-notes',
|
||||
'--sticky-quickopen',
|
||||
'--disable-telemetry',
|
||||
'--disable-updates',
|
||||
'--disable-crash-reporter',
|
||||
`--extensions-dir=${options.extensionsPath}`,
|
||||
`--user-data-dir=${options.userDataDir}`,
|
||||
'--driver', handle
|
||||
];
|
||||
|
||||
const env = process.env;
|
||||
|
||||
if (options.remote) {
|
||||
// Replace workspace path with URI
|
||||
args[0] = `--${options.workspacePath.endsWith('.code-workspace') ? 'file' : 'folder'}-uri=vscode-remote://test+test/${URI.file(options.workspacePath).path}`;
|
||||
|
||||
if (codePath) {
|
||||
// running against a build: copy the test resolver extension
|
||||
const testResolverExtPath = path.join(options.extensionsPath, 'vscode-test-resolver');
|
||||
if (!fs.existsSync(testResolverExtPath)) {
|
||||
const orig = path.join(repoPath, 'extensions', 'vscode-test-resolver');
|
||||
await new Promise((c, e) => ncp(orig, testResolverExtPath, err => err ? e(err) : c()));
|
||||
}
|
||||
}
|
||||
args.push('--enable-proposed-api=vscode.vscode-test-resolver');
|
||||
const remoteDataDir = `${options.userDataDir}-server`;
|
||||
mkdirp.sync(remoteDataDir);
|
||||
env['TESTRESOLVER_DATA_FOLDER'] = remoteDataDir;
|
||||
}
|
||||
|
||||
if (!codePath) {
|
||||
args.unshift(repoPath);
|
||||
}
|
||||
|
||||
if (options.verbose) {
|
||||
args.push('--driver-verbose');
|
||||
}
|
||||
|
||||
if (options.log) {
|
||||
args.push('--log', options.log);
|
||||
}
|
||||
|
||||
if (options.extraArgs) {
|
||||
args.push(...options.extraArgs);
|
||||
}
|
||||
|
||||
let child: cp.ChildProcess | undefined;
|
||||
let connectDriver: typeof connectElectronDriver;
|
||||
|
||||
if (options.web) {
|
||||
await launch(args);
|
||||
connectDriver = connectPuppeteerDriver.bind(connectPuppeteerDriver, !!options.headless);
|
||||
await launch(options.userDataDir, options.workspacePath, options.codePath);
|
||||
connectDriver = connectPlaywrightDriver.bind(connectPlaywrightDriver, !!options.headless, options.browser);
|
||||
} else {
|
||||
const env = process.env;
|
||||
|
||||
const args = [
|
||||
options.workspacePath,
|
||||
'--skip-getting-started',
|
||||
'--skip-release-notes',
|
||||
'--sticky-quickopen',
|
||||
'--disable-telemetry',
|
||||
'--disable-updates',
|
||||
'--disable-crash-reporter',
|
||||
`--extensions-dir=${options.extensionsPath}`,
|
||||
`--user-data-dir=${options.userDataDir}`,
|
||||
'--driver', handle
|
||||
];
|
||||
|
||||
if (options.remote) {
|
||||
// Replace workspace path with URI
|
||||
args[0] = `--${options.workspacePath.endsWith('.code-workspace') ? 'file' : 'folder'}-uri=vscode-remote://test+test/${URI.file(options.workspacePath).path}`;
|
||||
|
||||
if (codePath) {
|
||||
// running against a build: copy the test resolver extension
|
||||
const testResolverExtPath = path.join(options.extensionsPath, 'vscode-test-resolver');
|
||||
if (!fs.existsSync(testResolverExtPath)) {
|
||||
const orig = path.join(repoPath, 'extensions', 'vscode-test-resolver');
|
||||
await new Promise((c, e) => ncp(orig, testResolverExtPath, err => err ? e(err) : c()));
|
||||
}
|
||||
}
|
||||
args.push('--enable-proposed-api=vscode.vscode-test-resolver');
|
||||
const remoteDataDir = `${options.userDataDir}-server`;
|
||||
mkdirp.sync(remoteDataDir);
|
||||
env['TESTRESOLVER_DATA_FOLDER'] = remoteDataDir;
|
||||
}
|
||||
|
||||
if (!codePath) {
|
||||
args.unshift(repoPath);
|
||||
}
|
||||
|
||||
if (options.verbose) {
|
||||
args.push('--driver-verbose');
|
||||
}
|
||||
|
||||
if (options.log) {
|
||||
args.push('--log', options.log);
|
||||
}
|
||||
|
||||
if (options.extraArgs) {
|
||||
args.push(...options.extraArgs);
|
||||
}
|
||||
|
||||
const spawnOptions: cp.SpawnOptions = { env };
|
||||
child = cp.spawn(electronPath, args, spawnOptions);
|
||||
instances.add(child);
|
||||
child.once('exit', () => instances.delete(child!));
|
||||
connectDriver = connectElectronDriver;
|
||||
}
|
||||
|
||||
return connect(connectDriver, child, outPath, handle, options.logger);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,17 +3,18 @@
|
|||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
import * as puppeteer from 'puppeteer';
|
||||
import * as playwright from 'playwright';
|
||||
import { ChildProcess, spawn } from 'child_process';
|
||||
import { join } from 'path';
|
||||
import { mkdir } from 'fs';
|
||||
import { promisify } from 'util';
|
||||
import { IDriver, IDisposable } from './driver';
|
||||
import { URI } from 'vscode-uri';
|
||||
|
||||
const width = 1200;
|
||||
const height = 800;
|
||||
|
||||
const vscodeToPuppeteerKey: { [key: string]: string } = {
|
||||
const vscodeToPlaywrightKey: { [key: string]: string } = {
|
||||
cmd: 'Meta',
|
||||
ctrl: 'Control',
|
||||
shift: 'Shift',
|
||||
|
|
@ -26,7 +27,7 @@ const vscodeToPuppeteerKey: { [key: string]: string } = {
|
|||
home: 'Home'
|
||||
};
|
||||
|
||||
function buildDriver(browser: puppeteer.Browser, page: puppeteer.Page): IDriver {
|
||||
function buildDriver(browser: playwright.Browser, page: playwright.Page): IDriver {
|
||||
const driver: IDriver = {
|
||||
_serviceBrand: undefined,
|
||||
getWindowIds: () => {
|
||||
|
|
@ -45,8 +46,8 @@ function buildDriver(browser: puppeteer.Browser, page: puppeteer.Page): IDriver
|
|||
const keys = chord.split('+');
|
||||
const keysDown: string[] = [];
|
||||
for (let i = 0; i < keys.length; i++) {
|
||||
if (keys[i] in vscodeToPuppeteerKey) {
|
||||
keys[i] = vscodeToPuppeteerKey[keys[i]];
|
||||
if (keys[i] in vscodeToPlaywrightKey) {
|
||||
keys[i] = vscodeToPlaywrightKey[keys[i]];
|
||||
}
|
||||
await page.keyboard.down(keys[i]);
|
||||
keysDown.push(keys[i]);
|
||||
|
|
@ -68,7 +69,7 @@ function buildDriver(browser: puppeteer.Browser, page: puppeteer.Page): IDriver
|
|||
await driver.click(windowId, selector, 0, 0);
|
||||
await timeout(100);
|
||||
},
|
||||
setValue: async (windowId, selector, text) => page.evaluate(`window.driver.setValue('${selector}', '${text}')`),
|
||||
setValue: async (windowId, selector, text) => page.evaluate(`window.driver.setValue('${selector}', '${text}')`).then(undefined),
|
||||
getTitle: (windowId) => page.evaluate(`window.driver.getTitle()`),
|
||||
isActiveElement: (windowId, selector) => page.evaluate(`window.driver.isActiveElement('${selector}')`),
|
||||
getElements: (windowId, selector, recursive) => page.evaluate(`window.driver.getElements('${selector}', ${recursive})`),
|
||||
|
|
@ -86,31 +87,32 @@ function timeout(ms: number): Promise<void> {
|
|||
|
||||
// function runInDriver(call: string, args: (string | boolean)[]): Promise<any> {}
|
||||
|
||||
let args: string[] | undefined;
|
||||
let server: ChildProcess | undefined;
|
||||
let endpoint: string | undefined;
|
||||
let workspacePath: string | undefined;
|
||||
|
||||
export async function launch(_args: string[]): Promise<void> {
|
||||
args = _args;
|
||||
const agentFolder = args.filter(e => e.includes('--user-data-dir='))[0].replace('--user-data-dir=', '');
|
||||
export async function launch(userDataDir: string, _workspacePath: string, codeServerPath = process.env.VSCODE_REMOTE_SERVER_PATH): Promise<void> {
|
||||
workspacePath = _workspacePath;
|
||||
const agentFolder = userDataDir;
|
||||
await promisify(mkdir)(agentFolder);
|
||||
const env = {
|
||||
VSCODE_AGENT_FOLDER: agentFolder,
|
||||
VSCODE_REMOTE_SERVER_PATH: codeServerPath,
|
||||
...process.env
|
||||
};
|
||||
let serverLocation: string | undefined;
|
||||
if (process.env.VSCODE_REMOTE_SERVER_PATH) {
|
||||
serverLocation = join(process.env.VSCODE_REMOTE_SERVER_PATH, `server.${process.platform === 'win32' ? 'cmd' : 'sh'}`);
|
||||
if (codeServerPath) {
|
||||
serverLocation = join(codeServerPath, `server.${process.platform === 'win32' ? 'cmd' : 'sh'}`);
|
||||
} else {
|
||||
serverLocation = join(args[0], `resources/server/web.${process.platform === 'win32' ? 'bat' : 'sh'}`);
|
||||
serverLocation = join(__dirname, '..', '..', '..', `resources/server/web.${process.platform === 'win32' ? 'bat' : 'sh'}`);
|
||||
}
|
||||
server = spawn(
|
||||
serverLocation,
|
||||
['--browser', 'none', '--driver', 'web'],
|
||||
{ env }
|
||||
);
|
||||
server.stderr.on('data', e => console.log('Server stderr: ' + e));
|
||||
server.stdout.on('data', e => console.log('Server stdout: ' + e));
|
||||
server.stderr?.on('data', e => console.log('Server stderr: ' + e));
|
||||
server.stdout?.on('data', e => console.log('Server stdout: ' + e));
|
||||
process.on('exit', teardown);
|
||||
process.on('SIGINT', teardown);
|
||||
process.on('SIGTERM', teardown);
|
||||
|
|
@ -126,7 +128,7 @@ function teardown(): void {
|
|||
|
||||
function waitForEndpoint(): Promise<string> {
|
||||
return new Promise<string>(r => {
|
||||
server!.stdout.on('data', (d: Buffer) => {
|
||||
server!.stdout?.on('data', (d: Buffer) => {
|
||||
const matches = d.toString('ascii').match(/Web UI available at (.+)/);
|
||||
if (matches !== null) {
|
||||
r(matches[1]);
|
||||
|
|
@ -135,20 +137,18 @@ function waitForEndpoint(): Promise<string> {
|
|||
});
|
||||
}
|
||||
|
||||
export function connect(headless: boolean, outPath: string, handle: string): Promise<{ client: IDisposable, driver: IDriver }> {
|
||||
export function connect(headless: boolean, engine: 'chromium' | 'webkit' | 'firefox' = 'chromium'): Promise<{ client: IDisposable, driver: IDriver }> {
|
||||
return new Promise(async (c) => {
|
||||
const browser = await puppeteer.launch({
|
||||
const browser = await playwright[engine].launch({
|
||||
// Run in Edge dev on macOS
|
||||
// executablePath: '/Applications/Microsoft\ Edge\ Dev.app/Contents/MacOS/Microsoft\ Edge\ Dev',
|
||||
headless,
|
||||
slowMo: 80,
|
||||
args: [`--window-size=${width},${height}`]
|
||||
headless
|
||||
});
|
||||
const page = (await browser.pages())[0];
|
||||
const page = (await browser.defaultContext().pages())[0];
|
||||
await page.setViewport({ width, height });
|
||||
await page.goto(`${endpoint}&folder=vscode-remote://localhost:9888${args![1]}`);
|
||||
await page.goto(`${endpoint}&folder=vscode-remote://localhost:9888${URI.file(workspacePath!).path}`);
|
||||
const result = {
|
||||
client: { dispose: () => teardown },
|
||||
client: { dispose: () => teardown() },
|
||||
driver: buildDriver(browser, page)
|
||||
};
|
||||
c(result);
|
||||
|
|
@ -16,6 +16,6 @@
|
|||
"exclude": [
|
||||
"node_modules",
|
||||
"out",
|
||||
"tools",
|
||||
"tools"
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,6 +2,11 @@
|
|||
# yarn lockfile v1
|
||||
|
||||
|
||||
"@types/debug@4.1.5":
|
||||
version "4.1.5"
|
||||
resolved "https://registry.yarnpkg.com/@types/debug/-/debug-4.1.5.tgz#b14efa8852b7768d898906613c23f688713e02cd"
|
||||
integrity sha512-Q1y515GcOdTHgagaVFhHnIFQ38ygs/kmxdNpvpou+raI9UO3YZcHDngBSYKQklcKlvA7iuQlmIKbzvmxcOE9CQ==
|
||||
|
||||
"@types/mkdirp@0.5.1":
|
||||
version "0.5.1"
|
||||
resolved "https://registry.yarnpkg.com/@types/mkdirp/-/mkdirp-0.5.1.tgz#ea887cd024f691c1ca67cce20b7606b053e43b0f"
|
||||
|
|
@ -21,17 +26,10 @@
|
|||
resolved "https://registry.yarnpkg.com/@types/node/-/node-12.7.1.tgz#3b5c3a26393c19b400844ac422bd0f631a94d69d"
|
||||
integrity sha512-aK9jxMypeSrhiYofWWBf/T7O+KwaiAHzM4sveCdWPn71lzUSMimRnKzhXDKfKwV1kWoBo2P1aGgaIYGLf9/ljw==
|
||||
|
||||
"@types/node@8.0.33":
|
||||
version "8.0.33"
|
||||
resolved "https://registry.yarnpkg.com/@types/node/-/node-8.0.33.tgz#1126e94374014e54478092830704f6ea89df04cd"
|
||||
integrity sha512-vmCdO8Bm1ExT+FWfC9sd9r4jwqM7o97gGy2WBshkkXbf/2nLAJQUrZfIhw27yVOtLUev6kSZc4cav/46KbDd8A==
|
||||
|
||||
"@types/puppeteer@^1.19.0":
|
||||
version "1.19.1"
|
||||
resolved "https://registry.yarnpkg.com/@types/puppeteer/-/puppeteer-1.19.1.tgz#942ca62288953a0f5fbbc25c103b5f2ba28b60ab"
|
||||
integrity sha512-ReWZvoEfMiJIA3AG+eM+nCx5GKrU2ANVYY5TC0nbpeiTCtnJbcqnmBbR8TkXMBTvLBYcuTOAELbTcuX73siDNQ==
|
||||
dependencies:
|
||||
"@types/node" "*"
|
||||
"@types/node@^12.11.7":
|
||||
version "12.12.26"
|
||||
resolved "https://registry.yarnpkg.com/@types/node/-/node-12.12.26.tgz#213e153babac0ed169d44a6d919501e68f59dea9"
|
||||
integrity sha512-UmUm94/QZvU5xLcUlNR8hA7Ac+fGpO1EG/a8bcWVz0P0LqtxFmun9Y2bbtuckwGboWJIT70DoWq1r3hb56n3DA==
|
||||
|
||||
"@types/tmp@0.1.0":
|
||||
version "0.1.0"
|
||||
|
|
@ -729,10 +727,10 @@ hosted-git-info@^2.1.4:
|
|||
resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.4.tgz#44119abaf4bc64692a16ace34700fed9c03e2546"
|
||||
integrity sha512-pzXIvANXEFrc5oFFXRMkbLPQ2rXRoDERwDLyrcUxGhaZhgP54BBSl9Oheh7Vv0T090cszWBxPjkQQ5Sq1PbBRQ==
|
||||
|
||||
https-proxy-agent@^2.2.1:
|
||||
version "2.2.3"
|
||||
resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-2.2.3.tgz#fb6cd98ed5b9c35056b5a73cd01a8a721d7193d1"
|
||||
integrity sha512-Ytgnz23gm2DVftnzqRRz2dOXZbGd2uiajSw/95bPp6v53zPRspQjLm/AfBgqbJ2qfeRXWIOMVLpp86+/5yX39Q==
|
||||
https-proxy-agent@^3.0.0:
|
||||
version "3.0.1"
|
||||
resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-3.0.1.tgz#b8c286433e87602311b01c8ea34413d856a4af81"
|
||||
integrity sha512-+ML2Rbh6DAuee7d07tYGEKOEi2voWPUGan+ExdPbPW6Z3svq+JCqr0v8WmKPOkz1vOVykPCBSuobe7G8GJUtVg==
|
||||
dependencies:
|
||||
agent-base "^4.3.0"
|
||||
debug "^3.1.0"
|
||||
|
|
@ -938,6 +936,11 @@ isobject@^3.0.0, isobject@^3.0.1:
|
|||
resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df"
|
||||
integrity sha1-TkMekrEalzFjaqH5yNHMvP2reN8=
|
||||
|
||||
jpeg-js@^0.3.6:
|
||||
version "0.3.6"
|
||||
resolved "https://registry.yarnpkg.com/jpeg-js/-/jpeg-js-0.3.6.tgz#c40382aac9506e7d1f2d856eb02f6c7b2a98b37c"
|
||||
integrity sha512-MUj2XlMB8kpe+8DJUGH/3UJm4XpI8XEgZQ+CiHDeyrGoKPdW/8FJv6ku+3UiYm5Fz3CWaL+iXmD8Q4Ap6aC1Jw==
|
||||
|
||||
json-parse-better-errors@^1.0.1:
|
||||
version "1.0.2"
|
||||
resolved "https://registry.yarnpkg.com/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz#bb867cfb3450e69107c131d1c514bab3dc8bcaa9"
|
||||
|
|
@ -1320,6 +1323,35 @@ pify@^3.0.0:
|
|||
resolved "https://registry.yarnpkg.com/pify/-/pify-3.0.0.tgz#e5a4acd2c101fdf3d9a4d07f0dbc4db49dd28176"
|
||||
integrity sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=
|
||||
|
||||
playwright-core@=0.10.0:
|
||||
version "0.10.0"
|
||||
resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-0.10.0.tgz#86699c9cc3e613d733e6635a54aceea1993013d5"
|
||||
integrity sha512-yernA6yrrBhmb8M5eO6GZsJOrBKWOZszlu65Luz8LP7ryaDExN1sE9XjQBNbiwJ5Gfs8cehtAO7GfTDJt+Z2cQ==
|
||||
dependencies:
|
||||
debug "^4.1.0"
|
||||
extract-zip "^1.6.6"
|
||||
https-proxy-agent "^3.0.0"
|
||||
jpeg-js "^0.3.6"
|
||||
mime "^2.0.3"
|
||||
pngjs "^3.4.0"
|
||||
progress "^2.0.3"
|
||||
proxy-from-env "^1.0.0"
|
||||
rimraf "^2.6.1"
|
||||
uuid "^3.4.0"
|
||||
ws "^6.1.0"
|
||||
|
||||
playwright@0.10.0:
|
||||
version "0.10.0"
|
||||
resolved "https://registry.yarnpkg.com/playwright/-/playwright-0.10.0.tgz#d37f7e42e0e868dcc4ec35cb0a8dbc6248457642"
|
||||
integrity sha512-f3VRME/PIO5NbcWnlCDfXwPC0DAZJ7ETkcAdE+sensLCOkfDtLh97E71ZuxNCaPYsUA6FIPi5syD8pHJW/4hQQ==
|
||||
dependencies:
|
||||
playwright-core "=0.10.0"
|
||||
|
||||
pngjs@^3.4.0:
|
||||
version "3.4.0"
|
||||
resolved "https://registry.yarnpkg.com/pngjs/-/pngjs-3.4.0.tgz#99ca7d725965fb655814eaf65f38f12bbdbf555f"
|
||||
integrity sha512-NCrCHhWmnQklfH4MtJMRjZ2a8c80qXeMlQMv2uVp9ISJMTt562SbGd6n2oq0PaPgKm7Z6pL9E2UlLIhC+SHL3w==
|
||||
|
||||
posix-character-classes@^0.1.0:
|
||||
version "0.1.1"
|
||||
resolved "https://registry.yarnpkg.com/posix-character-classes/-/posix-character-classes-0.1.1.tgz#01eac0fe3b5af71a2a6c02feabb8c1fef7e00eab"
|
||||
|
|
@ -1335,7 +1367,7 @@ process-nextick-args@~2.0.0:
|
|||
resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2"
|
||||
integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==
|
||||
|
||||
progress@^2.0.1:
|
||||
progress@^2.0.3:
|
||||
version "2.0.3"
|
||||
resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8"
|
||||
integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==
|
||||
|
|
@ -1345,20 +1377,6 @@ proxy-from-env@^1.0.0:
|
|||
resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.0.0.tgz#33c50398f70ea7eb96d21f7b817630a55791c7ee"
|
||||
integrity sha1-M8UDmPcOp+uW0h97gXYwpVeRx+4=
|
||||
|
||||
puppeteer@^1.19.0:
|
||||
version "1.19.0"
|
||||
resolved "https://registry.yarnpkg.com/puppeteer/-/puppeteer-1.19.0.tgz#e3b7b448c2c97933517078d7a2c53687361bebea"
|
||||
integrity sha512-2S6E6ygpoqcECaagDbBopoSOPDv0pAZvTbnBgUY+6hq0/XDFDOLEMNlHF/SKJlzcaZ9ckiKjKDuueWI3FN/WXw==
|
||||
dependencies:
|
||||
debug "^4.1.0"
|
||||
extract-zip "^1.6.6"
|
||||
https-proxy-agent "^2.2.1"
|
||||
mime "^2.0.3"
|
||||
progress "^2.0.1"
|
||||
proxy-from-env "^1.0.0"
|
||||
rimraf "^2.6.1"
|
||||
ws "^6.1.0"
|
||||
|
||||
randomatic@^3.0.0:
|
||||
version "3.1.1"
|
||||
resolved "https://registry.yarnpkg.com/randomatic/-/randomatic-3.1.1.tgz#b776efc59375984e36c537b2f51a1f0aff0da1ed"
|
||||
|
|
@ -1751,10 +1769,10 @@ typedarray@^0.0.6:
|
|||
resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777"
|
||||
integrity sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=
|
||||
|
||||
typescript@2.9.2:
|
||||
version "2.9.2"
|
||||
resolved "https://registry.yarnpkg.com/typescript/-/typescript-2.9.2.tgz#1cbf61d05d6b96269244eb6a3bce4bd914e0f00c"
|
||||
integrity sha512-Gr4p6nFNaoufRIY4NMdpQRNmgxVIGMs4Fcu/ujdYk3nAZqk7supzBE9idmvfZIlH/Cuj//dvi+019qEue9lV0w==
|
||||
typescript@3.7.5:
|
||||
version "3.7.5"
|
||||
resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.7.5.tgz#0692e21f65fd4108b9330238aac11dd2e177a1ae"
|
||||
integrity sha512-/P5lkRXkWHNAbcJIiHPfRoKqyd7bsyCma1hZNUGfn20qm64T6ZBlrzprymeu918H+mB/0rIg2gGK/BXkhhYgBw==
|
||||
|
||||
union-value@^1.0.0:
|
||||
version "1.0.1"
|
||||
|
|
@ -1789,6 +1807,11 @@ util-deprecate@~1.0.1:
|
|||
resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
|
||||
integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=
|
||||
|
||||
uuid@^3.4.0:
|
||||
version "3.4.0"
|
||||
resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee"
|
||||
integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==
|
||||
|
||||
validate-npm-package-license@^3.0.1:
|
||||
version "3.0.4"
|
||||
resolved "https://registry.yarnpkg.com/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz#fc91f6b9c7ba15c857f4cb2c5defeec39d4f410a"
|
||||
|
|
|
|||
|
|
@ -7,15 +7,22 @@ Make sure you are on **Node v10.x**.
|
|||
```bash
|
||||
# Install Dependencies and Compile
|
||||
yarn --cwd test/smoke
|
||||
yarn --cwd test/automation
|
||||
|
||||
# Dev
|
||||
# Dev (Electron)
|
||||
yarn smoketest
|
||||
|
||||
# Build
|
||||
yarn smoketest --build PATH_TO_NEW_BUILD_PARENT_FOLDER --stable-build PATH_TO_LAST_STABLE_BUILD_PARENT_FOLDER
|
||||
# Dev (Web)
|
||||
yarn smoketest --web --browser <chromium|firefox|webkit>
|
||||
|
||||
# Remote
|
||||
yarn smoketest --build PATH_TO_NEW_BUILD_PARENT_FOLDER --remote
|
||||
# Build (Electron)
|
||||
yarn smoketest --build <path latest built version> --stable-build <path to previous stable version>
|
||||
|
||||
# Build (Web - read instructions below)
|
||||
yarn smoketest --build <path to web server folder> --web --browser <chromium|firefox|webkit>
|
||||
|
||||
# Remote (Electron)
|
||||
yarn smoketest --build <path latest built version> --remote
|
||||
```
|
||||
|
||||
### Run for a release
|
||||
|
|
@ -27,18 +34,33 @@ git checkout release/1.22
|
|||
yarn --cwd test/smoke
|
||||
```
|
||||
|
||||
#### Electron
|
||||
|
||||
In addition to the new build to be released you will need the previous stable build so that the smoketest can test the data migration.
|
||||
The recommended way to make these builds available for the smoketest is by downloading their archive version (\*.zip) and extracting
|
||||
them into two folders. Pass the folder paths to the smoketest as follows:
|
||||
|
||||
```bash
|
||||
yarn smoketest --build PATH_TO_NEW_RELEASE_PARENT_FOLDER --stable-build PATH_TO_LAST_STABLE_RELEASE_PARENT_FOLDER
|
||||
yarn smoketest --build <path latest built version> --stable-build <path to previous stable version>
|
||||
```
|
||||
|
||||
#### Web
|
||||
|
||||
**macOS**: if you have downloaded the server with web bits, make sure to run the following command before unzipping it to avoid security issues on startup:
|
||||
|
||||
```bash
|
||||
xattr -d com.apple.quarantine <path to server with web folder zip>
|
||||
```
|
||||
|
||||
There is no support for testing an old version to a new one yet, so simply configure the `--build` command line argument to point to
|
||||
the web server folder which includes the web client bits (e.g. `vscode-server-darwin-web` for macOS).
|
||||
|
||||
**Note**: make sure to point to the server that includes the client bits!
|
||||
|
||||
### Debug
|
||||
|
||||
- `--verbose` logs all the low level driver calls made to Code;
|
||||
- `-f PATTERN` filters the tests to be run. You can also use pretty much any mocha argument;
|
||||
- `-f PATTERN` (alias `-g PATTERN`) filters the tests to be run. You can also use pretty much any mocha argument;
|
||||
- `--screenshots SCREENSHOT_DIR` captures screenshots when tests fail.
|
||||
|
||||
### Develop
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@
|
|||
"rimraf": "^2.6.1",
|
||||
"strip-json-comments": "^2.0.1",
|
||||
"tmp": "0.0.33",
|
||||
"typescript": "2.9.2",
|
||||
"typescript": "3.7.5",
|
||||
"watch": "^1.0.2"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ import { join } from 'path';
|
|||
|
||||
export function setup(stableCodePath: string, testDataPath: string) {
|
||||
|
||||
|
||||
describe('Data Migration: This test MUST run before releasing by providing the --stable-build command line argument', () => {
|
||||
it(`verifies opened editors are restored`, async function () {
|
||||
if (!stableCodePath) {
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ process.once('exit', () => rimraf.sync(testDataPath));
|
|||
const [, , ...args] = process.argv;
|
||||
const opts = minimist(args, {
|
||||
string: [
|
||||
'browser',
|
||||
'build',
|
||||
'stable-build',
|
||||
'wait-time',
|
||||
|
|
@ -58,7 +59,8 @@ const opts = minimist(args, {
|
|||
'verbose',
|
||||
'remote',
|
||||
'web',
|
||||
'headless'
|
||||
'headless',
|
||||
'ci'
|
||||
],
|
||||
default: {
|
||||
verbose: false
|
||||
|
|
@ -82,42 +84,46 @@ function fail(errorMessage): void {
|
|||
|
||||
const repoPath = path.join(__dirname, '..', '..', '..');
|
||||
|
||||
function getDevElectronPath(): string {
|
||||
const buildPath = path.join(repoPath, '.build');
|
||||
const product = require(path.join(repoPath, 'product.json'));
|
||||
|
||||
switch (process.platform) {
|
||||
case 'darwin':
|
||||
return path.join(buildPath, 'electron', `${product.nameLong}.app`, 'Contents', 'MacOS', 'Electron');
|
||||
case 'linux':
|
||||
return path.join(buildPath, 'electron', `${product.applicationName}`);
|
||||
case 'win32':
|
||||
return path.join(buildPath, 'electron', `${product.nameShort}.exe`);
|
||||
default:
|
||||
throw new Error('Unsupported platform.');
|
||||
}
|
||||
}
|
||||
|
||||
function getBuildElectronPath(root: string): string {
|
||||
switch (process.platform) {
|
||||
case 'darwin':
|
||||
return path.join(root, 'Contents', 'MacOS', 'Electron');
|
||||
case 'linux': {
|
||||
const product = require(path.join(root, 'resources', 'app', 'product.json'));
|
||||
return path.join(root, product.applicationName);
|
||||
}
|
||||
case 'win32': {
|
||||
const product = require(path.join(root, 'resources', 'app', 'product.json'));
|
||||
return path.join(root, `${product.nameShort}.exe`);
|
||||
}
|
||||
default:
|
||||
throw new Error('Unsupported platform.');
|
||||
}
|
||||
}
|
||||
|
||||
let quality: Quality;
|
||||
|
||||
//
|
||||
// #### Electron Smoke Tests ####
|
||||
//
|
||||
if (!opts.web) {
|
||||
|
||||
function getDevElectronPath(): string {
|
||||
const buildPath = path.join(repoPath, '.build');
|
||||
const product = require(path.join(repoPath, 'product.json'));
|
||||
|
||||
switch (process.platform) {
|
||||
case 'darwin':
|
||||
return path.join(buildPath, 'electron', `${product.nameLong}.app`, 'Contents', 'MacOS', 'Electron');
|
||||
case 'linux':
|
||||
return path.join(buildPath, 'electron', `${product.applicationName}`);
|
||||
case 'win32':
|
||||
return path.join(buildPath, 'electron', `${product.nameShort}.exe`);
|
||||
default:
|
||||
throw new Error('Unsupported platform.');
|
||||
}
|
||||
}
|
||||
|
||||
function getBuildElectronPath(root: string): string {
|
||||
switch (process.platform) {
|
||||
case 'darwin':
|
||||
return path.join(root, 'Contents', 'MacOS', 'Electron');
|
||||
case 'linux': {
|
||||
const product = require(path.join(root, 'resources', 'app', 'product.json'));
|
||||
return path.join(root, product.applicationName);
|
||||
}
|
||||
case 'win32': {
|
||||
const product = require(path.join(root, 'resources', 'app', 'product.json'));
|
||||
return path.join(root, `${product.nameShort}.exe`);
|
||||
}
|
||||
default:
|
||||
throw new Error('Unsupported platform.');
|
||||
}
|
||||
}
|
||||
|
||||
let testCodePath = opts.build;
|
||||
let stableCodePath = opts['stable-build'];
|
||||
let electronPath: string;
|
||||
|
|
@ -152,8 +158,13 @@ if (!opts.web) {
|
|||
} else {
|
||||
quality = Quality.Stable;
|
||||
}
|
||||
} else {
|
||||
let testCodeServerPath = process.env.VSCODE_REMOTE_SERVER_PATH;
|
||||
}
|
||||
|
||||
//
|
||||
// #### Web Smoke Tests ####
|
||||
//
|
||||
else {
|
||||
const testCodeServerPath = opts.build || process.env.VSCODE_REMOTE_SERVER_PATH;
|
||||
|
||||
if (typeof testCodeServerPath === 'string' && !fs.existsSync(testCodeServerPath)) {
|
||||
fail(`Can't find Code server at ${testCodeServerPath}.`);
|
||||
|
|
@ -236,13 +247,13 @@ function createOptions(): ApplicationOptions {
|
|||
screenshotsPath,
|
||||
remote: opts.remote,
|
||||
web: opts.web,
|
||||
browser: opts.browser,
|
||||
headless: opts.headless
|
||||
};
|
||||
}
|
||||
|
||||
before(async function () {
|
||||
// allow two minutes for setup
|
||||
this.timeout(2 * 60 * 1000);
|
||||
this.timeout(2 * 60 * 1000); // allow two minutes for setup
|
||||
await setup();
|
||||
this.defaultOptions = createOptions();
|
||||
});
|
||||
|
|
@ -259,11 +270,7 @@ after(async function () {
|
|||
await new Promise((c, e) => rimraf(testDataPath, { maxBusyTries: 10 }, err => err ? e(err) : c()));
|
||||
});
|
||||
|
||||
if (!opts.web) {
|
||||
setupDataMigrationTests(opts['stable-build'], testDataPath);
|
||||
}
|
||||
|
||||
describe('Running Code', () => {
|
||||
describe(`VSCode Smoke Tests (${opts.web ? 'Web' : 'Electron'})`, () => {
|
||||
before(async function () {
|
||||
const app = new Application(this.defaultOptions);
|
||||
await app!.start(opts.web ? false : undefined);
|
||||
|
|
@ -295,19 +302,25 @@ describe('Running Code', () => {
|
|||
});
|
||||
}
|
||||
|
||||
if (!opts.web) { setupDataLossTests(); }
|
||||
setupDataExplorerTests();
|
||||
if (!opts.web) { setupDataPreferencesTests(); }
|
||||
setupDataSearchTests();
|
||||
setupDataCSSTests();
|
||||
setupDataEditorTests();
|
||||
setupDataStatusbarTests(!!opts.web);
|
||||
setupDataExtensionTests();
|
||||
setupTerminalTests();
|
||||
if (!opts.web) { setupDataMultirootTests(); }
|
||||
setupDataLocalizationTests();
|
||||
});
|
||||
// CI only tests (must be reliable)
|
||||
if (opts.ci) {
|
||||
// TODO@Ben figure out tests that can run continously and reliably
|
||||
}
|
||||
|
||||
if (!opts.web) {
|
||||
setupLaunchTests();
|
||||
}
|
||||
// Non-CI execution (all tests)
|
||||
else {
|
||||
if (!opts.web) { setupDataMigrationTests(opts['stable-build'], testDataPath); }
|
||||
if (!opts.web) { setupDataLossTests(); }
|
||||
setupDataExplorerTests();
|
||||
if (!opts.web) { setupDataPreferencesTests(); }
|
||||
setupDataSearchTests();
|
||||
setupDataCSSTests();
|
||||
setupDataEditorTests();
|
||||
setupDataStatusbarTests(!!opts.web);
|
||||
setupDataExtensionTests();
|
||||
setupTerminalTests();
|
||||
if (!opts.web) { setupDataMultirootTests(); }
|
||||
setupDataLocalizationTests();
|
||||
if (!opts.web) { setupLaunchTests(); }
|
||||
}
|
||||
});
|
||||
|
|
|
|||
|
|
@ -11,16 +11,14 @@ const suite = 'Smoke Tests';
|
|||
|
||||
const [, , ...args] = process.argv;
|
||||
const opts = minimist(args, {
|
||||
string: [
|
||||
'f'
|
||||
]
|
||||
string: ['f', 'g']
|
||||
});
|
||||
|
||||
const options = {
|
||||
useColors: true,
|
||||
color: true,
|
||||
timeout: 60000,
|
||||
slow: 30000,
|
||||
grep: opts['f']
|
||||
grep: opts['f'] || opts['g']
|
||||
};
|
||||
|
||||
if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) {
|
||||
|
|
|
|||
|
|
@ -2122,10 +2122,10 @@ tree-kill@^1.1.0:
|
|||
resolved "https://registry.yarnpkg.com/tree-kill/-/tree-kill-1.2.0.tgz#5846786237b4239014f05db156b643212d4c6f36"
|
||||
integrity sha512-DlX6dR0lOIRDFxI0mjL9IYg6OTncLm/Zt+JiBhE5OlFcAR8yc9S7FFXU9so0oda47frdM/JFsk7UjNt9vscKcg==
|
||||
|
||||
typescript@2.9.2:
|
||||
version "2.9.2"
|
||||
resolved "https://registry.yarnpkg.com/typescript/-/typescript-2.9.2.tgz#1cbf61d05d6b96269244eb6a3bce4bd914e0f00c"
|
||||
integrity sha512-Gr4p6nFNaoufRIY4NMdpQRNmgxVIGMs4Fcu/ujdYk3nAZqk7supzBE9idmvfZIlH/Cuj//dvi+019qEue9lV0w==
|
||||
typescript@3.7.5:
|
||||
version "3.7.5"
|
||||
resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.7.5.tgz#0692e21f65fd4108b9330238aac11dd2e177a1ae"
|
||||
integrity sha512-/P5lkRXkWHNAbcJIiHPfRoKqyd7bsyCma1hZNUGfn20qm64T6ZBlrzprymeu918H+mB/0rIg2gGK/BXkhhYgBw==
|
||||
|
||||
union-value@^1.0.0:
|
||||
version "1.0.1"
|
||||
|
|
|
|||
Loading…
Reference in a new issue