maxmustermann/vscode
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Smoke test tweaks (#137809)

Browse source 
* smoke - move data migration tests into one and align

* fix app starting

* `createWorkspaceFile` is not async

* 💄

* support screenshot on failure even for stable app

* smoke - try to remove timeout (#137847)

* improve exit call
This commit is contained in:
Benjamin Pasero 2021-11-25 14:37:22 +01:00 committed by GitHub
parent 2f8fb0b32e
commit 6f2239307b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 159 additions and 128 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
test/automation/src/application.ts
View file

@ -75,10 +75,9 @@ export class Application {
await this.code.waitForElement('.explorer-folders-view');
}
async restart(options: { workspaceOrFolder?: string, extraArgs?: string[] }): Promise<any> {
async restart(options?: { workspaceOrFolder?: string, extraArgs?: string[] }): Promise<any> {
await this.stop();
await new Promise(c => setTimeout(c, 1000));
await this._start(options.workspaceOrFolder, options.extraArgs);
await this._start(options?.workspaceOrFolder, options?.extraArgs);
}
private async _start(workspaceOrFolder = this.workspacePathOrFolder, extraArgs: string[] = []): Promise<any> {
@ -87,15 +86,6 @@ export class Application {
await this.checkWindowReady();
}
async reload(): Promise<any> {
this.code.reload()
.catch(err => null); // ignore the connection drop errors
// needs to be enough to propagate the 'Reload Window' command
await new Promise(c => setTimeout(c, 1500));
await this.checkWindowReady();
}
async stop(): Promise<any> {
if (this._code) {
await this._code.exit();

69
test/automation/src/code.ts
View file

@ -290,34 +290,55 @@ export class Code {
await this.driver.dispatchKeybinding(windowId, keybinding);
}
async reload(): Promise<void> {
const windowId = await this.getActiveWindowId();
await this.driver.reloadWindow(windowId);
}
async exit(): Promise<void> {
const exitPromise = this.driver.exitApplication();
return new Promise<void>((resolve, reject) => {
let done = false;
// If we know the `pid`, use that to await the
// process to terminate (desktop).
const pid = this.pid;
if (typeof pid === 'number') {
await (async () => {
while (true) {
try {
process.kill(pid, 0); // throws an exception if the main process doesn't exist anymore.
await new Promise(c => setTimeout(c, 100));
} catch (error) {
return;
}
// Start the exit flow via driver
const exitPromise = this.driver.exitApplication().then(veto => {
if (veto) {
done = true;
reject(new Error('Smoke test exit call resulted in unexpected veto'));
}
})();
}
});
// Otherwise await the exit promise (web).
else {
await exitPromise;
}
// If we know the `pid` of the smoke tested application
// use that as way to detect the exit of the application
const pid = this.pid;
if (typeof pid === 'number') {
(async () => {
let killCounter = 0;
while (!done) {
killCounter++;
if (killCounter > 40) {
done = true;
reject(new Error('Smoke test exit call did not terminate main process after 20s, giving up'));
}
try {
process.kill(pid, 0); // throws an exception if the main process doesn't exist anymore.
await new Promise(resolve => setTimeout(resolve, 500));
} catch (error) {
done = true;
resolve();
}
}
})();
}
// Otherwise await the exit promise (web).
else {
(async () => {
try {
await exitPromise;
resolve();
} catch (error) {
reject(new Error(`Smoke test exit call resulted in error: ${error}`));
}
})();
}
});
}
async waitForTextContent(selector: string, textContent?: string, accept?: (result: string) => boolean, retryCount?: number): Promise<string> {

4
test/smoke/src/areas/multiroot/multiroot.test.ts
View file

@ -17,7 +17,7 @@ function toUri(path: string): string {
return `${path}`;
}
async function createWorkspaceFile(workspacePath: string): Promise<string> {
function createWorkspaceFile(workspacePath: string): string {
const workspaceFilePath = path.join(path.dirname(workspacePath), 'smoketest.code-workspace');
const workspace = {
folders: [
@ -39,7 +39,7 @@ async function createWorkspaceFile(workspacePath: string): Promise<string> {
export function setup(opts: minimist.ParsedArgs) {
describe('Multiroot', () => {
beforeSuite(opts, async opts => {
const workspacePath = await createWorkspaceFile(opts.workspacePath);
const workspacePath = createWorkspaceFile(opts.workspacePath);
return { ...opts, workspacePath };
});

39
test/smoke/src/areas/workbench/data-loss.test.ts
View file

@ -1,39 +0,0 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import minimist = require('minimist');
import { Application } from '../../../../automation';
import { afterSuite, beforeSuite } from '../../utils';
export function setup(opts: minimist.ParsedArgs) {
describe('Dataloss', () => {
beforeSuite(opts);
afterSuite(opts);
it(`verifies that 'hot exit' works for dirty files`, async function () {
const app = this.app as Application;
await app.workbench.editors.newUntitledFile();
const untitled = 'Untitled-1';
const textToTypeInUntitled = 'Hello from Untitled';
await app.workbench.editor.waitForTypeInEditor(untitled, textToTypeInUntitled);
const readmeMd = 'readme.md';
const textToType = 'Hello, Code';
await app.workbench.quickaccess.openFile(readmeMd);
await app.workbench.editor.waitForTypeInEditor(readmeMd, textToType);
await app.reload();
await app.workbench.editors.waitForActiveTab(readmeMd, true);
await app.workbench.editor.waitForEditorContents(readmeMd, c => c.indexOf(textToType) > -1);
await app.workbench.editors.waitForTab(untitled);
await app.workbench.editors.selectTab(untitled);
await app.workbench.editor.waitForEditorContents(untitled, c => c.indexOf(textToTypeInUntitled) > -1);
});
});
}

70
test/smoke/src/areas/workbench/data-migration.test.ts
View file

@ -6,20 +6,78 @@
import { Application, ApplicationOptions, Quality } from '../../../../automation';
import { join } from 'path';
import { ParsedArgs } from 'minimist';
import { afterSuite, timeout } from '../../utils';
import { afterSuite, startApp } from '../../utils';
export function setup(opts: ParsedArgs, testDataPath: string) {
describe('Datamigration', () => {
describe('Data Migration (insiders -> insiders)', () => {
let app: Application | undefined = undefined;
afterSuite(opts, () => app);
it(`verifies opened editors are restored`, async function () {
app = await startApp(opts, this.defaultOptions);
// Open 3 editors and pin 2 of them
await app.workbench.quickaccess.openFile('www');
await app.workbench.quickaccess.runCommand('View: Keep Editor');
await app.workbench.quickaccess.openFile('app.js');
await app.workbench.quickaccess.runCommand('View: Keep Editor');
await app.workbench.editors.newUntitledFile();
await app.restart();
// Verify 3 editors are open
await app.workbench.editors.selectTab('Untitled-1');
await app.workbench.editors.selectTab('app.js');
await app.workbench.editors.selectTab('www');
await app.stop();
app = undefined;
});
it(`verifies that 'hot exit' works for dirty files`, async function () {
app = await startApp(opts, this.defaultOptions);
await app.workbench.editors.newUntitledFile();
const untitled = 'Untitled-1';
const textToTypeInUntitled = 'Hello from Untitled';
await app.workbench.editor.waitForTypeInEditor(untitled, textToTypeInUntitled);
const readmeMd = 'readme.md';
const textToType = 'Hello, Code';
await app.workbench.quickaccess.openFile(readmeMd);
await app.workbench.editor.waitForTypeInEditor(readmeMd, textToType);
await app.restart();
await app.workbench.editors.waitForTab(readmeMd, true);
await app.workbench.editors.selectTab(readmeMd);
await app.workbench.editor.waitForEditorContents(readmeMd, c => c.indexOf(textToType) > -1);
await app.workbench.editors.waitForTab(untitled, true);
await app.workbench.editors.selectTab(untitled);
await app.workbench.editor.waitForEditorContents(untitled, c => c.indexOf(textToTypeInUntitled) > -1);
await app.stop();
app = undefined;
});
});
describe('Data Migration (stable -> insiders)', () => {
let insidersApp: Application | undefined = undefined;
let stableApp: Application | undefined = undefined;
afterSuite(opts, () => insidersApp, async () => stableApp?.stop());
afterSuite(opts, () => insidersApp ?? stableApp, async () => stableApp?.stop());
it(`verifies opened editors are restored`, async function () {
const stableCodePath = opts['stable-build'];
if (!stableCodePath) {
if (!stableCodePath || opts.remote) {
this.skip();
}
@ -69,7 +127,7 @@ export function setup(opts: ParsedArgs, testDataPath: string) {
it(`verifies that 'hot exit' works for dirty files`, async function () {
const stableCodePath = opts['stable-build'];
if (!stableCodePath) {
if (!stableCodePath || opts.remote) {
this.skip();
}
@ -94,8 +152,6 @@ export function setup(opts: ParsedArgs, testDataPath: string) {
await stableApp.workbench.quickaccess.openFile(readmeMd);
await stableApp.workbench.editor.waitForTypeInEditor(readmeMd, textToType);
await timeout(2000); // give time to store the backup before stopping the app
await stableApp.stop();
stableApp = undefined;

21
test/smoke/src/areas/workbench/launch.test.ts
View file

@ -4,23 +4,24 @@
*--------------------------------------------------------------------------------------------*/
import minimist = require('minimist');
import * as path from 'path';
import { Application, ApplicationOptions } from '../../../../automation';
import { afterSuite } from '../../utils';
import { join } from 'path';
import { Application } from '../../../../automation';
import { afterSuite, startApp } from '../../utils';
export function setup(opts: minimist.ParsedArgs) {
export function setup(args: minimist.ParsedArgs) {
describe('Launch', () => {
let app: Application;
let app: Application | undefined;
afterSuite(opts, () => app);
afterSuite(args, () => app);
it(`verifies that application launches when user data directory has non-ascii characters`, async function () {
const defaultOptions = this.defaultOptions as ApplicationOptions;
const options: ApplicationOptions = { ...defaultOptions, userDataDir: path.join(defaultOptions.userDataDir, 'abcdø') };
app = new Application(options);
await app.start();
const massagedOptions = { ...this.defaultOptions, userDataDir: join(this.defaultOptions.userDataDir, 'ø') };
app = await startApp(args, massagedOptions);
await app.stop();
app = undefined;
});
});
}

20
test/smoke/src/areas/workbench/localization.test.ts
View file

@ -5,20 +5,23 @@
import minimist = require('minimist');
import { Application, Quality } from '../../../../automation';
import { afterSuite, beforeSuite } from '../../utils';
import { afterSuite, startApp } from '../../utils';
export function setup(args: minimist.ParsedArgs) {
export function setup(opts: minimist.ParsedArgs) {
describe('Localization', () => {
beforeSuite(opts);
afterSuite(opts);
let app: Application | undefined = undefined;
afterSuite(args, () => app);
it(`starts with 'DE' locale and verifies title and viewlets text is in German`, async function () {
const app = this.app as Application;
if (app.quality === Quality.Dev || app.remote) {
if (this.defaultOptions.quality === Quality.Dev || this.defaultOptions.remote) {
return this.skip();
}
app = await startApp(args, this.defaultOptions);
await app.workbench.extensions.openExtensionsViewlet();
await app.workbench.extensions.installExtension('ms-ceintl.vscode-language-pack-de', false);
await app.restart({ extraArgs: ['--locale=DE'] });
@ -26,6 +29,9 @@ export function setup(opts: minimist.ParsedArgs) {
const result = await app.workbench.localization.getLocalizedStrings();
const localeInfo = await app.workbench.localization.getLocaleInfo();
await app.stop();
app = undefined;
if (localeInfo.locale === undefined || localeInfo.locale.toLowerCase() !== 'de') {
throw new Error(`The requested locale for VS Code was not German. The received value is: ${localeInfo.locale === undefined ? 'not set' : localeInfo.locale}`);
}

9
test/smoke/src/main.ts
View file

@ -17,7 +17,6 @@ import fetch from 'node-fetch';
import { Quality, ApplicationOptions, MultiLogger, Logger, ConsoleLogger, FileLogger } from '../../automation';
import { setup as setupDataMigrationTests } from './areas/workbench/data-migration.test';
import { setup as setupDataLossTests } from './areas/workbench/data-loss.test';
import { setup as setupPreferencesTests } from './areas/preferences/preferences.test';
import { setup as setupSearchTests } from './areas/search/search.test';
import { setup as setupNotebookTests } from './areas/notebook/notebook.test';
@ -346,14 +345,8 @@ after(async function () {
await new Promise((c, e) => rimraf(testDataPath, { maxBusyTries: 10 }, err => err ? e(err) : c(undefined)));
});
if (!opts.web && opts['build'] && !opts['remote']) {
describe(`Stable vs Insiders Smoke Tests: This test MUST run before releasing`, () => {
setupDataMigrationTests(opts, testDataPath);
});
}
describe(`VSCode Smoke Tests (${opts.web ? 'Web' : 'Electron'})`, () => {
if (!opts.web) { setupDataLossTests(opts); }
if (!opts.web) { setupDataMigrationTests(opts, testDataPath); }
if (!opts.web) { setupPreferencesTests(opts); }
setupSearchTests(opts);
setupNotebookTests(opts);

41
test/smoke/src/utils.ts
View file

@ -19,29 +19,32 @@ export function itRepeat(n: number, description: string, callback: (this: Contex
}
}
export function beforeSuite(opts: minimist.ParsedArgs, optionsTransform?: (opts: ApplicationOptions) => Promise<ApplicationOptions>) {
export function beforeSuite(args: minimist.ParsedArgs, optionsTransform?: (opts: ApplicationOptions) => Promise<ApplicationOptions>) {
before(async function () {
let options: ApplicationOptions = { ...this.defaultOptions };
if (optionsTransform) {
options = await optionsTransform(options);
}
// https://github.com/microsoft/vscode/issues/34988
const userDataPathSuffix = [...Array(8)].map(() => Math.random().toString(36)[3]).join('');
const userDataDir = options.userDataDir.concat(`-${userDataPathSuffix}`);
const app = new Application({ ...options, userDataDir });
await app.start();
this.app = app;
if (opts.log) {
const title = this.currentTest!.fullTitle();
app.logger.log('*** Test start:', title);
}
this.app = await startApp(args, this.defaultOptions, optionsTransform);
});
}
export async function startApp(args: minimist.ParsedArgs, options: ApplicationOptions, optionsTransform?: (opts: ApplicationOptions) => Promise<ApplicationOptions>): Promise<Application> {
if (optionsTransform) {
options = await optionsTransform({ ...options });
}
// https://github.com/microsoft/vscode/issues/34988
const userDataPathSuffix = [...Array(8)].map(() => Math.random().toString(36)[3]).join('');
const userDataDir = options.userDataDir.concat(`-${userDataPathSuffix}`);
const app = new Application({ ...options, userDataDir });
await app.start();
if (args.log) {
const title = this.currentTest!.fullTitle();
app.logger.log('*** Test start:', title);
}
return app;
}
export function afterSuite(opts: minimist.ParsedArgs, appFn?: () => Application | undefined, joinFn?: () => Promise<unknown>) {
after(async function () {
const app: Application = appFn?.() ?? this.app;