maxmustermann/TypeScript
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Add support for sharding tests across multiple workers (#32173)

Browse source 
* Add support for sharding tests across multiple workers

* Disable unittests when runners are expressly provided (unless they contain the unittest runner)
This commit is contained in:
Wesley Wigham 2019-07-01 14:56:57 -07:00 committed by GitHub
parent 055a07ea4a
commit 3e6856137a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 48 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
Gulpfile.js
View file

@ -415,6 +415,8 @@ task("runtests").flags = {
" --no-lint": "Disables lint",
" --timeout=<ms>": "Overrides the default test timeout.",
" --built": "Compile using the built version of the compiler.",
" --shards": "Total number of shards running tests (default: 1)",
" --shardId": "1-based ID of this shard (default: 1)",
}
const runTestsParallel = () => runConsoleTests("built/local/run.js", "min", /*runInParallel*/ true, /*watchMode*/ false);
@ -430,6 +432,8 @@ task("runtests-parallel").flags = {
" --timeout=<ms>": "Overrides the default test timeout.",
" --built": "Compile using the built version of the compiler.",
" --skipPercent=<number>": "Skip expensive tests with <percent> chance to miss an edit. Default 5%.",
" --shards": "Total number of shards running tests (default: 1)",
" --shardId": "1-based ID of this shard (default: 1)",
};
task("diff", () => exec(getDiffTool(), [refBaseline, localBaseline], { ignoreExitCode: true }));

2
scripts/build/options.js
View file

@ -5,7 +5,7 @@ const os = require("os");
/** @type {CommandLineOptions} */
module.exports = minimist(process.argv.slice(2), {
boolean: ["debug", "dirty", "inspect", "light", "colors", "lint", "lkg", "soft", "fix", "failed", "keepFailed", "force", "built"],
string: ["browser", "tests", "host", "reporter", "stackTraceLimit", "timeout"],
string: ["browser", "tests", "host", "reporter", "stackTraceLimit", "timeout", "shards", "shardId"],
alias: {
"b": "browser",
"d": "debug", "debug-brk": "debug",

14
scripts/build/tests.js
View file

@ -36,6 +36,8 @@ async function runConsoleTests(runJs, defaultReporter, runInParallel, watchMode,
const testConfigFile = "test.config";
const failed = cmdLineOptions.failed;
const keepFailed = cmdLineOptions.keepFailed;
const shards = +cmdLineOptions.shards || undefined;
const shardId = +cmdLineOptions.shardId || undefined;
if (!cmdLineOptions.dirty) {
await cleanTestDirs();
cancelToken.throwIfCancellationRequested();
@ -63,8 +65,8 @@ async function runConsoleTests(runJs, defaultReporter, runInParallel, watchMode,
testTimeout = 400000;
}
if (tests || runners || light || testTimeout || taskConfigsFolder || keepFailed || skipPercent !== undefined) {
writeTestConfigFile(tests, runners, light, skipPercent, taskConfigsFolder, workerCount, stackTraceLimit, testTimeout, keepFailed);
if (tests || runners || light || testTimeout || taskConfigsFolder || keepFailed || skipPercent !== undefined || shards || shardId) {
writeTestConfigFile(tests, runners, light, skipPercent, taskConfigsFolder, workerCount, stackTraceLimit, testTimeout, keepFailed, shards, shardId);
}
const colors = cmdLineOptions.colors;
@ -165,8 +167,10 @@ exports.cleanTestDirs = cleanTestDirs;
* @param {string} [stackTraceLimit]
* @param {string | number} [timeout]
* @param {boolean} [keepFailed]
* @param {number | undefined} [shards]
* @param {number | undefined} [shardId]
*/
function writeTestConfigFile(tests, runners, light, skipPercent, taskConfigsFolder, workerCount, stackTraceLimit, timeout, keepFailed) {
function writeTestConfigFile(tests, runners, light, skipPercent, taskConfigsFolder, workerCount, stackTraceLimit, timeout, keepFailed, shards, shardId) {
const testConfigContents = JSON.stringify({
test: tests ? [tests] : undefined,
runners: runners ? runners.split(",") : undefined,
@ -177,7 +181,9 @@ function writeTestConfigFile(tests, runners, light, skipPercent, taskConfigsFold
taskConfigsFolder,
noColor: !cmdLineOptions.colors,
timeout,
keepFailed
keepFailed,
shards,
shardId
});
log.info("Running tests with config: " + testConfigContents);
fs.writeFileSync("test.config", testConfigContents);

2
src/harness/harness.ts
View file

@ -501,7 +501,7 @@ namespace Harness {
}
function enumerateTestFiles(runner: RunnerBase) {
return runner.enumerateTestFiles();
return runner.getTestFiles();
}
function listFiles(path: string, spec: RegExp, options: { recursive?: boolean } = {}) {

11
src/harness/runnerbase.ts
View file

@ -2,6 +2,9 @@ type TestRunnerKind = CompilerTestKind | FourslashTestKind | "project" | "rwc" |
type CompilerTestKind = "conformance" | "compiler";
type FourslashTestKind = "fourslash" | "fourslash-shims" | "fourslash-shims-pp" | "fourslash-server";
let shards = 1;
let shardId = 1;
abstract class RunnerBase {
// contains the tests to run
public tests: (string | Harness.FileBasedTest)[] = [];
@ -19,6 +22,14 @@ abstract class RunnerBase {
abstract enumerateTestFiles(): (string | Harness.FileBasedTest)[];
getTestFiles(): ReturnType<this["enumerateTestFiles"]> {
const all = this.enumerateTestFiles();
if (shards === 1) {
return all as ReturnType<this["enumerateTestFiles"]>;
}
return all.filter((_val, idx) => idx % shards === (shardId - 1)) as ReturnType<this["enumerateTestFiles"]>;
}
/** The working directory where tests are found. Needed for batch testing where the input path will differ from the output path inside baselines */
public workingDirectory = "";

4
src/testRunner/externalCompileRunner.ts
View file

@ -24,7 +24,7 @@ abstract class ExternalCompileRunnerBase extends RunnerBase {
*/
initializeTests(): void {
// Read in and evaluate the test list
const testList = this.tests && this.tests.length ? this.tests : this.enumerateTestFiles();
const testList = this.tests && this.tests.length ? this.tests : this.getTestFiles();
// tslint:disable-next-line:no-this-assignment
const cls = this;
@ -113,7 +113,7 @@ class DockerfileRunner extends ExternalCompileRunnerBase {
}
initializeTests(): void {
// Read in and evaluate the test list
const testList = this.tests && this.tests.length ? this.tests : this.enumerateTestFiles();
const testList = this.tests && this.tests.length ? this.tests : this.getTestFiles();
// tslint:disable-next-line:no-this-assignment
const cls = this;

2
src/testRunner/parallel/host.ts
View file

@ -222,7 +222,7 @@ namespace Harness.Parallel.Host {
console.log("Discovering runner-based tests...");
const discoverStart = +(new Date());
for (const runner of runners) {
for (const test of runner.enumerateTestFiles()) {
for (const test of runner.getTestFiles()) {
const file = typeof test === "string" ? test : test.file;
let size: number;
if (!perfData) {

6
src/testRunner/projectsRunner.ts
View file

@ -32,7 +32,11 @@ namespace project {
export class ProjectRunner extends RunnerBase {
public enumerateTestFiles() {
return this.enumerateFiles("tests/cases/project", /\.json$/, { recursive: true });
const all = this.enumerateFiles("tests/cases/project", /\.json$/, { recursive: true });
if (shards === 1) {
return all;
}
return all.filter((_val, idx) => idx % shards === (shardId - 1));
}
public kind(): TestRunnerKind {

11
src/testRunner/runner.ts
View file

@ -80,6 +80,8 @@ interface TestConfig {
timeout?: number;
keepFailed?: boolean;
skipPercent?: number;
shardId?: number;
shards?: number;
}
interface TaskSet {
@ -114,6 +116,12 @@ function handleTestConfig() {
if (testConfig.skipPercent !== undefined) {
skipPercent = testConfig.skipPercent;
}
if (testConfig.shardId) {
shardId = testConfig.shardId;
}
if (testConfig.shards) {
shards = testConfig.shards;
}
if (testConfig.stackTraceLimit === "full") {
(<any>Error).stackTraceLimit = Infinity;
@ -129,6 +137,9 @@ function handleTestConfig() {
const runnerConfig = testConfig.runners || testConfig.test;
if (runnerConfig && runnerConfig.length > 0) {
if (testConfig.runners) {
runUnitTests = runnerConfig.indexOf("unittest") !== -1;
}
for (const option of runnerConfig) {
if (!option) {
continue;

2
src/testRunner/rwcRunner.ts
View file

@ -225,7 +225,7 @@ class RWCRunner extends RunnerBase {
*/
public initializeTests(): void {
// Read in and evaluate the test list
for (const test of this.tests && this.tests.length ? this.tests : this.enumerateTestFiles()) {
for (const test of this.tests && this.tests.length ? this.tests : this.getTestFiles()) {
this.runTest(typeof test === "string" ? test : test.file);
}
}

2
src/testRunner/test262Runner.ts
View file

@ -97,7 +97,7 @@ class Test262BaselineRunner extends RunnerBase {
public initializeTests() {
// this will set up a series of describe/it blocks to run between the setup and cleanup phases
if (this.tests.length === 0) {
const testFiles = this.enumerateTestFiles();
const testFiles = this.getTestFiles();
testFiles.forEach(fn => {
this.runTest(fn);
});