maxmustermann/TypeScript
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge pull request #3820 from Microsoft/newMochaReporters

Browse source 
New mocha reporter
This commit is contained in:
Dan Quirk 2015-07-13 19:02:25 -07:00
parent c5aa3ad8a3 15a8544bfe
commit 649e40b171
5 changed files with 190 additions and 181 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Jakefile.js
View file

@ -566,7 +566,7 @@ task("runtests", ["tests", builtLocalDirectory], function() {
colors = process.env.colors || process.env.color
colors = colors ? ' --no-colors ' : ' --colors ';
tests = tests ? ' -g ' + tests : '';
reporter = process.env.reporter || process.env.r || 'dot';
reporter = process.env.reporter || process.env.r || 'mocha-fivemat-progress-reporter';
// timeout normally isn't necessary but Travis-CI has been timing out on compiler baselines occasionally
// default timeout is 2sec which really should be enough, but maybe we just need a small amount longer
var cmd = host + " -R " + reporter + tests + colors + ' -t ' + testTimeout + ' ' + run;

1
package.json
View file

@ -34,6 +34,7 @@
"chai": "latest",
"browserify": "latest",
"istanbul": "latest",
"mocha-fivemat-progress-reporter": "latest",
"tslint": "latest"
},
"scripts": {

56
src/harness/compilerRunner.ts
View file

@ -10,6 +10,7 @@ const enum CompilerTestType {
class CompilerBaselineRunner extends RunnerBase {
private basePath = 'tests/cases';
private testSuiteName: string;
private errors: boolean;
private emit: boolean;
private decl: boolean;
@ -24,16 +25,17 @@ class CompilerBaselineRunner extends RunnerBase {
this.decl = true;
this.output = true;
if (testType === CompilerTestType.Conformance) {
this.basePath += '/conformance';
this.testSuiteName = 'conformance';
}
else if (testType === CompilerTestType.Regressions) {
this.basePath += '/compiler';
this.testSuiteName = 'compiler';
}
else if (testType === CompilerTestType.Test262) {
this.basePath += '/test262';
this.testSuiteName = 'test262';
} else {
this.basePath += '/compiler'; // default to this for historical reasons
this.testSuiteName = 'compiler'; // default to this for historical reasons
}
this.basePath += '/' + this.testSuiteName;
}
public checkTestCodeOutput(fileName: string) {
@ -100,7 +102,7 @@ class CompilerBaselineRunner extends RunnerBase {
});
beforeEach(() => {
/* The compiler doesn't handle certain flags flipping during a single compilation setting. Tests on these flags will need
/* The compiler doesn't handle certain flags flipping during a single compilation setting. Tests on these flags will need
a fresh compiler instance for themselves and then create a fresh one for the next test. Would be nice to get dev fixes
eventually to remove this limitation. */
for (var i = 0; i < tcSettings.length; ++i) {
@ -261,19 +263,19 @@ class CompilerBaselineRunner extends RunnerBase {
// NEWTODO: Type baselines
if (result.errors.length === 0) {
// The full walker simulates the types that you would get from doing a full
// The full walker simulates the types that you would get from doing a full
// compile. The pull walker simulates the types you get when you just do
// a type query for a random node (like how the LS would do it). Most of the
// time, these will be the same. However, occasionally, they can be different.
// Specifically, when the compiler internally depends on symbol IDs to order
// things, then we may see different results because symbols can be created in a
// things, then we may see different results because symbols can be created in a
// different order with 'pull' operations, and thus can produce slightly differing
// output.
//
// For example, with a full type check, we may see a type outputed as: number | string
// But with a pull type check, we may see it as: string | number
//
// These types are equivalent, but depend on what order the compiler observed
// These types are equivalent, but depend on what order the compiler observed
// certain parts of the program.
let allFiles = toBeCompiled.concat(otherFiles).filter(file => !!program.getSourceFile(file.unitName));
@ -384,25 +386,27 @@ class CompilerBaselineRunner extends RunnerBase {
}
public initializeTests() {
describe("Setup compiler for compiler baselines", () => {
var harnessCompiler = Harness.Compiler.getCompiler();
this.parseOptions();
});
// this will set up a series of describe/it blocks to run between the setup and cleanup phases
if (this.tests.length === 0) {
var testFiles = this.enumerateFiles(this.basePath, /\.tsx?$/, { recursive: true });
testFiles.forEach(fn => {
fn = fn.replace(/\\/g, "/");
this.checkTestCodeOutput(fn);
describe(this.testSuiteName + ' tests', () => {
describe("Setup compiler for compiler baselines", () => {
var harnessCompiler = Harness.Compiler.getCompiler();
this.parseOptions();
});
}
else {
this.tests.forEach(test => this.checkTestCodeOutput(test));
}
describe("Cleanup after compiler baselines", () => {
var harnessCompiler = Harness.Compiler.getCompiler();
// this will set up a series of describe/it blocks to run between the setup and cleanup phases
if (this.tests.length === 0) {
var testFiles = this.enumerateFiles(this.basePath, /\.tsx?$/, { recursive: true });
testFiles.forEach(fn => {
fn = fn.replace(/\\/g, "/");
this.checkTestCodeOutput(fn);
});
}
else {
this.tests.forEach(test => this.checkTestCodeOutput(test));
}
describe("Cleanup after compiler baselines", () => {
var harnessCompiler = Harness.Compiler.getCompiler();
});
});
}
@ -434,4 +438,4 @@ class CompilerBaselineRunner extends RunnerBase {
}
}
}
}
}

124
src/harness/fourslashRunner.ts
View file

@ -2,7 +2,7 @@
///<reference path='harness.ts'/>
///<reference path='runnerbase.ts' />
const enum FourSlashTestType {
const enum FourSlashTestType {
Native,
Shims,
Server
@ -35,70 +35,72 @@ class FourSlashRunner extends RunnerBase {
this.tests = this.enumerateFiles(this.basePath, /\.ts/i, { recursive: false });
}
this.tests.forEach((fn: string) => {
describe(fn, () => {
fn = ts.normalizeSlashes(fn);
var justName = fn.replace(/^.*[\\\/]/, '');
describe(this.testSuiteName + ' tests', () => {
this.tests.forEach((fn: string) => {
describe(fn, () => {
fn = ts.normalizeSlashes(fn);
var justName = fn.replace(/^.*[\\\/]/, '');
// Convert to relative path
var testIndex = fn.indexOf('tests/');
if (testIndex >= 0) fn = fn.substr(testIndex);
// Convert to relative path
var testIndex = fn.indexOf('tests/');
if (testIndex >= 0) fn = fn.substr(testIndex);
if (justName && !justName.match(/fourslash\.ts$/i) && !justName.match(/\.d\.ts$/i)) {
it(this.testSuiteName + ' test ' + justName + ' runs correctly',() => {
FourSlash.runFourSlashTest(this.basePath, this.testType, fn);
});
}
});
});
if (justName && !justName.match(/fourslash\.ts$/i) && !justName.match(/\.d\.ts$/i)) {
it(this.testSuiteName + ' test ' + justName + ' runs correctly', () => {
FourSlash.runFourSlashTest(this.basePath, this.testType, fn);
});
describe('Generate Tao XML', () => {
var invalidReasons: any = {};
FourSlash.xmlData.forEach(xml => {
if (xml.invalidReason !== null) {
invalidReasons[xml.invalidReason] = (invalidReasons[xml.invalidReason] || 0) + 1;
}
});
var invalidReport: { reason: string; count: number }[] = [];
for (var reason in invalidReasons) {
if (invalidReasons.hasOwnProperty(reason)) {
invalidReport.push({ reason: reason, count: invalidReasons[reason] });
}
}
});
});
invalidReport.sort((lhs, rhs) => lhs.count > rhs.count ? -1 : lhs.count === rhs.count ? 0 : 1);
describe('Generate Tao XML', () => {
var invalidReasons: any = {};
FourSlash.xmlData.forEach(xml => {
if (xml.invalidReason !== null) {
invalidReasons[xml.invalidReason] = (invalidReasons[xml.invalidReason] || 0) + 1;
}
var lines: string[] = [];
lines.push('<!-- Blocked Test Report');
invalidReport.forEach((reasonAndCount) => {
lines.push(reasonAndCount.count + ' tests blocked by ' + reasonAndCount.reason);
});
lines.push('-->');
lines.push('<TaoTest xmlns="http://microsoft.com/schemas/VSLanguages/TAO">');
lines.push(' <InitTest>');
lines.push(' <StartTarget />');
lines.push(' </InitTest>');
lines.push(' <ScenarioList>');
FourSlash.xmlData.forEach(xml => {
if (xml.invalidReason !== null) {
lines.push('<!-- Skipped ' + xml.originalName + ', reason: ' + xml.invalidReason + ' -->');
} else {
lines.push(' <Scenario Name="' + xml.originalName + '">');
xml.actions.forEach(action => {
lines.push(' ' + action);
});
lines.push(' </Scenario>');
}
});
lines.push(' </ScenarioList>');
lines.push(' <CleanupScenario>');
lines.push(' <CloseAllDocuments />');
lines.push(' <CleanupCreatedFiles />');
lines.push(' </CleanupScenario>');
lines.push(' <CleanupTest>');
lines.push(' <CloseTarget />');
lines.push(' </CleanupTest>');
lines.push('</TaoTest>');
Harness.IO.writeFile('built/local/fourslash.xml', lines.join('\r\n'));
});
var invalidReport: { reason: string; count: number }[] = [];
for (var reason in invalidReasons) {
if (invalidReasons.hasOwnProperty(reason)) {
invalidReport.push({ reason: reason, count: invalidReasons[reason] });
}
}
invalidReport.sort((lhs, rhs) => lhs.count > rhs.count ? -1 : lhs.count === rhs.count ? 0 : 1);
var lines: string[] = [];
lines.push('<!-- Blocked Test Report');
invalidReport.forEach((reasonAndCount) => {
lines.push(reasonAndCount.count + ' tests blocked by ' + reasonAndCount.reason);
});
lines.push('-->');
lines.push('<TaoTest xmlns="http://microsoft.com/schemas/VSLanguages/TAO">');
lines.push(' <InitTest>');
lines.push(' <StartTarget />');
lines.push(' </InitTest>');
lines.push(' <ScenarioList>');
FourSlash.xmlData.forEach(xml => {
if (xml.invalidReason !== null) {
lines.push('<!-- Skipped ' + xml.originalName + ', reason: ' + xml.invalidReason + ' -->');
} else {
lines.push(' <Scenario Name="' + xml.originalName + '">');
xml.actions.forEach(action => {
lines.push(' ' + action);
});
lines.push(' </Scenario>');
}
});
lines.push(' </ScenarioList>');
lines.push(' <CleanupScenario>');
lines.push(' <CloseAllDocuments />');
lines.push(' <CleanupCreatedFiles />');
lines.push(' </CleanupScenario>');
lines.push(' <CleanupTest>');
lines.push(' <CloseTarget />');
lines.push(' </CleanupTest>');
lines.push('</TaoTest>');
Harness.IO.writeFile('built/local/fourslash.xml', lines.join('\r\n'));
});
}
}
@ -108,4 +110,4 @@ class GeneratedFourslashRunner extends FourSlashRunner {
super(testType);
this.basePath += '/generated/';
}
}
}

188
src/harness/projectsRunner.ts
View file

@ -89,7 +89,7 @@ class ProjectRunner extends RunnerBase {
}
// When test case output goes to tests/baselines/local/projectOutput/testCaseName/moduleKind/
// We have these two separate locations because when comparing baselines the baseline verifier will delete the existing file
// We have these two separate locations because when comparing baselines the baseline verifier will delete the existing file
// so even if it was created by compiler in that location, the file will be deleted by verified before we can read it
// so lets keep these two locations separate
function getProjectOutputFolder(fileName: string, moduleKind: ts.ModuleKind) {
@ -194,7 +194,7 @@ class ProjectRunner extends RunnerBase {
};
}
}
function batchCompilerProjectTestCase(moduleKind: ts.ModuleKind): BatchCompileProjectTestCaseResult{
var nonSubfolderDiskFiles = 0;
@ -230,7 +230,7 @@ class ProjectRunner extends RunnerBase {
var diskRelativeName = ts.getRelativePathToDirectoryOrUrl(testCase.projectRoot, diskFileName,
getCurrentDirectory(), Harness.Compiler.getCanonicalFileName, /*isAbsolutePathAnUrl*/ false);
if (ts.isRootedDiskPath(diskRelativeName) || diskRelativeName.substr(0, 3) === "../") {
// If the generated output file resides in the parent folder or is rooted path,
// If the generated output file resides in the parent folder or is rooted path,
// we need to instead create files that can live in the project reference folder
// but make sure extension of these files matches with the fileName the compiler asked to write
diskRelativeName = "diskFile" + nonSubfolderDiskFiles++ +
@ -330,108 +330,110 @@ class ProjectRunner extends RunnerBase {
var name = 'Compiling project for ' + testCase.scenario + ': testcase ' + testCaseFileName;
describe(name, () => {
function verifyCompilerResults(moduleKind: ts.ModuleKind) {
function getCompilerResolutionInfo() {
var resolutionInfo: ProjectRunnerTestCaseResolutionInfo = {
scenario: testCase.scenario,
projectRoot: testCase.projectRoot,
inputFiles: testCase.inputFiles,
out: testCase.out,
outDir: testCase.outDir,
sourceMap: testCase.sourceMap,
mapRoot: testCase.mapRoot,
resolveMapRoot: testCase.resolveMapRoot,
sourceRoot: testCase.sourceRoot,
resolveSourceRoot: testCase.resolveSourceRoot,
declaration: testCase.declaration,
baselineCheck: testCase.baselineCheck,
runTest: testCase.runTest,
bug: testCase.bug,
rootDir: testCase.rootDir,
resolvedInputFiles: ts.map(compilerResult.program.getSourceFiles(), inputFile => inputFile.fileName),
emittedFiles: ts.map(compilerResult.outputFiles, outputFile => outputFile.emittedFileName)
};
describe('Projects tests', () => {
describe(name, () => {
function verifyCompilerResults(moduleKind: ts.ModuleKind) {
function getCompilerResolutionInfo() {
var resolutionInfo: ProjectRunnerTestCaseResolutionInfo = {
scenario: testCase.scenario,
projectRoot: testCase.projectRoot,
inputFiles: testCase.inputFiles,
out: testCase.out,
outDir: testCase.outDir,
sourceMap: testCase.sourceMap,
mapRoot: testCase.mapRoot,
resolveMapRoot: testCase.resolveMapRoot,
sourceRoot: testCase.sourceRoot,
resolveSourceRoot: testCase.resolveSourceRoot,
declaration: testCase.declaration,
baselineCheck: testCase.baselineCheck,
runTest: testCase.runTest,
bug: testCase.bug,
rootDir: testCase.rootDir,
resolvedInputFiles: ts.map(compilerResult.program.getSourceFiles(), inputFile => inputFile.fileName),
emittedFiles: ts.map(compilerResult.outputFiles, outputFile => outputFile.emittedFileName)
};
return resolutionInfo;
}
return resolutionInfo;
}
var compilerResult: BatchCompileProjectTestCaseResult;
var compilerResult: BatchCompileProjectTestCaseResult;
it(name + ": " + moduleNameToString(moduleKind) , () => {
// Compile using node
compilerResult = batchCompilerProjectTestCase(moduleKind);
});
it('Resolution information of (' + moduleNameToString(moduleKind) + '): ' + testCaseFileName, () => {
Harness.Baseline.runBaseline('Resolution information of (' + moduleNameToString(compilerResult.moduleKind) + '): ' + testCaseFileName, getBaselineFolder(compilerResult.moduleKind) + testCaseJustName + '.json', () => {
return JSON.stringify(getCompilerResolutionInfo(), undefined, " ");
it(name + ": " + moduleNameToString(moduleKind) , () => {
// Compile using node
compilerResult = batchCompilerProjectTestCase(moduleKind);
});
});
it('Errors for (' + moduleNameToString(moduleKind) + '): ' + testCaseFileName, () => {
if (compilerResult.errors.length) {
Harness.Baseline.runBaseline('Errors for (' + moduleNameToString(compilerResult.moduleKind) + '): ' + testCaseFileName, getBaselineFolder(compilerResult.moduleKind) + testCaseJustName + '.errors.txt', () => {
return getErrorsBaseline(compilerResult);
it('Resolution information of (' + moduleNameToString(moduleKind) + '): ' + testCaseFileName, () => {
Harness.Baseline.runBaseline('Resolution information of (' + moduleNameToString(compilerResult.moduleKind) + '): ' + testCaseFileName, getBaselineFolder(compilerResult.moduleKind) + testCaseJustName + '.json', () => {
return JSON.stringify(getCompilerResolutionInfo(), undefined, " ");
});
}
});
});
it('Baseline of emitted result (' + moduleNameToString(moduleKind) + '): ' + testCaseFileName, () => {
if (testCase.baselineCheck) {
ts.forEach(compilerResult.outputFiles, outputFile => {
Harness.Baseline.runBaseline('Baseline of emitted result (' + moduleNameToString(compilerResult.moduleKind) + '): ' + testCaseFileName, getBaselineFolder(compilerResult.moduleKind) + outputFile.fileName, () => {
try {
return ts.sys.readFile(getProjectOutputFolder(outputFile.fileName, compilerResult.moduleKind));
}
catch (e) {
return undefined;
}
});
});
}
});
it('SourceMapRecord for (' + moduleNameToString(moduleKind) + '): ' + testCaseFileName, () => {
if (compilerResult.sourceMapData) {
Harness.Baseline.runBaseline('SourceMapRecord for (' + moduleNameToString(compilerResult.moduleKind) + '): ' + testCaseFileName, getBaselineFolder(compilerResult.moduleKind) + testCaseJustName + '.sourcemap.txt', () => {
return Harness.SourceMapRecoder.getSourceMapRecord(compilerResult.sourceMapData, compilerResult.program,
ts.filter(compilerResult.outputFiles, outputFile => Harness.Compiler.isJS(outputFile.emittedFileName)));
});
}
});
// Verify that all the generated .d.ts files compile
it('Errors in generated Dts files for (' + moduleNameToString(moduleKind) + '): ' + testCaseFileName, () => {
if (!compilerResult.errors.length && testCase.declaration) {
var dTsCompileResult = compileCompileDTsFiles(compilerResult);
if (dTsCompileResult.errors.length) {
Harness.Baseline.runBaseline('Errors in generated Dts files for (' + moduleNameToString(compilerResult.moduleKind) + '): ' + testCaseFileName, getBaselineFolder(compilerResult.moduleKind) + testCaseJustName + '.dts.errors.txt', () => {
return getErrorsBaseline(dTsCompileResult);
it('Errors for (' + moduleNameToString(moduleKind) + '): ' + testCaseFileName, () => {
if (compilerResult.errors.length) {
Harness.Baseline.runBaseline('Errors for (' + moduleNameToString(compilerResult.moduleKind) + '): ' + testCaseFileName, getBaselineFolder(compilerResult.moduleKind) + testCaseJustName + '.errors.txt', () => {
return getErrorsBaseline(compilerResult);
});
}
}
});
});
it('Baseline of emitted result (' + moduleNameToString(moduleKind) + '): ' + testCaseFileName, () => {
if (testCase.baselineCheck) {
ts.forEach(compilerResult.outputFiles, outputFile => {
Harness.Baseline.runBaseline('Baseline of emitted result (' + moduleNameToString(compilerResult.moduleKind) + '): ' + testCaseFileName, getBaselineFolder(compilerResult.moduleKind) + outputFile.fileName, () => {
try {
return ts.sys.readFile(getProjectOutputFolder(outputFile.fileName, compilerResult.moduleKind));
}
catch (e) {
return undefined;
}
});
});
}
});
it('SourceMapRecord for (' + moduleNameToString(moduleKind) + '): ' + testCaseFileName, () => {
if (compilerResult.sourceMapData) {
Harness.Baseline.runBaseline('SourceMapRecord for (' + moduleNameToString(compilerResult.moduleKind) + '): ' + testCaseFileName, getBaselineFolder(compilerResult.moduleKind) + testCaseJustName + '.sourcemap.txt', () => {
return Harness.SourceMapRecoder.getSourceMapRecord(compilerResult.sourceMapData, compilerResult.program,
ts.filter(compilerResult.outputFiles, outputFile => Harness.Compiler.isJS(outputFile.emittedFileName)));
});
}
});
// Verify that all the generated .d.ts files compile
it('Errors in generated Dts files for (' + moduleNameToString(moduleKind) + '): ' + testCaseFileName, () => {
if (!compilerResult.errors.length && testCase.declaration) {
var dTsCompileResult = compileCompileDTsFiles(compilerResult);
if (dTsCompileResult.errors.length) {
Harness.Baseline.runBaseline('Errors in generated Dts files for (' + moduleNameToString(compilerResult.moduleKind) + '): ' + testCaseFileName, getBaselineFolder(compilerResult.moduleKind) + testCaseJustName + '.dts.errors.txt', () => {
return getErrorsBaseline(dTsCompileResult);
});
}
}
});
after(() => {
compilerResult = undefined;
});
}
verifyCompilerResults(ts.ModuleKind.CommonJS);
verifyCompilerResults(ts.ModuleKind.AMD);
after(() => {
compilerResult = undefined;
// Mocha holds onto the closure environment of the describe callback even after the test is done.
// Therefore we have to clean out large objects after the test is done.
testCase = undefined;
testFileText = undefined;
testCaseJustName = undefined;
});
}
verifyCompilerResults(ts.ModuleKind.CommonJS);
verifyCompilerResults(ts.ModuleKind.AMD);
after(() => {
// Mocha holds onto the closure environment of the describe callback even after the test is done.
// Therefore we have to clean out large objects after the test is done.
testCase = undefined;
testFileText = undefined;
testCaseJustName = undefined;
});
});
}
}
}