Produce logs during test runs (integration, smoke) and publish as artefacts (#126065)
* first cut upload log files from integration test runs * always publish log files * move logs into integration-tests folder * also produce logs for smoke tests * store remote log files * wire in logspath arg properly * always publish logs * enable logs for browser based smoke tests * log only in verbose
This commit is contained in:
parent
71be3b3641
commit
2ddc1e4fc5
|
@ -253,6 +253,14 @@ steps:
|
||||||
continueOnError: true
|
continueOnError: true
|
||||||
condition: failed()
|
condition: failed()
|
||||||
|
|
||||||
|
- task: PublishPipelineArtifact@0
|
||||||
|
inputs:
|
||||||
|
artifactName: logs-macos-$(VSCODE_ARCH)
|
||||||
|
targetPath: .build/logs
|
||||||
|
displayName: "Publish Log Files"
|
||||||
|
continueOnError: true
|
||||||
|
condition: succeededOrFailed()
|
||||||
|
|
||||||
- task: PublishTestResults@2
|
- task: PublishTestResults@2
|
||||||
displayName: Publish Tests Results
|
displayName: Publish Tests Results
|
||||||
inputs:
|
inputs:
|
||||||
|
|
|
@ -205,12 +205,20 @@ steps:
|
||||||
|
|
||||||
- task: PublishPipelineArtifact@0
|
- task: PublishPipelineArtifact@0
|
||||||
inputs:
|
inputs:
|
||||||
artifactName: "crash-dump-linux-$(VSCODE_ARCH)"
|
artifactName: crash-dump-linux-$(VSCODE_ARCH)
|
||||||
targetPath: .build/crashes
|
targetPath: .build/crashes
|
||||||
displayName: "Publish Crash Reports"
|
displayName: "Publish Crash Reports"
|
||||||
continueOnError: true
|
continueOnError: true
|
||||||
condition: failed()
|
condition: failed()
|
||||||
|
|
||||||
|
- task: PublishPipelineArtifact@0
|
||||||
|
inputs:
|
||||||
|
artifactName: logs-linux-$(VSCODE_ARCH)
|
||||||
|
targetPath: .build/logs
|
||||||
|
displayName: "Publish Log Files"
|
||||||
|
continueOnError: true
|
||||||
|
condition: succeededOrFailed()
|
||||||
|
|
||||||
- task: PublishTestResults@2
|
- task: PublishTestResults@2
|
||||||
displayName: Publish Tests Results
|
displayName: Publish Tests Results
|
||||||
inputs:
|
inputs:
|
||||||
|
|
|
@ -195,6 +195,14 @@ steps:
|
||||||
continueOnError: true
|
continueOnError: true
|
||||||
condition: failed()
|
condition: failed()
|
||||||
|
|
||||||
|
- task: PublishPipelineArtifact@0
|
||||||
|
inputs:
|
||||||
|
artifactName: logs-windows-$(VSCODE_ARCH)
|
||||||
|
targetPath: .build\logs
|
||||||
|
displayName: "Publish Log Files"
|
||||||
|
continueOnError: true
|
||||||
|
condition: succeededOrFailed()
|
||||||
|
|
||||||
- task: PublishTestResults@2
|
- task: PublishTestResults@2
|
||||||
displayName: Publish Tests Results
|
displayName: Publish Tests Results
|
||||||
inputs:
|
inputs:
|
||||||
|
|
|
@ -5,6 +5,7 @@ pushd %~dp0\..
|
||||||
|
|
||||||
set VSCODEUSERDATADIR=%TEMP%\vscodeuserfolder-%RANDOM%-%TIME:~6,2%
|
set VSCODEUSERDATADIR=%TEMP%\vscodeuserfolder-%RANDOM%-%TIME:~6,2%
|
||||||
set VSCODECRASHDIR=%~dp0\..\.build\crashes
|
set VSCODECRASHDIR=%~dp0\..\.build\crashes
|
||||||
|
set VSCODELOGSDIR=%~dp0\..\.build\logs\integration-tests
|
||||||
|
|
||||||
:: Figure out which Electron to use for running tests
|
:: Figure out which Electron to use for running tests
|
||||||
if "%INTEGRATION_TEST_ELECTRON_PATH%"=="" (
|
if "%INTEGRATION_TEST_ELECTRON_PATH%"=="" (
|
||||||
|
@ -14,6 +15,7 @@ if "%INTEGRATION_TEST_ELECTRON_PATH%"=="" (
|
||||||
set VSCODE_BUILD_BUILTIN_EXTENSIONS_SILENCE_PLEASE=1
|
set VSCODE_BUILD_BUILTIN_EXTENSIONS_SILENCE_PLEASE=1
|
||||||
|
|
||||||
echo Storing crash reports into '%VSCODECRASHDIR%'.
|
echo Storing crash reports into '%VSCODECRASHDIR%'.
|
||||||
|
echo Storing log files into '%VSCODELOGSDIR%'.
|
||||||
echo Running integration tests out of sources.
|
echo Running integration tests out of sources.
|
||||||
) else (
|
) else (
|
||||||
:: Run from a built: need to compile all test extensions
|
:: Run from a built: need to compile all test extensions
|
||||||
|
@ -37,6 +39,7 @@ if "%INTEGRATION_TEST_ELECTRON_PATH%"=="" (
|
||||||
set ELECTRON_ENABLE_LOGGING=1
|
set ELECTRON_ENABLE_LOGGING=1
|
||||||
|
|
||||||
echo Storing crash reports into '%VSCODECRASHDIR%'.
|
echo Storing crash reports into '%VSCODECRASHDIR%'.
|
||||||
|
echo Storing log files into '%VSCODELOGSDIR%'.
|
||||||
echo Running integration tests with '%INTEGRATION_TEST_ELECTRON_PATH%' as build.
|
echo Running integration tests with '%INTEGRATION_TEST_ELECTRON_PATH%' as build.
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -46,7 +49,7 @@ if %errorlevel% neq 0 exit /b %errorlevel%
|
||||||
|
|
||||||
:: Tests in the extension host
|
:: Tests in the extension host
|
||||||
|
|
||||||
set ALL_PLATFORMS_API_TESTS_EXTRA_ARGS=--disable-telemetry --skip-welcome --crash-reporter-directory=%VSCODECRASHDIR% --no-cached-data --disable-updates --disable-keytar --disable-extensions --disable-workspace-trust --user-data-dir=%VSCODEUSERDATADIR%
|
set ALL_PLATFORMS_API_TESTS_EXTRA_ARGS=--disable-telemetry --skip-welcome --crash-reporter-directory=%VSCODECRASHDIR% --logsPath=%VSCODELOGSDIR% --no-cached-data --disable-updates --disable-keytar --disable-extensions --disable-workspace-trust --user-data-dir=%VSCODEUSERDATADIR%
|
||||||
|
|
||||||
call "%INTEGRATION_TEST_ELECTRON_PATH%" %~dp0\..\extensions\vscode-api-tests\testWorkspace --enable-proposed-api=vscode.vscode-api-tests --extensionDevelopmentPath=%~dp0\..\extensions\vscode-api-tests --extensionTestsPath=%~dp0\..\extensions\vscode-api-tests\out\singlefolder-tests %ALL_PLATFORMS_API_TESTS_EXTRA_ARGS%
|
call "%INTEGRATION_TEST_ELECTRON_PATH%" %~dp0\..\extensions\vscode-api-tests\testWorkspace --enable-proposed-api=vscode.vscode-api-tests --extensionDevelopmentPath=%~dp0\..\extensions\vscode-api-tests --extensionTestsPath=%~dp0\..\extensions\vscode-api-tests\out\singlefolder-tests %ALL_PLATFORMS_API_TESTS_EXTRA_ARGS%
|
||||||
if %errorlevel% neq 0 exit /b %errorlevel%
|
if %errorlevel% neq 0 exit /b %errorlevel%
|
||||||
|
|
|
@ -14,6 +14,7 @@ fi
|
||||||
|
|
||||||
VSCODEUSERDATADIR=`mktemp -d 2>/dev/null`
|
VSCODEUSERDATADIR=`mktemp -d 2>/dev/null`
|
||||||
VSCODECRASHDIR=$ROOT/.build/crashes
|
VSCODECRASHDIR=$ROOT/.build/crashes
|
||||||
|
VSCODELOGSDIR=$ROOT/.build/logs/integration-tests
|
||||||
cd $ROOT
|
cd $ROOT
|
||||||
|
|
||||||
# Figure out which Electron to use for running tests
|
# Figure out which Electron to use for running tests
|
||||||
|
@ -23,6 +24,7 @@ then
|
||||||
INTEGRATION_TEST_ELECTRON_PATH="./scripts/code.sh"
|
INTEGRATION_TEST_ELECTRON_PATH="./scripts/code.sh"
|
||||||
|
|
||||||
echo "Storing crash reports into '$VSCODECRASHDIR'."
|
echo "Storing crash reports into '$VSCODECRASHDIR'."
|
||||||
|
echo "Storing log files into '$VSCODELOGSDIR'."
|
||||||
echo "Running integration tests out of sources."
|
echo "Running integration tests out of sources."
|
||||||
else
|
else
|
||||||
# Run from a built: need to compile all test extensions
|
# Run from a built: need to compile all test extensions
|
||||||
|
@ -47,6 +49,7 @@ else
|
||||||
export ELECTRON_ENABLE_LOGGING=1
|
export ELECTRON_ENABLE_LOGGING=1
|
||||||
|
|
||||||
echo "Storing crash reports into '$VSCODECRASHDIR'."
|
echo "Storing crash reports into '$VSCODECRASHDIR'."
|
||||||
|
echo "Storing log files into '$VSCODELOGSDIR'."
|
||||||
echo "Running integration tests with '$INTEGRATION_TEST_ELECTRON_PATH' as build."
|
echo "Running integration tests with '$INTEGRATION_TEST_ELECTRON_PATH' as build."
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
@ -62,7 +65,7 @@ after_suite
|
||||||
|
|
||||||
# Tests in the extension host
|
# Tests in the extension host
|
||||||
|
|
||||||
ALL_PLATFORMS_API_TESTS_EXTRA_ARGS="--disable-telemetry --skip-welcome --crash-reporter-directory=$VSCODECRASHDIR --no-cached-data --disable-updates --disable-keytar --disable-extensions --disable-workspace-trust --user-data-dir=$VSCODEUSERDATADIR"
|
ALL_PLATFORMS_API_TESTS_EXTRA_ARGS="--disable-telemetry --skip-welcome --crash-reporter-directory=$VSCODECRASHDIR --logsPath=$VSCODELOGSDIR --no-cached-data --disable-updates --disable-keytar --disable-extensions --disable-workspace-trust --user-data-dir=$VSCODEUSERDATADIR"
|
||||||
|
|
||||||
"$INTEGRATION_TEST_ELECTRON_PATH" $LINUX_EXTRA_ARGS $ROOT/extensions/vscode-api-tests/testWorkspace --enable-proposed-api=vscode.vscode-api-tests --extensionDevelopmentPath=$ROOT/extensions/vscode-api-tests --extensionTestsPath=$ROOT/extensions/vscode-api-tests/out/singlefolder-tests $ALL_PLATFORMS_API_TESTS_EXTRA_ARGS
|
"$INTEGRATION_TEST_ELECTRON_PATH" $LINUX_EXTRA_ARGS $ROOT/extensions/vscode-api-tests/testWorkspace --enable-proposed-api=vscode.vscode-api-tests --extensionDevelopmentPath=$ROOT/extensions/vscode-api-tests --extensionTestsPath=$ROOT/extensions/vscode-api-tests/out/singlefolder-tests $ALL_PLATFORMS_API_TESTS_EXTRA_ARGS
|
||||||
after_suite
|
after_suite
|
||||||
|
|
|
@ -123,7 +123,7 @@ export async function spawn(options: SpawnOptions): Promise<Code> {
|
||||||
copyExtension(options.extensionsPath, 'vscode-notebook-tests');
|
copyExtension(options.extensionsPath, 'vscode-notebook-tests');
|
||||||
|
|
||||||
if (options.web) {
|
if (options.web) {
|
||||||
await launch(options.userDataDir, options.workspacePath, options.codePath, options.extensionsPath);
|
await launch(options.userDataDir, options.workspacePath, options.codePath, options.extensionsPath, Boolean(options.verbose));
|
||||||
connectDriver = connectPlaywrightDriver.bind(connectPlaywrightDriver, options.browser);
|
connectDriver = connectPlaywrightDriver.bind(connectPlaywrightDriver, options.browser);
|
||||||
return connect(connectDriver, child, '', handle, options.logger);
|
return connect(connectDriver, child, '', handle, options.logger);
|
||||||
}
|
}
|
||||||
|
@ -144,6 +144,7 @@ export async function spawn(options: SpawnOptions): Promise<Code> {
|
||||||
'--disable-workspace-trust',
|
'--disable-workspace-trust',
|
||||||
`--extensions-dir=${options.extensionsPath}`,
|
`--extensions-dir=${options.extensionsPath}`,
|
||||||
`--user-data-dir=${options.userDataDir}`,
|
`--user-data-dir=${options.userDataDir}`,
|
||||||
|
`--logsPath=${path.join(repoPath, '.build', 'logs', 'smoke-tests')}`,
|
||||||
'--driver', handle
|
'--driver', handle
|
||||||
];
|
];
|
||||||
|
|
||||||
|
|
|
@ -94,7 +94,7 @@ let server: ChildProcess | undefined;
|
||||||
let endpoint: string | undefined;
|
let endpoint: string | undefined;
|
||||||
let workspacePath: string | undefined;
|
let workspacePath: string | undefined;
|
||||||
|
|
||||||
export async function launch(userDataDir: string, _workspacePath: string, codeServerPath = process.env.VSCODE_REMOTE_SERVER_PATH, extPath: string): Promise<void> {
|
export async function launch(userDataDir: string, _workspacePath: string, codeServerPath = process.env.VSCODE_REMOTE_SERVER_PATH, extPath: string, verbose: boolean): Promise<void> {
|
||||||
workspacePath = _workspacePath;
|
workspacePath = _workspacePath;
|
||||||
|
|
||||||
const agentFolder = userDataDir;
|
const agentFolder = userDataDir;
|
||||||
|
@ -104,25 +104,42 @@ export async function launch(userDataDir: string, _workspacePath: string, codeSe
|
||||||
VSCODE_REMOTE_SERVER_PATH: codeServerPath,
|
VSCODE_REMOTE_SERVER_PATH: codeServerPath,
|
||||||
...process.env
|
...process.env
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const root = join(__dirname, '..', '..', '..');
|
||||||
|
const logsPath = join(root, '.build', 'logs', 'smoke-tests-browser');
|
||||||
|
|
||||||
const args = ['--browser', 'none', '--driver', 'web', '--extensions-dir', extPath];
|
const args = ['--browser', 'none', '--driver', 'web', '--extensions-dir', extPath];
|
||||||
|
|
||||||
let serverLocation: string | undefined;
|
let serverLocation: string | undefined;
|
||||||
if (codeServerPath) {
|
if (codeServerPath) {
|
||||||
serverLocation = join(codeServerPath, `server.${process.platform === 'win32' ? 'cmd' : 'sh'}`);
|
serverLocation = join(codeServerPath, `server.${process.platform === 'win32' ? 'cmd' : 'sh'}`);
|
||||||
|
args.push(`--logsPath=${logsPath}`);
|
||||||
|
|
||||||
console.log(`Starting built server from '${serverLocation}'`);
|
console.log(`Starting built server from '${serverLocation}'`);
|
||||||
|
console.log(`Storing log files into '${logsPath}'`);
|
||||||
} else {
|
} else {
|
||||||
serverLocation = join(__dirname, '..', '..', '..', `resources/server/web.${process.platform === 'win32' ? 'bat' : 'sh'}`);
|
serverLocation = join(root, `resources/server/web.${process.platform === 'win32' ? 'bat' : 'sh'}`);
|
||||||
|
args.push('--logsPath', logsPath);
|
||||||
|
|
||||||
console.log(`Starting server out of sources from '${serverLocation}'`);
|
console.log(`Starting server out of sources from '${serverLocation}'`);
|
||||||
|
console.log(`Storing log files into '${logsPath}'`);
|
||||||
}
|
}
|
||||||
|
|
||||||
server = spawn(
|
server = spawn(
|
||||||
serverLocation,
|
serverLocation,
|
||||||
args,
|
args,
|
||||||
{ env }
|
{ env }
|
||||||
);
|
);
|
||||||
server.stderr?.on('data', error => console.log(`Server stderr: ${error}`));
|
|
||||||
server.stdout?.on('data', data => console.log(`Server stdout: ${data}`));
|
if (verbose) {
|
||||||
|
server.stderr?.on('data', error => console.log(`Server stderr: ${error}`));
|
||||||
|
server.stdout?.on('data', data => console.log(`Server stdout: ${data}`));
|
||||||
|
}
|
||||||
|
|
||||||
process.on('exit', teardown);
|
process.on('exit', teardown);
|
||||||
process.on('SIGINT', teardown);
|
process.on('SIGINT', teardown);
|
||||||
process.on('SIGTERM', teardown);
|
process.on('SIGTERM', teardown);
|
||||||
|
|
||||||
endpoint = await waitForEndpoint();
|
endpoint = await waitForEndpoint();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -92,27 +92,35 @@ async function launchServer(browserType: BrowserType): Promise<{ endpoint: url.U
|
||||||
...process.env
|
...process.env
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const root = path.join(__dirname, '..', '..', '..', '..');
|
||||||
|
const logsPath = path.join(root, '.build', 'logs', 'integration-tests-browser');
|
||||||
|
|
||||||
|
const serverArgs = ['--browser', 'none', '--driver', 'web', '--enable-proposed-api'];
|
||||||
|
|
||||||
let serverLocation: string;
|
let serverLocation: string;
|
||||||
if (process.env.VSCODE_REMOTE_SERVER_PATH) {
|
if (process.env.VSCODE_REMOTE_SERVER_PATH) {
|
||||||
serverLocation = path.join(process.env.VSCODE_REMOTE_SERVER_PATH, `server.${process.platform === 'win32' ? 'cmd' : 'sh'}`);
|
serverLocation = path.join(process.env.VSCODE_REMOTE_SERVER_PATH, `server.${process.platform === 'win32' ? 'cmd' : 'sh'}`);
|
||||||
|
serverArgs.push(`--logsPath=${logsPath}`);
|
||||||
|
|
||||||
console.log(`Starting built server from '${serverLocation}'`);
|
console.log(`Starting built server from '${serverLocation}'`);
|
||||||
|
console.log(`Storing log files into '${logsPath}'`);
|
||||||
} else {
|
} else {
|
||||||
serverLocation = path.join(__dirname, '..', '..', '..', '..', `resources/server/web.${process.platform === 'win32' ? 'bat' : 'sh'}`);
|
serverLocation = path.join(root, `resources/server/web.${process.platform === 'win32' ? 'bat' : 'sh'}`);
|
||||||
|
serverArgs.push('--logsPath', logsPath);
|
||||||
process.env.VSCODE_DEV = '1';
|
process.env.VSCODE_DEV = '1';
|
||||||
|
|
||||||
console.log(`Starting server out of sources from '${serverLocation}'`);
|
console.log(`Starting server out of sources from '${serverLocation}'`);
|
||||||
|
console.log(`Storing log files into '${logsPath}'`);
|
||||||
}
|
}
|
||||||
|
|
||||||
let serverProcess = cp.spawn(
|
let serverProcess = cp.spawn(
|
||||||
serverLocation,
|
serverLocation,
|
||||||
['--browser', 'none', '--driver', 'web', '--enable-proposed-api'],
|
serverArgs,
|
||||||
{ env }
|
{ env }
|
||||||
);
|
);
|
||||||
|
|
||||||
serverProcess?.stderr?.on('data', error => console.log(`Server stderr: ${error}`));
|
|
||||||
|
|
||||||
if (optimist.argv.debug) {
|
if (optimist.argv.debug) {
|
||||||
|
serverProcess?.stderr?.on('data', error => console.log(`Server stderr: ${error}`));
|
||||||
serverProcess?.stdout?.on('data', data => console.log(`Server stdout: ${data}`));
|
serverProcess?.stdout?.on('data', data => console.log(`Server stdout: ${data}`));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue