take mooncake upload out of critical path
This commit is contained in:
parent
f5a513b40a
commit
3bd3d74a2c
|
@ -64,7 +64,7 @@ interface Asset {
|
|||
platform: string;
|
||||
type: string;
|
||||
url: string;
|
||||
mooncakeUrl: string;
|
||||
mooncakeUrl?: string;
|
||||
hash: string;
|
||||
sha256hash: string;
|
||||
size: number;
|
||||
|
@ -187,40 +187,18 @@ async function publish(commit: string, quality: string, platform: string, type:
|
|||
const blobService = azure.createBlobService(storageAccount, process.env['AZURE_STORAGE_ACCESS_KEY_2']!)
|
||||
.withFilter(new azure.ExponentialRetryPolicyFilter(20));
|
||||
|
||||
const mooncakeBlobService = azure.createBlobService(storageAccount, process.env['MOONCAKE_STORAGE_ACCESS_KEY']!, `${storageAccount}.blob.core.chinacloudapi.cn`)
|
||||
.withFilter(new azure.ExponentialRetryPolicyFilter(20));
|
||||
await assertContainer(blobService, quality);
|
||||
|
||||
// mooncake is fussy and far away, this is needed!
|
||||
mooncakeBlobService.defaultClientRequestTimeoutInMs = 10 * 60 * 1000;
|
||||
const blobExists = await doesAssetExist(blobService, quality, blobName);
|
||||
|
||||
await Promise.all([
|
||||
assertContainer(blobService, quality),
|
||||
assertContainer(mooncakeBlobService, quality)
|
||||
]);
|
||||
|
||||
const [blobExists, moooncakeBlobExists] = await Promise.all([
|
||||
doesAssetExist(blobService, quality, blobName),
|
||||
doesAssetExist(mooncakeBlobService, quality, blobName)
|
||||
]);
|
||||
|
||||
const promises: Array<Promise<void>> = [];
|
||||
|
||||
if (!blobExists) {
|
||||
promises.push(uploadBlob(blobService, quality, blobName, file));
|
||||
}
|
||||
|
||||
if (!moooncakeBlobExists) {
|
||||
promises.push(uploadBlob(mooncakeBlobService, quality, blobName, file));
|
||||
}
|
||||
|
||||
if (promises.length === 0) {
|
||||
if (blobExists) {
|
||||
console.log(`Blob ${quality}, ${blobName} already exists, not publishing again.`);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log('Uploading blobs to Azure storage...');
|
||||
|
||||
await Promise.all(promises);
|
||||
await uploadBlob(blobService, quality, blobName, file);
|
||||
|
||||
console.log('Blobs successfully uploaded.');
|
||||
|
||||
|
@ -232,7 +210,6 @@ async function publish(commit: string, quality: string, platform: string, type:
|
|||
platform: platform,
|
||||
type: type,
|
||||
url: `${process.env['AZURE_CDN_URL']}/${quality}/${blobName}`,
|
||||
mooncakeUrl: `${process.env['MOONCAKE_CDN_URL']}/${quality}/${blobName}`,
|
||||
hash: sha1hash,
|
||||
sha256hash,
|
||||
size
|
||||
|
|
176
build/azure-pipelines/common/sync-mooncake.ts
Normal file
176
build/azure-pipelines/common/sync-mooncake.ts
Normal file
|
@ -0,0 +1,176 @@
|
|||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as url from 'url';
|
||||
import * as azure from 'azure-storage';
|
||||
import * as mime from 'mime';
|
||||
import { DocumentClient, RetrievedDocument } from 'documentdb';
|
||||
|
||||
function log(...args: any[]) {
|
||||
console.log(...[`[${new Date().toISOString()}]`, ...args]);
|
||||
}
|
||||
|
||||
function error(...args: any[]) {
|
||||
console.error(...[`[${new Date().toISOString()}]`, ...args]);
|
||||
}
|
||||
|
||||
if (process.argv.length < 3) {
|
||||
error('Usage: node sync-mooncake.js <quality>');
|
||||
process.exit(-1);
|
||||
}
|
||||
|
||||
interface Build extends RetrievedDocument {
|
||||
assets: Asset[];
|
||||
}
|
||||
|
||||
interface Asset {
|
||||
platform: string;
|
||||
type: string;
|
||||
url: string;
|
||||
mooncakeUrl: string;
|
||||
hash: string;
|
||||
sha256hash: string;
|
||||
size: number;
|
||||
supportsFastUpdate?: boolean;
|
||||
}
|
||||
|
||||
function updateBuild(commit: string, quality: string, platform: string, type: string, asset: Asset): Promise<void> {
|
||||
const client = new DocumentClient(process.env['AZURE_DOCUMENTDB_ENDPOINT']!, { masterKey: process.env['AZURE_DOCUMENTDB_MASTERKEY'] });
|
||||
const collection = 'dbs/builds/colls/' + quality;
|
||||
const updateQuery = {
|
||||
query: 'SELECT TOP 1 * FROM c WHERE c.id = @id',
|
||||
parameters: [{ name: '@id', value: commit }]
|
||||
};
|
||||
|
||||
let updateTries = 0;
|
||||
|
||||
function _update(): Promise<void> {
|
||||
updateTries++;
|
||||
|
||||
return new Promise<void>((c, e) => {
|
||||
client.queryDocuments(collection, updateQuery).toArray((err, results) => {
|
||||
if (err) { return e(err); }
|
||||
if (results.length !== 1) { return e(new Error('No documents')); }
|
||||
|
||||
const release = results[0];
|
||||
|
||||
release.assets = [
|
||||
...release.assets.filter((a: any) => !(a.platform === platform && a.type === type)),
|
||||
asset
|
||||
];
|
||||
|
||||
client.replaceDocument(release._self, release, err => {
|
||||
if (err && err.code === 409 && updateTries < 5) { return c(_update()); }
|
||||
if (err) { return e(err); }
|
||||
|
||||
log('Build successfully updated.');
|
||||
c();
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
return _update();
|
||||
}
|
||||
|
||||
async function sync(commit: string, quality: string): Promise<void> {
|
||||
log(`Synchronizing Mooncake assets for ${quality}, ${commit}...`);
|
||||
|
||||
const cosmosdb = new DocumentClient(process.env['AZURE_DOCUMENTDB_ENDPOINT']!, { masterKey: process.env['AZURE_DOCUMENTDB_MASTERKEY'] });
|
||||
const collection = `dbs/builds/colls/${quality}`;
|
||||
const query = {
|
||||
query: 'SELECT TOP 1 * FROM c WHERE c.id = @id',
|
||||
parameters: [{ name: '@id', value: commit }]
|
||||
};
|
||||
|
||||
const build = await new Promise<Build>((c, e) => {
|
||||
cosmosdb.queryDocuments(collection, query).toArray((err, results) => {
|
||||
if (err) { return e(err); }
|
||||
if (results.length !== 1) { return e(new Error('No documents')); }
|
||||
c(results[0] as Build);
|
||||
});
|
||||
});
|
||||
|
||||
log(`Found build for ${commit}, with ${build.assets.length} assets`);
|
||||
|
||||
const storageAccount = process.env['AZURE_STORAGE_ACCOUNT_2']!;
|
||||
|
||||
const blobService = azure.createBlobService(storageAccount, process.env['AZURE_STORAGE_ACCESS_KEY_2']!)
|
||||
.withFilter(new azure.ExponentialRetryPolicyFilter(20));
|
||||
|
||||
const mooncakeBlobService = azure.createBlobService(storageAccount, process.env['MOONCAKE_STORAGE_ACCESS_KEY']!, `${storageAccount}.blob.core.chinacloudapi.cn`)
|
||||
.withFilter(new azure.ExponentialRetryPolicyFilter(20));
|
||||
|
||||
// mooncake is fussy and far away, this is needed!
|
||||
blobService.defaultClientRequestTimeoutInMs = 10 * 60 * 1000;
|
||||
mooncakeBlobService.defaultClientRequestTimeoutInMs = 10 * 60 * 1000;
|
||||
|
||||
for (const asset of build.assets) {
|
||||
try {
|
||||
const blobPath = url.parse(asset.url).path;
|
||||
|
||||
if (!blobPath) {
|
||||
throw new Error(`Failed to parse URL: ${asset.url}`);
|
||||
}
|
||||
|
||||
const blobName = blobPath.replace(/^\/\w+\//, '');
|
||||
|
||||
log(`Found ${blobName}`);
|
||||
|
||||
if (asset.mooncakeUrl) {
|
||||
log(` Already in Mooncake ✔️`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const readStream = blobService.createReadStream(quality, blobName, undefined!);
|
||||
const blobOptions: azure.BlobService.CreateBlockBlobRequestOptions = {
|
||||
contentSettings: {
|
||||
contentType: mime.lookup(blobPath),
|
||||
cacheControl: 'max-age=31536000, public'
|
||||
}
|
||||
};
|
||||
|
||||
const writeStream = mooncakeBlobService.createWriteStreamToBlockBlob(quality, blobName, blobOptions, undefined);
|
||||
|
||||
log(` Uploading to Mooncake...`);
|
||||
await new Promise((c, e) => readStream.pipe(writeStream).on('finish', c).on('error', e));
|
||||
|
||||
log(` Updating build in DB...`);
|
||||
asset.mooncakeUrl = `${process.env['MOONCAKE_CDN_URL']}${blobPath}`;
|
||||
await updateBuild(commit, quality, asset.platform, asset.type, asset);
|
||||
|
||||
log(` Done ✔️`);
|
||||
} catch (err) {
|
||||
error(err);
|
||||
}
|
||||
}
|
||||
|
||||
log(`All done ✔️`);
|
||||
}
|
||||
|
||||
function main(): void {
|
||||
if (process.env['VSCODE_BUILD_SKIP_PUBLISH']) {
|
||||
error('Skipping publish due to VSCODE_BUILD_SKIP_PUBLISH');
|
||||
return;
|
||||
}
|
||||
|
||||
const commit = process.env['BUILD_SOURCEVERSION'];
|
||||
|
||||
if (!commit) {
|
||||
error('Skipping publish due to missing BUILD_SOURCEVERSION');
|
||||
return;
|
||||
}
|
||||
|
||||
const quality = process.argv[2];
|
||||
|
||||
sync(commit, quality).catch(err => {
|
||||
error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
main();
|
|
@ -78,7 +78,6 @@ steps:
|
|||
VERSION=`node -p "require(\"$PACKAGEJSON\").version"`
|
||||
AZURE_DOCUMENTDB_MASTERKEY="$(AZURE_DOCUMENTDB_MASTERKEY)" \
|
||||
AZURE_STORAGE_ACCESS_KEY_2="$(AZURE_STORAGE_ACCESS_KEY_2)" \
|
||||
MOONCAKE_STORAGE_ACCESS_KEY="$(MOONCAKE_STORAGE_ACCESS_KEY)" \
|
||||
node build/azure-pipelines/common/publish.js \
|
||||
"$(VSCODE_QUALITY)" \
|
||||
darwin \
|
||||
|
|
|
@ -68,7 +68,6 @@ steps:
|
|||
|
||||
AZURE_DOCUMENTDB_MASTERKEY="$(AZURE_DOCUMENTDB_MASTERKEY)" \
|
||||
AZURE_STORAGE_ACCESS_KEY_2="$(AZURE_STORAGE_ACCESS_KEY_2)" \
|
||||
MOONCAKE_STORAGE_ACCESS_KEY="$(MOONCAKE_STORAGE_ACCESS_KEY)" \
|
||||
node build/azure-pipelines/common/publish.js "$VSCODE_QUALITY" "$PLATFORM_LINUX" archive-unsigned "$TARBALL_FILENAME" "$VERSION" true "$TARBALL_PATH"
|
||||
|
||||
# Publish hockeyapp symbols
|
||||
|
@ -83,7 +82,6 @@ steps:
|
|||
|
||||
AZURE_DOCUMENTDB_MASTERKEY="$(AZURE_DOCUMENTDB_MASTERKEY)" \
|
||||
AZURE_STORAGE_ACCESS_KEY_2="$(AZURE_STORAGE_ACCESS_KEY_2)" \
|
||||
MOONCAKE_STORAGE_ACCESS_KEY="$(MOONCAKE_STORAGE_ACCESS_KEY)" \
|
||||
node build/azure-pipelines/common/publish.js "$VSCODE_QUALITY" "$PLATFORM_DEB" package "$DEB_FILENAME" "$VERSION" true "$DEB_PATH"
|
||||
|
||||
# Publish RPM
|
||||
|
@ -95,7 +93,6 @@ steps:
|
|||
|
||||
AZURE_DOCUMENTDB_MASTERKEY="$(AZURE_DOCUMENTDB_MASTERKEY)" \
|
||||
AZURE_STORAGE_ACCESS_KEY_2="$(AZURE_STORAGE_ACCESS_KEY_2)" \
|
||||
MOONCAKE_STORAGE_ACCESS_KEY="$(MOONCAKE_STORAGE_ACCESS_KEY)" \
|
||||
node build/azure-pipelines/common/publish.js "$VSCODE_QUALITY" "$PLATFORM_RPM" package "$RPM_FILENAME" "$VERSION" true "$RPM_PATH"
|
||||
|
||||
# Publish Snap
|
||||
|
|
|
@ -46,5 +46,4 @@ steps:
|
|||
# Publish snap package
|
||||
AZURE_DOCUMENTDB_MASTERKEY="$(AZURE_DOCUMENTDB_MASTERKEY)" \
|
||||
AZURE_STORAGE_ACCESS_KEY_2="$(AZURE_STORAGE_ACCESS_KEY_2)" \
|
||||
MOONCAKE_STORAGE_ACCESS_KEY="$(MOONCAKE_STORAGE_ACCESS_KEY)" \
|
||||
node build/azure-pipelines/common/publish.js "$VSCODE_QUALITY" "linux-snap-$ARCH" package "$SNAP_FILENAME" "$VERSION" true "$SNAP_PATH"
|
|
@ -62,4 +62,18 @@ jobs:
|
|||
pool:
|
||||
vmImage: macOS 10.13
|
||||
steps:
|
||||
- template: darwin/product-build-darwin.yml
|
||||
- template: darwin/product-build-darwin.yml
|
||||
|
||||
- job: Mooncake
|
||||
pool:
|
||||
vmImage: 'Ubuntu-16.04'
|
||||
condition: true
|
||||
dependsOn:
|
||||
- Windows
|
||||
- Windows32
|
||||
- Linux
|
||||
- LinuxSnap
|
||||
- Linux32
|
||||
- macOS
|
||||
steps:
|
||||
- template: sync-mooncake.yml
|
18
build/azure-pipelines/sync-mooncake.yml
Normal file
18
build/azure-pipelines/sync-mooncake.yml
Normal file
|
@ -0,0 +1,18 @@
|
|||
steps:
|
||||
- task: NodeTool@0
|
||||
inputs:
|
||||
versionSpec: "10.15.1"
|
||||
|
||||
- task: geeklearningio.gl-vsts-tasks-yarn.yarn-installer-task.YarnInstaller@2
|
||||
inputs:
|
||||
versionSpec: "1.10.1"
|
||||
|
||||
- script: |
|
||||
set -e
|
||||
|
||||
(cd build ; yarn)
|
||||
|
||||
AZURE_DOCUMENTDB_MASTERKEY="$(AZURE_DOCUMENTDB_MASTERKEY)" \
|
||||
AZURE_STORAGE_ACCESS_KEY_2="$(AZURE_STORAGE_ACCESS_KEY_2)" \
|
||||
MOONCAKE_STORAGE_ACCESS_KEY="$(MOONCAKE_STORAGE_ACCESS_KEY)" \
|
||||
node build/azure-pipelines/common/sync-mooncake.js "$VSCODE_QUALITY"
|
|
@ -133,7 +133,6 @@ steps:
|
|||
$Version = $PackageJson.version
|
||||
$Quality = "$env:VSCODE_QUALITY"
|
||||
$env:AZURE_STORAGE_ACCESS_KEY_2 = "$(AZURE_STORAGE_ACCESS_KEY_2)"
|
||||
$env:MOONCAKE_STORAGE_ACCESS_KEY = "$(MOONCAKE_STORAGE_ACCESS_KEY)"
|
||||
$env:AZURE_DOCUMENTDB_MASTERKEY = "$(AZURE_DOCUMENTDB_MASTERKEY)"
|
||||
|
||||
$assetPlatform = if ("$(VSCODE_ARCH)" -eq "ia32") { "win32" } else { "win32-x64" }
|
||||
|
|
Loading…
Reference in a new issue