split compile & hygiene

build/azure-pipelines/product-build.yml

@@ -24,7 +24,20 @@ resources:
       image: snapcore/snapcraft:stable
 
 stages:
+  - stage: Dependencies
+    jobs:
+      - job: Dependencies
+        pool:
+          vmImage: "Ubuntu-18.04"
+        container: vscode-x64
+        variables:
+          VSCODE_ARCH: x64
+        steps:
+          - template: product-dependencies.yml
+
   - stage: Compile
+    dependsOn:
+      - Dependencies
     jobs:
       - job: Compile
         pool:
@@ -34,6 +47,14 @@ stages:
           VSCODE_ARCH: x64
         steps:
           - template: product-compile.yml
+      - job: Hygiene
+        pool:
+          vmImage: "Ubuntu-18.04"
+        container: vscode-x64
+        variables:
+          VSCODE_ARCH: x64
+        steps:
+          - template: product-hygiene.yml
 
   - stage: Windows
     dependsOn:

build/azure-pipelines/product-compile.yml

@@ -70,25 +70,12 @@ steps:
       vstsFeed: "npm-vscode"
     condition: and(succeeded(), ne(variables['CacheExists-Compilation'], 'true'))
 
+  # TODO@Joao: remove
   - script: |
       set -e
-      export CHILD_CONCURRENCY="1"
-      for i in {1..3}; do # try 3 times, for Terrapin
-        yarn --frozen-lockfile && break
-        if [ $i -eq 3 ]; then
-          echo "Yarn failed too many times" >&2
-          exit 1
-        fi
-        echo "Yarn failed $i, trying again..."
-      done
-    displayName: Install dependencies
-    condition: and(succeeded(), ne(variables['CacheExists-Compilation'], 'true'), ne(variables['CacheRestored'], 'true'))
-
-  - task: 1ESLighthouseEng.PipelineArtifactCaching.SaveCacheV1.SaveCache@1
-    inputs:
-      keyfile: ".build/arch, .build/terrapin, build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock"
-      targetfolder: "**/node_modules, !**/node_modules/**/node_modules"
-      vstsFeed: "npm-vscode"
+      echo "Restore cache should not have failed"
+      exit 1
+    displayName: Check cache restore
     condition: and(succeeded(), ne(variables['CacheExists-Compilation'], 'true'), ne(variables['CacheRestored'], 'true'))
 
   - script: |
@@ -104,27 +91,6 @@ steps:
     displayName: Mix in quality
     condition: and(succeeded(), ne(variables['CacheExists-Compilation'], 'true'))
 
-  - script: |
-      set -e
-      yarn gulp hygiene
-      yarn monaco-compile-check
-      yarn valid-layers-check
-    displayName: Run hygiene, monaco compile & valid layers checks
-    condition: and(succeeded(), ne(variables['CacheExists-Compilation'], 'true'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
-
-  - script: |
-      set -
-      ./build/azure-pipelines/common/extract-telemetry.sh
-    displayName: Extract Telemetry
-    condition: and(succeeded(), ne(variables['CacheExists-Compilation'], 'true'), ne(variables['VSCODE_PUBLISH'], 'false'))
-
-  - script: |
-      set -e
-      AZURE_WEBVIEW_STORAGE_ACCESS_KEY="$(vscode-webview-storage-key)" \
-      ./build/azure-pipelines/common/publish-webview.sh
-    displayName: Publish Webview
-    condition: and(succeeded(), ne(variables['CacheExists-Compilation'], 'true'), ne(variables['VSCODE_PUBLISH'], 'false'))
-
   - script: |
       set -e
       yarn gulp compile-build

build/azure-pipelines/product-dependencies.yml

@@ -0,0 +1,92 @@
+steps:
+  - script: |
+      mkdir -p .build
+      echo -n $BUILD_SOURCEVERSION > .build/commit
+      echo -n $VSCODE_QUALITY > .build/quality
+      echo -n $ENABLE_TERRAPIN > .build/terrapin
+    displayName: Prepare compilation cache flag
+
+  - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1
+    inputs:
+      keyfile: "build/.cachesalt, .build/commit, .build/quality, .build/terrapin"
+      targetfolder: ".build, out-build, out-vscode-min, out-vscode-reh-min, out-vscode-reh-web-min"
+      vstsFeed: "npm-vscode"
+      platformIndependent: true
+      alias: "Compilation"
+      dryRun: true
+
+  - task: NodeTool@0
+    inputs:
+      versionSpec: "12.18.3"
+    condition: and(succeeded(), ne(variables['CacheExists-Compilation'], 'true'))
+
+  - task: geeklearningio.gl-vsts-tasks-yarn.yarn-installer-task.YarnInstaller@2
+    inputs:
+      versionSpec: "1.x"
+    condition: and(succeeded(), ne(variables['CacheExists-Compilation'], 'true'))
+
+  - task: AzureKeyVault@1
+    displayName: "Azure Key Vault: Get Secrets"
+    inputs:
+      azureSubscription: "vscode-builds-subscription"
+      KeyVaultName: vscode
+    condition: and(succeeded(), ne(variables['CacheExists-Compilation'], 'true'))
+
+  - script: |
+      set -e
+      cat << EOF > ~/.netrc
+      machine github.com
+      login vscode
+      password $(github-distro-mixin-password)
+      EOF
+
+      git config user.email "vscode@microsoft.com"
+      git config user.name "VSCode"
+    displayName: Prepare tooling
+    condition: and(succeeded(), ne(variables['CacheExists-Compilation'], 'true'))
+
+  - script: |
+      set -e
+      git remote add distro "https://github.com/$(VSCODE_MIXIN_REPO).git"
+      git fetch distro
+      git merge $(node -p "require('./package.json').distro")
+    displayName: Merge distro
+    condition: and(succeeded(), ne(variables['CacheExists-Compilation'], 'true'))
+
+  - script: |
+      npx https://aka.ms/enablesecurefeed standAlone
+    displayName: Switch to Terrapin packages
+    timeoutInMinutes: 5
+    condition: and(succeeded(), ne(variables['CacheExists-Compilation'], 'true'), eq(variables['ENABLE_TERRAPIN'], 'true'))
+
+  - script: |
+      echo -n $(VSCODE_ARCH) > .build/arch
+    displayName: Prepare yarn cache flags
+
+  - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1
+    inputs:
+      keyfile: ".build/arch, .build/terrapin, build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock"
+      targetfolder: "**/node_modules, !**/node_modules/**/node_modules"
+      vstsFeed: "npm-vscode"
+    condition: and(succeeded(), ne(variables['CacheExists-Compilation'], 'true'))
+
+  - script: |
+      set -e
+      export CHILD_CONCURRENCY="1"
+      for i in {1..3}; do # try 3 times, for Terrapin
+        yarn --frozen-lockfile && break
+        if [ $i -eq 3 ]; then
+          echo "Yarn failed too many times" >&2
+          exit 1
+        fi
+        echo "Yarn failed $i, trying again..."
+      done
+    displayName: Install dependencies
+    condition: and(succeeded(), ne(variables['CacheExists-Compilation'], 'true'), ne(variables['CacheRestored'], 'true'))
+
+  - task: 1ESLighthouseEng.PipelineArtifactCaching.SaveCacheV1.SaveCache@1
+    inputs:
+      keyfile: ".build/arch, .build/terrapin, build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock"
+      targetfolder: "**/node_modules, !**/node_modules/**/node_modules"
+      vstsFeed: "npm-vscode"
+    condition: and(succeeded(), ne(variables['CacheExists-Compilation'], 'true'), ne(variables['CacheRestored'], 'true'))

build/azure-pipelines/product-hygiene.yml

@@ -0,0 +1,89 @@
+steps:
+  - script: |
+      mkdir -p .build
+      echo -n $ENABLE_TERRAPIN > .build/terrapin
+    displayName: Prepare compilation cache flag
+
+  - task: NodeTool@0
+    inputs:
+      versionSpec: "12.18.3"
+
+  - task: geeklearningio.gl-vsts-tasks-yarn.yarn-installer-task.YarnInstaller@2
+    inputs:
+      versionSpec: "1.x"
+
+  - task: AzureKeyVault@1
+    displayName: "Azure Key Vault: Get Secrets"
+    inputs:
+      azureSubscription: "vscode-builds-subscription"
+      KeyVaultName: vscode
+
+  - script: |
+      set -e
+      cat << EOF > ~/.netrc
+      machine github.com
+      login vscode
+      password $(github-distro-mixin-password)
+      EOF
+
+      git config user.email "vscode@microsoft.com"
+      git config user.name "VSCode"
+    displayName: Prepare tooling
+
+  - script: |
+      set -e
+      git remote add distro "https://github.com/$(VSCODE_MIXIN_REPO).git"
+      git fetch distro
+      git merge $(node -p "require('./package.json').distro")
+    displayName: Merge distro
+
+  - script: |
+      echo -n $(VSCODE_ARCH) > .build/arch
+    displayName: Prepare yarn cache flags
+
+  - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1
+    inputs:
+      keyfile: ".build/arch, .build/terrapin, build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock"
+      targetfolder: "**/node_modules, !**/node_modules/**/node_modules"
+      vstsFeed: "npm-vscode"
+
+  # TODO@Joao: remove
+  - script: |
+      set -e
+      echo "Restore cache should not have failed"
+      exit 1
+    displayName: Check cache restore
+    condition: and(succeeded(), ne(variables['CacheRestored'], 'true'))
+
+  - script: |
+      set -e
+      yarn postinstall
+    displayName: Run postinstall scripts
+    condition: and(succeeded(), eq(variables['CacheRestored'], 'true'))
+
+  # Mixin must run before optimize, because the CSS loader will inline small SVGs
+  - script: |
+      set -e
+      node build/azure-pipelines/mixin
+    displayName: Mix in quality
+
+  - script: |
+      set -e
+      yarn gulp hygiene
+      yarn monaco-compile-check
+      yarn valid-layers-check
+    displayName: Run hygiene, monaco compile & valid layers checks
+    condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
+
+  - script: |
+      set -
+      ./build/azure-pipelines/common/extract-telemetry.sh
+    displayName: Extract Telemetry
+    condition: and(succeeded(), ne(variables['VSCODE_PUBLISH'], 'false'))
+
+  - script: |
+      set -e
+      AZURE_WEBVIEW_STORAGE_ACCESS_KEY="$(vscode-webview-storage-key)" \
+      ./build/azure-pipelines/common/publish-webview.sh
+    displayName: Publish Webview
+    condition: and(succeeded(), ne(variables['VSCODE_PUBLISH'], 'false'))