From 7f76703f07cea406ef44fc8829d2c2f313d13c39 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9my=20Coutable?= <remy@rymai.me>
Date: Wed, 13 Oct 2021 17:46:14 +0200
Subject: [PATCH] ci: Move long static-analysis tasks to dedicated jobs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Rémy Coutable <remy@rymai.me>
---
 .gitlab-ci.yml                           |  1 +
 .gitlab/ci/docs.gitlab-ci.yml            |  6 +-
 .gitlab/ci/frontend.gitlab-ci.yml        |  9 ---
 .gitlab/ci/global.gitlab-ci.yml          | 11 ++-
 .gitlab/ci/rails.gitlab-ci.yml           |  6 +-
 .gitlab/ci/reports.gitlab-ci.yml         |  5 ++
 .gitlab/ci/rules.gitlab-ci.yml           |  6 ++
 .gitlab/ci/static-analysis.gitlab-ci.yml | 96 +++++++++++++++++++-----
 .gitlab/ci/yaml.gitlab-ci.yml            |  2 +-
 .haml-lint.yml                           |  1 +
 doc/development/pipelines.md             |  7 +-
 lib/tasks/haml-lint.rake                 | 11 ---
 scripts/static-analysis                  |  6 --
 13 files changed, 113 insertions(+), 54 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d119de421ef7..e810edd81339 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -3,6 +3,7 @@ stages:
   - prepare
   - build-images
   - fixtures
+  - lint
   - test
   - post-test
   - review
diff --git a/.gitlab/ci/docs.gitlab-ci.yml b/.gitlab/ci/docs.gitlab-ci.yml
index f4d8698f22da..43a2cd6dbcb0 100644
--- a/.gitlab/ci/docs.gitlab-ci.yml
+++ b/.gitlab/ci/docs.gitlab-ci.yml
@@ -45,7 +45,7 @@ docs-lint markdown:
     - .docs:rules:docs-lint
   # When updating the image version here, update it in /scripts/lint-doc.sh too.
   image: registry.gitlab.com/gitlab-org/gitlab-docs/lint-markdown:alpine-3.14-vale-2.10.4-markdownlint-0.28.1
-  stage: test
+  stage: lint
   needs: []
   script:
     - scripts/lint-doc.sh
@@ -54,7 +54,7 @@ docs-lint links:
   extends:
     - .docs:rules:docs-lint
   image: registry.gitlab.com/gitlab-org/gitlab-docs/lint-html:alpine-3.14-ruby-2.7.4-db71f027
-  stage: test
+  stage: lint
   needs: []
   script:
     # Prepare docs for build
@@ -71,7 +71,7 @@ ui-docs-links lint:
   extends:
     - .docs:rules:docs-lint
     - .static-analysis-base
-  stage: test
+  stage: lint
   needs: []
   script:
     - bundle exec haml-lint -i DocumentationLinks
diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml
index 139888c12fcd..bd2c68068d46 100644
--- a/.gitlab/ci/frontend.gitlab-ci.yml
+++ b/.gitlab/ci/frontend.gitlab-ci.yml
@@ -197,15 +197,6 @@ graphql-schema-dump:
     - *yarn-install
   stage: test
 
-eslint-as-if-foss:
-  extends:
-    - .frontend-test-base
-    - .frontend:rules:eslint-as-if-foss
-    - .as-if-foss
-  needs: []
-  script:
-    - run_timed_command "yarn run lint:eslint:all"
-
 .jest-base:
   extends: .frontend-test-base
   script:
diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml
index f840d83bf971..22bd3b1e2e90 100644
--- a/.gitlab/ci/global.gitlab-ci.yml
+++ b/.gitlab/ci/global.gitlab-ci.yml
@@ -140,6 +140,10 @@
     - <<: *gitaly-binaries-cache
       policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
 
+.ruby-cache:
+  cache:
+    - *ruby-gems-cache
+
 .rails-cache:
   cache:
     - *ruby-gems-cache
@@ -151,7 +155,12 @@
     - *node-modules-cache
     - *rubocop-cache
 
-.static-analysis-cache-push:
+.rubocop-job-cache:
+  cache:
+    - *ruby-gems-cache
+    - *rubocop-cache
+
+.rubocop-job-cache-push:
   cache:
     - *ruby-gems-cache  # We don't push this cache as it's already rebuilt by `update-setup-test-env-cache`
     - *rubocop-cache-push
diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml
index d5f56139444a..5ff387ee3a7e 100644
--- a/.gitlab/ci/rails.gitlab-ci.yml
+++ b/.gitlab/ci/rails.gitlab-ci.yml
@@ -507,7 +507,11 @@ rspec:feature-flags:
     - .coverage-base
     - .rails:rules:rspec-feature-flags
   stage: post-test
-  needs: ["static-analysis"]
+  needs:
+    - job: "feature-flags-usage"
+    - job: "haml-lint foss"
+    - job: "haml-lint ee"
+      optional: true
   script:
     - !reference [.minimal-bundle-install, script]
     - if [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then
diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml
index b581cf83d56f..40d9c55c5bdc 100644
--- a/.gitlab/ci/reports.gitlab-ci.yml
+++ b/.gitlab/ci/reports.gitlab-ci.yml
@@ -9,6 +9,7 @@ code_quality:
   extends:
     - .default-retry
     - .use-docker-in-docker
+  stage: lint
   artifacts:
     paths:
       - gl-code-quality-report.json  # GitLab-specific
@@ -20,6 +21,7 @@ code_quality:
   extends:
     - .default-retry
     - sast
+  stage: lint
   needs: []
   artifacts:
     paths:
@@ -54,6 +56,7 @@ gosec-sast:
 
 .secret-analyzer:
   extends: .default-retry
+  stage: lint
   needs: []
   artifacts:
     paths:
@@ -68,6 +71,7 @@ secret_detection:
   extends:
     - .default-retry
     - dependency_scanning
+  stage: lint
   needs: []
   variables:
     DS_EXCLUDED_PATHS: "qa/qa/ee/fixtures/secure_premade_reports, spec, ee/spec, tmp"  # GitLab-specific
@@ -134,6 +138,7 @@ package_hunter-bundler:
 
 license_scanning:
   extends: .default-retry
+  stage: lint
   needs: []
   artifacts:
     expire_in: 1 week  # GitLab-specific
diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml
index a2b8aca811b3..8a782ec2b684 100644
--- a/.gitlab/ci/rules.gitlab-ci.yml
+++ b/.gitlab/ci/rules.gitlab-ci.yml
@@ -1354,6 +1354,12 @@
   rules:
     - changes: *code-backstage-qa-patterns
 
+.static-analysis:rules:ee:
+  rules:
+    - <<: *if-not-ee
+      when: never
+    - changes: *code-backstage-qa-patterns
+
 .static-analysis:rules:as-if-foss:
   rules:
     - <<: *if-not-ee
diff --git a/.gitlab/ci/static-analysis.gitlab-ci.yml b/.gitlab/ci/static-analysis.gitlab-ci.yml
index 85df68e90306..8824d3d753f5 100644
--- a/.gitlab/ci/static-analysis.gitlab-ci.yml
+++ b/.gitlab/ci/static-analysis.gitlab-ci.yml
@@ -2,7 +2,7 @@
   extends:
     - .default-retry
     - .default-before_script
-    - .static-analysis-cache
+  stage: lint
   needs: []
   variables:
     SETUP_DB: "false"
@@ -14,7 +14,7 @@
 update-static-analysis-cache:
   extends:
     - .static-analysis-base
-    - .static-analysis-cache-push
+    - .rubocop-job-cache-push
     - .shared:rules:update-cache
   stage: prepare
   script:
@@ -23,31 +23,87 @@ update-static-analysis-cache:
 static-analysis:
   extends:
     - .static-analysis-base
+    - .static-analysis-cache
     - .static-analysis:rules:ee-and-foss
-  stage: test
-  parallel: 4
+  parallel: 2
   script:
     - run_timed_command "retry yarn install --frozen-lockfile"
     - scripts/static-analysis
-  artifacts:
-    expire_in: 31d
-    when: always
-    paths:
-      - tmp/feature_flags/
-
-static-analysis-with-database:
-  extends:
-    - .static-analysis-base
-    - .static-analysis:rules:ee-and-foss
-    - .use-pg12
-  stage: test
-  script:
-    - bundle exec rake lint:static_verification_with_database
-  variables:
-    SETUP_DB: "true"
 
 static-analysis as-if-foss:
   extends:
     - static-analysis
     - .static-analysis:rules:as-if-foss
     - .as-if-foss
+
+static-verification-with-database:
+  extends:
+    - .static-analysis-base
+    - .rubocop-job-cache
+    - .static-analysis:rules:ee-and-foss
+    - .use-pg12
+  script:
+    - bundle exec rake lint:static_verification_with_database
+  variables:
+    SETUP_DB: "true"
+
+eslint:
+  extends:
+    - .static-analysis-base
+    - .yarn-cache
+    - .static-analysis:rules:ee
+  needs: []
+  variables:
+    USE_BUNDLE_INSTALL: "false"
+  script:
+    - run_timed_command "retry yarn install --frozen-lockfile"
+    - run_timed_command "yarn run lint:eslint:all"
+
+eslint as-if-foss:
+  extends:
+    - eslint
+    - .frontend:rules:eslint-as-if-foss
+    - .as-if-foss
+
+haml-lint foss:
+  extends:
+    - .static-analysis-base
+    - .ruby-cache
+    - .static-analysis:rules:ee-and-foss
+  script:
+    - run_timed_command "bin/rake 'haml_lint[app/views]'"
+  artifacts:
+    expire_in: 31d
+    when: always
+    paths:
+      - tmp/feature_flags/
+
+haml-lint ee:
+  extends:
+    - "haml-lint foss"
+    - .static-analysis:rules:ee
+  script:
+    - run_timed_command "bin/rake 'haml_lint[ee/app/views]'"
+
+rubocop:
+  extends:
+    - .static-analysis-base
+    - .rubocop-job-cache
+    - .static-analysis:rules:ee-and-foss
+  script:
+    - run_timed_command "bundle exec rubocop --parallel"
+
+feature-flags-usage:
+  extends:
+    - .static-analysis-base
+    - .rubocop-job-cache
+    - .static-analysis:rules:ee-and-foss
+  script:
+    # We need to disable the cache for this cop since it creates files under tmp/feature_flags/*.used,
+    # the cache would prevent these files from being created.
+    - run_timed_command "bundle exec rubocop --only Gitlab/MarkUsedFeatureFlags --cache false"
+  artifacts:
+    expire_in: 31d
+    when: always
+    paths:
+      - tmp/feature_flags/
diff --git a/.gitlab/ci/yaml.gitlab-ci.yml b/.gitlab/ci/yaml.gitlab-ci.yml
index b25ad55e0ce6..590593b9d75c 100644
--- a/.gitlab/ci/yaml.gitlab-ci.yml
+++ b/.gitlab/ci/yaml.gitlab-ci.yml
@@ -5,7 +5,7 @@ lint-yaml:
     - .default-retry
     - .yaml-lint:rules
   image: pipelinecomponents/yamllint:latest
-  stage: test
+  stage: lint
   needs: []
   variables:
     LINT_PATHS: .gitlab-ci.yml .gitlab/ci lib/gitlab/ci/templates
diff --git a/.haml-lint.yml b/.haml-lint.yml
index 1d1c0fa1de23..8d084a0c356a 100644
--- a/.haml-lint.yml
+++ b/.haml-lint.yml
@@ -6,6 +6,7 @@ skip_frontmatter: false
 exclude:
   - 'vendor/**/*'
   - 'spec/**/*'
+  - 'ee/spec/**/*'
 require:
   - './haml_lint/linter/no_plain_nodes.rb'
   - './haml_lint/linter/documentation_links.rb'
diff --git a/doc/development/pipelines.md b/doc/development/pipelines.md
index d19c7d896f86..aef29bd4f219 100644
--- a/doc/development/pipelines.md
+++ b/doc/development/pipelines.md
@@ -590,7 +590,8 @@ The current stages are:
 - `build-images`: This stage includes jobs that prepare Docker images
   that are needed by jobs in subsequent stages or downstream pipelines.
 - `fixtures`: This stage includes jobs that prepare fixtures needed by frontend tests.
-- `test`: This stage includes most of the tests, DB/migration jobs, and static analysis jobs.
+- `lint`: This stage includes linting and static analysis jobs.
+- `test`: This stage includes most of the tests, and DB/migration jobs.
 - `post-test`: This stage includes jobs that build reports or gather data from
   the `test` stage's jobs (for example, coverage, Knapsack metadata, and so on).
 - `review-prepare`: This stage includes a job that build the CNG images that are
@@ -741,8 +742,10 @@ request, be sure to start the `dont-interrupt-me` job before pushing.
    [`.gitlab/ci/global.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/global.gitlab-ci.yml),
    with fixed keys:
    - `.setup-test-env-cache`
+   - `.ruby-cache`
    - `.rails-cache`
    - `.static-analysis-cache`
+   - `.rubocop-cache`
    - `.coverage-cache`
    - `.danger-review-cache`
    - `.qa-cache`
@@ -752,7 +755,7 @@ request, be sure to start the `dont-interrupt-me` job before pushing.
 1. Only the following jobs, running in 2-hourly scheduled pipelines, are pushing (that is, updating) to the caches:
    - `update-setup-test-env-cache`, defined in [`.gitlab/ci/rails.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/rails.gitlab-ci.yml).
    - `update-gitaly-binaries-cache`, defined in [`.gitlab/ci/rails.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/rails.gitlab-ci.yml).
-   - `update-static-analysis-cache`, defined in [`.gitlab/ci/rails.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/rails.gitlab-ci.yml).
+   - `update-rubocop-cache`, defined in [`.gitlab/ci/rails.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/rails.gitlab-ci.yml).
    - `update-qa-cache`, defined in [`.gitlab/ci/qa.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/qa.gitlab-ci.yml).
    - `update-assets-compile-production-cache`, defined in [`.gitlab/ci/frontend.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/frontend.gitlab-ci.yml).
    - `update-assets-compile-test-cache`, defined in [`.gitlab/ci/frontend.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/frontend.gitlab-ci.yml).
diff --git a/lib/tasks/haml-lint.rake b/lib/tasks/haml-lint.rake
index 270793359e10..71e84d3795f9 100644
--- a/lib/tasks/haml-lint.rake
+++ b/lib/tasks/haml-lint.rake
@@ -4,16 +4,5 @@ unless Rails.env.production?
   require 'haml_lint/rake_task'
   require Rails.root.join('haml_lint/inline_javascript')
 
-  # Workaround for warnings from parser/current
-  # Keep it even if it no longer emits any warnings,
-  # because we'll still see warnings in console/server anyway,
-  # and we don't need to break static-analysis for this.
-  task :haml_lint do
-    require 'parser'
-    def Parser.warn(*args)
-      puts(*args) # static-analysis ignores stdout if status is 0
-    end
-  end
-
   HamlLint::RakeTask.new
 end
diff --git a/scripts/static-analysis b/scripts/static-analysis
index f50e4a24b585..9c6a948adc1c 100755
--- a/scripts/static-analysis
+++ b/scripts/static-analysis
@@ -43,13 +43,7 @@ class StaticAnalysis
   # contain values that a FOSS installation won't find. To work
   # around this we will only enable this task on EE installations.
   TASKS_WITH_DURATIONS_SECONDS = [
-    Task.new(%w[bin/rake lint:haml], 562),
-    # We need to disable the cache for this cop since it creates files under tmp/feature_flags/*.used,
-    # the cache would prevent these files from being created.
-    Task.new(%w[bundle exec rubocop --only Gitlab/MarkUsedFeatureFlags --cache false], 400),
     (Gitlab.ee? ? Task.new(%w[bin/rake gettext:updated_check], 360) : nil),
-    Task.new(%w[yarn run lint:eslint:all], 312),
-    Task.new(%w[bundle exec rubocop --parallel], 60),
     Task.new(%w[yarn run lint:prettier], 160),
     Task.new(%w[bin/rake gettext:lint], 85),
     Task.new(%W[bundle exec license_finder --decisions-file config/dependency_decisions.yml --project-path #{project_path}], 20),