maxmustermann/gitlab
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

ci: Move long static-analysis tasks to dedicated jobs

Browse source 
Signed-off-by: Rémy Coutable <remy@rymai.me>
This commit is contained in:
Rémy Coutable 2021-10-13 17:46:14 +02:00 committed by Albert Salim
parent 91cbaab4d0
commit 7f76703f07
13 changed files with 113 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitlab-ci.yml
View file

@ -3,6 +3,7 @@ stages:
- prepare
- build-images
- fixtures
- lint
- test
- post-test
- review

6
.gitlab/ci/docs.gitlab-ci.yml
View file

@ -45,7 +45,7 @@ docs-lint markdown:
- .docs:rules:docs-lint
# When updating the image version here, update it in /scripts/lint-doc.sh too.
image: registry.gitlab.com/gitlab-org/gitlab-docs/lint-markdown:alpine-3.14-vale-2.10.4-markdownlint-0.28.1
stage: test
stage: lint
needs: []
script:
- scripts/lint-doc.sh
@ -54,7 +54,7 @@ docs-lint links:
extends:
- .docs:rules:docs-lint
image: registry.gitlab.com/gitlab-org/gitlab-docs/lint-html:alpine-3.14-ruby-2.7.4-db71f027
stage: test
stage: lint
needs: []
script:
# Prepare docs for build
@ -71,7 +71,7 @@ ui-docs-links lint:
extends:
- .docs:rules:docs-lint
- .static-analysis-base
stage: test
stage: lint
needs: []
script:
- bundle exec haml-lint -i DocumentationLinks

9
.gitlab/ci/frontend.gitlab-ci.yml
View file

@ -197,15 +197,6 @@ graphql-schema-dump:
- *yarn-install
stage: test
eslint-as-if-foss:
extends:
- .frontend-test-base
- .frontend:rules:eslint-as-if-foss
- .as-if-foss
needs: []
script:
- run_timed_command "yarn run lint:eslint:all"
.jest-base:
extends: .frontend-test-base
script:

11
.gitlab/ci/global.gitlab-ci.yml
View file

@ -140,6 +140,10 @@
- <<: *gitaly-binaries-cache
policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
.ruby-cache:
cache:
- *ruby-gems-cache
.rails-cache:
cache:
- *ruby-gems-cache
@ -151,7 +155,12 @@
- *node-modules-cache
- *rubocop-cache
.static-analysis-cache-push:
.rubocop-job-cache:
cache:
- *ruby-gems-cache
- *rubocop-cache
.rubocop-job-cache-push:
cache:
- *ruby-gems-cache # We don't push this cache as it's already rebuilt by `update-setup-test-env-cache`
- *rubocop-cache-push

6
.gitlab/ci/rails.gitlab-ci.yml
View file

@ -507,7 +507,11 @@ rspec:feature-flags:
- .coverage-base
- .rails:rules:rspec-feature-flags
stage: post-test
needs: ["static-analysis"]
needs:
- job: "feature-flags-usage"
- job: "haml-lint foss"
- job: "haml-lint ee"
optional: true
script:
- !reference [.minimal-bundle-install, script]
- if [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then

5
.gitlab/ci/reports.gitlab-ci.yml
View file

@ -9,6 +9,7 @@ code_quality:
extends:
- .default-retry
- .use-docker-in-docker
stage: lint
artifacts:
paths:
- gl-code-quality-report.json # GitLab-specific
@ -20,6 +21,7 @@ code_quality:
extends:
- .default-retry
- sast
stage: lint
needs: []
artifacts:
paths:
@ -54,6 +56,7 @@ gosec-sast:
.secret-analyzer:
extends: .default-retry
stage: lint
needs: []
artifacts:
paths:
@ -68,6 +71,7 @@ secret_detection:
extends:
- .default-retry
- dependency_scanning
stage: lint
needs: []
variables:
DS_EXCLUDED_PATHS: "qa/qa/ee/fixtures/secure_premade_reports, spec, ee/spec, tmp" # GitLab-specific
@ -134,6 +138,7 @@ package_hunter-bundler:
license_scanning:
extends: .default-retry
stage: lint
needs: []
artifacts:
expire_in: 1 week # GitLab-specific

6
.gitlab/ci/rules.gitlab-ci.yml
View file

@ -1354,6 +1354,12 @@
rules:
- changes: *code-backstage-qa-patterns
.static-analysis:rules:ee:
rules:
- <<: *if-not-ee
when: never
- changes: *code-backstage-qa-patterns
.static-analysis:rules:as-if-foss:
rules:
- <<: *if-not-ee

96
.gitlab/ci/static-analysis.gitlab-ci.yml
View file

@ -2,7 +2,7 @@
extends:
- .default-retry
- .default-before_script
- .static-analysis-cache
stage: lint
needs: []
variables:
SETUP_DB: "false"
@ -14,7 +14,7 @@
update-static-analysis-cache:
extends:
- .static-analysis-base
- .static-analysis-cache-push
- .rubocop-job-cache-push
- .shared:rules:update-cache
stage: prepare
script:
@ -23,31 +23,87 @@ update-static-analysis-cache:
static-analysis:
extends:
- .static-analysis-base
- .static-analysis-cache
- .static-analysis:rules:ee-and-foss
stage: test
parallel: 4
parallel: 2
script:
- run_timed_command "retry yarn install --frozen-lockfile"
- scripts/static-analysis
artifacts:
expire_in: 31d
when: always
paths:
- tmp/feature_flags/
static-analysis-with-database:
extends:
- .static-analysis-base
- .static-analysis:rules:ee-and-foss
- .use-pg12
stage: test
script:
- bundle exec rake lint:static_verification_with_database
variables:
SETUP_DB: "true"
static-analysis as-if-foss:
extends:
- static-analysis
- .static-analysis:rules:as-if-foss
- .as-if-foss
static-verification-with-database:
extends:
- .static-analysis-base
- .rubocop-job-cache
- .static-analysis:rules:ee-and-foss
- .use-pg12
script:
- bundle exec rake lint:static_verification_with_database
variables:
SETUP_DB: "true"
eslint:
extends:
- .static-analysis-base
- .yarn-cache
- .static-analysis:rules:ee
needs: []
variables:
USE_BUNDLE_INSTALL: "false"
script:
- run_timed_command "retry yarn install --frozen-lockfile"
- run_timed_command "yarn run lint:eslint:all"
eslint as-if-foss:
extends:
- eslint
- .frontend:rules:eslint-as-if-foss
- .as-if-foss
haml-lint foss:
extends:
- .static-analysis-base
- .ruby-cache
- .static-analysis:rules:ee-and-foss
script:
- run_timed_command "bin/rake 'haml_lint[app/views]'"
artifacts:
expire_in: 31d
when: always
paths:
- tmp/feature_flags/
haml-lint ee:
extends:
- "haml-lint foss"
- .static-analysis:rules:ee
script:
- run_timed_command "bin/rake 'haml_lint[ee/app/views]'"
rubocop:
extends:
- .static-analysis-base
- .rubocop-job-cache
- .static-analysis:rules:ee-and-foss
script:
- run_timed_command "bundle exec rubocop --parallel"
feature-flags-usage:
extends:
- .static-analysis-base
- .rubocop-job-cache
- .static-analysis:rules:ee-and-foss
script:
# We need to disable the cache for this cop since it creates files under tmp/feature_flags/*.used,
# the cache would prevent these files from being created.
- run_timed_command "bundle exec rubocop --only Gitlab/MarkUsedFeatureFlags --cache false"
artifacts:
expire_in: 31d
when: always
paths:
- tmp/feature_flags/

2
.gitlab/ci/yaml.gitlab-ci.yml
View file

@ -5,7 +5,7 @@ lint-yaml:
- .default-retry
- .yaml-lint:rules
image: pipelinecomponents/yamllint:latest
stage: test
stage: lint
needs: []
variables:
LINT_PATHS: .gitlab-ci.yml .gitlab/ci lib/gitlab/ci/templates

1
.haml-lint.yml
View file

@ -6,6 +6,7 @@ skip_frontmatter: false
exclude:
- 'vendor/**/*'
- 'spec/**/*'
- 'ee/spec/**/*'
require:
- './haml_lint/linter/no_plain_nodes.rb'
- './haml_lint/linter/documentation_links.rb'

7
doc/development/pipelines.md
View file

@ -590,7 +590,8 @@ The current stages are:
- `build-images`: This stage includes jobs that prepare Docker images
that are needed by jobs in subsequent stages or downstream pipelines.
- `fixtures`: This stage includes jobs that prepare fixtures needed by frontend tests.
- `test`: This stage includes most of the tests, DB/migration jobs, and static analysis jobs.
- `lint`: This stage includes linting and static analysis jobs.
- `test`: This stage includes most of the tests, and DB/migration jobs.
- `post-test`: This stage includes jobs that build reports or gather data from
the `test` stage's jobs (for example, coverage, Knapsack metadata, and so on).
- `review-prepare`: This stage includes a job that build the CNG images that are
@ -741,8 +742,10 @@ request, be sure to start the `dont-interrupt-me` job before pushing.
[`.gitlab/ci/global.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/global.gitlab-ci.yml),
with fixed keys:
- `.setup-test-env-cache`
- `.ruby-cache`
- `.rails-cache`
- `.static-analysis-cache`
- `.rubocop-cache`
- `.coverage-cache`
- `.danger-review-cache`
- `.qa-cache`
@ -752,7 +755,7 @@ request, be sure to start the `dont-interrupt-me` job before pushing.
1. Only the following jobs, running in 2-hourly scheduled pipelines, are pushing (that is, updating) to the caches:
- `update-setup-test-env-cache`, defined in [`.gitlab/ci/rails.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/rails.gitlab-ci.yml).
- `update-gitaly-binaries-cache`, defined in [`.gitlab/ci/rails.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/rails.gitlab-ci.yml).
- `update-static-analysis-cache`, defined in [`.gitlab/ci/rails.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/rails.gitlab-ci.yml).
- `update-rubocop-cache`, defined in [`.gitlab/ci/rails.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/rails.gitlab-ci.yml).
- `update-qa-cache`, defined in [`.gitlab/ci/qa.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/qa.gitlab-ci.yml).
- `update-assets-compile-production-cache`, defined in [`.gitlab/ci/frontend.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/frontend.gitlab-ci.yml).
- `update-assets-compile-test-cache`, defined in [`.gitlab/ci/frontend.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/frontend.gitlab-ci.yml).

11
lib/tasks/haml-lint.rake
View file

@ -4,16 +4,5 @@ unless Rails.env.production?
require 'haml_lint/rake_task'
require Rails.root.join('haml_lint/inline_javascript')
# Workaround for warnings from parser/current
# Keep it even if it no longer emits any warnings,
# because we'll still see warnings in console/server anyway,
# and we don't need to break static-analysis for this.
task :haml_lint do
require 'parser'
def Parser.warn(*args)
puts(*args) # static-analysis ignores stdout if status is 0
end
end
HamlLint::RakeTask.new
end

6
scripts/static-analysis
View file

@ -43,13 +43,7 @@ class StaticAnalysis
# contain values that a FOSS installation won't find. To work
# around this we will only enable this task on EE installations.
TASKS_WITH_DURATIONS_SECONDS = [
Task.new(%w[bin/rake lint:haml], 562),
# We need to disable the cache for this cop since it creates files under tmp/feature_flags/*.used,
# the cache would prevent these files from being created.
Task.new(%w[bundle exec rubocop --only Gitlab/MarkUsedFeatureFlags --cache false], 400),
(Gitlab.ee? ? Task.new(%w[bin/rake gettext:updated_check], 360) : nil),
Task.new(%w[yarn run lint:eslint:all], 312),
Task.new(%w[bundle exec rubocop --parallel], 60),
Task.new(%w[yarn run lint:prettier], 160),
Task.new(%w[bin/rake gettext:lint], 85),
Task.new(%W[bundle exec license_finder --decisions-file config/dependency_decisions.yml --project-path #{project_path}], 20),