Extract sidekiq-cluster to top-level directory
This prevents name clashes with library code that is auto-loaded in workers from lib/
This commit is contained in:
parent
64d174b2bb
commit
f52c7ef5be
|
@ -1,13 +1,7 @@
|
||||||
#!/usr/bin/env ruby
|
#!/usr/bin/env ruby
|
||||||
# frozen_string_literal: true
|
# frozen_string_literal: true
|
||||||
|
|
||||||
require 'optparse'
|
require_relative '../sidekiq_cluster/cli'
|
||||||
require_relative '../lib/gitlab'
|
|
||||||
require_relative '../lib/gitlab/utils'
|
|
||||||
require_relative '../lib/gitlab/sidekiq_config/cli_methods'
|
|
||||||
require_relative '../lib/gitlab/sidekiq_config/worker_matcher'
|
|
||||||
require_relative '../lib/gitlab/sidekiq_cluster'
|
|
||||||
require_relative '../lib/gitlab/sidekiq_cluster/cli'
|
|
||||||
|
|
||||||
Thread.abort_on_exception = true
|
Thread.abort_on_exception = true
|
||||||
|
|
||||||
|
|
|
@ -4,27 +4,21 @@
|
||||||
require 'logger'
|
require 'logger'
|
||||||
require 'time'
|
require 'time'
|
||||||
|
|
||||||
|
# In environments where code is preloaded and cached such as `spring`,
|
||||||
|
# we may run into "already initialized" warnings, hence the check.
|
||||||
|
require_relative '../lib/gitlab' unless Object.const_defined?('Gitlab')
|
||||||
|
require_relative '../lib/gitlab/utils'
|
||||||
|
require_relative '../lib/gitlab/sidekiq_config/cli_methods'
|
||||||
|
require_relative '../lib/gitlab/sidekiq_config/worker_matcher'
|
||||||
|
require_relative '../lib/gitlab/sidekiq_logging/json_formatter'
|
||||||
|
require_relative 'sidekiq_cluster'
|
||||||
|
|
||||||
module Gitlab
|
module Gitlab
|
||||||
module SidekiqCluster
|
module SidekiqCluster
|
||||||
class CLI
|
class CLI
|
||||||
CHECK_TERMINATE_INTERVAL_SECONDS = 1
|
|
||||||
|
|
||||||
# How long to wait when asking for a clean termination.
|
|
||||||
# It maps the Sidekiq default timeout:
|
|
||||||
# https://github.com/mperham/sidekiq/wiki/Signals#term
|
|
||||||
#
|
|
||||||
# This value is passed to Sidekiq's `-t` if none
|
|
||||||
# is given through arguments.
|
|
||||||
DEFAULT_SOFT_TIMEOUT_SECONDS = 25
|
|
||||||
|
|
||||||
# After surpassing the soft timeout.
|
|
||||||
DEFAULT_HARD_TIMEOUT_SECONDS = 5
|
|
||||||
|
|
||||||
CommandError = Class.new(StandardError)
|
CommandError = Class.new(StandardError)
|
||||||
|
|
||||||
def initialize(log_output = $stderr)
|
def initialize(log_output = $stderr)
|
||||||
require_relative '../../../lib/gitlab/sidekiq_logging/json_formatter'
|
|
||||||
|
|
||||||
# As recommended by https://github.com/mperham/sidekiq/wiki/Advanced-Options#concurrency
|
# As recommended by https://github.com/mperham/sidekiq/wiki/Advanced-Options#concurrency
|
||||||
@max_concurrency = 50
|
@max_concurrency = 50
|
||||||
@min_concurrency = 0
|
@min_concurrency = 0
|
6
sidekiq_cluster/dependencies.rb
Normal file
6
sidekiq_cluster/dependencies.rb
Normal file
|
@ -0,0 +1,6 @@
|
||||||
|
# rubocop:disable Naming/FileName
|
||||||
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
require 'shellwords'
|
||||||
|
|
||||||
|
# rubocop:enable Naming/FileName
|
|
@ -1,9 +1,22 @@
|
||||||
# frozen_string_literal: true
|
# frozen_string_literal: true
|
||||||
|
|
||||||
require 'shellwords'
|
require_relative 'dependencies'
|
||||||
|
|
||||||
module Gitlab
|
module Gitlab
|
||||||
module SidekiqCluster
|
module SidekiqCluster
|
||||||
|
CHECK_TERMINATE_INTERVAL_SECONDS = 1
|
||||||
|
|
||||||
|
# How long to wait when asking for a clean termination.
|
||||||
|
# It maps the Sidekiq default timeout:
|
||||||
|
# https://github.com/mperham/sidekiq/wiki/Signals#term
|
||||||
|
#
|
||||||
|
# This value is passed to Sidekiq's `-t` if none
|
||||||
|
# is given through arguments.
|
||||||
|
DEFAULT_SOFT_TIMEOUT_SECONDS = 25
|
||||||
|
|
||||||
|
# After surpassing the soft timeout.
|
||||||
|
DEFAULT_HARD_TIMEOUT_SECONDS = 5
|
||||||
|
|
||||||
# The signals that should terminate both the master and workers.
|
# The signals that should terminate both the master and workers.
|
||||||
TERMINATE_SIGNALS = %i(INT TERM).freeze
|
TERMINATE_SIGNALS = %i(INT TERM).freeze
|
||||||
|
|
||||||
|
@ -62,7 +75,7 @@ def self.signal_processes(pids, signal)
|
||||||
# directory - The directory of the Rails application.
|
# directory - The directory of the Rails application.
|
||||||
#
|
#
|
||||||
# Returns an Array containing the PIDs of the started processes.
|
# Returns an Array containing the PIDs of the started processes.
|
||||||
def self.start(queues, env: :development, directory: Dir.pwd, max_concurrency: 50, min_concurrency: 0, timeout: CLI::DEFAULT_SOFT_TIMEOUT_SECONDS, dryrun: false)
|
def self.start(queues, env: :development, directory: Dir.pwd, max_concurrency: 50, min_concurrency: 0, timeout: DEFAULT_SOFT_TIMEOUT_SECONDS, dryrun: false)
|
||||||
queues.map.with_index do |pair, index|
|
queues.map.with_index do |pair, index|
|
||||||
start_sidekiq(pair, env: env,
|
start_sidekiq(pair, env: env,
|
||||||
directory: directory,
|
directory: directory,
|
|
@ -3,9 +3,11 @@
|
||||||
require 'fast_spec_helper'
|
require 'fast_spec_helper'
|
||||||
require 'rspec-parameterized'
|
require 'rspec-parameterized'
|
||||||
|
|
||||||
RSpec.describe Gitlab::SidekiqCluster::CLI do
|
require_relative '../../../sidekiq_cluster/cli'
|
||||||
|
|
||||||
|
RSpec.describe Gitlab::SidekiqCluster::CLI do # rubocop:disable RSpec/FilePath
|
||||||
let(:cli) { described_class.new('/dev/null') }
|
let(:cli) { described_class.new('/dev/null') }
|
||||||
let(:timeout) { described_class::DEFAULT_SOFT_TIMEOUT_SECONDS }
|
let(:timeout) { Gitlab::SidekiqCluster::DEFAULT_SOFT_TIMEOUT_SECONDS }
|
||||||
let(:default_options) do
|
let(:default_options) do
|
||||||
{ env: 'test', directory: Dir.pwd, max_concurrency: 50, min_concurrency: 0, dryrun: false, timeout: timeout }
|
{ env: 'test', directory: Dir.pwd, max_concurrency: 50, min_concurrency: 0, dryrun: false, timeout: timeout }
|
||||||
end
|
end
|
||||||
|
@ -103,7 +105,7 @@
|
||||||
|
|
||||||
it 'when not given', 'starts Sidekiq workers with default timeout' do
|
it 'when not given', 'starts Sidekiq workers with default timeout' do
|
||||||
expect(Gitlab::SidekiqCluster).to receive(:start)
|
expect(Gitlab::SidekiqCluster).to receive(:start)
|
||||||
.with([['foo']], default_options.merge(timeout: described_class::DEFAULT_SOFT_TIMEOUT_SECONDS))
|
.with([['foo']], default_options.merge(timeout: Gitlab::SidekiqCluster::DEFAULT_SOFT_TIMEOUT_SECONDS))
|
||||||
|
|
||||||
cli.run(%w(foo))
|
cli.run(%w(foo))
|
||||||
end
|
end
|
||||||
|
@ -271,7 +273,7 @@
|
||||||
expect(Gitlab::SidekiqCluster).to receive(:signal_processes)
|
expect(Gitlab::SidekiqCluster).to receive(:signal_processes)
|
||||||
.with([], "-KILL")
|
.with([], "-KILL")
|
||||||
|
|
||||||
stub_const("Gitlab::SidekiqCluster::CLI::CHECK_TERMINATE_INTERVAL_SECONDS", 0.1)
|
stub_const("Gitlab::SidekiqCluster::CHECK_TERMINATE_INTERVAL_SECONDS", 0.1)
|
||||||
allow(cli).to receive(:terminate_timeout_seconds) { 1 }
|
allow(cli).to receive(:terminate_timeout_seconds) { 1 }
|
||||||
|
|
||||||
cli.wait_for_termination
|
cli.wait_for_termination
|
||||||
|
@ -301,7 +303,7 @@
|
||||||
|
|
||||||
cli.run(%w(foo))
|
cli.run(%w(foo))
|
||||||
|
|
||||||
stub_const("Gitlab::SidekiqCluster::CLI::CHECK_TERMINATE_INTERVAL_SECONDS", 0.1)
|
stub_const("Gitlab::SidekiqCluster::CHECK_TERMINATE_INTERVAL_SECONDS", 0.1)
|
||||||
allow(cli).to receive(:terminate_timeout_seconds) { 1 }
|
allow(cli).to receive(:terminate_timeout_seconds) { 1 }
|
||||||
|
|
||||||
cli.wait_for_termination
|
cli.wait_for_termination
|
|
@ -1,9 +1,10 @@
|
||||||
# frozen_string_literal: true
|
# frozen_string_literal: true
|
||||||
|
|
||||||
require 'fast_spec_helper'
|
|
||||||
require 'rspec-parameterized'
|
require 'rspec-parameterized'
|
||||||
|
|
||||||
RSpec.describe Gitlab::SidekiqCluster do
|
require_relative '../../sidekiq_cluster/sidekiq_cluster'
|
||||||
|
|
||||||
|
RSpec.describe Gitlab::SidekiqCluster do # rubocop:disable RSpec/FilePath
|
||||||
describe '.trap_signals' do
|
describe '.trap_signals' do
|
||||||
it 'traps the given signals' do
|
it 'traps the given signals' do
|
||||||
expect(described_class).to receive(:trap).ordered.with(:INT)
|
expect(described_class).to receive(:trap).ordered.with(:INT)
|
|
@ -49,7 +49,7 @@
|
||||||
context 'when level is integration' do
|
context 'when level is integration' do
|
||||||
it 'returns a pattern' do
|
it 'returns a pattern' do
|
||||||
expect(subject.pattern(:integration))
|
expect(subject.pattern(:integration))
|
||||||
.to eq("spec/{controllers,mailers,requests}{,/**/}*_spec.rb")
|
.to eq("spec/{commands,controllers,mailers,requests}{,/**/}*_spec.rb")
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@ -131,7 +131,7 @@
|
||||||
context 'when level is integration' do
|
context 'when level is integration' do
|
||||||
it 'returns a regexp' do
|
it 'returns a regexp' do
|
||||||
expect(subject.regexp(:integration))
|
expect(subject.regexp(:integration))
|
||||||
.to eq(%r{spec/(controllers|mailers|requests)})
|
.to eq(%r{spec/(commands|controllers|mailers|requests)})
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@ -204,6 +204,10 @@
|
||||||
expect(subject.level_for('spec/mailers/abuse_report_mailer_spec.rb')).to eq(:integration)
|
expect(subject.level_for('spec/mailers/abuse_report_mailer_spec.rb')).to eq(:integration)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
it 'returns the correct level for an integration test in a subfolder' do
|
||||||
|
expect(subject.level_for('spec/commands/sidekiq_cluster/cli.rb')).to eq(:integration)
|
||||||
|
end
|
||||||
|
|
||||||
it 'returns the correct level for a system test' do
|
it 'returns the correct level for a system test' do
|
||||||
expect(subject.level_for('spec/features/abuse_report_spec.rb')).to eq(:system)
|
expect(subject.level_for('spec/features/abuse_report_spec.rb')).to eq(:system)
|
||||||
end
|
end
|
||||||
|
|
|
@ -54,6 +54,7 @@ class TestLevel
|
||||||
tooling
|
tooling
|
||||||
],
|
],
|
||||||
integration: %w[
|
integration: %w[
|
||||||
|
commands
|
||||||
controllers
|
controllers
|
||||||
mailers
|
mailers
|
||||||
requests
|
requests
|
||||||
|
|
Loading…
Reference in a new issue