diff --git a/Gemfile b/Gemfile index 40194cd65b389fadaceedc2044c837476e7c5169..f92bc38598820e3ededabf22180142e0b8877855 100644 --- a/Gemfile +++ b/Gemfile @@ -41,6 +41,7 @@ gem 'gitlab-safe_request_store', path: 'gems/gitlab-safe_request_store' # ruboco group :monorepo do gem 'gitlab-utils', path: 'gems/gitlab-utils' # rubocop:todo Gemfile/MissingFeatureCategory gem 'gitlab-backup-cli', path: 'gems/gitlab-backup-cli', feature_category: :backup_restore + gem 'gitlab-database-load_balancing', path: 'gems/gitlab-database-load_balancing', feature_category: :database end gem 'gitlab-secret_detection', path: 'gems/gitlab-secret_detection', feature_category: :secret_detection diff --git a/Gemfile.lock b/Gemfile.lock index 9d40284e424ff9bf20d780b8fffcc3f06c3f0d1b..70ae3c079711ea6dcbdac11c4ce4b264df08e979 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -29,6 +29,14 @@ PATH gitlab-backup-cli (0.0.1) thor (~> 1.3) +PATH + remote: gems/gitlab-database-load_balancing + specs: + gitlab-database-load_balancing (0.1.0) + gitlab-net-dns (~> 0.9.2) + pg (~> 1.5.4) + rails (~> 7.0.8) + PATH remote: gems/gitlab-http specs: @@ -1866,6 +1874,7 @@ DEPENDENCIES gitlab-backup-cli! gitlab-chronic (~> 0.10.5) gitlab-dangerfiles (~> 4.6.0) + gitlab-database-load_balancing! gitlab-experiment (~> 0.8.0) gitlab-fog-azure-rm (~> 1.8.0) gitlab-http! diff --git a/config/initializers/load_balancing.rb b/config/initializers/load_balancing.rb index aa6216025732ce0c4b197527d264a77730bd63a1..65295b400d9cd703197cd03faccba6a44300be8e 100644 --- a/config/initializers/load_balancing.rb +++ b/config/initializers/load_balancing.rb @@ -1,5 +1,31 @@ # frozen_string_literal: true +# Configure Load Balancing +Gitlab::Database::LoadBalancing.configure! do |load_balancer| + load_balancer.enabled = !Gitlab::Runtime.rake? + load_balancer.default_pool_size = Gitlab::Database.default_pool_size + load_balancer.base_models = ::Gitlab::Database.database_base_models_using_load_balancing.values.freeze + + Gitlab::Cluster::LifecycleEvents.on_worker_start do + load_balancer.enabled = !Gitlab::Runtime.rake? + load_balancer.default_pool_size = Gitlab::Database.default_pool_size + end +end + +# Configure Load Balancing interface callbacks +Gitlab::Database::LoadBalancing::Callbacks.configure! do |callbacks| + hosts_gauge = Gitlab::Metrics.gauge(:db_load_balancing_hosts, 'Current number of load balancing hosts') + + callbacks.logger_proc = -> { Gitlab::Database::LoadBalancing::Logger.build } + callbacks.metrics_host_gauge_proc = ->(labels, value) { hosts_gauge.set(labels, value) } + callbacks.track_exception_proc = ->(exception) { Gitlab::ErrorTracking.track_exception(exception) } + callbacks.get_wal_for_proc = ->(key) { Gitlab::Redis::DbLoadBalancing.with { |redis| redis.get(key) } } + callbacks.del_wal_for_proc = ->(key) { Gitlab::Redis::DbLoadBalancing.with { |redis| redis.del(key) } } + callbacks.set_wal_for_proc = ->(key, value, ex:) do + Gitlab::Redis::DbLoadBalancing.with { |redis| redis.set(key, value, ex: ex) } + end +end + Gitlab::Application.configure do |config| config.middleware.use(Gitlab::Database::LoadBalancing::RackMiddleware) diff --git a/gems/gitlab-database-load_balancing/.rubocop.yml b/gems/gitlab-database-load_balancing/.rubocop.yml index 583fa8227eec03099bb349d080ecc17feceb32ea..26b40847d087e92993181c911e010f0aa113b7a2 100644 --- a/gems/gitlab-database-load_balancing/.rubocop.yml +++ b/gems/gitlab-database-load_balancing/.rubocop.yml @@ -1,6 +1,54 @@ inherit_from: + - .rubocop_todo.yml - ../config/rubocop.yml Gemfile/MissingFeatureCategory: Exclude: - 'Gemfile' + +Database/MultipleDatabases: + Exclude: + - 'lib/gitlab/database/load_balancing/load_balancer.rb' + - 'spec/gitlab/database/load_balancing_spec.rb' + - 'spec/gitlab/database/load_balancing/host_list_spec.rb' + - 'spec/gitlab/database/load_balancing/host_spec.rb' + - 'spec/gitlab/database/load_balancing/load_balancer_spec.rb' + - 'spec/gitlab/database/load_balancing/setup_spec.rb' + - 'spec/gitlab/database/load_balancing/sticking_spec.rb' + +Graphql/ResolverType: + Exclude: + - 'lib/gitlab/database/load_balancing/resolver.rb' + - 'lib/gitlab/database/load_balancing/srv_resolver.rb' + +Rails/SkipsModelValidations: + Exclude: + - 'spec/gitlab/database/load_balancing/connection_proxy_spec.rb' + - 'spec/gitlab/database/load_balancing_spec.rb' + +RSpec/ExpectInHook: + Exclude: + - 'spec/gitlab/database/load_balancing/host_spec.rb' + - 'spec/gitlab/database/load_balancing/load_balancer_spec.rb' + +RSpec/ImplicitSubject: + Exclude: + - 'spec/gitlab/database/load_balancing/configuration_spec.rb' + +RSpec/MultipleMemoizedHelpers: + Max: 9 + +RSpec/VerifiedDoubles: + Exclude: + - 'spec/gitlab/database/load_balancing/configuration_spec.rb' + - 'spec/gitlab/database/load_balancing/connection_proxy_spec.rb' + - 'spec/gitlab/database/load_balancing/host_list_spec.rb' + - 'spec/gitlab/database/load_balancing/host_spec.rb' + - 'spec/gitlab/database/load_balancing/load_balancer_spec.rb' + - 'spec/gitlab/database/load_balancing/resolver_spec.rb' + - 'spec/gitlab/database/load_balancing/service_discovery_spec.rb' + - 'spec/gitlab/database/load_balancing/setup_spec.rb' + - 'spec/gitlab/database/load_balancing_spec.rb' + +Layout/LineLength: + Max: 129 diff --git a/gems/gitlab-database-load_balancing/.rubocop_todo.yml b/gems/gitlab-database-load_balancing/.rubocop_todo.yml new file mode 100644 index 0000000000000000000000000000000000000000..94be7317466be827329501d3cb252391fc9d204f --- /dev/null +++ b/gems/gitlab-database-load_balancing/.rubocop_todo.yml @@ -0,0 +1,42 @@ +Lint/AssignmentInCondition: + Exclude: + - 'lib/gitlab/database/load_balancing/configuration.rb' + - 'lib/gitlab/database/load_balancing/host.rb' + - 'lib/gitlab/database/load_balancing/load_balancer.rb' + +Lint/DuplicateBranch: + Exclude: + - 'lib/gitlab/database/load_balancing/load_balancer.rb' + +Lint/EmptyBlock: + Exclude: + - 'spec/gitlab/database/load_balancing/load_balancer_spec.rb' + +Lint/RedundantCopDisableDirective: + Exclude: + - 'lib/gitlab/database/load_balancing.rb' + +Lint/UnusedMethodArgument: + Exclude: + - 'lib/gitlab/database/load_balancing/connection_proxy.rb' + - 'lib/gitlab/database/load_balancing/primary_host.rb' + +Naming/RescuedExceptionsVariableName: + Exclude: + - 'lib/gitlab/database/load_balancing/load_balancer.rb' + - 'lib/gitlab/database/load_balancing/service_discovery.rb' + - 'spec/gitlab/database/load_balancing/host_spec.rb' + - 'spec/gitlab/database/load_balancing/load_balancer_spec.rb' + +Style/GuardClause: + Exclude: + - 'lib/gitlab/database/load_balancing/load_balancer.rb' + +Style/Lambda: + Exclude: + - 'lib/gitlab/database/load_balancing/action_cable_callbacks.rb' + +Style/RedundantSelf: + Exclude: + - 'spec/gitlab/database/load_balancing_spec.rb' + - 'lib/gitlab/database/load_balancing/service_discovery.rb' diff --git a/gems/gitlab-database-load_balancing/README.md b/gems/gitlab-database-load_balancing/README.md index e1a963dba5087cc767dc11a29c5817e7234d22a4..fb05fa373fe9dd1bf70a3c5b063aebc7992cebdd 100644 --- a/gems/gitlab-database-load_balancing/README.md +++ b/gems/gitlab-database-load_balancing/README.md @@ -1,3 +1,3 @@ # GitLab Database Load Balancing -This gem is a stub for a move of all `Gitlab::Database::LoadBalancing` code. +TODO diff --git a/lib/gitlab/database/load_balancing.rb b/gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing.rb similarity index 58% rename from lib/gitlab/database/load_balancing.rb rename to gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing.rb index 5f9416fb4db160bfd357a5edbd2cf1925303f04f..083af88d7698e11739624842b0c412759d85c697 100644 --- a/lib/gitlab/database/load_balancing.rb +++ b/gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing.rb @@ -1,5 +1,15 @@ # frozen_string_literal: true +require 'pg' +require 'active_support/time_with_zone' +require 'active_record' +require 'active_record/database_configurations' +require 'active_job' + +Dir['load_balancing/**/*.rb', base: File.dirname(__FILE__)].each do |file| + require_relative file +end + module Gitlab module Database module LoadBalancing @@ -18,8 +28,12 @@ module LoadBalancing ActiveRecord::ConnectionNotEstablished ].freeze - def self.base_models - @base_models ||= ::Gitlab::Database.database_base_models_using_load_balancing.values.freeze + mattr_accessor :base_models + mattr_accessor :default_pool_size + mattr_accessor :enabled, default: true + + def self.configure! + yield(self) end def self.each_load_balancer @@ -49,7 +63,7 @@ def self.release_hosts def self.db_role_for_connection(connection) return ROLE_UNKNOWN if connection.is_a?(::Gitlab::Database::LoadBalancing::ConnectionProxy) - db_config = Database.db_config_for_connection(connection) + db_config = db_config_for_connection(connection) return ROLE_UNKNOWN unless db_config if db_config.name.ends_with?(LoadBalancer::REPLICA_SUFFIX) @@ -58,6 +72,23 @@ def self.db_role_for_connection(connection) ROLE_PRIMARY end end + + def self.db_config_for_connection(connection) + return unless connection + + # For a ConnectionProxy we want to avoid ambiguous db_config as it may + # sometimes default to replica so we always return the primary config + # instead. + if connection.is_a?(::Gitlab::Database::LoadBalancing::ConnectionProxy) # rubocop:disable Style/IfUnlessModifier + return connection.load_balancer.configuration.db_config + end + + # During application init we might receive `NullPool` + return unless connection.respond_to?(:pool) && + connection.pool.respond_to?(:db_config) + + connection.pool.db_config + end end end end diff --git a/lib/gitlab/database/load_balancing/action_cable_callbacks.rb b/gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/action_cable_callbacks.rb similarity index 100% rename from lib/gitlab/database/load_balancing/action_cable_callbacks.rb rename to gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/action_cable_callbacks.rb diff --git a/gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/callbacks.rb b/gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/callbacks.rb new file mode 100644 index 0000000000000000000000000000000000000000..086819e016ba931af57f9ba5c673ce7e69c47d19 --- /dev/null +++ b/gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/callbacks.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LoadBalancing + module Callbacks + mattr_accessor :logger_proc + mattr_accessor :metrics_host_gauge_proc, :track_exception_proc + mattr_accessor :set_wal_for_proc, :get_wal_for_proc, :del_wal_for_proc + + NULL_LOGGER = Logger.new(File::NULL) + + def self.configure! + yield(self) + end + + def self.set_wal_for(key, wal, ex:) + set_wal_for_proc&.call(key, wal, ex: ex) + end + + def self.get_wal_for(key) + get_wal_for_proc&.call(key) + end + + def self.del_wal_for(key) + del_wal_for_proc&.call(key) + end + + def self.logger + logger_proc&.call || NULL_LOGGER + end + + def self.metrics_host_gauge(labels, value) + metrics_host_gauge_proc&.call(labels, value) + end + + def self.track_exception(ex) + track_exception_proc&.call(ex) + end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/configuration.rb b/gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/configuration.rb similarity index 90% rename from lib/gitlab/database/load_balancing/configuration.rb rename to gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/configuration.rb index 50472bd5780c6068b7db9b98f0673de11a8d6e3c..4bcc3fde99c0baa341c8b3038ad47f876ec65209 100644 --- a/lib/gitlab/database/load_balancing/configuration.rb +++ b/gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/configuration.rb @@ -1,13 +1,15 @@ # frozen_string_literal: true +require 'active_support/core_ext/numeric/bytes' + module Gitlab module Database module LoadBalancing # Configuration settings for a single LoadBalancer instance. class Configuration attr_accessor :hosts, :max_replication_difference, - :max_replication_lag_time, :replica_check_interval, - :service_discovery + :max_replication_lag_time, :replica_check_interval, + :service_discovery # Creates a configuration object for the given ActiveRecord model. def self.for_model(model) @@ -82,7 +84,7 @@ def pool_size # To support this scenario, we always attempt to read the pool size # from the model's configuration. @model.connection_db_config.configuration_hash[:pool] || - Database.default_pool_size + Gitlab::Database::LoadBalancing.default_pool_size end # Returns `true` if the use of load balancing replicas should be @@ -91,7 +93,7 @@ def pool_size # This is disabled for Rake tasks to ensure e.g. database migrations # always produce consistent results. def load_balancing_enabled? - return false if Gitlab::Runtime.rake? + return false unless Gitlab::Database::LoadBalancing.enabled hosts.any? || service_discovery_enabled? end @@ -99,7 +101,7 @@ def load_balancing_enabled? # This is disabled for Rake tasks to ensure e.g. database migrations # always produce consistent results. def service_discovery_enabled? - return false if Gitlab::Runtime.rake? + return false unless Gitlab::Database::LoadBalancing.enabled service_discovery[:record].present? end diff --git a/lib/gitlab/database/load_balancing/connection_proxy.rb b/gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/connection_proxy.rb similarity index 99% rename from lib/gitlab/database/load_balancing/connection_proxy.rb rename to gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/connection_proxy.rb index 67cde3e687d3a41fb396980a6586c91de1b350a8..7ae3e706b391d79050af33f0a8522c7ec5df82db 100644 --- a/lib/gitlab/database/load_balancing/connection_proxy.rb +++ b/gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/connection_proxy.rb @@ -149,3 +149,5 @@ def read_only_transaction? end end end + +# rubocop:enable GitlabSecurity/PublicSend diff --git a/lib/gitlab/database/load_balancing/host.rb b/gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/host.rb similarity index 95% rename from lib/gitlab/database/load_balancing/host.rb rename to gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/host.rb index 880324f3ae467573d0453f3e09af1f832a64840c..8acc7c01c19dfc1d70cd7ec29104731a65f82b52 100644 --- a/lib/gitlab/database/load_balancing/host.rb +++ b/gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/host.rb @@ -7,7 +7,8 @@ module LoadBalancing class Host attr_reader :pool, :last_checked_at, :intervals, :load_balancer, :host, :port - delegate :connection, :release_connection, :enable_query_cache!, :disable_query_cache!, :query_cache_enabled, to: :pool + delegate :connection, :release_connection, :enable_query_cache!, :disable_query_cache!, :query_cache_enabled, + to: :pool CONNECTION_ERRORS = [ ActionView::Template::Error, @@ -77,9 +78,9 @@ def initialize(host, load_balancer, port: nil) # timeout - The time after which the pool should be forcefully # disconnected. def disconnect!(timeout: 120) - start_time = ::Gitlab::Metrics::System.monotonic_time + start_time = ::Gitlab::Utils::System.monotonic_time - while (::Gitlab::Metrics::System.monotonic_time - start_time) <= timeout + while (::Gitlab::Utils::System.monotonic_time - start_time) <= timeout return if try_disconnect sleep(2) @@ -104,7 +105,7 @@ def force_disconnect! end def offline! - ::Gitlab::Database::LoadBalancing::Logger.warn( + ::Gitlab::Database::LoadBalancing::Callbacks.logger.warn( event: :host_offline, message: 'Marking host as offline', db_host: @host, @@ -124,7 +125,7 @@ def online? # Log that the host came back online if it was previously offline if @online && !was_online - ::Gitlab::Database::LoadBalancing::Logger.info( + ::Gitlab::Database::LoadBalancing::Callbacks.logger.info( event: :host_online, message: 'Host is online after replica status check', db_host: @host, @@ -132,7 +133,7 @@ def online? ) # Always log if the host goes offline elsif !@online - ::Gitlab::Database::LoadBalancing::Logger.warn( + ::Gitlab::Database::LoadBalancing::Callbacks.logger.warn( event: :host_offline, message: 'Host is offline after replica status check', db_host: @host, diff --git a/lib/gitlab/database/load_balancing/host_list.rb b/gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/host_list.rb similarity index 92% rename from lib/gitlab/database/load_balancing/host_list.rb rename to gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/host_list.rb index fb3175c7d5d899a0ac277aab20862ce6c90ae83c..7a3750f637b43081e7cc17c57557977507b9e834 100644 --- a/lib/gitlab/database/load_balancing/host_list.rb +++ b/gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/host_list.rb @@ -10,7 +10,6 @@ def initialize(hosts = []) @hosts = hosts.shuffle @index = 0 @mutex = Mutex.new - @hosts_gauge = Gitlab::Metrics.gauge(:db_load_balancing_hosts, 'Current number of load balancing hosts') set_metrics! end @@ -80,7 +79,7 @@ def next_host end def set_metrics! - @hosts_gauge.set({}, @hosts.length) + Callbacks.metrics_host_gauge({}, @hosts.length) end end end diff --git a/lib/gitlab/database/load_balancing/load_balancer.rb b/gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/load_balancer.rb similarity index 96% rename from lib/gitlab/database/load_balancing/load_balancer.rb rename to gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/load_balancer.rb index 55a27f89b36c8a2c2b2250dada435c6027b99f23..c89491b0338ae92016a598facb9b3fce55d55926 100644 --- a/lib/gitlab/database/load_balancing/load_balancer.rb +++ b/gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/load_balancer.rb @@ -76,7 +76,7 @@ def read(&block) # times before using the primary instead. will_retry = conflict_retried < @host_list.length * 3 - ::Gitlab::Database::LoadBalancing::Logger.warn( + ::Gitlab::Database::LoadBalancing::Callbacks.logger.warn( event: :host_query_conflict, message: 'Query conflict on host', conflict_retried: conflict_retried, @@ -101,7 +101,7 @@ def read(&block) end end - ::Gitlab::Database::LoadBalancing::Logger.warn( + ::Gitlab::Database::LoadBalancing::Callbacks.logger.warn( event: :no_secondaries_available, message: 'No secondaries were available, using primary instead', conflict_retried: conflict_retried, @@ -132,7 +132,7 @@ def read_write transaction_open = connection.transaction_open? if attempt && attempt > 1 - ::Gitlab::Database::LoadBalancing::Logger.warn( + ::Gitlab::Database::LoadBalancing::Callbacks.logger.warn( event: :read_write_retry, message: 'A read_write block was retried because of connection error' ) @@ -143,7 +143,7 @@ def read_write # No leaking will happen on the final attempt. Leaks are caused by subsequent retries not_final_attempt = attempt && attempt < attempts if transaction_open && connection_error?(e) && not_final_attempt - ::Gitlab::Database::LoadBalancing::Logger.warn( + ::Gitlab::Database::LoadBalancing::Callbacks.logger.warn( event: :transaction_leak, message: 'A write transaction has leaked during database fail-over' ) @@ -311,7 +311,6 @@ def create_replica_connection_pool(pool_size, host = nil, port = nil) # ActiveRecord::ConnectionAdapters::ConnectionHandler handles fetching, # and caching for connections pools for each "connection", so we # leverage that. - # rubocop:disable Database/MultipleDatabases def pool ActiveRecord::Base.connection_handler.retrieve_connection_pool( @configuration.connection_specification_name, @@ -319,7 +318,6 @@ def pool shard: ActiveRecord::Base.default_shard ) || raise(::ActiveRecord::ConnectionNotEstablished) end - # rubocop:enable Database/MultipleDatabases def wal_diff(location1, location2) read_write do |connection| @@ -368,7 +366,7 @@ def get_write_location(ar_connection) END AS location; NEWSQL else - <<~SQL + <<~SQL.squish SELECT pg_current_wal_insert_lsn()::text AS location SQL end @@ -380,7 +378,8 @@ def get_write_location(ar_connection) def raise_if_concurrent_ruby! Gitlab::Utils.raise_if_concurrent_ruby!(:db) rescue StandardError => e - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) + ::Gitlab::Database::LoadBalancing::Callbacks.track_exception(e) + raise if Rails.env.development? || Rails.env.test? end end end diff --git a/lib/gitlab/database/load_balancing/primary_host.rb b/gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/primary_host.rb similarity index 96% rename from lib/gitlab/database/load_balancing/primary_host.rb rename to gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/primary_host.rb index fb52b384ddb4a496ba57e56fee2339fc60c2464b..48a6c034fe6b021ba407c85da339cda7ecb70e60 100644 --- a/lib/gitlab/database/load_balancing/primary_host.rb +++ b/gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/primary_host.rb @@ -49,7 +49,7 @@ def disconnect!(timeout: 120) end def offline! - ::Gitlab::Database::LoadBalancing::Logger.warn( + ::Gitlab::Database::LoadBalancing::Callbacks.logger.warn( event: :host_offline, message: 'Marking primary host as offline' ) diff --git a/lib/gitlab/database/load_balancing/resolver.rb b/gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/resolver.rb similarity index 100% rename from lib/gitlab/database/load_balancing/resolver.rb rename to gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/resolver.rb diff --git a/lib/gitlab/database/load_balancing/service_discovery.rb b/gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/service_discovery.rb similarity index 94% rename from lib/gitlab/database/load_balancing/service_discovery.rb rename to gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/service_discovery.rb index 3d11f0f88c1515498d30df14eaea7deeefda8bf1..2d2a0cb3f3b35d7a741a9b05a796d3b0051279f7 100644 --- a/lib/gitlab/database/load_balancing/service_discovery.rb +++ b/gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/service_discovery.rb @@ -17,8 +17,7 @@ class ServiceDiscovery attr_accessor :refresh_thread, :refresh_thread_last_run, :refresh_thread_interruption_logged - attr_reader :interval, :record, :record_type, :disconnect_timeout, - :load_balancer + attr_reader :interval, :record, :record_type, :disconnect_timeout, :load_balancer MAX_SLEEP_ADJUSTMENT = 10 MAX_DISCOVERY_RETRIES = 3 @@ -96,9 +95,9 @@ def perform_service_discovery rescue StandardError => error # Any exceptions that might occur should be reported to # Sentry, instead of silently terminating this thread. - Gitlab::ErrorTracking.track_exception(error) + ::Gitlab::Database::LoadBalancing::Callbacks.track_exception(error) - Gitlab::Database::LoadBalancing::Logger.error( + ::Gitlab::Database::LoadBalancing::Callbacks.logger.error( event: :service_discovery_failure, message: "Service discovery encountered an error: #{error.message}", host_list_length: load_balancer.host_list.length @@ -123,7 +122,7 @@ def refresh_if_necessary current = addresses_from_load_balancer if from_dns != current - ::Gitlab::Database::LoadBalancing::Logger.info( + ::Gitlab::Database::LoadBalancing::Callbacks.logger.info( event: :host_list_update, message: "Updating the host list for service discovery", host_list_length: from_dns.length, @@ -211,7 +210,7 @@ def log_refresh_thread_interruption return if refresh_thread_last_run.blank? || refresh_thread_interruption_logged || (refresh_thread_last_run + DISCOVERY_THREAD_REFRESH_DELTA.minutes).future? - Gitlab::Database::LoadBalancing::Logger.error( + ::Gitlab::Database::LoadBalancing::Callbacks.logger.error( event: :service_discovery_refresh_thread_interrupt, refresh_thread_last_run: refresh_thread_last_run, thread_status: refresh_thread&.status&.to_s, @@ -232,12 +231,12 @@ def record_type_for(type) def addresses_from_srv_record(response) srv_resolver = SrvResolver.new(resolver, response.additional) - response.answer.map do |r| + response.answer.filter_map do |r| address = srv_resolver.address_for(r.host.to_s) next unless address Address.new(address.to_s, r.port) - end.compact + end end def addresses_from_a_record(resources) @@ -252,13 +251,13 @@ def sampler def disconnect_old_hosts(hosts) return unless hosts.present? - gentle_disconnect_start = ::Gitlab::Metrics::System.monotonic_time + gentle_disconnect_start = ::Gitlab::Utils::System.monotonic_time gentle_disconnect_deadline = gentle_disconnect_start + disconnect_timeout hosts_to_disconnect = hosts gentle_disconnect_duration = Benchmark.realtime do - while ::Gitlab::Metrics::System.monotonic_time < gentle_disconnect_deadline + while ::Gitlab::Utils::System.monotonic_time < gentle_disconnect_deadline hosts_to_disconnect = hosts_to_disconnect.reject(&:try_disconnect) break if hosts_to_disconnect.empty? @@ -275,7 +274,7 @@ def disconnect_old_hosts(hosts) formatted_hosts = hosts_to_disconnect.map { |h| "#{h.host}:#{h.port}" } total_disconnect_duration = gentle_disconnect_duration + force_disconnect_duration - ::Gitlab::Database::LoadBalancing::Logger.info( + ::Gitlab::Database::LoadBalancing::Callbacks.logger.info( event: :host_list_disconnection, message: "Disconnected #{formatted_hosts} old load balancing hosts after #{total_disconnect_duration}s", gentle_disconnect_duration_s: gentle_disconnect_duration, diff --git a/lib/gitlab/database/load_balancing/service_discovery/sampler.rb b/gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/service_discovery/sampler.rb similarity index 93% rename from lib/gitlab/database/load_balancing/service_discovery/sampler.rb rename to gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/service_discovery/sampler.rb index 4fc11cc1035f9239dfa1dca206dee54c021ed4e6..55e320fd42ecc7c456cadb601b580e0c082b97a7 100644 --- a/lib/gitlab/database/load_balancing/service_discovery/sampler.rb +++ b/gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/service_discovery/sampler.rb @@ -16,7 +16,7 @@ def initialize(max_replica_pools:, seed: Random.new_seed) def sample(addresses) return addresses if @max_replica_pools.nil? || addresses.count <= @max_replica_pools - ::Gitlab::Database::LoadBalancing::Logger.debug( + ::Gitlab::Database::LoadBalancing::Callbacks.logger.debug( event: :host_list_limit_exceeded, message: "Host list length exceeds max_replica_pools so random hosts will be chosen.", max_replica_pools: @max_replica_pools, @@ -38,7 +38,7 @@ def sample(addresses) while selected_addresses.count < @max_replica_pools # Loop over all hostnames grabbing one address at a time to # evenly distribute across all hostnames - addresses_by_host.each do |host, addresses| + addresses_by_host.each do |_host, addresses| next if addresses.empty? selected_addresses << addresses.pop diff --git a/lib/gitlab/database/load_balancing/session.rb b/gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/session.rb similarity index 95% rename from lib/gitlab/database/load_balancing/session.rb rename to gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/session.rb index 3682c9265c22127611db67ca05ea1679986fd0ee..8f5f1b06c19396568f738422f5d8dadbe9582f8f 100644 --- a/lib/gitlab/database/load_balancing/session.rb +++ b/gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/session.rb @@ -41,7 +41,7 @@ def use_primary! @use_primary = true end - def use_primary(&blk) + def use_primary(&_blk) used_primary = @use_primary @use_primary = true yield @@ -49,7 +49,7 @@ def use_primary(&blk) @use_primary = used_primary || @performed_write end - def ignore_writes(&block) + def ignore_writes(&_block) @ignore_writes = true yield @@ -66,7 +66,7 @@ def ignore_writes(&block) # # Write and ambiguous queries inside this block are still handled by # the primary. - def use_replicas_for_read_queries(&blk) + def use_replicas_for_read_queries(&_blk) previous_flag = @use_replicas_for_read_queries @use_replicas_for_read_queries = true yield @@ -89,7 +89,7 @@ def use_replicas_for_read_queries? # - If the queries are about to write # - The current session already performed writes # - It prefers to use primary, aka, use_primary or use_primary! were called - def fallback_to_replicas_for_ambiguous_queries(&blk) + def fallback_to_replicas_for_ambiguous_queries(&_blk) previous_flag = @fallback_to_replicas_for_ambiguous_queries @fallback_to_replicas_for_ambiguous_queries = true yield diff --git a/lib/gitlab/database/load_balancing/setup.rb b/gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/setup.rb similarity index 94% rename from lib/gitlab/database/load_balancing/setup.rb rename to gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/setup.rb index 2e65e1c8e56faefab1b0e58c091beb8978579bc6..d5be4800c6d7f16ca3c4ec74bf3db5baf958e41c 100644 --- a/lib/gitlab/database/load_balancing/setup.rb +++ b/gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/setup.rb @@ -18,7 +18,7 @@ def setup setup_connection_proxy setup_service_discovery - ::Gitlab::Database::LoadBalancing::Logger.debug( + ::Gitlab::Database::LoadBalancing::Callbacks.logger.debug( event: :setup, model: model.name, start_service_discovery: @start_service_discovery @@ -30,7 +30,7 @@ def configure_connection hash = db_config_object.configuration_hash.merge( prepared_statements: false, - pool: Gitlab::Database.default_pool_size + pool: Gitlab::Database::LoadBalancing.default_pool_size ) hash_config = ActiveRecord::DatabaseConfigurations::HashConfig.new( diff --git a/lib/gitlab/database/load_balancing/srv_resolver.rb b/gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/srv_resolver.rb similarity index 84% rename from lib/gitlab/database/load_balancing/srv_resolver.rb rename to gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/srv_resolver.rb index 1f599ef4a273f9dc48d7cc727d23a96eefd75208..312c95dd0283e550e766028f666c076ee6944b7d 100644 --- a/lib/gitlab/database/load_balancing/srv_resolver.rb +++ b/gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/srv_resolver.rb @@ -28,12 +28,11 @@ def address_for(host) private def addresses_from_additional - strong_memoize(:addresses_from_additional) do - additional.each_with_object({}) do |rr, h| - h[rr.name] = rr.address if rr.is_a?(Net::DNS::RR::A) || rr.is_a?(Net::DNS::RR::AAAA) - end + additional.each_with_object({}) do |rr, h| + h[rr.name] = rr.address if rr.is_a?(Net::DNS::RR::A) || rr.is_a?(Net::DNS::RR::AAAA) end end + strong_memoize_attr :addresses_from_additional def resolve_host(host) record = resolver.search(host, Net::DNS::ANY).answer.find do |rr| diff --git a/lib/gitlab/database/load_balancing/sticking.rb b/gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/sticking.rb similarity index 89% rename from lib/gitlab/database/load_balancing/sticking.rb rename to gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/sticking.rb index df07f510c8d2f1fcd7ee5fc9a1b24194b4784f55..dfeb30c47ecc8196c2609d7b1f0e52503c9bedcb 100644 --- a/lib/gitlab/database/load_balancing/sticking.rb +++ b/gems/gitlab-database-load_balancing/lib/gitlab/database/load_balancing/sticking.rb @@ -74,21 +74,15 @@ def with_primary_write_location end def unstick(namespace, id) - with_redis do |redis| - redis.del(redis_key_for(namespace, id)) - end + Callbacks.del_wal_for(redis_key_for(namespace, id)) end def set_write_location_for(namespace, id, location) - with_redis do |redis| - redis.set(redis_key_for(namespace, id), location, ex: EXPIRATION) - end + Callbacks.set_wal_for(redis_key_for(namespace, id), location, ex: EXPIRATION) end def last_write_location_for(namespace, id) - with_redis do |redis| - redis.get(redis_key_for(namespace, id)) - end + Callbacks.get_wal_for(redis_key_for(namespace, id)) end def redis_key_for(namespace, id) @@ -96,10 +90,6 @@ def redis_key_for(namespace, id) "database-load-balancing/write-location/#{name}/#{namespace}/#{id}" end - - def with_redis(&block) - Gitlab::Redis::DbLoadBalancing.with(&block) - end end end end diff --git a/spec/fixtures/dns/a_rr.json b/gems/gitlab-database-load_balancing/spec/fixtures/dns/a_rr.json similarity index 100% rename from spec/fixtures/dns/a_rr.json rename to gems/gitlab-database-load_balancing/spec/fixtures/dns/a_rr.json diff --git a/spec/fixtures/dns/a_with_aaaa_rr_in_additional_section.json b/gems/gitlab-database-load_balancing/spec/fixtures/dns/a_with_aaaa_rr_in_additional_section.json similarity index 100% rename from spec/fixtures/dns/a_with_aaaa_rr_in_additional_section.json rename to gems/gitlab-database-load_balancing/spec/fixtures/dns/a_with_aaaa_rr_in_additional_section.json diff --git a/spec/fixtures/dns/aaaa_rr.json b/gems/gitlab-database-load_balancing/spec/fixtures/dns/aaaa_rr.json similarity index 100% rename from spec/fixtures/dns/aaaa_rr.json rename to gems/gitlab-database-load_balancing/spec/fixtures/dns/aaaa_rr.json diff --git a/spec/fixtures/dns/srv_with_a_rr_in_additional_section.json b/gems/gitlab-database-load_balancing/spec/fixtures/dns/srv_with_a_rr_in_additional_section.json similarity index 100% rename from spec/fixtures/dns/srv_with_a_rr_in_additional_section.json rename to gems/gitlab-database-load_balancing/spec/fixtures/dns/srv_with_a_rr_in_additional_section.json diff --git a/spec/lib/gitlab/database/load_balancing/action_cable_callbacks_spec.rb b/gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/action_cable_callbacks_spec.rb similarity index 85% rename from spec/lib/gitlab/database/load_balancing/action_cable_callbacks_spec.rb rename to gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/action_cable_callbacks_spec.rb index a57f02b22dfcbfbdf7d39faed9ecea867fc398bb..2c69f39f131c5fd107e658d967f9a999f4bc8699 100644 --- a/spec/lib/gitlab/database/load_balancing/action_cable_callbacks_spec.rb +++ b/gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/action_cable_callbacks_spec.rb @@ -8,7 +8,7 @@ expect(Gitlab::Database::LoadBalancing).to receive(:release_hosts) expect(Gitlab::Database::LoadBalancing::Session).to receive(:clear_session) - described_class.wrapper.call(nil, lambda {}) + described_class.wrapper.call(nil, -> {}) end context 'with an exception' do @@ -17,7 +17,7 @@ expect(Gitlab::Database::LoadBalancing::Session).to receive(:clear_session) expect do - described_class.wrapper.call(nil, lambda { raise 'test_exception' }) + described_class.wrapper.call(nil, -> { raise 'test_exception' }) end.to raise_error('test_exception') end end diff --git a/spec/lib/gitlab/database/load_balancing/configuration_spec.rb b/gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/configuration_spec.rb similarity index 94% rename from spec/lib/gitlab/database/load_balancing/configuration_spec.rb rename to gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/configuration_spec.rb index 7dc2e0be3e5e90b6ede8c9554bdd4fbedac42bf7..41534c17c0629310e0a54654ccd49093b3115bf0 100644 --- a/spec/lib/gitlab/database/load_balancing/configuration_spec.rb +++ b/gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/configuration_spec.rb @@ -26,7 +26,7 @@ use_tcp: false, max_replica_pools: nil ) - expect(config.pool_size).to eq(Gitlab::Database.default_pool_size) + expect(config.pool_size).to eq(Gitlab::Database::LoadBalancing.default_pool_size) end end @@ -110,7 +110,7 @@ it 'returns false when running inside a Rake task' do config = described_class.new(ActiveRecord::Base, %w[foo bar]) - allow(Gitlab::Runtime).to receive(:rake?).and_return(true) + allow(Gitlab::Database::LoadBalancing).to receive(:enabled).and_return(false) expect(config.load_balancing_enabled?).to eq(false) end @@ -137,7 +137,7 @@ describe '#service_discovery_enabled?' do it 'returns false when running inside a Rake task' do - allow(Gitlab::Runtime).to receive(:rake?).and_return(true) + allow(Gitlab::Database::LoadBalancing).to receive(:enabled).and_return(false) config = described_class.new(ActiveRecord::Base) config.service_discovery[:record] = 'foo' @@ -184,7 +184,7 @@ it 'returns the default pool size' do config = described_class.new(model) - expect(config.pool_size).to eq(Gitlab::Database.default_pool_size) + expect(config.pool_size).to eq(Gitlab::Database::LoadBalancing.default_pool_size) end end end diff --git a/spec/lib/gitlab/database/load_balancing/connection_proxy_spec.rb b/gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/connection_proxy_spec.rb similarity index 97% rename from spec/lib/gitlab/database/load_balancing/connection_proxy_spec.rb rename to gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/connection_proxy_spec.rb index a2076f5b95037d875e0b235ce5b52532a83960ea..d879f46c6148f9f9f21fd2c70ffbf21604122fef 100644 --- a/spec/lib/gitlab/database/load_balancing/connection_proxy_spec.rb +++ b/gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/connection_proxy_spec.rb @@ -16,7 +16,7 @@ end describe '#select_all' do - let(:override_proxy) { ActiveRecord::Base.connection.class } + let(:override_proxy) { ApplicationRecord.connection.class } # We can't use :Gitlab::Utils::Override because this method is dynamically prepended it 'method signatures match' do @@ -141,7 +141,7 @@ .and_return(session) end - context 'session fallbacks ambiguous queries to replicas' do + context 'when session fallbacks ambiguous queries to replicas' do let(:replica) { double(:connection) } before do @@ -174,7 +174,7 @@ end end - context 'session does not fallback to replicas for ambiguous queries' do + context 'when session does not fallback to replicas for ambiguous queries' do let(:primary) { double(:connection) } before do @@ -227,7 +227,7 @@ .not_to raise_error end - context 'current session prefers to fallback ambiguous queries to replicas' do + context 'when current session prefers to fallback ambiguous queries to replicas' do let(:session) { double(:session) } before do diff --git a/spec/lib/gitlab/database/load_balancing/host_list_spec.rb b/gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/host_list_spec.rb similarity index 93% rename from spec/lib/gitlab/database/load_balancing/host_list_spec.rb rename to gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/host_list_spec.rb index 9bb8116c434e258ecb6e0f12b3e9ed19364ea723..0f4e21bb6d5e4839738520656dcdbb2d21485624 100644 --- a/spec/lib/gitlab/database/load_balancing/host_list_spec.rb +++ b/gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/host_list_spec.rb @@ -19,6 +19,8 @@ allow(load_balancer).to receive(:create_replica_connection_pool) do double(:replica_connection_pool) end + + allow(Gitlab::Database::LoadBalancing::Callbacks).to receive(:metrics_host_gauge) end describe '#initialize' do @@ -136,6 +138,8 @@ end def expect_metrics(hosts) - expect(Gitlab::Metrics.registry.get(:db_load_balancing_hosts).get({})).to eq(hosts) + expect(Gitlab::Database::LoadBalancing::Callbacks).to have_received(:metrics_host_gauge) + .with({}, hosts) + .at_least(1).times end end diff --git a/spec/lib/gitlab/database/load_balancing/host_spec.rb b/gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/host_spec.rb similarity index 96% rename from spec/lib/gitlab/database/load_balancing/host_spec.rb rename to gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/host_spec.rb index 89cecaff075ebf5b48c0de9841ad13d9fcd84523..f3008bd528097d78182d2dec0aea03d39a315c9a 100644 --- a/spec/lib/gitlab/database/load_balancing/host_spec.rb +++ b/gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/host_spec.rb @@ -83,7 +83,7 @@ def wrapped_exception(wrapper, original) it 'marks the host as offline' do expect(host.pool).to receive(:disconnect!) - expect(Gitlab::Database::LoadBalancing::Logger).to receive(:warn) + expect(Gitlab::Database::LoadBalancing::Callbacks.logger).to receive(:warn) .with(hash_including(event: :host_offline)) .and_call_original @@ -99,8 +99,8 @@ def wrapped_exception(wrapper, original) it 'returns the latest status' do expect(host).not_to receive(:refresh_status) - expect(Gitlab::Database::LoadBalancing::Logger).not_to receive(:info) - expect(Gitlab::Database::LoadBalancing::Logger).not_to receive(:warn) + expect(Gitlab::Database::LoadBalancing::Callbacks.logger).not_to receive(:info) + expect(Gitlab::Database::LoadBalancing::Callbacks.logger).not_to receive(:warn) expect(host).to be_online end @@ -109,8 +109,8 @@ def wrapped_exception(wrapper, original) host.offline! expect(host).not_to receive(:refresh_status) - expect(Gitlab::Database::LoadBalancing::Logger).not_to receive(:info) - expect(Gitlab::Database::LoadBalancing::Logger).not_to receive(:warn) + expect(Gitlab::Database::LoadBalancing::Callbacks.logger).not_to receive(:info) + expect(Gitlab::Database::LoadBalancing::Callbacks.logger).not_to receive(:warn) expect(host).not_to be_online end @@ -131,7 +131,7 @@ def wrapped_exception(wrapper, original) # Hosts are online by default it 'does not log the online event' do - expect(Gitlab::Database::LoadBalancing::Logger) + expect(Gitlab::Database::LoadBalancing::Callbacks.logger) .not_to receive(:info) .with(hash_including(event: :host_online)) @@ -145,7 +145,7 @@ def wrapped_exception(wrapper, original) end it 'logs the online event' do - expect(Gitlab::Database::LoadBalancing::Logger) + expect(Gitlab::Database::LoadBalancing::Callbacks.logger) .to receive(:info) .with(hash_including(event: :host_online)) .and_call_original @@ -160,7 +160,7 @@ def wrapped_exception(wrapper, original) end it 'marks the host offline' do - expect(Gitlab::Database::LoadBalancing::Logger).to receive(:warn) + expect(Gitlab::Database::LoadBalancing::Callbacks.logger).to receive(:warn) .with(hash_including(event: :host_offline)) .and_call_original diff --git a/spec/lib/gitlab/database/load_balancing/load_balancer_spec.rb b/gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/load_balancer_spec.rb similarity index 98% rename from spec/lib/gitlab/database/load_balancing/load_balancer_spec.rb rename to gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/load_balancer_spec.rb index 3c14dc23a8083ad3cd5c29e2a8711c970da7b45b..24bdb7ee6f1a93a371ed7c45756acb2188db20d1 100644 --- a/spec/lib/gitlab/database/load_balancing/load_balancer_spec.rb +++ b/gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/load_balancer_spec.rb @@ -191,7 +191,7 @@ def twice_wrapped_exception(top, middle, original) lb.read { raise conflict_error } end - context 'only primary is configured' do + context 'when only primary is configured' do let(:lb) do config = Gitlab::Database::LoadBalancing::Configuration.new(ActiveRecord::Base) allow(config).to receive(:load_balancing_enabled?).and_return(false) @@ -230,7 +230,7 @@ def twice_wrapped_exception(top, middle, original) # When no hosts are configured, we don't want to produce any warnings, as # they aren't useful/too noisy. - expect(Gitlab::Database::LoadBalancing::Logger).not_to receive(:warn) + expect(Gitlab::Database::LoadBalancing::Callbacks.logger).not_to receive(:warn) expect { |b| lb.read(&b) } .to yield_with_args(ActiveRecord::Base.retrieve_connection) @@ -594,7 +594,11 @@ def with_replica_pool(*args) it 'returns the diff between two write locations' do loc1 = lb.send(:get_write_location, lb.pool.connection) - create(:user) # This ensures we get a new WAL location + ActiveRecord::Schema.define do + create_table :_test_load_balancing_test, force: true do |t| + t.string :name, null: true + end + end loc2 = lb.send(:get_write_location, lb.pool.connection) diff = lb.wal_diff(loc2, loc1) diff --git a/spec/lib/gitlab/database/load_balancing/primary_host_spec.rb b/gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/primary_host_spec.rb similarity index 96% rename from spec/lib/gitlab/database/load_balancing/primary_host_spec.rb rename to gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/primary_host_spec.rb index 53605d14c1726d81b14c4c7d0b27430eceb6578c..d1e1b3eda46106b61008f5a004a701c87a81467b 100644 --- a/spec/lib/gitlab/database/load_balancing/primary_host_spec.rb +++ b/gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/primary_host_spec.rb @@ -52,7 +52,7 @@ describe '#offline!' do it 'logs the event but does nothing else' do - expect(Gitlab::Database::LoadBalancing::Logger).to receive(:warn) + expect(Gitlab::Database::LoadBalancing::Callbacks.logger).to receive(:warn) .with(hash_including(event: :host_offline)) .and_call_original diff --git a/spec/lib/gitlab/database/load_balancing/resolver_spec.rb b/gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/resolver_spec.rb similarity index 100% rename from spec/lib/gitlab/database/load_balancing/resolver_spec.rb rename to gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/resolver_spec.rb diff --git a/spec/lib/gitlab/database/load_balancing/service_discovery/sampler_spec.rb b/gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/service_discovery/sampler_spec.rb similarity index 88% rename from spec/lib/gitlab/database/load_balancing/service_discovery/sampler_spec.rb rename to gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/service_discovery/sampler_spec.rb index 1a49aa2871fab4645fef63d06a5a1f3ff22a5e16..b29ea1bc46c779f2c51ca230ed2fafcb9608d6e3 100644 --- a/spec/lib/gitlab/database/load_balancing/service_discovery/sampler_spec.rb +++ b/gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/service_discovery/sampler_spec.rb @@ -26,10 +26,10 @@ it 'samples random ports across all hosts' do expect(sampler.sample(addresses)).to eq([ - address_class.new("127.0.0.1", 6432), - address_class.new("127.0.0.2", 6435), - address_class.new("127.0.0.1", 6435) - ]) + address_class.new("127.0.0.1", 6432), + address_class.new("127.0.0.2", 6435), + address_class.new("127.0.0.1", 6435) + ]) end it 'returns the same answer for the same input when called multiple times' do diff --git a/spec/lib/gitlab/database/load_balancing/service_discovery_spec.rb b/gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/service_discovery_spec.rb similarity index 96% rename from spec/lib/gitlab/database/load_balancing/service_discovery_spec.rb rename to gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/service_discovery_spec.rb index 442fa678d4efc071c60462169280f1b4c03c4c5d..1ff6cb381dd1ada04cc88b45fd0112ab977d3afa 100644 --- a/spec/lib/gitlab/database/load_balancing/service_discovery_spec.rb +++ b/gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/service_discovery_spec.rb @@ -88,7 +88,7 @@ expect(service).not_to receive(:sleep) - expect(Gitlab::ErrorTracking).not_to receive(:track_exception) + expect(Gitlab::Database::LoadBalancing::Callbacks).not_to receive(:track_exception) service.perform_service_discovery end @@ -96,7 +96,7 @@ context 'with StandardError' do before do - allow(Gitlab::ErrorTracking).to receive(:track_exception) + allow(Gitlab::Database::LoadBalancing::Callbacks).to receive(:track_exception) allow(service).to receive(:sleep) end @@ -139,7 +139,7 @@ .to receive(:refresh_if_necessary) .and_raise(error).exactly(described_class::MAX_DISCOVERY_RETRIES).times - expect(Gitlab::ErrorTracking) + expect(Gitlab::Database::LoadBalancing::Callbacks) .to receive(:track_exception) .with(error).exactly(described_class::MAX_DISCOVERY_RETRIES).times @@ -251,7 +251,7 @@ end it 'does not log any load balancing event' do - expect(::Gitlab::Database::LoadBalancing::Logger).not_to receive(:info) + expect(::Gitlab::Database::LoadBalancing::Callbacks.logger).not_to receive(:info) service.replace_hosts([address_foo, address_bar]) end @@ -460,7 +460,7 @@ it 'does not log any interruption' do expect(service.refresh_thread_last_run).to be_nil - expect(Gitlab::Database::LoadBalancing::Logger).not_to receive(:error) + expect(Gitlab::Database::LoadBalancing::Callbacks.logger).not_to receive(:error) subject end @@ -470,7 +470,7 @@ let(:last_run_timestamp) { Time.current } it 'does not log if last run time plus delta is in future' do - expect(Gitlab::Database::LoadBalancing::Logger).not_to receive(:error) + expect(Gitlab::Database::LoadBalancing::Callbacks.logger).not_to receive(:error) subject end @@ -482,7 +482,7 @@ it 'does not log if the interruption is already logged' do service.refresh_thread_interruption_logged = true - expect(Gitlab::Database::LoadBalancing::Logger).not_to receive(:error) + expect(Gitlab::Database::LoadBalancing::Callbacks.logger).not_to receive(:error) subject end @@ -490,7 +490,7 @@ it 'logs the error if the interruption was not logged before' do expect(service.refresh_thread_interruption_logged).not_to be_present - expect(Gitlab::Database::LoadBalancing::Logger).to receive(:error).with( + expect(Gitlab::Database::LoadBalancing::Callbacks.logger).to receive(:error).with( event: :service_discovery_refresh_thread_interrupt, refresh_thread_last_run: last_run_timestamp, thread_status: refresh_thread.status.to_s, diff --git a/spec/lib/gitlab/database/load_balancing/session_spec.rb b/gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/session_spec.rb similarity index 100% rename from spec/lib/gitlab/database/load_balancing/session_spec.rb rename to gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/session_spec.rb diff --git a/spec/lib/gitlab/database/load_balancing/setup_spec.rb b/gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/setup_spec.rb similarity index 97% rename from spec/lib/gitlab/database/load_balancing/setup_spec.rb rename to gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/setup_spec.rb index fa6d71bca7fab962ba1910564db63b562324a02a..2e62dd0f95d4439965e2bd72c3cfd86604a94c92 100644 --- a/spec/lib/gitlab/database/load_balancing/setup_spec.rb +++ b/gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/setup_spec.rb @@ -30,7 +30,7 @@ .with('test', 'main', { host: 'localhost', prepared_statements: false, - pool: Gitlab::Database.default_pool_size + pool: Gitlab::Database::LoadBalancing.default_pool_size }) .and_call_original @@ -119,7 +119,7 @@ end end - context 'uses correct base models', :reestablished_active_record_base do + context 'when using correct base models', :reestablished_active_record_base do using RSpec::Parameterized::TableSyntax let(:ci_class) do diff --git a/spec/lib/gitlab/database/load_balancing/srv_resolver_spec.rb b/gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/srv_resolver_spec.rb similarity index 91% rename from spec/lib/gitlab/database/load_balancing/srv_resolver_spec.rb rename to gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/srv_resolver_spec.rb index 6ac0608d4851de8bc46072513861c4c2939fb589..27c8a0b7c26420305ccf351175eb69e9c8d70c94 100644 --- a/spec/lib/gitlab/database/load_balancing/srv_resolver_spec.rb +++ b/gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/srv_resolver_spec.rb @@ -52,8 +52,9 @@ end def dns_response_packet_from_fixture(fixture_name) - fixture = File.read(Rails.root + "spec/fixtures/dns/#{fixture_name}.json") - encoded_payload = Gitlab::Json.parse(fixture)['payload'] + path = File.join(File.dirname(__FILE__), "../../../fixtures/dns/#{fixture_name}.json") + fixture = File.read(path) + encoded_payload = JSON.parse(fixture)['payload'] payload = Base64.decode64(encoded_payload) Net::DNS::Packet.parse(payload) diff --git a/spec/lib/gitlab/database/load_balancing/sticking_spec.rb b/gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/sticking_spec.rb similarity index 87% rename from spec/lib/gitlab/database/load_balancing/sticking_spec.rb rename to gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/sticking_spec.rb index 8c2901c3b8994e58c870ffc6a25a757987f76ccc..0e6dd4b354a6e82c3591b3b6819072c5a52b802d 100644 --- a/spec/lib/gitlab/database/load_balancing/sticking_spec.rb +++ b/gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/sticking_spec.rb @@ -11,16 +11,12 @@ described_class.new(load_balancer) end - let(:redis) { instance_double(::Gitlab::Redis::MultiStore) } - before do - allow(::Gitlab::Redis::DbLoadBalancing).to receive(:with).and_yield(redis) - allow(ActiveRecord::Base.load_balancer) .to receive(:primary_write_location) .and_return(primary_write_location) - allow(redis).to receive(:get) + allow(Gitlab::Database::LoadBalancing::Callbacks).to receive(:get_wal_for) .with("database-load-balancing/write-location/#{load_balancer.name}/user/42") .and_return(last_write_location) end @@ -47,7 +43,7 @@ it 'returns false, does not unstick and calls use_primary!' do expect(load_balancer).not_to receive(:select_up_to_date_host) - expect(redis).not_to receive(:del) + expect(Gitlab::Database::LoadBalancing::Callbacks).not_to receive(:del_wal_for) expect(::Gitlab::Database::LoadBalancing::Session.current).to receive(:use_primary!) expect(sticking.find_caught_up_replica(:user, 42, use_primary_on_empty_location: true)).to eq(false) @@ -60,8 +56,7 @@ expect(load_balancer).to receive(:select_up_to_date_host).with(last_write_location) .and_return(::Gitlab::Database::LoadBalancing::LoadBalancer::ALL_CAUGHT_UP) - expect(redis) - .to receive(:del) + expect(Gitlab::Database::LoadBalancing::Callbacks).to receive(:del_wal_for) .with("database-load-balancing/write-location/#{load_balancer.name}/user/42") expect(sticking.find_caught_up_replica(:user, 42)).to eq(true) @@ -73,7 +68,7 @@ expect(load_balancer).to receive(:select_up_to_date_host).with(last_write_location) .and_return(::Gitlab::Database::LoadBalancing::LoadBalancer::ANY_CAUGHT_UP) - expect(redis).not_to receive(:del) + expect(Gitlab::Database::LoadBalancing::Callbacks).not_to receive(:del_wal_for) expect(sticking.find_caught_up_replica(:user, 42)).to eq(true) end @@ -86,7 +81,7 @@ end it 'returns false, does not unstick and calls use_primary!' do - expect(redis).not_to receive(:del) + expect(Gitlab::Database::LoadBalancing::Callbacks).not_to receive(:del_wal_for) expect(::Gitlab::Database::LoadBalancing::Session.current).to receive(:use_primary!) expect(sticking.find_caught_up_replica(:user, 42)).to eq(false) @@ -94,7 +89,7 @@ context 'when use_primary_on_failure is false' do it 'does not call use_primary!' do - expect(redis).not_to receive(:del) + expect(Gitlab::Database::LoadBalancing::Callbacks).not_to receive(:del_wal_for) expect(::Gitlab::Database::LoadBalancing::Session.current).not_to receive(:use_primary!) expect(sticking.find_caught_up_replica(:user, 42, use_primary_on_failure: false)).to eq(false) @@ -110,8 +105,7 @@ .and_return(false) ids.each do |id| - expect(redis) - .to receive(:set) + expect(Gitlab::Database::LoadBalancing::Callbacks).to receive(:set_wal_for) .with("database-load-balancing/write-location/#{load_balancer.name}/user/#{id}", 'the-primary-lsn', ex: 30) end diff --git a/spec/lib/gitlab/database/load_balancing/transaction_leaking_spec.rb b/gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/transaction_leaking_spec.rb similarity index 87% rename from spec/lib/gitlab/database/load_balancing/transaction_leaking_spec.rb rename to gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/transaction_leaking_spec.rb index 56fbaef031dd26f551dd79b7f2c42854478df87d..16a09912df823b0b80938bb5c88628b54b029930 100644 --- a/spec/lib/gitlab/database/load_balancing/transaction_leaking_spec.rb +++ b/gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing/transaction_leaking_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe 'Load balancer behavior with errors inside a transaction', :redis, :delete, feature_category: :database do # rubocop:disable Layout/LineLength +RSpec.describe 'Load balancer behavior with errors inside a transaction', :redis, :delete, feature_category: :database do include StubENV let(:model) { ActiveRecord::Base } let(:db_host) { model.connection_pool.db_config.host } @@ -17,7 +17,7 @@ Gitlab::Database::LoadBalancing::Setup.new(model).setup - model.connection.execute(<<~SQL) + model.connection.execute(<<~SQL.squish) CREATE TABLE IF NOT EXISTS #{test_table_name} (id SERIAL PRIMARY KEY, value INTEGER) SQL @@ -27,7 +27,7 @@ end after do - model.connection.execute(<<~SQL) + model.connection.execute(<<~SQL.squish) DROP TABLE IF EXISTS #{test_table_name} SQL @@ -43,7 +43,7 @@ def execute(conn) # This will result in a PG error, which is not raised. # Instead, we retry the statement on a fresh connection (where the pid is different and it does nothing) # and the load balancer continues with a fresh connection and no transaction if a transaction was open previously - conn.execute(<<~SQL) + conn.execute(<<~SQL.squish) SELECT CASE WHEN pg_backend_pid() = #{backend_pid} THEN pg_terminate_backend(#{backend_pid}) @@ -57,7 +57,7 @@ def execute(conn) context 'in a transaction' do it 'raises an exception when a retry would occur' do - expect(::Gitlab::Database::LoadBalancing::Logger) + expect(::Gitlab::Database::LoadBalancing::Callbacks.logger) .not_to receive(:warn).with(hash_including(event: :transaction_leak)) expect do @@ -70,9 +70,9 @@ def execute(conn) context 'without a transaction' do it 'retries' do - expect(::Gitlab::Database::LoadBalancing::Logger) + expect(::Gitlab::Database::LoadBalancing::Callbacks.logger) .not_to receive(:warn).with(hash_including(event: :transaction_leak)) - expect(::Gitlab::Database::LoadBalancing::Logger) + expect(::Gitlab::Database::LoadBalancing::Callbacks.logger) .to receive(:warn).with(hash_including(event: :read_write_retry)) expect { execute(model.connection) }.not_to raise_error diff --git a/spec/lib/gitlab/database/load_balancing_spec.rb b/gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing_spec.rb similarity index 96% rename from spec/lib/gitlab/database/load_balancing_spec.rb rename to gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing_spec.rb index f65c27688b8e2a9e084d9bf8f10af7017f41ba46..e0e11ed7e079f332d718b7c218aad38765fc4e9a 100644 --- a/spec/lib/gitlab/database/load_balancing_spec.rb +++ b/gems/gitlab-database-load_balancing/spec/gitlab/database/load_balancing_spec.rb @@ -3,22 +3,6 @@ require 'spec_helper' RSpec.describe Gitlab::Database::LoadBalancing, :suppress_gitlab_schemas_validate_connection, feature_category: :cell do - describe '.base_models' do - it 'returns the models to apply load balancing to' do - models = described_class.base_models - - expect(models).to include(ActiveRecord::Base) - - if Gitlab::Database.has_config?(:ci) - expect(models).to include(Ci::ApplicationRecord) - end - end - - it 'returns the models as a frozen array' do - expect(described_class.base_models).to be_frozen - end - end - describe '.each_load_balancer' do it 'yields every load balancer to the supplied block' do lbs = [] @@ -493,7 +477,7 @@ end end - context 'custom connection handling' do + context 'with custom connection handling' do where(:queries, :expected_role) do [ # Reload cache. The schema loading queries should be handled by @@ -558,7 +542,7 @@ end end - context 'a write inside a transaction inside fallback_to_replicas_for_ambiguous_queries block' do + context 'with a write inside a transaction inside fallback_to_replicas_for_ambiguous_queries block' do it 'raises an exception' do expect do ::Gitlab::Database::LoadBalancing::Session.current.fallback_to_replicas_for_ambiguous_queries do diff --git a/gems/gitlab-database-load_balancing/spec/spec_helper.rb b/gems/gitlab-database-load_balancing/spec/spec_helper.rb index 71faf49cce0c789072005377ad4e8e977a6144a9..835ce299e823c29ef811309794f748d8cdac41ca 100644 --- a/gems/gitlab-database-load_balancing/spec/spec_helper.rb +++ b/gems/gitlab-database-load_balancing/spec/spec_helper.rb @@ -8,6 +8,11 @@ require 'gitlab/utils/all' require 'gitlab/safe_request_store' +require "gitlab/database/load_balancing" + +require_relative 'support/database_replica' +require_relative 'support/application_record' + RSpec.configure do |config| include StubRails include NextInstanceOf @@ -25,4 +30,15 @@ config.around(:example, :request_store) do |example| ::Gitlab::SafeRequestStore.ensure_request_store { example.run } end + + config.before(:all) do + Gitlab::Database::LoadBalancing.configure! do |load_balancing| + load_balancing.default_pool_size = 20 + load_balancing.base_models = [ActiveRecord::Base, Ci::ApplicationRecord] + + load_balancing.base_models.each do |model| + Gitlab::Database::LoadBalancing::Setup.new(model).setup + end + end + end end diff --git a/gems/gitlab-database-load_balancing/spec/support/application_record.rb b/gems/gitlab-database-load_balancing/spec/support/application_record.rb new file mode 100644 index 0000000000000000000000000000000000000000..acea00b5072cc1b7ab0a59e38244e0797bd75ed0 --- /dev/null +++ b/gems/gitlab-database-load_balancing/spec/support/application_record.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class ApplicationRecord < ActiveRecord::Base + self.abstract_class = true +end + +module Ci + class ApplicationRecord < ApplicationRecord + self.abstract_class = true + end +end diff --git a/gems/gitlab-database-load_balancing/spec/support/database_replica.rb b/gems/gitlab-database-load_balancing/spec/support/database_replica.rb new file mode 100644 index 0000000000000000000000000000000000000000..b37540105a8e1097e72e77710a4d18c042abea9f --- /dev/null +++ b/gems/gitlab-database-load_balancing/spec/support/database_replica.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +RSpec.configure do |config| + config.around(:each, :database_replica) do |example| + old_proxies = {} + + Gitlab::Database::LoadBalancing.base_models.each do |model| + old_proxies[model] = [model.load_balancer, model.connection, model.sticking] + + config = Gitlab::Database::LoadBalancing::Configuration + .new(model, [model.connection_db_config.configuration_hash[:host]]) + + model.load_balancer = Gitlab::Database::LoadBalancing::LoadBalancer.new(config) + model.sticking = Gitlab::Database::LoadBalancing::Sticking.new(model.load_balancer) + model.connection = Gitlab::Database::LoadBalancing::ConnectionProxy.new(model.load_balancer) + end + + Gitlab::Database::LoadBalancing::Session.clear_session + + example.run + + Gitlab::Database::LoadBalancing::Session.clear_session + + old_proxies.each do |model, proxy| + model.load_balancer, model.connection, model.sticking = proxy + end + end +end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 6222155d812f78eb0879e040926d0ec7f059299b..ee47762fc2e690f34ce88ddc47047c51f4a2a7d0 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'gitlab/database/load_balancing' + module Gitlab module Database MAIN_DATABASE_NAME = 'main' @@ -269,20 +271,7 @@ def self.gitlab_schemas_for_connection(connection) end def self.db_config_for_connection(connection) - return unless connection - - # For a ConnectionProxy we want to avoid ambiguous db_config as it may - # sometimes default to replica so we always return the primary config - # instead. - if connection.is_a?(::Gitlab::Database::LoadBalancing::ConnectionProxy) - return connection.load_balancer.configuration.db_config - end - - # During application init we might receive `NullPool` - return unless connection.respond_to?(:pool) && - connection.pool.respond_to?(:db_config) - - connection.pool.db_config + Gitlab::Database::LoadBalancing.db_config_for_connection(connection) end # At the moment, the connection can only be retrieved by diff --git a/spec/initializers/load_balancing_spec.rb b/spec/initializers/load_balancing_spec.rb index eddedcb2f386fee42c64b37afb3ee4ac2a1df972..2f7fd712f2b6a5eb78916da69ac13f3abd0d0681 100644 --- a/spec/initializers/load_balancing_spec.rb +++ b/spec/initializers/load_balancing_spec.rb @@ -2,18 +2,16 @@ require 'spec_helper' -RSpec.describe 'load_balancing', :delete, :reestablished_active_record_base, feature_category: :cell do - subject(:initialize_load_balancer) do - load Rails.root.join('config/initializers/load_balancing.rb') - end - - before do - # Stub out middleware call, as not idempotent - allow(Gitlab::Application.instance.middleware).to receive(:use) - end +RSpec.describe 'load_balancing', feature_category: :cell do + context 'with replica hosts configured', :delete, :reestablished_active_record_base do + subject(:initialize_load_balancer) do + load Rails.root.join('config/initializers/load_balancing.rb') + end - context 'with replica hosts configured' do before do + # Stub out middleware call, as not idempotent + allow(Gitlab::Application.instance.middleware).to receive(:use) + # Setup host-based load balancing # Patch in our load balancer config, simply pointing at the test database twice allow(Gitlab::Database::LoadBalancing::Configuration).to receive(:for_model) do |base_model| @@ -97,4 +95,91 @@ def simulate_puma_worker end end end + + describe Gitlab::Database::LoadBalancing do + describe '.base_models' do + it 'returns the models to apply load balancing to' do + models = described_class.base_models + + expect(models).to include(ActiveRecord::Base) + expect(models).to include(Ci::ApplicationRecord) if Gitlab::Database.has_config?(:ci) + end + + it 'returns the models as a frozen array' do + expect(described_class.base_models).to be_frozen + end + end + end + + describe Gitlab::Database::LoadBalancing::Callbacks do + describe '.metrics_host_gauge' do + it 'sets prometheus metric', :reestablished_active_record_base do + expect(described_class).to receive(:metrics_host_gauge) + .with({}, 1).and_call_original + + # Metric will be set once configured host list: + # - 1 host since we only have primary + # - this test will fail if running CI against replicas + model = Gitlab::Database::LoadBalancing.base_models.first + Gitlab::Database::LoadBalancing::Setup.new(model).setup + + metric = ::Prometheus::Client.registry.get(:db_load_balancing_hosts) + expect(metric).not_to be_nil + expect(metric.values[{}].get).to eq(1) + end + end + + context 'when sticking connection', :database_replica, :redis, :request_store do + let(:key) { 'database-load-balancing/write-location/main/user/100' } + let(:wal) { '0/D525E3A8' } + + it 'does set latest WAL and unstick once caught-up' do + allow(ApplicationRecord.load_balancer).to receive(:primary_write_location).and_return(wal) + + expect(described_class).to receive(:set_wal_for) + .with(key, wal, ex: 30) + .and_call_original + + expect { ApplicationRecord.sticking.stick(:user, 100) } + .to change { described_class.get_wal_for(key) }.from(nil).to(wal) + + expect(described_class).to receive(:del_wal_for) + .with(key).and_call_original + + expect(ApplicationRecord.load_balancer).to receive(:select_up_to_date_host) + .with(wal).and_return(Gitlab::Database::LoadBalancing::LoadBalancer::ALL_CAUGHT_UP) + + expect { ApplicationRecord.sticking.find_caught_up_replica(:user, 100) } + .to change { described_class.get_wal_for(key) }.from(wal).to(nil) + end + + describe '.set_wal_for' do + it 'does get latest set value' do + expect { described_class.set_wal_for(key, wal, ex: 30) } + .to change { described_class.get_wal_for(key) }.from(nil).to(wal) + end + end + + describe '.del_wal_for' do + it 'does delete set value' do + described_class.set_wal_for(key, wal, ex: 30) + + expect { described_class.del_wal_for(key) } + .to change { described_class.get_wal_for(key) }.from(wal).to(nil) + end + end + end + + describe '.track_exception' do + it 'forwards exception to ErrorTracking' do + expect(Gitlab::ErrorTracking).to receive(:track_exception) + .with(Gitlab::Utils::ConcurrentRubyThreadIsUsedError) + + Gitlab::Utils.restrict_within_concurrent_ruby do + expect { ApplicationRecord.connection.execute("SELECT 1") } + .to raise_error(Gitlab::Utils::ConcurrentRubyThreadIsUsedError) + end + end + end + end end