Merge pull request #3004 from alphagov/dependabot/bundler/sidekiq-7.3.2

Bump sidekiq from 6.5.12 to 7.3.2
alphagov · Oct 14, 2024 · aff014e · aff014e
2 parents df66a13 + 2eab4cf
commit aff014e
Show file tree

Hide file tree

Showing 36 changed files with 175 additions and 168 deletions.
diff --git a/Gemfile b/Gemfile
@@ -27,7 +27,7 @@ gem "rainbow"
 gem "rake"
 gem "rubyzip"
 gem "sentry-sidekiq"
-gem "sidekiq", "6.5.12"
+gem "sidekiq"
 gem "sidekiq-limit_fetch"
 gem "sinatra"
 gem "statsd-ruby"

diff --git a/Gemfile.lock b/Gemfile.lock
@@ -197,12 +197,11 @@ GEM
     govuk_schemas (5.0.4)
       faker (~> 3.4.1)
       json-schema (>= 2.8, < 4.4)
-    govuk_sidekiq (8.0.1)
+    govuk_sidekiq (9.0.0)
       gds-api-adapters (>= 19.1.0)
       govuk_app_config (>= 1.1)
-      redis (< 5)
-      redis-namespace (~> 1.6)
-      sidekiq (~> 6.5, >= 6.5.12)
+      redis-client (>= 0.22.2)
+      sidekiq (~> 7.0, < 8)
     grpc (1.66.0)
       google-protobuf (>= 3.25, < 5.0)
       googleapis-common-protos-types (~> 1.0)
@@ -544,9 +543,8 @@ GEM
     rbtree (0.4.6)
     rdoc (6.7.0)
       psych (>= 4.0.0)
-    redis (4.8.1)
-    redis-namespace (1.11.0)
-      redis (>= 4)
+    redis-client (0.22.2)
+      connection_pool
     regexp_parser (2.9.2)
     reline (0.5.10)
       io-console (~> 0.5)
@@ -621,10 +619,12 @@ GEM
       sentry-ruby (~> 5.21.0)
       sidekiq (>= 3.0)
     set (1.1.0)
-    sidekiq (6.5.12)
-      connection_pool (>= 2.2.5, < 3)
-      rack (~> 2.0)
-      redis (>= 4.5.0, < 5)
+    sidekiq (7.3.2)
+      concurrent-ruby (< 2)
+      connection_pool (>= 2.3.0)
+      logger
+      rack (>= 2.2.4)
+      redis-client (>= 0.22.2)
     sidekiq-limit_fetch (4.4.1)
       sidekiq (>= 6)
     signet (0.19.0)
@@ -714,7 +714,7 @@ DEPENDENCIES
   rubocop-govuk
   rubyzip
   sentry-sidekiq
-  sidekiq (= 6.5.12)
+  sidekiq
   sidekiq-limit_fetch
   simplecov
   sinatra

diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb
@@ -1,6 +1,5 @@
 require "govuk_sidekiq/sidekiq_initializer"
 
 GovukSidekiq::SidekiqInitializer.setup_sidekiq(
-  "search-api",
   { url: ENV.fetch("REDIS_URL", "redis://127.0.0.1:6379") },
 )
diff --git a/lib/base_job.rb b/lib/base_job.rb
@@ -0,0 +1,41 @@
+class BaseJob
+  include Sidekiq::Job
+
+  # How long to wait, by default, if the index is currently locked
+  LOCK_DELAY = 60 # seconds
+
+  # Default options: can be overridden with `sidekiq_options` in subclasses
+  sidekiq_options retry: 5, backtrace: 12
+
+  def self.notify_of_failures
+    sidekiq_retries_exhausted do |msg|
+      GovukError.notify(Indexer::FailedJobException.new, extra: msg)
+    end
+  end
+
+  # Wait for all tasks for the given queue/job class combination to be
+  # completed before continuing
+  def self.wait_until_processed(max_timeout: 2 * 60 * 60)
+    Timeout.timeout(max_timeout) do
+      # wait for all queued tasks to be started
+      sleep 1 while Sidekiq::Queue.new(self::QUEUE_NAME).any? { |job| job.display_class == to_s }
+
+      # wait for started tasks to be finished
+      sleep 1 while active_jobs?
+    end
+  end
+
+  def self.active_jobs?
+    Sidekiq::Job.jobs.any? do |_, _, work|
+      work["queue"] == self::QUEUE_NAME && work["payload"]["class"] == to_s
+    end
+  end
+
+private
+
+  def indexes(index_name)
+    SearchConfig.search_servers.map do |search_server|
+      search_server.index(index_name)
+    end
+  end
+end
diff --git a/lib/govuk_index/page_traffic_worker.rb → lib/govuk_index/page_traffic_job.rb b/lib/govuk_index/page_traffic_worker.rb → lib/govuk_index/page_traffic_job.rb
@@ -1,5 +1,5 @@
 module GovukIndex
-  class PageTrafficWorker < Indexer::BaseWorker
+  class PageTrafficJob < BaseJob
     BULK_INDEX_TIMEOUT = 60
     QUEUE_NAME = "bulk".freeze
     sidekiq_options queue: QUEUE_NAME

diff --git a/lib/govuk_index/page_traffic_loader.rb b/lib/govuk_index/page_traffic_loader.rb
@@ -18,10 +18,10 @@ def load_from(iostream)
         @logger.info "Indexing to #{new_index.real_name}"
 
         in_even_sized_batches(iostream) do |lines|
-          GovukIndex::PageTrafficWorker.perform_async(lines, new_index.real_name, cluster.key)
+          GovukIndex::PageTrafficJob.perform_async(lines, new_index.real_name, cluster.key)
         end
 
-        GovukIndex::PageTrafficWorker.wait_until_processed
+        GovukIndex::PageTrafficJob.wait_until_processed
         new_index.commit
       end
 

diff --git a/lib/govuk_index/popularity_worker.rb → lib/govuk_index/popularity_job.rb b/lib/govuk_index/popularity_worker.rb → lib/govuk_index/popularity_job.rb
@@ -1,5 +1,5 @@
 module GovukIndex
-  class PopularityWorker < Indexer::BaseWorker
+  class PopularityJob < BaseJob
     BULK_INDEX_TIMEOUT = 60
     QUEUE_NAME = "bulk".freeze
     sidekiq_options queue: QUEUE_NAME

diff --git a/lib/govuk_index/popularity_updater.rb b/lib/govuk_index/popularity_updater.rb
@@ -8,8 +8,8 @@ def self.update(index_name, process_all: false)
       ).run
     end
 
-    def self.worker
-      PopularityWorker
+    def self.job
+      PopularityJob
     end
 
     def initialize(source_index:, destination_index:, process_all: false)

diff --git a/lib/govuk_index/publishing_event_worker.rb → lib/govuk_index/publishing_event_job.rb b/lib/govuk_index/publishing_event_worker.rb → lib/govuk_index/publishing_event_job.rb
@@ -40,7 +40,7 @@ class MissingExternalUrl < StandardError; end
       worldwide_office_staff_role
     ].freeze
 
-  class PublishingEventWorker < Indexer::BaseWorker
+  class PublishingEventJob < BaseJob
     notify_of_failures
 
     def perform(messages)

diff --git a/lib/govuk_index/publishing_event_processor.rb b/lib/govuk_index/publishing_event_processor.rb
@@ -4,7 +4,7 @@ def process(messages)
       messages = Array(messages) # treat a single message as an array with one value
 
       Services.statsd_client.increment("govuk_index.rabbit-mq-consumed")
-      PublishingEventWorker.perform_async(messages.map { |msg| [msg.delivery_info[:routing_key], msg.payload] })
+      PublishingEventJob.perform_async(messages.map { |msg| [msg.delivery_info[:routing_key], msg.payload] })
       messages.each(&:ack)
     end
   end

diff --git a/lib/govuk_index/supertype_worker.rb → lib/govuk_index/supertype_job.rb b/lib/govuk_index/supertype_worker.rb → lib/govuk_index/supertype_job.rb
@@ -1,5 +1,5 @@
 module GovukIndex
-  class SupertypeWorker < Indexer::BaseWorker
+  class SupertypeJob < BaseJob
     BULK_INDEX_TIMEOUT = 60
     QUEUE_NAME = "bulk".freeze
     sidekiq_options queue: QUEUE_NAME

diff --git a/lib/govuk_index/supertype_updater.rb b/lib/govuk_index/supertype_updater.rb
@@ -7,8 +7,8 @@ def self.update(index_name)
       ).run
     end
 
-    def self.worker
-      SupertypeWorker
+    def self.job
+      SupertypeJob
     end
 
     def initialize(source_index:, destination_index:)

diff --git a/lib/govuk_index/updater.rb b/lib/govuk_index/updater.rb
@@ -15,15 +15,15 @@ def run(async: true)
       Clusters.active.each do |cluster|
         scroll_enumerator(cluster:).each_slice(PROCESSOR_BATCH_SIZE) do |document_id|
           if async
-            worker.perform_async(document_id, @destination_index)
+            job.perform_async(document_id, @destination_index)
           else
-            worker.new.perform(document_id, @destination_index)
+            job.new.perform(document_id, @destination_index)
           end
         end
       end
     end
 
-    def self.worker
+    def self.job
       raise ImplementationRequired
     end
 
@@ -33,8 +33,8 @@ def search_body
 
   private
 
-    def worker
-      self.class.worker
+    def job
+      self.class.job
     end
 
     def scroll_enumerator(cluster:)

diff --git a/lib/indexer/workers/amend_worker.rb → lib/indexer/amend_job.rb b/lib/indexer/workers/amend_worker.rb → lib/indexer/amend_job.rb
@@ -1,5 +1,5 @@
 module Indexer
-  class AmendWorker < BaseWorker
+  class AmendJob < BaseJob
     notify_of_failures
 
     def perform(index_name, document_link, updates)

diff --git a/lib/indexer/workers/bulk_index_worker.rb → lib/indexer/bulk_index_job.rb b/lib/indexer/workers/bulk_index_worker.rb → lib/indexer/bulk_index_job.rb
@@ -1,5 +1,5 @@
 module Indexer
-  class BulkIndexWorker < BaseWorker
+  class BulkIndexJob < BaseJob
     notify_of_failures
 
     def perform(index_name, document_hashes)

diff --git a/lib/indexer/change_notification_processor.rb b/lib/indexer/change_notification_processor.rb
@@ -15,7 +15,7 @@ def self.trigger(content_item)
       index_name = document["real_index_name"]
       document_id = document["_id"]
       updates = {}
-      Indexer::AmendWorker.perform_async(index_name, document_id, updates)
+      Indexer::AmendJob.perform_async(index_name, document_id, updates)
       :accepted
     end
 

diff --git a/lib/indexer/workers/delete_worker.rb → lib/indexer/delete_job.rb b/lib/indexer/workers/delete_worker.rb → lib/indexer/delete_job.rb
@@ -1,5 +1,5 @@
 module Indexer
-  class DeleteWorker < BaseWorker
+  class DeleteJob < BaseJob
     notify_of_failures
 
     def perform(index_name, elasticsearch_type, document_id)

diff --git a/lib/indexer/workers/base_worker.rb b/lib/indexer/workers/base_worker.rb
diff --git a/lib/missing_metadata/fetcher.rb b/lib/missing_metadata/fetcher.rb
@@ -34,7 +34,7 @@ def lookup_content_id(document_id)
     def update_metadata(content_id, index_name, document_id)
       response = publishing_api.get_content(content_id)
 
-      Indexer::AmendWorker.perform_async(
+      Indexer::AmendJob.perform_async(
         index_name,
         document_id,
         content_store_document_type: response["document_type"],

diff --git a/lib/rummager.rb b/lib/rummager.rb
@@ -23,7 +23,7 @@
 require "open-uri"
 require "ostruct"
 require "plek"
-require "redis"
+require "redis-client"
 require "securerandom"
 require "sidekiq"
 require "sidekiq-limit_fetch"
@@ -62,6 +62,8 @@
 require "content_item_publisher/finder_email_signup_publisher"
 require "content_item_publisher/finder_email_signup_presenter"
 
+require "base_job"
+
 require "indexer"
 require "indexer/amender"
 require "indexer/attachments_lookup"
@@ -76,25 +78,24 @@
 require "indexer/govuk_index_field_comparer"
 require "indexer/parts_lookup"
 require "indexer/popularity_lookup"
-require "indexer/workers/base_worker"
-require "indexer/workers/amend_worker"
-require "indexer/workers/bulk_index_worker"
-require "indexer/workers/delete_worker"
+require "indexer/amend_job"
+require "indexer/bulk_index_job"
+require "indexer/delete_job"
 require "index/client"
 require "index/elasticsearch_processor"
 require "index/response_validator"
 
 require "govuk_index/updater"
 require "govuk_index/client"
 require "govuk_index/document_type_mapper"
-require "govuk_index/page_traffic_worker"
+require "govuk_index/page_traffic_job"
 require "govuk_index/method_builder"
 require "govuk_index/indexable_content_sanitiser"
 require "govuk_index/migrated_formats"
 require "govuk_index/page_traffic_loader"
 require "govuk_index/payload_preparer"
 require "govuk_index/popularity_updater"
-require "govuk_index/popularity_worker"
+require "govuk_index/popularity_job"
 require "govuk_index/presenters/common_fields_presenter"
 require "govuk_index/presenters/details_presenter"
 require "govuk_index/presenters/elasticsearch_identity"
@@ -105,9 +106,9 @@
 require "govuk_index/presenters/parts_presenter"
 require "govuk_index/presenters/specialist_presenter"
 require "govuk_index/publishing_event_processor"
-require "govuk_index/publishing_event_worker"
+require "govuk_index/publishing_event_job"
 require "govuk_index/supertype_updater"
-require "govuk_index/supertype_worker"
+require "govuk_index/supertype_job"
 require "govuk_message_queue_consumer"
 
 require "evaluate/ndcg"