From 6048eeed045693be63ad34dc97e98f7b2e519a70 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 May 2022 03:27:02 +0900 Subject: Bump faker from 2.20.0 to 2.21.0 (#18434) Bumps [faker](https://github.com/faker-ruby/faker) from 2.20.0 to 2.21.0. - [Release notes](https://github.com/faker-ruby/faker/releases) - [Changelog](https://github.com/faker-ruby/faker/blob/master/CHANGELOG.md) - [Commits](https://github.com/faker-ruby/faker/compare/v2.20.0...v2.21.0) --- updated-dependencies: - dependency-name: faker dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Gemfile') diff --git a/Gemfile b/Gemfile index 002a44f7f..9d5fe51b6 100644 --- a/Gemfile +++ b/Gemfile @@ -114,7 +114,7 @@ end group :test do gem 'capybara', '~> 3.37' gem 'climate_control', '~> 0.2' - gem 'faker', '~> 2.20' + gem 'faker', '~> 2.21' gem 'microformats', '~> 4.2' gem 'rails-controller-testing', '~> 1.0' gem 'rspec-sidekiq', '~> 3.1' -- cgit From 6b4e2d5ce686140797dd207b1ba932a86665262f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 May 2022 03:27:55 +0900 Subject: Bump rubocop from 1.28.2 to 1.29.1 (#18433) Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.28.2 to 1.29.1. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.28.2...v1.29.1) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Gemfile | 2 +- Gemfile.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'Gemfile') diff --git a/Gemfile b/Gemfile index 9d5fe51b6..445b10496 100644 --- a/Gemfile +++ b/Gemfile @@ -132,7 +132,7 @@ group :development do gem 'letter_opener', '~> 1.8' gem 'letter_opener_web', '~> 2.0' gem 'memory_profiler' - gem 'rubocop', '~> 1.28', require: false + gem 'rubocop', '~> 1.29', require: false gem 'rubocop-rails', '~> 2.14', require: false gem 'brakeman', '~> 5.2', require: false gem 'bundler-audit', '~> 0.9', require: false diff --git a/Gemfile.lock b/Gemfile.lock index 640320702..0600862df 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -568,16 +568,16 @@ GEM rspec-support (3.11.0) rspec_junit_formatter (0.5.1) rspec-core (>= 2, < 4, != 2.12.0) - rubocop (1.28.2) + rubocop (1.29.1) parallel (~> 1.10) parser (>= 3.1.0.0) rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 1.8, < 3.0) - rexml + rexml (>= 3.2.5, < 4.0) rubocop-ast (>= 1.17.0, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 1.4.0, < 3.0) - rubocop-ast (1.17.0) + rubocop-ast (1.18.0) parser (>= 3.1.1.0) rubocop-rails (2.14.2) activesupport (>= 4.2.0) @@ -820,7 +820,7 @@ DEPENDENCIES rspec-rails (~> 5.1) rspec-sidekiq (~> 3.1) rspec_junit_formatter (~> 0.5) - rubocop (~> 1.28) + rubocop (~> 1.29) rubocop-rails (~> 2.14) ruby-progressbar (~> 1.11) sanitize (~> 6.0) -- cgit From 679b7158e3cd3881e8cbaf2d2c0c97725b3b5fd9 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 18 May 2022 23:29:14 +0200 Subject: Change search indexing to use batches to minimize resource usage (#18451) --- Gemfile | 2 +- app/chewy/accounts_index.rb | 2 +- app/chewy/statuses_index.rb | 2 +- app/chewy/tags_index.rb | 2 +- app/workers/scheduler/indexing_scheduler.rb | 26 ++++++++++++++++++++++++++ config/application.rb | 2 +- config/initializers/chewy.rb | 5 ++--- config/sidekiq.yml | 4 ++++ lib/chewy/strategy/custom_sidekiq.rb | 11 ----------- lib/chewy/strategy/mastodon.rb | 27 +++++++++++++++++++++++++++ 10 files changed, 64 insertions(+), 19 deletions(-) create mode 100644 app/workers/scheduler/indexing_scheduler.rb delete mode 100644 lib/chewy/strategy/custom_sidekiq.rb create mode 100644 lib/chewy/strategy/mastodon.rb (limited to 'Gemfile') diff --git a/Gemfile b/Gemfile index 445b10496..2e77fb42a 100644 --- a/Gemfile +++ b/Gemfile @@ -81,7 +81,7 @@ gem 'scenic', '~> 1.6' gem 'sidekiq', '~> 6.4' gem 'sidekiq-scheduler', '~> 4.0' gem 'sidekiq-unique-jobs', '~> 7.1' -gem 'sidekiq-bulk', '~>0.2.0' +gem 'sidekiq-bulk', '~> 0.2.0' gem 'simple-navigation', '~> 4.3' gem 'simple_form', '~> 5.1' gem 'sprockets-rails', '~> 3.4', require: 'sprockets/railtie' diff --git a/app/chewy/accounts_index.rb b/app/chewy/accounts_index.rb index 6f9ea76e9..763958a3f 100644 --- a/app/chewy/accounts_index.rb +++ b/app/chewy/accounts_index.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true class AccountsIndex < Chewy::Index - settings index: { refresh_interval: '5m' }, analysis: { + settings index: { refresh_interval: '30s' }, analysis: { analyzer: { content: { tokenizer: 'whitespace', diff --git a/app/chewy/statuses_index.rb b/app/chewy/statuses_index.rb index 1304aeedb..c20009879 100644 --- a/app/chewy/statuses_index.rb +++ b/app/chewy/statuses_index.rb @@ -3,7 +3,7 @@ class StatusesIndex < Chewy::Index include FormattingHelper - settings index: { refresh_interval: '15m' }, analysis: { + settings index: { refresh_interval: '30s' }, analysis: { filter: { english_stop: { type: 'stop', diff --git a/app/chewy/tags_index.rb b/app/chewy/tags_index.rb index f9db2b03a..a5b139bca 100644 --- a/app/chewy/tags_index.rb +++ b/app/chewy/tags_index.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true class TagsIndex < Chewy::Index - settings index: { refresh_interval: '15m' }, analysis: { + settings index: { refresh_interval: '30s' }, analysis: { analyzer: { content: { tokenizer: 'keyword', diff --git a/app/workers/scheduler/indexing_scheduler.rb b/app/workers/scheduler/indexing_scheduler.rb new file mode 100644 index 000000000..3a6f47a29 --- /dev/null +++ b/app/workers/scheduler/indexing_scheduler.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class Scheduler::IndexingScheduler + include Sidekiq::Worker + include Redisable + + sidekiq_options retry: 0 + + def perform + indexes.each do |type| + with_redis do |redis| + ids = redis.smembers("chewy:queue:#{type.name}") + + type.import!(ids) + + redis.pipelined do |pipeline| + ids.each { |id| pipeline.srem("chewy:queue:#{type.name}", id) } + end + end + end + end + + def indexes + [AccountsIndex, TagsIndex, StatusesIndex] + end +end diff --git a/config/application.rb b/config/application.rb index 64987cfe7..24fa2a978 100644 --- a/config/application.rb +++ b/config/application.rb @@ -38,7 +38,7 @@ require_relative '../lib/mastodon/version' require_relative '../lib/mastodon/rack_middleware' require_relative '../lib/devise/two_factor_ldap_authenticatable' require_relative '../lib/devise/two_factor_pam_authenticatable' -require_relative '../lib/chewy/strategy/custom_sidekiq' +require_relative '../lib/chewy/strategy/mastodon' require_relative '../lib/webpacker/manifest_extensions' require_relative '../lib/webpacker/helper_extensions' require_relative '../lib/rails/engine_extensions' diff --git a/config/initializers/chewy.rb b/config/initializers/chewy.rb index f303fc54d..752fc3c6d 100644 --- a/config/initializers/chewy.rb +++ b/config/initializers/chewy.rb @@ -13,15 +13,14 @@ Chewy.settings = { journal: false, user: user, password: password, - sidekiq: { queue: 'pull' }, } # We use our own async strategy even outside the request-response # cycle, which takes care of checking if Elasticsearch is enabled # or not. However, mind that for the Rails console, the :urgent # strategy is set automatically with no way to override it. -Chewy.root_strategy = :custom_sidekiq -Chewy.request_strategy = :custom_sidekiq +Chewy.root_strategy = :mastodon +Chewy.request_strategy = :mastodon Chewy.use_after_commit_callbacks = false module Chewy diff --git a/config/sidekiq.yml b/config/sidekiq.yml index 26be26326..2a3871468 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -21,6 +21,10 @@ every: '6h' class: Scheduler::Trends::ReviewNotificationsScheduler queue: scheduler + indexing_scheduler: + every: '5m' + class: Scheduler::IndexingScheduler + queue: scheduler media_cleanup_scheduler: cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *' class: Scheduler::MediaCleanupScheduler diff --git a/lib/chewy/strategy/custom_sidekiq.rb b/lib/chewy/strategy/custom_sidekiq.rb deleted file mode 100644 index 794ae4ed4..000000000 --- a/lib/chewy/strategy/custom_sidekiq.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module Chewy - class Strategy - class CustomSidekiq < Sidekiq - def update(_type, _objects, _options = {}) - super if Chewy.enabled? - end - end - end -end diff --git a/lib/chewy/strategy/mastodon.rb b/lib/chewy/strategy/mastodon.rb new file mode 100644 index 000000000..ee8b92186 --- /dev/null +++ b/lib/chewy/strategy/mastodon.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Chewy + class Strategy + class Mastodon < Base + def initialize + super + + @stash = Hash.new { |hash, key| hash[key] = [] } + end + + def update(type, objects, _options = {}) + @stash[type].concat(type.root.id ? Array.wrap(objects) : type.adapter.identify(objects)) if Chewy.enabled? + end + + def leave + RedisConfiguration.with do |redis| + redis.pipelined do |pipeline| + @stash.each do |type, ids| + pipeline.sadd("chewy:queue:#{type.name}", ids) + end + end + end + end + end + end +end -- cgit