From 014065913ca59ae58559239eabd49f5c06e62dbc Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 4 May 2022 00:57:42 +0200 Subject: Bump version to 3.5.2 (#18295) * Bump version to 3.5.2 * Change some entries to be more clear * Add some extra notes * Fix line wrap Co-authored-by: Eugen Rochko --- lib/mastodon/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index 23e164b2e..73024e612 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -13,7 +13,7 @@ module Mastodon end def patch - 1 + 2 end def flags -- cgit From f714e24ff104c3525a9a31be442364d2be1273fd Mon Sep 17 00:00:00 2001 From: Claire Date: Mon, 9 May 2022 23:19:11 +0200 Subject: Fix redis configuration not being changed by mastodon:setup (#18383) Fixes #18342 --- lib/tasks/mastodon.rake | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'lib') diff --git a/lib/tasks/mastodon.rake b/lib/tasks/mastodon.rake index a89af6778..d652468b3 100644 --- a/lib/tasks/mastodon.rake +++ b/lib/tasks/mastodon.rake @@ -8,6 +8,14 @@ namespace :mastodon do prompt = TTY::Prompt.new env = {} + # When the application code gets loaded, it runs `lib/mastodon/redis_configuration.rb`. + # This happens before application environment configuration and sets REDIS_URL etc. + # These variables are then used even when REDIS_HOST etc. are changed, so clear them + # out so they don't interfer with our new configuration. + ENV.delete('REDIS_URL') + ENV.delete('CACHE_REDIS_URL') + ENV.delete('SIDEKIQ_REDIS_URL') + begin prompt.say('Your instance is identified by its domain name. Changing it afterward will break things.') env['LOCAL_DOMAIN'] = prompt.ask('Domain name:') do |q| -- cgit From 679b7158e3cd3881e8cbaf2d2c0c97725b3b5fd9 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 18 May 2022 23:29:14 +0200 Subject: Change search indexing to use batches to minimize resource usage (#18451) --- Gemfile | 2 +- app/chewy/accounts_index.rb | 2 +- app/chewy/statuses_index.rb | 2 +- app/chewy/tags_index.rb | 2 +- app/workers/scheduler/indexing_scheduler.rb | 26 ++++++++++++++++++++++++++ config/application.rb | 2 +- config/initializers/chewy.rb | 5 ++--- config/sidekiq.yml | 4 ++++ lib/chewy/strategy/custom_sidekiq.rb | 11 ----------- lib/chewy/strategy/mastodon.rb | 27 +++++++++++++++++++++++++++ 10 files changed, 64 insertions(+), 19 deletions(-) create mode 100644 app/workers/scheduler/indexing_scheduler.rb delete mode 100644 lib/chewy/strategy/custom_sidekiq.rb create mode 100644 lib/chewy/strategy/mastodon.rb (limited to 'lib') diff --git a/Gemfile b/Gemfile index 445b10496..2e77fb42a 100644 --- a/Gemfile +++ b/Gemfile @@ -81,7 +81,7 @@ gem 'scenic', '~> 1.6' gem 'sidekiq', '~> 6.4' gem 'sidekiq-scheduler', '~> 4.0' gem 'sidekiq-unique-jobs', '~> 7.1' -gem 'sidekiq-bulk', '~>0.2.0' +gem 'sidekiq-bulk', '~> 0.2.0' gem 'simple-navigation', '~> 4.3' gem 'simple_form', '~> 5.1' gem 'sprockets-rails', '~> 3.4', require: 'sprockets/railtie' diff --git a/app/chewy/accounts_index.rb b/app/chewy/accounts_index.rb index 6f9ea76e9..763958a3f 100644 --- a/app/chewy/accounts_index.rb +++ b/app/chewy/accounts_index.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true class AccountsIndex < Chewy::Index - settings index: { refresh_interval: '5m' }, analysis: { + settings index: { refresh_interval: '30s' }, analysis: { analyzer: { content: { tokenizer: 'whitespace', diff --git a/app/chewy/statuses_index.rb b/app/chewy/statuses_index.rb index 1304aeedb..c20009879 100644 --- a/app/chewy/statuses_index.rb +++ b/app/chewy/statuses_index.rb @@ -3,7 +3,7 @@ class StatusesIndex < Chewy::Index include FormattingHelper - settings index: { refresh_interval: '15m' }, analysis: { + settings index: { refresh_interval: '30s' }, analysis: { filter: { english_stop: { type: 'stop', diff --git a/app/chewy/tags_index.rb b/app/chewy/tags_index.rb index f9db2b03a..a5b139bca 100644 --- a/app/chewy/tags_index.rb +++ b/app/chewy/tags_index.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true class TagsIndex < Chewy::Index - settings index: { refresh_interval: '15m' }, analysis: { + settings index: { refresh_interval: '30s' }, analysis: { analyzer: { content: { tokenizer: 'keyword', diff --git a/app/workers/scheduler/indexing_scheduler.rb b/app/workers/scheduler/indexing_scheduler.rb new file mode 100644 index 000000000..3a6f47a29 --- /dev/null +++ b/app/workers/scheduler/indexing_scheduler.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class Scheduler::IndexingScheduler + include Sidekiq::Worker + include Redisable + + sidekiq_options retry: 0 + + def perform + indexes.each do |type| + with_redis do |redis| + ids = redis.smembers("chewy:queue:#{type.name}") + + type.import!(ids) + + redis.pipelined do |pipeline| + ids.each { |id| pipeline.srem("chewy:queue:#{type.name}", id) } + end + end + end + end + + def indexes + [AccountsIndex, TagsIndex, StatusesIndex] + end +end diff --git a/config/application.rb b/config/application.rb index 64987cfe7..24fa2a978 100644 --- a/config/application.rb +++ b/config/application.rb @@ -38,7 +38,7 @@ require_relative '../lib/mastodon/version' require_relative '../lib/mastodon/rack_middleware' require_relative '../lib/devise/two_factor_ldap_authenticatable' require_relative '../lib/devise/two_factor_pam_authenticatable' -require_relative '../lib/chewy/strategy/custom_sidekiq' +require_relative '../lib/chewy/strategy/mastodon' require_relative '../lib/webpacker/manifest_extensions' require_relative '../lib/webpacker/helper_extensions' require_relative '../lib/rails/engine_extensions' diff --git a/config/initializers/chewy.rb b/config/initializers/chewy.rb index f303fc54d..752fc3c6d 100644 --- a/config/initializers/chewy.rb +++ b/config/initializers/chewy.rb @@ -13,15 +13,14 @@ Chewy.settings = { journal: false, user: user, password: password, - sidekiq: { queue: 'pull' }, } # We use our own async strategy even outside the request-response # cycle, which takes care of checking if Elasticsearch is enabled # or not. However, mind that for the Rails console, the :urgent # strategy is set automatically with no way to override it. -Chewy.root_strategy = :custom_sidekiq -Chewy.request_strategy = :custom_sidekiq +Chewy.root_strategy = :mastodon +Chewy.request_strategy = :mastodon Chewy.use_after_commit_callbacks = false module Chewy diff --git a/config/sidekiq.yml b/config/sidekiq.yml index 26be26326..2a3871468 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -21,6 +21,10 @@ every: '6h' class: Scheduler::Trends::ReviewNotificationsScheduler queue: scheduler + indexing_scheduler: + every: '5m' + class: Scheduler::IndexingScheduler + queue: scheduler media_cleanup_scheduler: cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *' class: Scheduler::MediaCleanupScheduler diff --git a/lib/chewy/strategy/custom_sidekiq.rb b/lib/chewy/strategy/custom_sidekiq.rb deleted file mode 100644 index 794ae4ed4..000000000 --- a/lib/chewy/strategy/custom_sidekiq.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module Chewy - class Strategy - class CustomSidekiq < Sidekiq - def update(_type, _objects, _options = {}) - super if Chewy.enabled? - end - end - end -end diff --git a/lib/chewy/strategy/mastodon.rb b/lib/chewy/strategy/mastodon.rb new file mode 100644 index 000000000..ee8b92186 --- /dev/null +++ b/lib/chewy/strategy/mastodon.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Chewy + class Strategy + class Mastodon < Base + def initialize + super + + @stash = Hash.new { |hash, key| hash[key] = [] } + end + + def update(type, objects, _options = {}) + @stash[type].concat(type.root.id ? Array.wrap(objects) : type.adapter.identify(objects)) if Chewy.enabled? + end + + def leave + RedisConfiguration.with do |redis| + redis.pipelined do |pipeline| + @stash.each do |type, ids| + pipeline.sadd("chewy:queue:#{type.name}", ids) + end + end + end + end + end + end +end -- cgit From a9b64b24d6c076cb96a66307c07d4f0158dc07da Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sun, 22 May 2022 22:16:43 +0200 Subject: Change algorithm of `tootctl search deploy` to improve performance (#18463) --- app/chewy/accounts_index.rb | 6 +- app/chewy/statuses_index.rb | 2 + app/chewy/tags_index.rb | 8 +- app/lib/importer/accounts_index_importer.rb | 30 +++++++ app/lib/importer/base_importer.rb | 87 +++++++++++++++++++ app/lib/importer/statuses_index_importer.rb | 89 +++++++++++++++++++ app/lib/importer/tags_index_importer.rb | 26 ++++++ app/models/trends/history.rb | 20 +++-- lib/mastodon/search_cli.rb | 129 +++++++++------------------- 9 files changed, 294 insertions(+), 103 deletions(-) create mode 100644 app/lib/importer/accounts_index_importer.rb create mode 100644 app/lib/importer/base_importer.rb create mode 100644 app/lib/importer/statuses_index_importer.rb create mode 100644 app/lib/importer/tags_index_importer.rb (limited to 'lib') diff --git a/app/chewy/accounts_index.rb b/app/chewy/accounts_index.rb index 763958a3f..e38e14a10 100644 --- a/app/chewy/accounts_index.rb +++ b/app/chewy/accounts_index.rb @@ -23,7 +23,7 @@ class AccountsIndex < Chewy::Index }, } - index_scope ::Account.searchable.includes(:account_stat), delete_if: ->(account) { account.destroyed? || !account.searchable? } + index_scope ::Account.searchable.includes(:account_stat) root date_detection: false do field :id, type: 'long' @@ -36,8 +36,8 @@ class AccountsIndex < Chewy::Index field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'content' end - field :following_count, type: 'long', value: ->(account) { account.following.local.count } - field :followers_count, type: 'long', value: ->(account) { account.followers.local.count } + field :following_count, type: 'long', value: ->(account) { account.following_count } + field :followers_count, type: 'long', value: ->(account) { account.followers_count } field :last_status_at, type: 'date', value: ->(account) { account.last_status_at || account.created_at } end end diff --git a/app/chewy/statuses_index.rb b/app/chewy/statuses_index.rb index c20009879..6dd4fb18b 100644 --- a/app/chewy/statuses_index.rb +++ b/app/chewy/statuses_index.rb @@ -33,6 +33,8 @@ class StatusesIndex < Chewy::Index }, } + # We do not use delete_if option here because it would call a method that we + # expect to be called with crutches without crutches, causing n+1 queries index_scope ::Status.unscoped.kept.without_reblogs.includes(:media_attachments, :preloadable_poll) crutch :mentions do |collection| diff --git a/app/chewy/tags_index.rb b/app/chewy/tags_index.rb index a5b139bca..df3d9e4cc 100644 --- a/app/chewy/tags_index.rb +++ b/app/chewy/tags_index.rb @@ -23,7 +23,11 @@ class TagsIndex < Chewy::Index }, } - index_scope ::Tag.listable, delete_if: ->(tag) { tag.destroyed? || !tag.listable? } + index_scope ::Tag.listable + + crutch :time_period do + 7.days.ago.to_date..0.days.ago.to_date + end root date_detection: false do field :name, type: 'text', analyzer: 'content' do @@ -31,7 +35,7 @@ class TagsIndex < Chewy::Index end field :reviewed, type: 'boolean', value: ->(tag) { tag.reviewed? } - field :usage, type: 'long', value: ->(tag) { tag.history.reduce(0) { |total, day| total + day.accounts } } + field :usage, type: 'long', value: ->(tag, crutches) { tag.history.aggregate(crutches.time_period).accounts } field :last_status_at, type: 'date', value: ->(tag) { tag.last_status_at || tag.created_at } end end diff --git a/app/lib/importer/accounts_index_importer.rb b/app/lib/importer/accounts_index_importer.rb new file mode 100644 index 000000000..792a31b1b --- /dev/null +++ b/app/lib/importer/accounts_index_importer.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class Importer::AccountsIndexImporter < Importer::BaseImporter + def import! + scope.includes(:account_stat).find_in_batches(batch_size: @batch_size) do |tmp| + in_work_unit(tmp) do |accounts| + bulk = Chewy::Index::Import::BulkBuilder.new(index, to_index: accounts).bulk_body + + indexed = bulk.select { |entry| entry[:index] }.size + deleted = bulk.select { |entry| entry[:delete] }.size + + Chewy::Index::Import::BulkRequest.new(index).perform(bulk) + + [indexed, deleted] + end + end + + wait! + end + + private + + def index + AccountsIndex + end + + def scope + Account.searchable + end +end diff --git a/app/lib/importer/base_importer.rb b/app/lib/importer/base_importer.rb new file mode 100644 index 000000000..ea522c600 --- /dev/null +++ b/app/lib/importer/base_importer.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +class Importer::BaseImporter + # @param [Integer] batch_size + # @param [Concurrent::ThreadPoolExecutor] executor + def initialize(batch_size:, executor:) + @batch_size = batch_size + @executor = executor + @wait_for = Concurrent::Set.new + end + + # Callback to run when a concurrent work unit completes + # @param [Proc] + def on_progress(&block) + @on_progress = block + end + + # Callback to run when a concurrent work unit fails + # @param [Proc] + def on_failure(&block) + @on_failure = block + end + + # Reduce resource usage during and improve speed of indexing + def optimize_for_import! + Chewy.client.indices.put_settings index: index.index_name, body: { index: { refresh_interval: -1 } } + end + + # Restore original index settings + def optimize_for_search! + Chewy.client.indices.put_settings index: index.index_name, body: { index: { refresh_interval: index.settings_hash[:settings][:index][:refresh_interval] } } + end + + # Estimate the amount of documents that would be indexed. Not exact! + # @returns [Integer] + def estimate! + ActiveRecord::Base.connection_pool.with_connection { |connection| connection.select_one("SELECT reltuples AS estimate FROM pg_class WHERE relname = '#{index.adapter.target.table_name}'")['estimate'].to_i } + end + + # Import data from the database into the index + def import! + raise NotImplementedError + end + + # Remove documents from the index that no longer exist in the database + def clean_up! + index.scroll_batches do |documents| + ids = documents.map { |doc| doc['_id'] } + existence_map = index.adapter.target.where(id: ids).pluck(:id).each_with_object({}) { |id, map| map[id.to_s] = true } + tmp = ids.reject { |id| existence_map[id] } + + next if tmp.empty? + + in_work_unit(tmp) do |deleted_ids| + bulk = Chewy::Index::Import::BulkBuilder.new(index, delete: deleted_ids).bulk_body + + Chewy::Index::Import::BulkRequest.new(index).perform(bulk) + + [0, bulk.size] + end + end + + wait! + end + + protected + + def in_work_unit(*args, &block) + work_unit = Concurrent::Promises.future_on(@executor, *args, &block) + + work_unit.on_fulfillment!(&@on_progress) + work_unit.on_rejection!(&@on_failure) + work_unit.on_resolution! { @wait_for.delete(work_unit) } + + @wait_for << work_unit + rescue Concurrent::RejectedExecutionError + sleep(0.1) && retry # Backpressure + end + + def wait! + Concurrent::Promises.zip(*@wait_for).wait + end + + def index + raise NotImplementedError + end +end diff --git a/app/lib/importer/statuses_index_importer.rb b/app/lib/importer/statuses_index_importer.rb new file mode 100644 index 000000000..7c6532560 --- /dev/null +++ b/app/lib/importer/statuses_index_importer.rb @@ -0,0 +1,89 @@ +# frozen_string_literal: true + +class Importer::StatusesIndexImporter < Importer::BaseImporter + def import! + # The idea is that instead of iterating over all statuses in the database + # and calculating the searchable_by for each of them (majority of which + # would be empty), we approach the index from the other end + + scopes.each do |scope| + # We could be tempted to keep track of status IDs we have already processed + # from a different scope to avoid indexing them multiple times, but that + # could end up being a very large array + + scope.find_in_batches(batch_size: @batch_size) do |tmp| + in_work_unit(tmp.map(&:status_id)) do |status_ids| + bulk = ActiveRecord::Base.connection_pool.with_connection do + Chewy::Index::Import::BulkBuilder.new(index, to_index: Status.includes(:media_attachments, :preloadable_poll).where(id: status_ids)).bulk_body + end + + indexed = 0 + deleted = 0 + + # We can't use the delete_if proc to do the filtering because delete_if + # is called before rendering the data and we need to filter based + # on the results of the filter, so this filtering happens here instead + bulk.map! do |entry| + new_entry = begin + if entry[:index] && entry.dig(:index, :data, 'searchable_by').blank? + { delete: entry[:index].except(:data) } + else + entry + end + end + + if new_entry[:index] + indexed += 1 + else + deleted += 1 + end + + new_entry + end + + Chewy::Index::Import::BulkRequest.new(index).perform(bulk) + + [indexed, deleted] + end + end + end + + wait! + end + + private + + def index + StatusesIndex + end + + def scopes + [ + local_statuses_scope, + local_mentions_scope, + local_favourites_scope, + local_votes_scope, + local_bookmarks_scope, + ] + end + + def local_mentions_scope + Mention.where(account: Account.local, silent: false).select(:id, :status_id) + end + + def local_favourites_scope + Favourite.where(account: Account.local).select(:id, :status_id) + end + + def local_bookmarks_scope + Bookmark.select(:id, :status_id) + end + + def local_votes_scope + Poll.joins(:votes).where(votes: { account: Account.local }).select('polls.id, polls.status_id') + end + + def local_statuses_scope + Status.local.select('id, coalesce(reblog_of_id, id) as status_id') + end +end diff --git a/app/lib/importer/tags_index_importer.rb b/app/lib/importer/tags_index_importer.rb new file mode 100644 index 000000000..f5bd8f052 --- /dev/null +++ b/app/lib/importer/tags_index_importer.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class Importer::TagsIndexImporter < Importer::BaseImporter + def import! + index.adapter.default_scope.find_in_batches(batch_size: @batch_size) do |tmp| + in_work_unit(tmp) do |tags| + bulk = Chewy::Index::Import::BulkBuilder.new(index, to_index: tags).bulk_body + + indexed = bulk.select { |entry| entry[:index] }.size + deleted = bulk.select { |entry| entry[:delete] }.size + + Chewy::Index::Import::BulkRequest.new(index).perform(bulk) + + [indexed, deleted] + end + end + + wait! + end + + private + + def index + TagsIndex + end +end diff --git a/app/models/trends/history.rb b/app/models/trends/history.rb index 608e33792..74723e35c 100644 --- a/app/models/trends/history.rb +++ b/app/models/trends/history.rb @@ -11,11 +11,11 @@ class Trends::History end def uses - redis.mget(*@days.map { |day| day.key_for(:uses) }).map(&:to_i).sum + with_redis { |redis| redis.mget(*@days.map { |day| day.key_for(:uses) }).map(&:to_i).sum } end def accounts - redis.pfcount(*@days.map { |day| day.key_for(:accounts) }) + with_redis { |redis| redis.pfcount(*@days.map { |day| day.key_for(:accounts) }) } end end @@ -33,19 +33,21 @@ class Trends::History attr_reader :day def accounts - redis.pfcount(key_for(:accounts)) + with_redis { |redis| redis.pfcount(key_for(:accounts)) } end def uses - redis.get(key_for(:uses))&.to_i || 0 + with_redis { |redis| redis.get(key_for(:uses))&.to_i || 0 } end def add(account_id) - redis.pipelined do - redis.incrby(key_for(:uses), 1) - redis.pfadd(key_for(:accounts), account_id) - redis.expire(key_for(:uses), EXPIRE_AFTER) - redis.expire(key_for(:accounts), EXPIRE_AFTER) + with_redis do |redis| + redis.pipelined do |pipeline| + pipeline.incrby(key_for(:uses), 1) + pipeline.pfadd(key_for(:accounts), account_id) + pipeline.expire(key_for(:uses), EXPIRE_AFTER) + pipeline.expire(key_for(:accounts), EXPIRE_AFTER) + end end end diff --git a/lib/mastodon/search_cli.rb b/lib/mastodon/search_cli.rb index 74f980ba1..b579ebc14 100644 --- a/lib/mastodon/search_cli.rb +++ b/lib/mastodon/search_cli.rb @@ -16,19 +16,21 @@ module Mastodon StatusesIndex, ].freeze - option :concurrency, type: :numeric, default: 2, aliases: [:c], desc: 'Workload will be split between this number of threads' - option :batch_size, type: :numeric, default: 1_000, aliases: [:b], desc: 'Number of records in each batch' + option :concurrency, type: :numeric, default: 5, aliases: [:c], desc: 'Workload will be split between this number of threads' + option :batch_size, type: :numeric, default: 100, aliases: [:b], desc: 'Number of records in each batch' option :only, type: :array, enum: %w(accounts tags statuses), desc: 'Only process these indices' + option :import, type: :boolean, default: true, desc: 'Import data from the database to the index' + option :clean, type: :boolean, default: true, desc: 'Remove outdated documents from the index' desc 'deploy', 'Create or upgrade Elasticsearch indices and populate them' long_desc <<~LONG_DESC If Elasticsearch is empty, this command will create the necessary indices and then import data from the database into those indices. This command will also upgrade indices if the underlying schema has been - changed since the last run. + changed since the last run. Index upgrades erase index data. Even if creating or upgrading indices is not necessary, data from the - database will be imported into the indices. + database will be imported into the indices, unless overriden with --no-import. LONG_DESC def deploy if options[:concurrency] < 1 @@ -49,7 +51,9 @@ module Mastodon end end - progress = ProgressBar.create(total: nil, format: '%t%c/%u |%b%i| %e (%r docs/s)', autofinish: false) + pool = Concurrent::FixedThreadPool.new(options[:concurrency], max_queue: options[:concurrency] * 10) + importers = indices.index_with { |index| "Importer::#{index.name}Importer".constantize.new(batch_size: options[:batch_size], executor: pool) } + progress = ProgressBar.create(total: nil, format: '%t%c/%u |%b%i| %e (%r docs/s)', autofinish: false) # First, ensure all indices are created and have the correct # structure, so that live data can already be written @@ -59,99 +63,46 @@ module Mastodon index.specification.lock! end + progress.title = 'Estimating workload ' + progress.total = indices.sum { |index| importers[index].estimate! } + reset_connection_pools! - pool = Concurrent::FixedThreadPool.new(options[:concurrency]) - added = Concurrent::AtomicFixnum.new(0) - removed = Concurrent::AtomicFixnum.new(0) + added = 0 + removed = 0 - progress.title = 'Estimating workload ' + indices.each do |index| + importer = importers[index] + importer.optimize_for_import! + + importer.on_progress do |(indexed, deleted)| + progress.total = nil if progress.progress + indexed + deleted > progress.total + progress.progress += indexed + deleted + added += indexed + removed += deleted + end - # Estimate the amount of data that has to be imported first - progress.total = indices.sum { |index| index.adapter.default_scope.count } + importer.on_failure do |reason| + progress.log(pastel.red("Error while importing #{index}: #{reason}")) + end - # Now import all the actual data. Mind that unlike chewy:sync, we don't - # fetch and compare all record IDs from the database and the index to - # find out which to add and which to remove from the index. Because with - # potentially millions of rows, the memory footprint of such a calculation - # is uneconomical. So we only ever add. - indices.each do |index| - progress.title = "Importing #{index} " - batch_size = options[:batch_size] - slice_size = (batch_size / options[:concurrency]).ceil - - index.adapter.default_scope.reorder(nil).find_in_batches(batch_size: batch_size) do |batch| - futures = [] - - batch.each_slice(slice_size) do |records| - futures << Concurrent::Future.execute(executor: pool) do - begin - if !progress.total.nil? && progress.progress + records.size > progress.total - # The number of items has changed between start and now, - # since there is no good way to predict the final count from - # here, just change the progress bar to an indeterminate one - - progress.total = nil - end - - grouped_records = nil - bulk_body = nil - index_count = 0 - delete_count = 0 - - ActiveRecord::Base.connection_pool.with_connection do - grouped_records = records.to_a.group_by do |record| - index.adapter.send(:delete_from_index?, record) ? :delete : :to_index - end - - bulk_body = Chewy::Index::Import::BulkBuilder.new(index, **grouped_records).bulk_body - end - - index_count = grouped_records[:to_index].size if grouped_records.key?(:to_index) - delete_count = grouped_records[:delete].size if grouped_records.key?(:delete) - - # The following is an optimization for statuses specifically, since - # we want to de-index statuses that cannot be searched by anybody, - # but can't use Chewy's delete_if logic because it doesn't use - # crutches and our searchable_by logic depends on them - if index == StatusesIndex - bulk_body.map! do |entry| - if entry[:to_index] && entry.dig(:to_index, :data, 'searchable_by').blank? - index_count -= 1 - delete_count += 1 - - { delete: entry[:to_index].except(:data) } - else - entry - end - end - end - - Chewy::Index::Import::BulkRequest.new(index).perform(bulk_body) - - progress.progress += records.size - - added.increment(index_count) - removed.increment(delete_count) - - sleep 1 - rescue => e - progress.log pastel.red("Error importing #{index}: #{e}") - ensure - RedisConfiguration.pool.checkin if Thread.current[:redis] - Thread.current[:redis] = nil - end - end - end - - futures.map(&:value) + if options[:import] + progress.title = "Importing #{index} " + importer.import! + end + + if options[:clean] + progress.title = "Cleaning #{index} " + importer.clean_up! end + ensure + importer.optimize_for_search! end - progress.title = '' - progress.stop + progress.title = 'Done! ' + progress.finish - say("Indexed #{added.value} records, de-indexed #{removed.value}", :green, true) + say("Indexed #{added} records, de-indexed #{removed}", :green, true) end end end -- cgit From fbcbf7898f000d9d1a21d52e8a8d3ed4602aa7db Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 26 May 2022 23:26:15 +0200 Subject: Bump version to 3.5.3 (#18530) --- CHANGELOG.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ lib/mastodon/version.rb | 2 +- 2 files changed, 48 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/CHANGELOG.md b/CHANGELOG.md index 841c01bfb..ed4cdd881 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,53 @@ Changelog All notable changes to this project will be documented in this file. +## [3.5.3] - 2022-05-26 +### Added + +- **Add language dropdown to compose form in web UI** ([Gargron](https://github.com/mastodon/mastodon/pull/18420), [ykzts](https://github.com/mastodon/mastodon/pull/18460)) +- **Add warning for limited accounts in web UI** ([Gargron](https://github.com/mastodon/mastodon/pull/18344)) +- Add `limited` attribute to accounts in REST API ([Gargron](https://github.com/mastodon/mastodon/pull/18344)) + +### Changed + +- **Change RSS feeds** ([Gargron](https://github.com/mastodon/mastodon/pull/18356), [tribela](https://github.com/mastodon/mastodon/pull/18406)) + - Titles are now date and time of post + - Bodies now render all content faithfully, including polls and emojis + - All media attachments are included with Media RSS +- Change "dangerous" to "sensitive" in privacy policy and web UI ([Gargron](https://github.com/mastodon/mastodon/pull/18515)) +- Change unconfirmed accounts to not be visible in REST API ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17530)) +- Change `tootctl search deploy` to improve performance ([Gargron](https://github.com/mastodon/mastodon/pull/18463), [Gargron](https://github.com/mastodon/mastodon/pull/18514)) +- Change search indexing to use batches to minimize resource usage ([Gargron](https://github.com/mastodon/mastodon/pull/18451)) + +### Fixed + +- Fix follower and other counters being able to go negative ([Gargron](https://github.com/mastodon/mastodon/pull/18517)) +- Fix unnecessary query on when creating a status ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17901)) +- Fix warning an account outside of a report closing all reports for that account ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/18387)) +- Fix error when resolving a link that redirects to a local post ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/18314)) +- Fix preferred posting language returning unusable value in REST API ([Gargron](https://github.com/mastodon/mastodon/pull/18428)) +- Fix race condition error when external status is reblogged ([ykzts](https://github.com/mastodon/mastodon/pull/18424)) +- Fix missing string for appeal validation error ([Gargron](https://github.com/mastodon/mastodon/pull/18410)) +- Fix block/mute lists showing a follow button in web UI ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/18364)) +- Fix Redis configuration not being changed by `mastodon:setup` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/18383)) +- Fix streaming notifications not using quick filter logic in web UI ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/18316)) +- Fix ambiguous wording on appeal actions in admin UI ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/18328)) +- Fix floating action button obscuring last element in web UI ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/18332)) +- Fix account warnings not being recorded in audit log ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/18338)) +- Fix leftover icons for direct visibility statuses ([Steffo99](https://github.com/mastodon/mastodon/pull/18305)) +- Fix link verification requiring case sensitivity on links ([sgolemon](https://github.com/mastodon/mastodon/pull/18320)) +- Fix embeds not setting their height correctly ([rinsuki](https://github.com/mastodon/mastodon/pull/18301)) + +### Security + +- Fix concurrent unfollowing decrementing follower count more than once ([Gargron](https://github.com/mastodon/mastodon/pull/18527)) +- Fix being able to appeal a strike unlimited times ([Gargron](https://github.com/mastodon/mastodon/pull/18529)) +- Fix being able to report otherwise inaccessible statuses ([Gargron](https://github.com/mastodon/mastodon/pull/18528)) +- Fix empty votes arbitrarily increasing voters count in polls ([Gargron](https://github.com/mastodon/mastodon/pull/18526)) +- Fix moderator identity leak when approving appeal of sensitive marked statuses ([Gargron](https://github.com/mastodon/mastodon/pull/18525)) +- Fix suspended users being able to access APIs that don't require a user ([Gargron](https://github.com/mastodon/mastodon/pull/18524)) +- Fix confirmation redirect to app without `Location` header ([Gargron](https://github.com/mastodon/mastodon/pull/18523)) + ## [3.5.2] - 2022-05-04 ### Added diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index 73024e612..8ed955aa0 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -13,7 +13,7 @@ module Mastodon end def patch - 2 + 3 end def flags -- cgit